mirror of
https://github.com/shy3130/tick-stock-panel.git
synced 2026-11-02 02:16:13 +00:00
feat(backtest): 增加未来数据检查与数据版本追溯
Signed-off-by: shy3130 <shy3130@qq.com>
This commit is contained in:
@@ -468,6 +468,20 @@ def _stamp_daily_data(result, revisions=None) -> dict:
|
||||
return {**payload, "daily_data_revisions": revisions if revisions is not None else _daily_data_revisions()}
|
||||
|
||||
|
||||
@router.get("/strategy/runs/{run_id}/evidence")
|
||||
def strategy_run_evidence(run_id: str):
|
||||
from app.backtest.evidence import load_evidence
|
||||
|
||||
try:
|
||||
return load_evidence(settings.data_dir, run_id)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(404, detail="该回测没有版本记录,旧结果可重新回测生成") from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, detail=str(exc)) from exc
|
||||
except OSError as exc:
|
||||
raise HTTPException(503, detail="暂时无法读取回测版本记录") from exc
|
||||
|
||||
|
||||
class _BacktestJob:
|
||||
"""单个回测任务的状态, 存模块级供重连使用。"""
|
||||
__slots__ = ("key", "cancel_event", "progress", "result", "error", "done", "finish_ts", "daily_data_revisions")
|
||||
|
||||
@@ -387,6 +387,7 @@ class BacktestEngine:
|
||||
def __init__(self, repo: KlineRepository) -> None:
|
||||
self.repo = repo
|
||||
self._cache = PanelCache()
|
||||
self._evidence = None
|
||||
|
||||
# ── 数据加载 ──────────────────────────────────────
|
||||
|
||||
@@ -484,14 +485,10 @@ class BacktestEngine:
|
||||
end: date,
|
||||
feature_plan,
|
||||
asset_type: str = "stock",
|
||||
*,
|
||||
evidence=None,
|
||||
) -> pl.DataFrame:
|
||||
"""按解析后的依赖加载窄基础列并计算回测所需特征。"""
|
||||
from app.indicators.pipeline import (
|
||||
compute_indicators,
|
||||
compute_limit_signals,
|
||||
compute_signals,
|
||||
)
|
||||
|
||||
df = self.load_panel(
|
||||
symbols,
|
||||
start,
|
||||
@@ -503,7 +500,6 @@ class BacktestEngine:
|
||||
return df
|
||||
|
||||
from app.backtest.fundamentals import (
|
||||
attach_fundamental_factors,
|
||||
load_fundamental_snapshot,
|
||||
)
|
||||
|
||||
@@ -511,25 +507,59 @@ class BacktestEngine:
|
||||
getattr(feature_plan, "fundamental_columns", frozenset())
|
||||
or frozenset()
|
||||
)
|
||||
if fundamental_names:
|
||||
# 财务因子列不落 enriched 存储, 在加载口按公告日门控并入。
|
||||
df = attach_fundamental_factors(
|
||||
df,
|
||||
load_fundamental_snapshot(
|
||||
self.repo.store.data_dir if self.repo is not None else None
|
||||
),
|
||||
fundamental_names,
|
||||
)
|
||||
|
||||
financials = load_fundamental_snapshot(
|
||||
self.repo.store.data_dir if self.repo is not None else None,
|
||||
) if fundamental_names else None
|
||||
instruments = (
|
||||
self.repo.get_instruments_asset(asset_type)
|
||||
if self.repo is not None
|
||||
else pl.DataFrame()
|
||||
)
|
||||
historical_shares = (
|
||||
self.repo.get_historical_shares()
|
||||
if asset_type == "stock" and self.repo is not None and not instruments.is_empty()
|
||||
else None
|
||||
)
|
||||
if evidence is not None:
|
||||
from app.factors.ext_factors import attach_ext_columns
|
||||
|
||||
# 扩展时序值也要固定在本次原始快照内;检查期间不重新读取磁盘。
|
||||
df = attach_ext_columns(df, include_snapshot=False, data_dir=self.repo.store.data_dir)
|
||||
|
||||
def prepare(raw):
|
||||
return self._prepare_panel_features(
|
||||
raw, feature_plan, instruments, historical_shares, financials,
|
||||
attach_extensions=evidence is None,
|
||||
)
|
||||
|
||||
if evidence is not None:
|
||||
evidence.capture_panel(df, prepare, {
|
||||
"instruments": instruments, "historical_shares": historical_shares,
|
||||
"financial_announcements": financials,
|
||||
})
|
||||
return prepare(df)
|
||||
|
||||
@staticmethod
|
||||
def _prepare_panel_features(
|
||||
df, feature_plan, instruments, historical_shares, financials, *, attach_extensions=True,
|
||||
):
|
||||
from app.backtest.fundamentals import attach_fundamental_factors
|
||||
from app.indicators.pipeline import (
|
||||
compute_indicators,
|
||||
compute_limit_signals,
|
||||
compute_signals,
|
||||
)
|
||||
|
||||
fundamental_names = sorted(getattr(feature_plan, "fundamental_columns", ()) or ())
|
||||
if fundamental_names:
|
||||
df = attach_fundamental_factors(df, financials, fundamental_names)
|
||||
matrix_native = feature_plan.execution_backend == "matrix_native"
|
||||
if not matrix_native:
|
||||
df = compute_indicators(df, needed=set(feature_plan.indicator_columns))
|
||||
df = compute_signals(df, needed=set(feature_plan.signal_columns))
|
||||
df = compute_signals(
|
||||
df, needed=set(feature_plan.signal_columns),
|
||||
**({"attach_extensions": False} if not attach_extensions else {}),
|
||||
)
|
||||
if not instruments.is_empty():
|
||||
df = compute_limit_signals(
|
||||
df,
|
||||
@@ -537,11 +567,7 @@ class BacktestEngine:
|
||||
needed={"signal_limit_up", "signal_limit_down"}
|
||||
if matrix_native
|
||||
else set(feature_plan.signal_columns),
|
||||
historical_shares=(
|
||||
self.repo.get_historical_shares()
|
||||
if asset_type == "stock" and self.repo is not None
|
||||
else None
|
||||
),
|
||||
historical_shares=historical_shares,
|
||||
)
|
||||
join_cols = ["symbol"] if "symbol" in instruments.columns else []
|
||||
join_cols.extend(
|
||||
@@ -595,6 +621,8 @@ class BacktestEngine:
|
||||
|
||||
parquet_root = self.repo.store.data_dir / enriched_dirname(asset_type)
|
||||
instruments = self.repo.get_instruments_asset(asset_type)
|
||||
if self._evidence is not None:
|
||||
self._evidence.record_frame("instruments", instruments)
|
||||
field_columns = (
|
||||
set(feature_plan.base_columns)
|
||||
| set(feature_plan.instrument_columns)
|
||||
@@ -673,6 +701,7 @@ class BacktestEngine:
|
||||
market,
|
||||
self.repo.store.data_dir if self.repo is not None else None,
|
||||
fundamental_names,
|
||||
evidence=self._evidence,
|
||||
)
|
||||
if ext_wanted:
|
||||
from app.factors.ext_factors import attach_matrix_ext_fields
|
||||
@@ -988,7 +1017,9 @@ class BacktestEngine:
|
||||
dates = {matrix.timestamp_labels[int(t)][:10] for t in trigger_times}
|
||||
symbols = {matrix.symbols[int(a)] for a in trigger_assets}
|
||||
if dates and symbols:
|
||||
loaded = self._load_minute_for_fills(self.repo, list(symbols), dates, config.asset_type)
|
||||
loaded = self._load_minute_for_fills(
|
||||
self.repo, list(symbols), dates, config.asset_type, evidence=self._evidence,
|
||||
)
|
||||
minute_cache = {key: value for key, value in loaded.items() if value is not None and len(value) > 0}
|
||||
|
||||
def _count(key: str) -> None:
|
||||
@@ -1398,6 +1429,7 @@ class BacktestEngine:
|
||||
if _trigger_dates and _trigger_symbols:
|
||||
_loaded = self._load_minute_for_fills(
|
||||
self.repo, list(_trigger_symbols), _trigger_dates, "stock",
|
||||
evidence=self._evidence,
|
||||
)
|
||||
for _key, _marr in _loaded.items():
|
||||
if _marr is not None and len(_marr) > 0:
|
||||
@@ -1848,6 +1880,7 @@ class BacktestEngine:
|
||||
symbols: list[str],
|
||||
dates_needed: set,
|
||||
asset_type: str,
|
||||
*, evidence=None,
|
||||
) -> dict:
|
||||
"""按触发日加载分钟K, 返回 {(symbol, date_str): float64 2D ndarray}。
|
||||
|
||||
@@ -1879,6 +1912,8 @@ class BacktestEngine:
|
||||
continue
|
||||
if df.is_empty():
|
||||
continue
|
||||
if evidence is not None:
|
||||
evidence.record_frame(f"minute_fill/{asset_type}/{batch[0]}/{batch[-1]}", df)
|
||||
# 按 (symbol, 日期) 分组, 每组转紧凑 float64 数组存入 cache
|
||||
df = df.sort(["symbol", "datetime"]).with_columns(
|
||||
pl.col("datetime").dt.strftime("%Y-%m-%d").alias("_d_str")
|
||||
@@ -1985,6 +2020,7 @@ class BacktestEngine:
|
||||
if trigger_dates and trigger_symbols:
|
||||
loaded = self._load_minute_for_fills(
|
||||
self.repo, list(trigger_symbols), trigger_dates, config.asset_type,
|
||||
evidence=self._evidence,
|
||||
)
|
||||
minute_cache = {key: value for key, value in loaded.items() if value is not None and len(value) > 0}
|
||||
|
||||
@@ -2601,6 +2637,7 @@ class BacktestEngine:
|
||||
) else "stock"
|
||||
loaded = self._load_minute_for_fills(
|
||||
self.repo, list(trigger_symbols), trigger_dates, asset_type,
|
||||
evidence=self._evidence,
|
||||
)
|
||||
for key, marr in loaded.items():
|
||||
if marr is not None and len(marr) > 0:
|
||||
|
||||
@@ -0,0 +1,538 @@
|
||||
"""回测输入指纹及有限次时间因果检查。只在正式回测 worker 中启用。
|
||||
|
||||
检查固定本次输入、参数和预热起点,比较截断/扰动未来后的历史输出;
|
||||
它不能证明上游供应商数据本身满足 point-in-time,也不替代样本外验证。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from app.backtest.matrix import MarketDataMatrix, SignalMatrix, slice_market_data_matrix
|
||||
|
||||
_SAMPLES = 3
|
||||
_EXAMPLES = 5
|
||||
_RUN_ID = re.compile(r"[0-9a-f]{10,32}\Z")
|
||||
_MAX_MANIFEST_BYTES = 4 * 1024 * 1024
|
||||
|
||||
|
||||
def json_digest(value: Any) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, ensure_ascii=False, default=str, allow_nan=False).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _array_digest(array: np.ndarray) -> str:
|
||||
digest = hashlib.sha256()
|
||||
digest.update(str(array.dtype).encode())
|
||||
digest.update(str(array.shape).encode())
|
||||
# memmap 和非连续视图分块处理,避免为整个回测矩阵再复制一份。
|
||||
for start in range(0, len(array), 256):
|
||||
block = np.ascontiguousarray(array[start:start + 256])
|
||||
digest.update(memoryview(block).cast("B"))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def fingerprint_matrix(market: MarketDataMatrix) -> dict:
|
||||
fields = {
|
||||
name: _array_digest(getattr(market, name))
|
||||
for name in (
|
||||
"timestamps", "session_ids", "open", "high", "low", "close", "volume",
|
||||
"tradable", "limit_up_locked", "limit_down_locked",
|
||||
)
|
||||
}
|
||||
fields.update({f"field:{key}": _array_digest(value) for key, value in sorted(market.fields.items())})
|
||||
identity = {"timestamps": market.timestamp_labels, "symbols": market.symbols, "names": market.names}
|
||||
return {
|
||||
"sha256": json_digest({"identity": identity, "fields": fields}),
|
||||
"fields": fields,
|
||||
"rows": market.shape[0] * market.shape[1],
|
||||
"symbols": len(market.symbols),
|
||||
"start": market.timestamp_labels[0] if market.timestamp_labels else None,
|
||||
"end": market.timestamp_labels[-1] if market.timestamp_labels else None,
|
||||
}
|
||||
|
||||
|
||||
def fingerprint_frame(frame: pl.DataFrame) -> dict:
|
||||
keys = [name for name in ("symbol", "date", "datetime") if name in frame.columns]
|
||||
# 公告/股本表同一标的有多行;补齐排序键,避免物理行序变化被误判为内容变化。
|
||||
frame = frame.sort(keys + sorted(set(frame.columns) - set(keys)))
|
||||
fields = {}
|
||||
for name in sorted(frame.columns):
|
||||
digest = hashlib.sha256(str(frame.schema[name]).encode())
|
||||
for block in frame.select(name).iter_slices(65536):
|
||||
buffer = io.BytesIO()
|
||||
block.rechunk().write_ipc(buffer)
|
||||
digest.update(buffer.getbuffer())
|
||||
fields[name] = digest.hexdigest()
|
||||
time_col = next((col for col in ("datetime", "date") if col in frame.columns), None)
|
||||
return {
|
||||
"sha256": json_digest({"rows": frame.height, "fields": fields}),
|
||||
"fields": fields,
|
||||
"rows": frame.height,
|
||||
"symbols": frame["symbol"].n_unique() if "symbol" in frame.columns else None,
|
||||
"start": str(frame[time_col].min()) if time_col and frame.height else None,
|
||||
"end": str(frame[time_col].max()) if time_col and frame.height else None,
|
||||
}
|
||||
|
||||
|
||||
def _cutoffs(values: list) -> list:
|
||||
if not values:
|
||||
return []
|
||||
indices = np.linspace(0, len(values) - 1, min(_SAMPLES, len(values)), dtype=int)
|
||||
return [values[int(i)] for i in indices]
|
||||
|
||||
|
||||
def _scalar(value):
|
||||
if isinstance(value, np.generic):
|
||||
value = value.item()
|
||||
if isinstance(value, float) and not np.isfinite(value):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _different(left: np.ndarray, right: np.ndarray) -> np.ndarray:
|
||||
if left.shape != right.shape:
|
||||
raise ValueError("重算输出形状与基线不一致")
|
||||
if left.dtype.kind in "iufb" and right.dtype.kind in "iufb":
|
||||
return ~np.isclose(left, right, rtol=1e-6, atol=1e-7, equal_nan=True)
|
||||
return ~((left == right) | ((left == None) & (right == None))) # noqa: E711
|
||||
|
||||
|
||||
def _report(checks: list[dict]) -> dict:
|
||||
differences = sum(c.get("differences", 0) for c in checks)
|
||||
status = (
|
||||
"error" if any(c["kind"] == "repeat" and c["status"] == "error" for c in checks)
|
||||
else "failed" if differences else "error" if any(c["status"] == "error" for c in checks)
|
||||
else "passed" if any(c.get("compared_values", 0) > 0 for c in checks) else "insufficient"
|
||||
)
|
||||
return {"status": status, "checks": checks, "differences": differences}
|
||||
|
||||
|
||||
def _check(kind: str, cutoff, compare: Callable[[], dict]) -> dict:
|
||||
try:
|
||||
return {"kind": kind, "cutoff": str(cutoff), "status": "completed", **compare()}
|
||||
except Exception as exc: # noqa: BLE001 — 用户策略异常必须转为可见的检查失败。
|
||||
return {
|
||||
"kind": kind, "cutoff": str(cutoff), "status": "error",
|
||||
"error": f"重算未完成({type(exc).__name__})",
|
||||
"differences": 0, "compared_values": 0, "examples": [],
|
||||
}
|
||||
|
||||
|
||||
def _perturb_matrix(market: MarketDataMatrix, cutoff: int) -> MarketDataMatrix:
|
||||
def change(array):
|
||||
if array.ndim != 2 or array.dtype.kind not in "fiu":
|
||||
return array
|
||||
values = array.copy()
|
||||
scale = 1.5 + np.arange(len(values) - cutoff - 1)[:, None] * 0.03
|
||||
future = values[cutoff + 1:]
|
||||
values[cutoff + 1:] = (future * scale + (future == 0)).astype(array.dtype)
|
||||
values.flags.writeable = False
|
||||
return values
|
||||
|
||||
return replace(
|
||||
market,
|
||||
**{name: change(getattr(market, name)) for name in ("open", "high", "low", "close", "volume")},
|
||||
fields={name: change(value) for name, value in market.fields.items()},
|
||||
_valid_bars=None,
|
||||
)
|
||||
|
||||
|
||||
def audit_matrix(
|
||||
market: MarketDataMatrix,
|
||||
baseline: SignalMatrix,
|
||||
evaluate: Callable[[MarketDataMatrix], SignalMatrix],
|
||||
*, start: str, end: str, cancel_event=None,
|
||||
) -> dict:
|
||||
candidates = [
|
||||
i for i, label in enumerate(market.timestamp_labels[:-1])
|
||||
if start <= label[:10] <= end
|
||||
]
|
||||
checks = []
|
||||
|
||||
def compare(kind, cutoff):
|
||||
label = market.timestamp_labels[cutoff]
|
||||
if kind == "repeat":
|
||||
data = market
|
||||
elif kind == "truncate":
|
||||
data = slice_market_data_matrix(market, 0, cutoff + 1)
|
||||
else:
|
||||
data = _perturb_matrix(market, cutoff)
|
||||
actual = evaluate(data)
|
||||
if actual.shape != data.shape:
|
||||
raise ValueError("重算信号形状不匹配")
|
||||
rows = np.array([start <= t[:10] <= label[:10] for t in market.timestamp_labels[:cutoff + 1]])
|
||||
count, compared, examples = 0, 0, []
|
||||
for name in ("entry", "exit", "score", "entry_signal_code", "exit_signal_code"):
|
||||
expected = getattr(baseline, name)[:cutoff + 1]
|
||||
observed = getattr(actual, name)[:cutoff + 1]
|
||||
diff = _different(expected, observed) & rows[:, None]
|
||||
count += int(diff.sum())
|
||||
compared += int(rows.sum()) * market.shape[1]
|
||||
for t, a in np.argwhere(diff)[:max(0, _EXAMPLES - len(examples))]:
|
||||
examples.append({
|
||||
"date": market.timestamp_labels[t], "symbol": market.symbols[a],
|
||||
"field": name, "baseline": _scalar(expected[t, a]),
|
||||
"recomputed": _scalar(observed[t, a]), "cutoff": label,
|
||||
})
|
||||
return {"differences": count, "compared_values": compared, "examples": examples}
|
||||
|
||||
for cutoff in _cutoffs(candidates):
|
||||
label = market.timestamp_labels[cutoff]
|
||||
for kind in ("truncate", "perturb"):
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
return {**_report(checks), "status": "cancelled"}
|
||||
|
||||
checks.append(_check(kind, label, lambda kind=kind, cutoff=cutoff: compare(kind, cutoff)))
|
||||
if any(check["differences"] for check in checks):
|
||||
# 先排除非确定性/带状态策略,避免把随机变化直接解释为未来数据依赖。
|
||||
kind = "repeat"
|
||||
cutoff = max(i for i, label in enumerate(market.timestamp_labels) if start <= label[:10] <= end)
|
||||
label = market.timestamp_labels[cutoff]
|
||||
control = _check(kind, label, lambda: compare("repeat", cutoff))
|
||||
if control["differences"]:
|
||||
control.update(status="error", error="相同输入重算结果不一致,无法归因于未来数据")
|
||||
checks.append(control)
|
||||
return _report(checks)
|
||||
|
||||
|
||||
def _perturb_frame(frame: pl.DataFrame, time_col: str, cutoff) -> pl.DataFrame:
|
||||
future = pl.col(time_col) > cutoff
|
||||
return frame.with_columns([
|
||||
pl.when(future).then(
|
||||
pl.col(name) * (1.5 + pl.col(time_col).rank("dense") * 0.03) + (pl.col(name) == 0)
|
||||
)
|
||||
.otherwise(pl.col(name)).cast(dtype).alias(name)
|
||||
for name, dtype in frame.schema.items()
|
||||
if dtype.is_numeric() and name != time_col
|
||||
])
|
||||
|
||||
|
||||
def _compare_frames(baseline, actual, time_col, start, cutoff):
|
||||
keys = ["symbol", time_col]
|
||||
bounds = (pl.col(time_col) >= start) & (pl.col(time_col) <= cutoff)
|
||||
left, right = baseline.filter(bounds), actual.filter(bounds)
|
||||
if left.select(keys).is_duplicated().any() or right.select(keys).is_duplicated().any():
|
||||
raise ValueError("检查输出包含重复时间和标的")
|
||||
names = [name for name, dtype in left.schema.items() if name not in keys and (dtype.is_numeric() or dtype == pl.Boolean)]
|
||||
if any(name not in right.columns for name in names):
|
||||
raise ValueError("重算输出缺少字段")
|
||||
# full join 能发现消失或新增的历史信号,而不只检查两边都存在的行。
|
||||
joined = (
|
||||
left.select(*keys, *names).with_columns(pl.lit(True).alias("_present"))
|
||||
.join(
|
||||
right.select(*keys, *names).with_columns(pl.lit(True).alias("_present")),
|
||||
on=keys, how="full", coalesce=True, suffix="__audit",
|
||||
).sort(keys)
|
||||
)
|
||||
count, examples = 0, []
|
||||
for name in ["_present", *names]:
|
||||
expected, observed = joined[name].to_numpy(), joined[f"{name}__audit"].to_numpy()
|
||||
diff = _different(expected, observed)
|
||||
count += int(diff.sum())
|
||||
for row in np.flatnonzero(diff)[:max(0, _EXAMPLES - len(examples))]:
|
||||
examples.append({
|
||||
"date": str(joined[time_col][int(row)]), "symbol": joined["symbol"][int(row)],
|
||||
"field": "signal_presence" if name == "_present" else name,
|
||||
"baseline": _scalar(expected[row]), "recomputed": _scalar(observed[row]),
|
||||
"cutoff": str(cutoff),
|
||||
})
|
||||
return {"differences": count, "compared_values": joined.height * (len(names) + 1), "examples": examples}
|
||||
|
||||
|
||||
def audit_frame(
|
||||
raw: pl.DataFrame, baseline: pl.DataFrame, evaluate: Callable[[pl.DataFrame], pl.DataFrame],
|
||||
*, start, end, time_col: str = "date", cancel_event=None,
|
||||
) -> dict:
|
||||
times = raw[time_col].unique().sort().to_list()
|
||||
if times and isinstance(start, str) and not isinstance(times[0], str):
|
||||
start = type(times[0]).fromisoformat(start)
|
||||
end = type(times[0]).fromisoformat(end)
|
||||
candidates = [t for t in times[:-1] if start <= t <= end]
|
||||
checks = []
|
||||
|
||||
def compare(kind, cutoff):
|
||||
if kind == "repeat":
|
||||
variant = raw
|
||||
elif kind == "truncate":
|
||||
variant = raw.filter(pl.col(time_col) <= cutoff)
|
||||
else:
|
||||
variant = _perturb_frame(raw, time_col, cutoff)
|
||||
return _compare_frames(baseline, evaluate(variant), time_col, start, cutoff)
|
||||
|
||||
for cutoff in _cutoffs(candidates):
|
||||
for kind in ("truncate", "perturb"):
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
return {**_report(checks), "status": "cancelled"}
|
||||
|
||||
checks.append(_check(kind, cutoff, lambda kind=kind, cutoff=cutoff: compare(kind, cutoff)))
|
||||
if any(check["differences"] for check in checks):
|
||||
kind, cutoff = "repeat", end
|
||||
control = _check(kind, cutoff, lambda: compare("repeat", cutoff))
|
||||
if control["differences"]:
|
||||
control.update(status="error", error="相同输入重算结果不一致,无法归因于未来数据")
|
||||
checks.append(control)
|
||||
return _report(checks)
|
||||
|
||||
|
||||
def _manifest_path(data_dir: Path, run_id: str) -> Path:
|
||||
if not _RUN_ID.fullmatch(run_id):
|
||||
raise ValueError("回测编号无效")
|
||||
root = Path(data_dir).resolve()
|
||||
path = root / "backtest_runs" / f"{run_id}.json"
|
||||
if path.resolve() != path:
|
||||
raise ValueError("回测记录目录无效")
|
||||
return path
|
||||
|
||||
|
||||
def save_evidence(data_dir: Path, evidence: dict) -> None:
|
||||
from app.services.fs_utils import atomic_write_text
|
||||
|
||||
path = _manifest_path(data_dir, evidence["run_id"])
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = json.dumps(evidence, ensure_ascii=False, sort_keys=True, allow_nan=False, default=str)
|
||||
if len(content.encode()) > _MAX_MANIFEST_BYTES:
|
||||
raise ValueError("回测版本记录超过大小限制")
|
||||
# 先占用同编号写锁,防止两个 worker 的检查与原子替换之间发生覆盖。
|
||||
lock_path = path.with_suffix(".lock")
|
||||
descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
os.close(descriptor)
|
||||
try:
|
||||
if path.exists():
|
||||
raise FileExistsError("回测版本记录已存在")
|
||||
atomic_write_text(path, content)
|
||||
finally:
|
||||
lock_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def load_evidence(data_dir: Path, run_id: str) -> dict:
|
||||
path = _manifest_path(data_dir, run_id)
|
||||
if path.stat().st_size > _MAX_MANIFEST_BYTES:
|
||||
raise ValueError("回测版本记录超过大小限制")
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict) or value.get("schema_version") != 1 or value.get("run_id") != run_id:
|
||||
raise ValueError("回测版本记录无效")
|
||||
return value
|
||||
|
||||
|
||||
def audit_trade_timing(trades: list[dict], config: dict, backend: str) -> dict:
|
||||
"""检查已记录的信号日与成交日,不把期末估值记录当成次日信号成交。"""
|
||||
differences, compared, examples = 0, 0, []
|
||||
for trade in trades:
|
||||
for side in ("entry", "exit"):
|
||||
signal = trade.get(f"{side}_signal_date")
|
||||
fill = trade.get(f"{side}_date")
|
||||
if not signal or not fill:
|
||||
continue
|
||||
signal, fill = str(signal)[:10], str(fill)[:10]
|
||||
next_open = config.get(f"{side}_fill") == "open_t+1"
|
||||
if side == "entry" and backend == "minute_filter":
|
||||
next_open = False # 分钟策略按触发分钟入场,忽略日线 entry_fill。
|
||||
if side == "exit" and trade.get("exit_reason") != "signal":
|
||||
next_open = False
|
||||
compared += 1
|
||||
if signal > fill or (next_open and signal == fill):
|
||||
differences += 1
|
||||
if len(examples) < _EXAMPLES:
|
||||
examples.append({
|
||||
"symbol": trade["symbol"], "side": side,
|
||||
"signal_date": signal, "fill_date": fill,
|
||||
})
|
||||
return {
|
||||
"status": "failed" if differences else "passed" if compared else "insufficient",
|
||||
"compared_values": compared, "differences": differences, "examples": examples,
|
||||
}
|
||||
|
||||
|
||||
def definition_versions(data_dir: Path, strategy) -> dict:
|
||||
"""只读取策略及定义目录;绝不扫描偏好、密钥或环境变量。"""
|
||||
files = {}
|
||||
if strategy.file_path is not None:
|
||||
files["strategy"] = _file_digest(Path(strategy.file_path))
|
||||
for directory in ("custom_factors", "custom_signals", "position_policies"):
|
||||
root = Path(data_dir) / "user_data" / directory
|
||||
for path in sorted(root.glob("*")):
|
||||
if path.is_file() and path.suffix in {".json", ".py"}:
|
||||
files[f"{directory}/{path.name}"] = _file_digest(path)
|
||||
return {"sha256": json_digest(files), "files": files}
|
||||
|
||||
|
||||
def _file_digest(path: Path) -> str | None:
|
||||
try:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def runtime_version() -> dict:
|
||||
from app import __version__
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
paths = [
|
||||
"backtest/evidence.py", "backtest/engine.py", "backtest/strategy.py",
|
||||
"backtest/matrix.py", "backtest/fundamentals.py", "backtest/minute_replay.py",
|
||||
"strategy/engine.py", "strategy/scoring.py", "indicators/pipeline.py",
|
||||
]
|
||||
files = {
|
||||
name: _file_digest(root / name) # 冻结发行版可能只携带字节码。
|
||||
for name in paths
|
||||
}
|
||||
return {
|
||||
"application": __version__, "python": sys.version.split()[0],
|
||||
"polars": pl.__version__, "numpy": np.__version__,
|
||||
"code_sha256": json_digest(files), "code_files": files,
|
||||
}
|
||||
|
||||
|
||||
class EvidenceRecorder:
|
||||
"""一次回测专属的观察器,不进入共享缓存,也不保存全量行情副本。"""
|
||||
|
||||
def __init__(self, data_dir: Path, config, strategy, strategy_engine, cancel_event=None):
|
||||
self.data_dir = data_dir
|
||||
self.config = config
|
||||
self.strategy = strategy
|
||||
self.cancel_event = cancel_event
|
||||
self.datasets: dict[str, dict] = {}
|
||||
self.outputs: dict[str, dict] = {}
|
||||
self.reports: list[dict] = []
|
||||
self.raw_panel: pl.DataFrame | None = None
|
||||
self.prepare_panel: Callable | None = None
|
||||
self.definitions = definition_versions(data_dir, strategy)
|
||||
self.runtime = runtime_version()
|
||||
self.children = []
|
||||
if strategy.composite is not None:
|
||||
for child in strategy.composite.children:
|
||||
child_def = strategy_engine.get(child.strategy_id)
|
||||
self.children.append({
|
||||
"id": child.strategy_id, "weight": child.weight,
|
||||
"definitions": definition_versions(data_dir, child_def),
|
||||
})
|
||||
|
||||
def record_frame(self, name: str, frame: pl.DataFrame) -> None:
|
||||
self.datasets[name] = fingerprint_frame(frame)
|
||||
|
||||
def capture_panel(self, raw, prepare, auxiliaries) -> None:
|
||||
self.raw_panel, self.prepare_panel = raw, prepare
|
||||
self.record_frame("daily_base", raw)
|
||||
for name, frame in auxiliaries.items():
|
||||
if frame is not None:
|
||||
self.record_frame(name, frame)
|
||||
|
||||
def check_matrix(self, market, signals, evaluate) -> None:
|
||||
self.datasets["daily_matrix"] = fingerprint_matrix(market)
|
||||
self.reports.append({
|
||||
"scope": "daily_signals",
|
||||
**audit_matrix(
|
||||
market, signals, evaluate, start=str(self.config.start), end=str(self.config.end),
|
||||
cancel_event=self.cancel_event,
|
||||
),
|
||||
})
|
||||
|
||||
def check_panel(self, baseline, evaluate) -> None:
|
||||
self.outputs["daily_features_and_signals"] = fingerprint_frame(baseline)
|
||||
if self.raw_panel is None or self.prepare_panel is None:
|
||||
self.reports.append({"scope": "daily_signals", "status": "error", "checks": [], "differences": 0})
|
||||
return
|
||||
self.reports.append({
|
||||
"scope": "daily_features_and_signals",
|
||||
**audit_frame(
|
||||
self.raw_panel, baseline, lambda raw: evaluate(self.prepare_panel(raw)),
|
||||
start=self.config.start, end=self.config.end, cancel_event=self.cancel_event,
|
||||
),
|
||||
})
|
||||
# 检查完成后释放对大面板及准备上下文的引用。
|
||||
self.raw_panel, self.prepare_panel = None, None
|
||||
|
||||
def check_minute(self, history, result, evaluate) -> None:
|
||||
dtype = history.schema["datetime"]
|
||||
|
||||
def decisions(value):
|
||||
rows = []
|
||||
for row in value.rows:
|
||||
timestamp = row.get("last_datetime")
|
||||
if timestamp is None:
|
||||
raise ValueError("分钟信号缺少触发时间,无法检查")
|
||||
if isinstance(timestamp, str):
|
||||
timestamp = datetime.fromisoformat(timestamp)
|
||||
rows.append({
|
||||
"symbol": str(row["symbol"]), "datetime": timestamp,
|
||||
"close": row.get("close"), "score": value.scores.get(str(row["symbol"]), 0.0),
|
||||
})
|
||||
return pl.DataFrame(rows, schema={
|
||||
"symbol": pl.Utf8, "datetime": dtype, "close": pl.Float64, "score": pl.Float64,
|
||||
})
|
||||
|
||||
try:
|
||||
report = audit_frame(
|
||||
history, decisions(result), lambda variant: decisions(evaluate(variant)),
|
||||
start=history["datetime"].min(), end=history["datetime"].max(),
|
||||
time_col="datetime", cancel_event=self.cancel_event,
|
||||
)
|
||||
except (ValueError, TypeError, pl.exceptions.PolarsError):
|
||||
report = {"status": "error", "checks": [], "differences": 0}
|
||||
self.reports.append({"scope": "minute_signals", **report})
|
||||
|
||||
@staticmethod
|
||||
def minute_sample_days(days):
|
||||
return set(_cutoffs(days))
|
||||
|
||||
def finish(self, result, effective_config: dict) -> dict:
|
||||
checks = [check for report in self.reports for check in report["checks"]]
|
||||
summary = _report(checks)
|
||||
if any(r["status"] == "cancelled" for r in self.reports):
|
||||
summary["status"] = "cancelled"
|
||||
elif any(r["status"] == "error" for r in self.reports) and summary["status"] != "failed":
|
||||
summary["status"] = "error"
|
||||
elif any(r["status"] == "insufficient" for r in self.reports) and summary["status"] == "passed":
|
||||
summary["status"] = "partial"
|
||||
definitions_after = definition_versions(self.data_dir, self.strategy)
|
||||
data_digest = json_digest({key: value["sha256"] for key, value in sorted(self.datasets.items())})
|
||||
config_digest = json_digest(effective_config)
|
||||
strategy = {
|
||||
"id": self.config.strategy_id, "backend": self.strategy.execution_backend,
|
||||
"meta": self.strategy.meta, "definitions": self.definitions, "children": self.children,
|
||||
"changed_during_run": definitions_after != self.definitions,
|
||||
}
|
||||
strategy["sha256"] = json_digest(strategy)
|
||||
runtime = self.runtime
|
||||
return {
|
||||
"schema_version": 1, "run_id": result.run_id,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"runtime": runtime,
|
||||
"config": effective_config, "config_sha256": config_digest,
|
||||
"strategy": strategy,
|
||||
"data": {
|
||||
"sha256": data_digest, "origin": "local_normalized_snapshot",
|
||||
"asset_type": self.config.asset_type, "daily_price_basis": "qfq",
|
||||
"datasets": self.datasets,
|
||||
},
|
||||
"reproducibility_sha256": json_digest({
|
||||
"data": data_digest, "strategy": strategy["sha256"], "config": config_digest, "runtime": runtime,
|
||||
}),
|
||||
"lookahead": {**summary, "reports": self.reports, "sampled": True},
|
||||
"execution_timing": audit_trade_timing(result.trades, effective_config, self.strategy.execution_backend),
|
||||
"outputs": self.outputs,
|
||||
"result_summary": {
|
||||
key: _scalar(result.stats.get(key)) for key in (
|
||||
"total_return", "annual_return", "max_drawdown", "sharpe", "win_rate", "n_trades",
|
||||
)
|
||||
},
|
||||
"limitations": [
|
||||
"按交易时间抽样检查本次加载的输入、特征和信号,不是全时点无未来数据证明。",
|
||||
"供应商历史修订、历史股票池及静态维表的时点正确性需另行核实。",
|
||||
"记录内容指纹和有效参数,不归档全量行情;原始数据被覆盖后不能仅凭指纹还原。",
|
||||
],
|
||||
}
|
||||
@@ -201,7 +201,7 @@ def build_fundamental_matrices(
|
||||
return result
|
||||
|
||||
|
||||
def attach_matrix_fundamental_fields(market: Any, data_dir: Path | None, names: Any) -> Any:
|
||||
def attach_matrix_fundamental_fields(market: Any, data_dir: Path | None, names: Any, *, evidence=None) -> Any:
|
||||
"""把财务因子作为 matrix fields 附加到 (frozen) MarketDataMatrix 副本。"""
|
||||
import dataclasses
|
||||
|
||||
@@ -209,6 +209,8 @@ def attach_matrix_fundamental_fields(market: Any, data_dir: Path | None, names:
|
||||
if not requested:
|
||||
return market
|
||||
snapshot = load_fundamental_snapshot(data_dir)
|
||||
if evidence is not None and snapshot is not None:
|
||||
evidence.record_frame("financial_announcements", snapshot)
|
||||
extra = build_fundamental_matrices(market, snapshot, requested)
|
||||
if not extra:
|
||||
return market
|
||||
|
||||
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import date, timedelta
|
||||
|
||||
import polars as pl
|
||||
@@ -135,6 +135,7 @@ class MinuteSignalReplayer:
|
||||
symbols: list[str] | None = None,
|
||||
progress_cb: Callable[[dict], None] | None = None,
|
||||
cancel_event=None,
|
||||
evidence=None,
|
||||
) -> MinuteReplayResult:
|
||||
t0 = time.perf_counter()
|
||||
result = MinuteReplayResult()
|
||||
@@ -158,6 +159,7 @@ class MinuteSignalReplayer:
|
||||
replay_days = [day for day in panel_dates if start <= day <= end]
|
||||
result.skipped_days = [day for day in replay_days if day not in minute_day_set]
|
||||
total = len(minute_days)
|
||||
audit_days = evidence.minute_sample_days(minute_days) if evidence is not None else set()
|
||||
|
||||
# 逐标的的 T-1 原始收盘/复权收盘查表 (涨停价与复权折算用)。
|
||||
prev_raw_close: dict[str, float] = {}
|
||||
@@ -179,6 +181,8 @@ class MinuteSignalReplayer:
|
||||
if history.is_empty():
|
||||
result.skipped_days.append(day)
|
||||
continue
|
||||
if evidence is not None:
|
||||
evidence.record_frame(f"minute_replay/{day}", history)
|
||||
|
||||
# 日线窗口: 截至 T-1 的完成态日K (index of last panel date < day)。
|
||||
prior = [d for d in panel_dates if d < day]
|
||||
@@ -223,6 +227,15 @@ class MinuteSignalReplayer:
|
||||
result.skipped_days.append(day)
|
||||
continue
|
||||
|
||||
if evidence is not None and day in audit_days:
|
||||
evidence.check_minute(
|
||||
history, run_result,
|
||||
lambda variant, context=context: self.strategy_engine.run(
|
||||
strategy.meta.get("id", ""), replace(context, history=variant),
|
||||
pool, params, overrides,
|
||||
),
|
||||
)
|
||||
|
||||
result.replayed_days += 1
|
||||
result.strategy_matches += len(run_result.rows)
|
||||
for row in run_result.rows:
|
||||
|
||||
@@ -680,6 +680,7 @@ class StrategyBacktestResult:
|
||||
factor_attribution: dict | None = None
|
||||
elapsed_ms: float = 0.0
|
||||
error: str | None = None
|
||||
evidence: dict | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -1259,6 +1260,40 @@ class StrategyBacktestService:
|
||||
sim_encoding=sim_encoding,
|
||||
)
|
||||
|
||||
def run_with_evidence(self, config, progress_cb=None, cancel_event=None) -> StrategyBacktestResult:
|
||||
"""正式交互回测启用检查和留档;优化器继续复用轻量 run。"""
|
||||
from app.backtest.evidence import EvidenceRecorder, save_evidence
|
||||
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
strategy = self.strategy_engine.get(config.strategy_id)
|
||||
except ValueError:
|
||||
return self.run(config, progress_cb, cancel_event)
|
||||
data_dir = self.engine.repo.store.data_dir
|
||||
recorder = EvidenceRecorder(data_dir, config, strategy, self.strategy_engine, cancel_event)
|
||||
# 每个正式回测 worker 拥有独立引擎;仅在本次运行内观察成交用分钟数据。
|
||||
previous = getattr(self.engine, "_evidence", None)
|
||||
self.engine._evidence = recorder
|
||||
try:
|
||||
result = self.run(config, progress_cb, cancel_event, evidence=recorder)
|
||||
finally:
|
||||
self.engine._evidence = previous
|
||||
if result.error == "cancelled":
|
||||
return result
|
||||
effective = {
|
||||
**result.config,
|
||||
"params": self._normalize_params(config.params or {}, strategy),
|
||||
}
|
||||
result.evidence = recorder.finish(result, effective)
|
||||
try:
|
||||
# 保存状态是结果的一部分;写入失败时不会宣称记录已经落盘。
|
||||
result.evidence["persistence"] = {"status": "saved"}
|
||||
save_evidence(data_dir, result.evidence)
|
||||
except (OSError, ValueError):
|
||||
result.evidence["persistence"] = {"status": "error", "error": "版本记录未能保存,请导出本次报告"}
|
||||
result.elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
return result
|
||||
|
||||
def run(
|
||||
self,
|
||||
config: StrategyBacktestConfig,
|
||||
@@ -1267,6 +1302,7 @@ class StrategyBacktestService:
|
||||
prepared: PreparedMatrixBacktest | None = None,
|
||||
result_policy: BacktestResultPolicy | None = None,
|
||||
prepared_panel: PreparedPanelBacktest | None = None,
|
||||
evidence=None,
|
||||
) -> StrategyBacktestResult:
|
||||
t0 = time.perf_counter()
|
||||
run_id = uuid.uuid4().hex[:10]
|
||||
@@ -1372,6 +1408,7 @@ class StrategyBacktestService:
|
||||
result_policy=result_policy,
|
||||
run_id=run_id,
|
||||
t0=t0,
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -1528,6 +1565,7 @@ class StrategyBacktestService:
|
||||
load_end,
|
||||
feature_plan,
|
||||
asset_type=config.asset_type,
|
||||
**({"evidence": evidence} if evidence is not None else {}),
|
||||
)
|
||||
except (ValueError, pl.exceptions.PolarsError) as e:
|
||||
return _err(f"回测特征准备失败: {e}")
|
||||
@@ -1594,6 +1632,7 @@ class StrategyBacktestService:
|
||||
getattr(getattr(self.engine.repo, "store", None), "data_dir", None),
|
||||
required_start=config.start,
|
||||
required_end=config.end,
|
||||
evidence=evidence,
|
||||
)
|
||||
except ValueError as e:
|
||||
return _err(str(e))
|
||||
@@ -1638,6 +1677,15 @@ class StrategyBacktestService:
|
||||
except ValueError as e:
|
||||
return _err(str(e))
|
||||
|
||||
if evidence is not None:
|
||||
evidence.check_matrix(
|
||||
market_data, signal_matrix,
|
||||
lambda data: self._generate_composite_signal_matrix(
|
||||
composite_children_resolved, data, merge_mode, min_confirm,
|
||||
composite_max_hold, {},
|
||||
),
|
||||
)
|
||||
|
||||
sim_market_data = slice_market_data_matrix(market_data, start_id, stop_id)
|
||||
sim_signal_matrix = slice_signal_matrix(signal_matrix, start_id, stop_id)
|
||||
sim_signal_matrix = apply_time_masks(
|
||||
@@ -1698,6 +1746,7 @@ class StrategyBacktestService:
|
||||
getattr(getattr(self.engine.repo, "store", None), "data_dir", None),
|
||||
required_start=config.start,
|
||||
required_end=config.end,
|
||||
evidence=evidence,
|
||||
)
|
||||
except ValueError as e:
|
||||
return _err(str(e))
|
||||
@@ -1756,6 +1805,14 @@ class StrategyBacktestService:
|
||||
except (TypeError, ValueError) as e:
|
||||
return _err(f"矩阵策略信号计算失败: {e}")
|
||||
|
||||
if evidence is not None:
|
||||
evidence.check_matrix(
|
||||
market_data, signal_matrix,
|
||||
lambda data: MatrixStrategyPipeline().run(
|
||||
s.matrix_strategy, data, params, pipeline_config,
|
||||
),
|
||||
)
|
||||
|
||||
sim_market_data = slice_market_data_matrix(market_data, start_id, stop_id)
|
||||
sim_signal_matrix = slice_signal_matrix(signal_matrix, start_id, stop_id)
|
||||
sim_signal_matrix = apply_time_masks(
|
||||
@@ -1810,31 +1867,25 @@ class StrategyBacktestService:
|
||||
else:
|
||||
if panel is None or formal_range is None:
|
||||
return _err("非矩阵策略不能使用共享基础矩阵")
|
||||
# basic_filter 只影响买入候选,不能删除持仓估值和卖出所需行情。
|
||||
basic_mask = pl.Series("_basic", [True] * len(panel), dtype=pl.Boolean)
|
||||
if basic_filter and basic_filter.get("enabled", True):
|
||||
expr = StrategyEngine._basic_filter_expr(panel, basic_filter)
|
||||
if expr is not None:
|
||||
try:
|
||||
basic_mask = panel.select(expr.alias("_basic"))["_basic"].fill_null(False).cast(pl.Boolean)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("basic_filter mask failed: %s", e)
|
||||
return _err(f"基础过滤计算失败: {e}")
|
||||
|
||||
candidate_filter_mask = self._build_candidate_filter_mask(panel, s, params)
|
||||
candidate_mask = basic_mask & candidate_filter_mask
|
||||
panel = self._apply_score(panel, s, overrides, universe_mask=candidate_mask, factor_snapshot=factor_snapshot)
|
||||
try:
|
||||
panel, candidate_mask, entry_mask, raw_exit_mask = self._evaluate_panel_decisions(
|
||||
panel, s, params, overrides, basic_filter, entry_signals, exit_signals,
|
||||
factor_snapshot=factor_snapshot,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return _err(str(exc))
|
||||
if evidence is not None:
|
||||
evidence.check_panel(
|
||||
self._decision_frame(panel, candidate_mask, entry_mask, raw_exit_mask),
|
||||
lambda data: self._decision_frame(*self._evaluate_panel_decisions(
|
||||
data, s, params, overrides, basic_filter, entry_signals, exit_signals,
|
||||
)),
|
||||
)
|
||||
formal_candidate_mask = candidate_mask & formal_range
|
||||
entry_mask = self._build_entry_mask_from_candidate(panel, candidate_mask, s, entry_signals)
|
||||
entry_mask = entry_mask & formal_range
|
||||
# 叠加条件: 在裁剪前的完整 panel (ext_*/因子列已在, polars 路径经
|
||||
# compute_signals 附带 ext 列) 上评估, 只 AND 进 entry — exit 掩码
|
||||
# 独立构建, 已持仓卖出不受影响 (与 basic_filter 同口径)。
|
||||
if overlay_conditions:
|
||||
overlay_mask, overlay_err = self._overlay_panel_mask(panel, overlay_conditions)
|
||||
if overlay_err is not None:
|
||||
return _err(overlay_err)
|
||||
entry_mask = entry_mask & overlay_mask
|
||||
if config.regime_filter:
|
||||
date_values = panel.get_column("date").unique().sort().to_list()
|
||||
date_labels = tuple(str(value)[:10] for value in date_values)
|
||||
@@ -1845,6 +1896,7 @@ class StrategyBacktestService:
|
||||
getattr(getattr(self.engine.repo, "store", None), "data_dir", None),
|
||||
required_start=config.start,
|
||||
required_end=config.end,
|
||||
evidence=evidence,
|
||||
)
|
||||
except ValueError as e:
|
||||
return _err(str(e))
|
||||
@@ -1856,7 +1908,6 @@ class StrategyBacktestService:
|
||||
regime_row_mask = panel.get_column("date").is_in(allowed_dates).fill_null(False)
|
||||
formal_candidate_mask = formal_candidate_mask & regime_row_mask
|
||||
entry_mask = entry_mask & regime_row_mask
|
||||
raw_exit_mask = self._build_signal_mask(panel, exit_signals, "_exit")
|
||||
exit_range = self._date_range_mask(panel, config.start, load_end) if config.mode == "full" else formal_range
|
||||
exit_mask = raw_exit_mask & exit_range
|
||||
timing_ms["signals_score"] = round((time.perf_counter() - t_signal) * 1000, 1)
|
||||
@@ -2076,6 +2127,7 @@ class StrategyBacktestService:
|
||||
result_policy: BacktestResultPolicy,
|
||||
run_id: str,
|
||||
t0: float,
|
||||
evidence=None,
|
||||
) -> StrategyBacktestResult:
|
||||
def _err(msg: str) -> StrategyBacktestResult:
|
||||
return StrategyBacktestResult(
|
||||
@@ -2117,12 +2169,15 @@ class StrategyBacktestService:
|
||||
load_end,
|
||||
feature_plan,
|
||||
asset_type="stock",
|
||||
**({"evidence": evidence} if evidence is not None else {}),
|
||||
)
|
||||
except (ValueError, OSError, pl.exceptions.PolarsError) as e:
|
||||
return _err(f"回测特征准备失败: {e}")
|
||||
timing_ms["load_panel"] = round((time.perf_counter() - t_load) * 1000, 1)
|
||||
if panel.is_empty():
|
||||
return _err("无日线数据, 请检查日期范围或先运行盘后管道")
|
||||
if evidence is not None:
|
||||
evidence.record_frame("minute_daily_features", panel)
|
||||
|
||||
replayer = MinuteSignalReplayer(self.engine, self.strategy_engine)
|
||||
replay = replayer.replay(
|
||||
@@ -2135,6 +2190,7 @@ class StrategyBacktestService:
|
||||
symbols=config.symbols,
|
||||
progress_cb=progress_cb,
|
||||
cancel_event=cancel_event,
|
||||
**({"evidence": evidence} if evidence is not None else {}),
|
||||
)
|
||||
timing_ms["minute_replay"] = replay.elapsed_ms
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
@@ -2516,6 +2572,7 @@ class StrategyBacktestService:
|
||||
*,
|
||||
required_start: date | None = None,
|
||||
required_end: date | None = None,
|
||||
evidence=None,
|
||||
) -> np.ndarray | None:
|
||||
"""构造逐日 T-1 regime mask, 保留历史静态入口兼容调用方。"""
|
||||
if not regime_filter:
|
||||
@@ -2531,6 +2588,8 @@ class StrategyBacktestService:
|
||||
from app.services import regime_builder
|
||||
|
||||
regime_df = regime_builder.load_regime_history(data_dir)
|
||||
if evidence is not None:
|
||||
evidence.record_frame("regime_history", regime_df)
|
||||
regime_by_date = {
|
||||
row["date"]: {
|
||||
"state": row.get("state", ""),
|
||||
@@ -2547,6 +2606,40 @@ class StrategyBacktestService:
|
||||
required_end=required_end,
|
||||
)
|
||||
|
||||
def _evaluate_panel_decisions(
|
||||
self, panel, strategy, params, overrides, basic_filter, entry_signals, exit_signals,
|
||||
*, factor_snapshot=None,
|
||||
):
|
||||
"""生产回测与时间切片检查共用信号管线,日期/成交延迟在其后应用。"""
|
||||
basic_mask = pl.Series("_basic", [True] * len(panel), dtype=pl.Boolean)
|
||||
if basic_filter and basic_filter.get("enabled", True):
|
||||
expr = StrategyEngine._basic_filter_expr(panel, basic_filter)
|
||||
if expr is not None:
|
||||
basic_mask = panel.select(expr.alias("_basic"))["_basic"].fill_null(False).cast(pl.Boolean)
|
||||
candidate = basic_mask & self._build_candidate_filter_mask(panel, strategy, params)
|
||||
panel = self._apply_score(
|
||||
panel, strategy, overrides, universe_mask=candidate, factor_snapshot=factor_snapshot,
|
||||
)
|
||||
entry = self._build_entry_mask_from_candidate(panel, candidate, strategy, entry_signals)
|
||||
overlay = StrategyEngine._effective_overlay(overrides)
|
||||
if overlay:
|
||||
mask, error = self._overlay_panel_mask(panel, overlay)
|
||||
if error is not None:
|
||||
raise ValueError(error)
|
||||
entry = entry & mask
|
||||
exits = self._build_signal_mask(panel, exit_signals, "_exit")
|
||||
return panel, candidate, entry, exits
|
||||
|
||||
@staticmethod
|
||||
def _decision_frame(panel, candidate, entry, exits):
|
||||
names = [
|
||||
name for name, dtype in panel.schema.items()
|
||||
if name in {"symbol", "date"} or dtype.is_numeric() or dtype == pl.Boolean
|
||||
]
|
||||
return panel.select(names).with_columns(
|
||||
candidate.rename("_audit_candidate"), entry.rename("_audit_entry"), exits.rename("_audit_exit"),
|
||||
)
|
||||
|
||||
def _build_candidate_filter_mask(
|
||||
self,
|
||||
panel: pl.DataFrame,
|
||||
@@ -2741,6 +2834,10 @@ class StrategyBacktestService:
|
||||
logger.warning("load benchmark %s failed: %s", BENCHMARK_SYMBOL, e)
|
||||
return []
|
||||
|
||||
evidence = getattr(self.engine, "_evidence", None)
|
||||
if evidence is not None:
|
||||
evidence.record_frame("benchmark", df)
|
||||
|
||||
if df.is_empty() or "close" not in df.columns:
|
||||
return []
|
||||
|
||||
@@ -2918,6 +3015,8 @@ class StrategyBacktestService:
|
||||
"initial_capital": c.initial_capital,
|
||||
"position_sizing": c.position_sizing,
|
||||
"position_policy": c.position_policy,
|
||||
"position_policy_params": c.position_policy_params,
|
||||
"asset_type": c.asset_type,
|
||||
"mode": c.mode,
|
||||
"holding_days": c.holding_days,
|
||||
"minute_fill": c.minute_fill,
|
||||
|
||||
@@ -235,7 +235,7 @@ def _worker_entry(task: dict[str, Any], event_queue, cancel_event) -> None:
|
||||
kind = task["kind"]
|
||||
if kind == "backtest":
|
||||
config = _decode_backtest_config(task["config"])
|
||||
result = asdict(service.run(config, _progress, cancel_event))
|
||||
result = asdict(service.run_with_evidence(config, _progress, cancel_event))
|
||||
elif kind == "robust_optimize":
|
||||
from app.backtest.robust_optimizer import RobustOptimizeService
|
||||
|
||||
|
||||
@@ -654,7 +654,9 @@ def _needed_references_limit_prices(needed: set[str]) -> bool:
|
||||
return True
|
||||
return False
|
||||
|
||||
def compute_signals(df: pl.DataFrame, needed: set[str] | None = None) -> pl.DataFrame:
|
||||
def compute_signals(
|
||||
df: pl.DataFrame, needed: set[str] | None = None, *, attach_extensions: bool = True,
|
||||
) -> pl.DataFrame:
|
||||
"""注入自定义信号布尔列 (含迁移自内置的 signal_* 定义)。
|
||||
|
||||
输入必须包含 compute_indicators() 产出的指标列。
|
||||
@@ -667,7 +669,8 @@ def compute_signals(df: pl.DataFrame, needed: set[str] | None = None) -> pl.Data
|
||||
# 都按列存在性解析。历史多日帧仅注入时序模式 —— 快照代表"最新值",
|
||||
# 历史回看注入会引入未来数据 (CONTRIBUTING §5.3)。
|
||||
from app.factors import ext_factors
|
||||
df = ext_factors.attach_ext_columns(df, include_snapshot=False)
|
||||
if attach_extensions:
|
||||
df = ext_factors.attach_ext_columns(df, include_snapshot=False)
|
||||
# 条件引用的注册表因子列先复用评分物化管线补算 (虚拟/自定义/复合均可)。
|
||||
from app.strategy import custom_signals
|
||||
exprs = _get_custom_signal_exprs()
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, timedelta
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
import pytest
|
||||
|
||||
from app.backtest.evidence import (
|
||||
audit_frame,
|
||||
audit_matrix,
|
||||
audit_trade_timing,
|
||||
fingerprint_frame,
|
||||
load_evidence,
|
||||
save_evidence,
|
||||
)
|
||||
from app.backtest.matrix import build_market_data_matrix, make_signal_matrix, rolling_mean
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service_factory(tmp_path, monkeypatch):
|
||||
from app.backtest.engine import BacktestEngine
|
||||
from app.backtest.strategy import StrategyBacktestService
|
||||
from app.config import settings
|
||||
from app.strategy.engine import StrategyEngine
|
||||
from app.tickflow.repository import DataStore, KlineRepository
|
||||
from tests.signal_seeds import install_pipeline_caches
|
||||
|
||||
monkeypatch.setattr(settings, "data_dir", tmp_path)
|
||||
install_pipeline_caches(monkeypatch)
|
||||
frame = _panel().with_columns(
|
||||
pl.col("close").alias("raw_close"), pl.col("high").alias("raw_high"),
|
||||
pl.col("low").alias("raw_low"), (pl.col("close") * pl.col("volume")).alias("amount"),
|
||||
pl.lit(1.0).alias("turnover_rate"),
|
||||
pl.lit(0).alias("consecutive_limit_ups"), pl.lit(0).alias("consecutive_limit_downs"),
|
||||
)
|
||||
for part in frame.partition_by("date"):
|
||||
path = tmp_path / "kline_daily_enriched" / f"date={part['date'][0]}" / "part.parquet"
|
||||
path.parent.mkdir(parents=True)
|
||||
part.write_parquet(path)
|
||||
instruments = tmp_path / "instruments" / "part.parquet"
|
||||
instruments.parent.mkdir()
|
||||
pl.DataFrame({
|
||||
"symbol": ["600000.SH", "000001.SZ"], "name": ["A", "B"],
|
||||
"float_shares": [1e8, 2e8], "total_shares": [2e8, 4e8],
|
||||
}).write_parquet(instruments)
|
||||
strategies = tmp_path / "strategies" / "custom"
|
||||
strategies.mkdir(parents=True)
|
||||
store = DataStore(tmp_path)
|
||||
repo = KlineRepository(store)
|
||||
|
||||
def build(backend="matrix_native", leak=False):
|
||||
if backend == "matrix_native":
|
||||
source = '''
|
||||
import numpy as np
|
||||
from app.backtest.matrix import make_signal_matrix
|
||||
META = {"id": "audit_test", "name": "audit", "params": [{"id": "offset", "type": "int", "default": 0}], "scoring": {}}
|
||||
EXECUTION_BACKEND = "matrix_native"
|
||||
class Strategy:
|
||||
def required_fields(self):
|
||||
return frozenset({"open", "high", "low", "close", "volume"})
|
||||
def required_warmup_bars(self, params):
|
||||
return 1
|
||||
def compute_signals(self, market, params):
|
||||
score = SCORE + params.get("offset", 0)
|
||||
return make_signal_matrix(market.shape, entry=np.ones(market.shape), score=score)
|
||||
MATRIX_STRATEGY = Strategy()
|
||||
'''.replace("SCORE", "np.broadcast_to(market.close[-1], market.shape)" if leak else "market.close")
|
||||
else:
|
||||
source = '''
|
||||
import polars as pl
|
||||
META = {"id": "audit_test", "name": "audit", "scoring": {}, "params": []}
|
||||
EXECUTION_BACKEND = "polars_expr"
|
||||
def filter(df, params):
|
||||
return EXPR
|
||||
'''.replace("EXPR", 'pl.col("close") > pl.col("close").mean().over("symbol")' if leak else 'pl.col("close") > 0')
|
||||
(strategies / "audit_test.py").write_text(source, encoding="utf-8")
|
||||
return StrategyBacktestService(BacktestEngine(repo), StrategyEngine(strategy_dirs=[strategies]))
|
||||
|
||||
yield build
|
||||
store.db.close()
|
||||
|
||||
|
||||
def _config(**kwargs):
|
||||
from app.backtest.strategy import StrategyBacktestConfig
|
||||
return StrategyBacktestConfig(
|
||||
strategy_id="audit_test", symbols=None, start=date(2024, 1, 4), end=date(2024, 1, 12),
|
||||
overrides={"basic_filter": {"enabled": False}}, **kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _panel() -> pl.DataFrame:
|
||||
return pl.DataFrame([
|
||||
{
|
||||
"symbol": symbol,
|
||||
"date": date(2024, 1, 1) + timedelta(days=i),
|
||||
"open": float(i + offset), "high": float(i + offset + 1),
|
||||
"low": float(i + offset - 1), "close": float(i + offset),
|
||||
"volume": float(100 + i),
|
||||
}
|
||||
for symbol, offset in [("600000.SH", 10), ("000001.SZ", 20)]
|
||||
for i in range(12)
|
||||
]).sort(["symbol", "date"])
|
||||
|
||||
|
||||
def test_matrix_causal_rolling_passes_without_modifying_inputs():
|
||||
market = build_market_data_matrix(_panel())
|
||||
original = market.close.copy()
|
||||
|
||||
def evaluate(data):
|
||||
score = rolling_mean(data.close, 3)
|
||||
return make_signal_matrix(data.shape, entry=data.close > score, score=np.nan_to_num(score))
|
||||
|
||||
report = audit_matrix(market, evaluate(market), evaluate, start="2024-01-04", end="2024-01-12")
|
||||
assert report["status"] == "passed"
|
||||
assert {c["kind"] for c in report["checks"]} == {"truncate", "perturb"}
|
||||
assert len(report["checks"]) == 6
|
||||
assert all(c["compared_values"] > 0 for c in report["checks"])
|
||||
np.testing.assert_equal(market.close, original)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("leak", ["next_bar", "global_mean", "last_bar"])
|
||||
def test_matrix_detects_future_dependency_with_location(leak):
|
||||
market = build_market_data_matrix(_panel())
|
||||
|
||||
def evaluate(data):
|
||||
if leak == "next_bar":
|
||||
score = np.roll(data.close, -1, axis=0)
|
||||
elif leak == "global_mean":
|
||||
score = np.broadcast_to(np.mean(data.close, axis=0), data.shape)
|
||||
else:
|
||||
score = np.broadcast_to(data.close[-1], data.shape)
|
||||
return make_signal_matrix(data.shape, entry=np.ones(data.shape), score=score)
|
||||
|
||||
report = audit_matrix(market, evaluate(market), evaluate, start="2024-01-04", end="2024-01-12")
|
||||
assert report["status"] == "failed"
|
||||
assert report["differences"] > 0
|
||||
example = next(c for c in report["checks"] if c["examples"])["examples"][0]
|
||||
assert example["field"] == "score"
|
||||
assert example["symbol"] in market.symbols
|
||||
assert example["date"] <= example["cutoff"]
|
||||
|
||||
|
||||
def test_frame_recomputes_features_on_prefix_and_perturbed_future():
|
||||
raw = _panel()
|
||||
|
||||
def evaluate(frame):
|
||||
return frame.with_columns(
|
||||
pl.col("close").shift(-1).over("symbol").alias("future_price"),
|
||||
pl.lit(True).alias("entry"),
|
||||
).select("symbol", "date", "future_price", "entry")
|
||||
|
||||
report = audit_frame(raw, evaluate(raw), evaluate, start="2024-01-04", end="2024-01-12")
|
||||
assert report["status"] == "failed"
|
||||
assert any(e["field"] == "future_price" for c in report["checks"] for e in c["examples"])
|
||||
assert raw.equals(_panel())
|
||||
|
||||
|
||||
def test_audit_insufficient_or_failed_recompute_never_passes():
|
||||
market = build_market_data_matrix(_panel().filter(pl.col("date") == date(2024, 1, 1)))
|
||||
signals = make_signal_matrix(market.shape, entry=np.ones(market.shape))
|
||||
report = audit_matrix(market, signals, lambda _: signals, start="2024-01-01", end="2024-01-01")
|
||||
assert report["status"] == "insufficient"
|
||||
|
||||
market = build_market_data_matrix(_panel())
|
||||
|
||||
def broken(_):
|
||||
raise ValueError("bad prefix")
|
||||
|
||||
report = audit_matrix(market, make_signal_matrix(market.shape), broken, start="2024-01-04", end="2024-01-12")
|
||||
assert report["status"] == "error"
|
||||
assert all(c["status"] == "error" for c in report["checks"])
|
||||
|
||||
|
||||
def test_fingerprint_stable_across_row_and_column_order_but_changes_with_values():
|
||||
frame = _panel()
|
||||
first = fingerprint_frame(frame)
|
||||
assert first["sha256"] == fingerprint_frame(frame.reverse().select(reversed(frame.columns)))["sha256"]
|
||||
changed = frame.with_columns((pl.col("volume") + 1).alias("volume"))
|
||||
second = fingerprint_frame(changed)
|
||||
assert first["sha256"] != second["sha256"]
|
||||
assert first["fields"]["close"] == second["fields"]["close"]
|
||||
assert first["fields"]["volume"] != second["fields"]["volume"]
|
||||
|
||||
|
||||
def test_announcement_fingerprint_ignores_physical_order_of_same_symbol_rows():
|
||||
frame = pl.DataFrame({
|
||||
"symbol": ["600000.SH", "600000.SH"],
|
||||
"announce_date": ["2024-01-04", "2024-01-05"], "value": [1.0, 2.0],
|
||||
})
|
||||
assert fingerprint_frame(frame)["sha256"] == fingerprint_frame(frame.reverse())["sha256"]
|
||||
|
||||
|
||||
def test_manifest_persists_and_rejects_unsafe_ids_and_overwrites(tmp_path):
|
||||
evidence = {"schema_version": 1, "run_id": "012345abcd", "data": {"sha256": "a" * 64}}
|
||||
save_evidence(tmp_path, evidence)
|
||||
assert load_evidence(tmp_path, "012345abcd") == evidence
|
||||
with pytest.raises(FileExistsError):
|
||||
save_evidence(tmp_path, evidence)
|
||||
with pytest.raises(ValueError):
|
||||
load_evidence(tmp_path, "../../secrets")
|
||||
with pytest.raises(FileNotFoundError):
|
||||
load_evidence(tmp_path, "0000000000")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["matrix_native", "polars_expr"])
|
||||
@pytest.mark.parametrize("leak", [False, True])
|
||||
def test_real_service_checks_and_persists_exact_result_inputs(service_factory, tmp_path, backend, leak):
|
||||
service = service_factory(backend, leak)
|
||||
result = service.run_with_evidence(_config())
|
||||
assert result.error is None
|
||||
assert result.evidence["lookahead"]["status"] == ("failed" if leak else "passed")
|
||||
assert result.evidence["persistence"]["status"] == "saved"
|
||||
assert load_evidence(tmp_path, result.run_id) == result.evidence
|
||||
assert result.evidence["data"]["datasets"]
|
||||
assert result.evidence["config"]["asset_type"] == "stock"
|
||||
assert service.engine._evidence is None
|
||||
assert result.evidence["execution_timing"]["status"] == "passed"
|
||||
|
||||
|
||||
def test_versions_separate_data_parameters_and_strategy_changes(service_factory, tmp_path):
|
||||
service = service_factory()
|
||||
first = service.run_with_evidence(_config()).evidence
|
||||
same = service.run_with_evidence(_config()).evidence
|
||||
assert first["reproducibility_sha256"] == same["reproducibility_sha256"]
|
||||
params = service.run_with_evidence(_config(params={"offset": 2})).evidence
|
||||
assert params["data"]["sha256"] == first["data"]["sha256"]
|
||||
assert params["config_sha256"] != first["config_sha256"]
|
||||
assert params["strategy"]["sha256"] == first["strategy"]["sha256"]
|
||||
|
||||
from app.enriched_generation import bump_enriched_generation
|
||||
path = tmp_path / "kline_daily_enriched" / "date=2024-01-08" / "part.parquet"
|
||||
pl.read_parquet(path).with_columns((pl.col("volume") + 5).alias("volume")).write_parquet(path)
|
||||
bump_enriched_generation(tmp_path, "stock")
|
||||
data = service.run_with_evidence(_config()).evidence
|
||||
assert data["data"]["sha256"] != first["data"]["sha256"]
|
||||
assert data["config_sha256"] == first["config_sha256"]
|
||||
assert data["strategy"]["sha256"] == first["strategy"]["sha256"]
|
||||
assert load_evidence(tmp_path, first["run_id"]) == first
|
||||
|
||||
changed_service = service_factory(leak=True)
|
||||
strategy = changed_service.run_with_evidence(_config()).evidence
|
||||
assert strategy["data"]["sha256"] == data["data"]["sha256"]
|
||||
assert strategy["strategy"]["sha256"] != data["strategy"]["sha256"]
|
||||
|
||||
|
||||
def test_unwritable_manifest_is_visible_and_normal_trials_do_not_audit(service_factory, monkeypatch):
|
||||
service = service_factory()
|
||||
assert service.run(_config()).evidence is None
|
||||
|
||||
def denied(*_args):
|
||||
raise PermissionError("synthetic")
|
||||
|
||||
monkeypatch.setattr("app.backtest.evidence.save_evidence", denied)
|
||||
result = service.run_with_evidence(_config())
|
||||
assert result.error is None
|
||||
assert result.evidence["persistence"]["status"] == "error"
|
||||
assert service.engine._evidence is None
|
||||
|
||||
|
||||
def test_nondeterministic_strategy_is_inconclusive_not_a_future_leak_verdict():
|
||||
market = build_market_data_matrix(_panel())
|
||||
calls = 0
|
||||
|
||||
def evaluate(data):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return make_signal_matrix(data.shape, score=np.full(data.shape, calls))
|
||||
|
||||
result = audit_matrix(market, evaluate(market), evaluate, start="2024-01-04", end="2024-01-12")
|
||||
assert result["status"] == "error"
|
||||
assert result["checks"][-1]["kind"] == "repeat"
|
||||
|
||||
|
||||
def test_evidence_api_reads_saved_manifest_and_handles_missing_invalid_and_corrupt(tmp_path, monkeypatch):
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.backtest import router
|
||||
from app.config import settings
|
||||
|
||||
monkeypatch.setattr(settings, "data_dir", tmp_path)
|
||||
app = FastAPI()
|
||||
app.include_router(router)
|
||||
client = TestClient(app)
|
||||
evidence = {"schema_version": 1, "run_id": "012345abcd"}
|
||||
save_evidence(tmp_path, evidence)
|
||||
assert client.get("/api/backtest/strategy/runs/012345abcd/evidence").json() == evidence
|
||||
assert client.get("/api/backtest/strategy/runs/0000000000/evidence").status_code == 404
|
||||
assert client.get("/api/backtest/strategy/runs/invalid/evidence").status_code == 400
|
||||
(tmp_path / "backtest_runs" / "012345abcd.json").write_text("{", encoding="utf-8")
|
||||
assert client.get("/api/backtest/strategy/runs/012345abcd/evidence").status_code == 400
|
||||
|
||||
|
||||
def test_real_post_api_returns_same_persisted_report_as_worker(service_factory, tmp_path):
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.backtest import router
|
||||
|
||||
service = service_factory()
|
||||
app = FastAPI()
|
||||
app.state.repo = service.engine.repo
|
||||
app.state.strategy_engine = service.strategy_engine
|
||||
app.include_router(router)
|
||||
client = TestClient(app)
|
||||
response = client.post("/api/backtest/strategy/run", json={
|
||||
"strategy_id": "audit_test", "start": "2024-01-04", "end": "2024-01-12",
|
||||
"overrides": {"basic_filter": {"enabled": False}},
|
||||
})
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
assert result["error"] is None
|
||||
assert result["evidence"]["lookahead"]["status"] == "passed"
|
||||
assert load_evidence(tmp_path, result["run_id"]) == result["evidence"]
|
||||
assert client.get(f"/api/backtest/strategy/runs/{result['run_id']}/evidence").json() == result["evidence"]
|
||||
|
||||
|
||||
def test_panel_audit_freezes_external_timeseries_values(service_factory, monkeypatch):
|
||||
from app.factors import ext_factors
|
||||
|
||||
service = service_factory("polars_expr")
|
||||
calls = 0
|
||||
|
||||
def attach(frame, **_kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return frame.with_columns(pl.lit(float(calls)).alias("ext_test_value"))
|
||||
|
||||
monkeypatch.setattr(ext_factors, "attach_ext_columns", attach)
|
||||
result = service.run_with_evidence(_config())
|
||||
assert result.error is None
|
||||
assert result.evidence["lookahead"]["status"] == "passed"
|
||||
assert calls == 1
|
||||
assert "ext_test_value" in result.evidence["data"]["datasets"]["daily_base"]["fields"]
|
||||
|
||||
|
||||
def test_report_path_rejects_symlink_even_inside_data_directory(tmp_path):
|
||||
target = tmp_path / "other"
|
||||
target.mkdir()
|
||||
(tmp_path / "backtest_runs").symlink_to(target, target_is_directory=True)
|
||||
with pytest.raises(ValueError, match="目录无效"):
|
||||
save_evidence(tmp_path, {"schema_version": 1, "run_id": "012345abcd"})
|
||||
|
||||
|
||||
def test_concurrent_manifest_writes_never_replace_an_existing_version(tmp_path):
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
def write(value):
|
||||
try:
|
||||
save_evidence(tmp_path, {"schema_version": 1, "run_id": "012345abcd", "value": value})
|
||||
return value
|
||||
except FileExistsError:
|
||||
return None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
written = [value for value in pool.map(write, range(8)) if value is not None]
|
||||
assert len(written) == 1
|
||||
assert load_evidence(tmp_path, "012345abcd")["value"] == written[0]
|
||||
|
||||
|
||||
def test_trade_timing_checks_future_dates_and_next_open_but_respects_minute_entries():
|
||||
trade = {"symbol": "600000.SH", "entry_signal_date": "2024-01-05", "entry_date": "2024-01-04"}
|
||||
assert audit_trade_timing([trade], {}, "matrix_native")["status"] == "failed"
|
||||
trade["entry_date"] = "2024-01-05"
|
||||
assert audit_trade_timing([trade], {"entry_fill": "open_t+1"}, "matrix_native")["status"] == "failed"
|
||||
assert audit_trade_timing([trade], {"entry_fill": "close_t"}, "matrix_native")["status"] == "passed"
|
||||
assert audit_trade_timing([trade], {"entry_fill": "open_t+1"}, "minute_filter")["status"] == "passed"
|
||||
trade.update(exit_signal_date="2024-01-08", exit_date="2024-01-08", exit_reason="signal")
|
||||
assert audit_trade_timing([trade], {"exit_fill": "open_t+1"}, "matrix_native")["status"] == "failed"
|
||||
trade["exit_reason"] = "forced_close"
|
||||
assert audit_trade_timing([trade], {"exit_fill": "open_t+1"}, "matrix_native")["status"] == "passed"
|
||||
|
||||
|
||||
def test_unknown_strategy_keeps_legacy_error_contract(service_factory):
|
||||
from dataclasses import replace
|
||||
|
||||
result = service_factory().run_with_evidence(replace(_config(), strategy_id="missing"))
|
||||
assert result.error
|
||||
assert result.evidence is None
|
||||
|
||||
|
||||
def test_empty_minute_signal_comparisons_are_insufficient_instead_of_passed():
|
||||
raw = _panel()
|
||||
empty = raw.head(0)
|
||||
result = audit_frame(raw, empty, lambda _: empty, start=date(2024, 1, 4), end=date(2024, 1, 12))
|
||||
assert result["status"] == "insufficient"
|
||||
|
||||
|
||||
def test_perturbation_also_exercises_zero_valued_future_fields():
|
||||
market = build_market_data_matrix(_panel().with_columns(pl.lit(0.0).alias("volume")))
|
||||
|
||||
def evaluate(data):
|
||||
return make_signal_matrix(data.shape, score=np.broadcast_to(data.volume.mean(axis=0), data.shape))
|
||||
|
||||
result = audit_matrix(market, evaluate(market), evaluate, start="2024-01-04", end="2024-01-12")
|
||||
assert result["status"] == "failed"
|
||||
assert any(check["kind"] == "perturb" and check["differences"] > 0 for check in result["checks"])
|
||||
|
||||
|
||||
def test_cancelled_checks_are_not_reported_as_passed():
|
||||
import threading
|
||||
|
||||
event = threading.Event()
|
||||
event.set()
|
||||
market = build_market_data_matrix(_panel())
|
||||
result = audit_matrix(market, make_signal_matrix(market.shape), lambda _: None,
|
||||
start="2024-01-04", end="2024-01-12", cancel_event=event)
|
||||
assert result["status"] == "cancelled"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("leak", [False, True])
|
||||
def test_minute_replay_checks_intraday_future_and_tracks_minute_inputs(tmp_path, monkeypatch, leak):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.config import settings
|
||||
from tests.backtest.test_minute_backtest import (
|
||||
TEST_STRATEGY_SOURCE,
|
||||
_daily_panel,
|
||||
_make_service,
|
||||
_minute_frame,
|
||||
_trading_days,
|
||||
)
|
||||
from tests.backtest.test_minute_backtest import (
|
||||
_config as minute_config,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(settings, "data_dir", tmp_path)
|
||||
days = _trading_days(10)
|
||||
frame = _daily_panel(days, ["600000.SH"])
|
||||
minutes = {
|
||||
day: _minute_frame(day, [("600000.SH", f"09:{31 + i}", 10.7 + i * 0.01) for i in range(6)])
|
||||
for day in days[5:9]
|
||||
}
|
||||
service = _make_service(tmp_path, frame, minutes)
|
||||
service.engine.repo.store = SimpleNamespace(data_dir=tmp_path)
|
||||
if leak:
|
||||
source = TEST_STRATEGY_SOURCE.replace(
|
||||
'joined = df.join(prev, on="symbol", how="inner")',
|
||||
'df = df.with_columns(pl.col("close").shift(-1).over("symbol").alias("close"))\n joined = df.join(prev, on="symbol", how="inner")',
|
||||
)
|
||||
(tmp_path / "strategies" / "test_minute_ping.py").write_text(source, encoding="utf-8")
|
||||
service.strategy_engine.reload()
|
||||
result = service.run_with_evidence(minute_config(days[5], days[8]))
|
||||
assert result.error is None
|
||||
assert result.evidence["lookahead"]["status"] == ("failed" if leak else "passed")
|
||||
assert len([key for key in result.evidence["data"]["datasets"] if key.startswith("minute_replay/")]) == 4
|
||||
@@ -161,6 +161,10 @@ def test_spawn_worker_returns_compact_result_and_memory_metrics(tmp_path):
|
||||
|
||||
assert result["error"] is None
|
||||
assert result["stats"]["execution_backend"] == "matrix_native"
|
||||
from app.backtest.evidence import load_evidence
|
||||
|
||||
assert result["evidence"]["lookahead"]["status"] == "passed"
|
||||
assert load_evidence(data_dir, result["run_id"]) == result["evidence"]
|
||||
worker = result["stats"]["worker"]
|
||||
assert worker["peak_rss_bytes"] > 0
|
||||
assert worker["serialized_result_bytes"] > 0
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# 回测未来数据检查与版本记录
|
||||
|
||||
从本功能起,通过策略回测页面、`POST /api/backtest/strategy/run` 或 SSE 发起的正式回测,会自动生成核查报告。日线表达式、历史 Python 策略、矩阵策略、叠加策略及分钟策略复用各自的生产信号管线;优化器的每个参数试算保持轻量,需将选中的参数载入正式回测生成报告。
|
||||
|
||||
## 时间检查
|
||||
|
||||
- 固定本次数据快照、策略参数、标的集合和预热起点,最多选择 3 个实际交易时点;分钟策略选择最多 3 个交易日,每日最多 3 个分钟截点。
|
||||
- 分别删除截点后的行情、扰动截点后的数值,重算后比较截点及之前的历史特征、候选、买卖信号和评分。矩阵策略检查其公开的信号输出;分钟策略检查触发时间、触发价及评分。
|
||||
- 若发现变化,再用完全相同的输入复算基线,区分未来数据依赖与非确定性/带状态策略。
|
||||
- 浮点比较容差为 `rtol=1e-6, atol=1e-7`,空值与空值相等。报告记录差异数量,并为每次重算保留最多 5 条日期、标的、字段和前后数值示例。
|
||||
- 状态分别为抽样未发现差异、检测到变化、数据不足、检查错误、部分未完成和取消。缺字段、重算异常、无可切分时间点都不会记为通过。发现变化只标记结果供排查,不擅自修改用户策略。
|
||||
|
||||
日线表达式路径从基础数据重新计算指标与信号,不是切割已经算好的信号。公告日门控、交易日、股票 T+1、费用、滑点和涨跌停成交约束继续由原有管线执行。
|
||||
|
||||
同时单独核对已完成交易的信号日和成交日:信号不能晚于成交,日线的「次日开盘」信号成交必须晚于信号日;分钟触发入场及期末估值按原有语义处理。时间核对不修改成交结果,异常在页面和报告中单独标出。扩展时序数据在准备时固定为快照,检查重算不会再次读取更新后的扩展数据。
|
||||
|
||||
这不是全时点无泄漏证明。检查不能证明供应商数据、复权历史修订、历史股票池、静态维表及外部 Python 导入满足 point-in-time;未触发的逻辑也可能未被覆盖。应结合公告时点验证、滚动样本外验证及模拟盘结果判断。
|
||||
|
||||
## 版本追溯
|
||||
|
||||
报告分别记录:
|
||||
|
||||
- 本次实际加载的本地标准化数据内容 SHA-256、字段指纹、行数和时间范围,区分股票/ETF。包括预热和全量模式退出尾部;分钟回放和分钟精确成交按实际读取批次记录。
|
||||
- 标的信息、历史股本、财务公告、市场环境和同期基准等实际使用的辅助数据。
|
||||
- 策略文件、叠加子策略及本地信号/因子/仓位定义目录的内容指纹。定义目录指纹覆盖整个注册目录,未使用定义的修改也会改变该部分指纹。
|
||||
- 包含默认参数的有效运行配置、独立参数指纹、应用/核心代码/Python/Polars/NumPy 版本及组合指纹。冻结发行版缺少源码文件时,对应代码文件指纹为 `null`,应用版本仍保留。
|
||||
- 检查报告、核心回测统计及策略定义在运行期间是否发生变化。
|
||||
|
||||
数据来源标注为本次读取的本地标准化快照,**不把当前配置的数据源冒充为历史入库来源**。不读取密钥、环境变量或完整偏好配置。
|
||||
|
||||
每次正式回测的报告保存在运行数据目录的 `backtest_runs/<run_id>.json`。按回测编号读取:
|
||||
|
||||
```text
|
||||
GET /api/backtest/strategy/runs/{run_id}/evidence
|
||||
```
|
||||
|
||||
策略回测结果的 `evidence` 字段包含同一份报告,页面可展开查看及导出 JSON;CSV 概要携带回测编号及三个版本指纹。保存失败会明确标记,仍可导出内存中的报告。旧结果缺少 `evidence` 时正常显示,重新回测后才生成记录。
|
||||
|
||||
**内容指纹不是全量行情备份。** 它用于确认两次运行的数据、策略或参数是否变化、定位变化字段。原始行情被覆盖后,不能仅凭指纹恢复旧数据或自动重放旧回测。
|
||||
|
||||
## 缓存与兼容边界
|
||||
|
||||
版本与检查仅在独立回测 worker 内收集。检查重用本次不可变输入,变体不写行情仓库、Parquet、共享矩阵缓存或策略结果缓存;报告不进入缓存键。SSE 重连复用原结果时携带原报告,不重新盖章。原有日K修订失效检查保持生效。
|
||||
|
||||
未修改行情存储 schema、数据源协议或历史配置。核查报告按版本 1 独立落盘,读写拒绝非法回测编号、符号链接越界和损坏记录;原有用户数据不迁移、不覆盖。
|
||||
@@ -1893,6 +1893,56 @@ export interface StrategyBacktestTrade {
|
||||
exit_signal_id?: string | null
|
||||
}
|
||||
|
||||
export interface BacktestEvidenceCheck {
|
||||
kind: string
|
||||
cutoff: string
|
||||
status: 'completed' | 'error'
|
||||
differences: number
|
||||
compared_values: number
|
||||
error?: string
|
||||
examples: {
|
||||
date: string; symbol: string; field: string; cutoff: string
|
||||
baseline: number | string | boolean | null
|
||||
recomputed: number | string | boolean | null
|
||||
}[]
|
||||
}
|
||||
|
||||
export interface BacktestEvidence {
|
||||
schema_version: number
|
||||
run_id: string
|
||||
created_at: string
|
||||
runtime: { application: string; python: string; polars: string; numpy: string; code_sha256: string }
|
||||
config: Record<string, unknown>
|
||||
config_sha256: string
|
||||
strategy: { id: string; backend: string; sha256: string; changed_during_run: boolean }
|
||||
data: {
|
||||
sha256: string
|
||||
origin: string
|
||||
asset_type: string
|
||||
daily_price_basis: string
|
||||
datasets: Record<string, {
|
||||
sha256: string; fields: Record<string, string>; rows: number; symbols: number | null
|
||||
start: string | null; end: string | null
|
||||
}>
|
||||
}
|
||||
reproducibility_sha256: string
|
||||
lookahead: {
|
||||
status: 'passed' | 'failed' | 'partial' | 'error' | 'insufficient' | 'cancelled'
|
||||
sampled: boolean
|
||||
differences: number
|
||||
checks: BacktestEvidenceCheck[]
|
||||
reports: { scope: string; status: string; checks: BacktestEvidenceCheck[]; differences: number }[]
|
||||
}
|
||||
persistence: { status: 'saved' | 'error'; error?: string }
|
||||
execution_timing?: {
|
||||
status: 'passed' | 'failed' | 'insufficient'
|
||||
compared_values: number
|
||||
differences: number
|
||||
examples: { symbol: string; side: string; signal_date: string; fill_date: string }[]
|
||||
}
|
||||
limitations: string[]
|
||||
}
|
||||
|
||||
export interface StrategyBacktestResult {
|
||||
run_id: string
|
||||
config: Record<string, any>
|
||||
@@ -1936,6 +1986,7 @@ export interface StrategyBacktestResult {
|
||||
}
|
||||
elapsed_ms: number
|
||||
error: string | null
|
||||
evidence?: BacktestEvidence | null
|
||||
}
|
||||
|
||||
// ===== Settings =====
|
||||
@@ -3744,6 +3795,9 @@ export const api = {
|
||||
method: 'DELETE',
|
||||
}),
|
||||
|
||||
strategyBacktestEvidence: (runId: string) =>
|
||||
request<BacktestEvidence>(`/api/backtest/strategy/runs/${encodeURIComponent(runId)}/evidence`),
|
||||
|
||||
strategyBacktestRun: (payload: {
|
||||
strategy_id: string
|
||||
symbols?: string[] | null
|
||||
|
||||
@@ -42,6 +42,7 @@ import { ConditionEditor } from '@/components/signals/ConditionEditor'
|
||||
import { WatchlistGroupMenu } from '@/components/WatchlistAddMenu'
|
||||
import { ScoringEditor } from '@/components/ScoringEditor'
|
||||
import { strategyResultCandidate } from './researchCandidates'
|
||||
import { BacktestEvidencePanel } from './components/BacktestEvidencePanel'
|
||||
import { buildDailyNavItems, buildTradeKeyMap, buildTradesNavItems, type TradeNavItem } from './tradeNav'
|
||||
|
||||
const formatDate = (date: Date) => date.toISOString().slice(0, 10)
|
||||
@@ -1345,6 +1346,13 @@ export function StrategyBacktest({ loadCandidate, onLoadConsumed }: {
|
||||
lines.push(`策略名称,${name}`)
|
||||
if (result.strategy_info?.id) lines.push(`策略ID,${result.strategy_info.id}`)
|
||||
lines.push(`回测区间,${start} ~ ${end}`)
|
||||
if (result.evidence) {
|
||||
lines.push(`回测编号,${result.run_id}`)
|
||||
lines.push(`数据版本,${result.evidence.data.sha256}`)
|
||||
lines.push(`策略版本,${result.evidence.strategy.sha256}`)
|
||||
lines.push(`参数版本,${result.evidence.config_sha256}`)
|
||||
lines.push(`时间检查状态,${result.evidence.lookahead.status}`)
|
||||
}
|
||||
lines.push(`净值曲线天数,${result.equity_curve?.length ?? 0}`)
|
||||
lines.push(`完成交易数,${result.trades?.length ?? 0}`)
|
||||
lines.push(`总收益,${pct(strategyReturn)}`)
|
||||
@@ -2350,6 +2358,8 @@ export function StrategyBacktest({ loadCandidate, onLoadConsumed }: {
|
||||
</motion.div>
|
||||
)}
|
||||
|
||||
{result && <BacktestEvidencePanel key={result.run_id} evidence={result.evidence} />}
|
||||
|
||||
{/* 旧全量模拟结果: 固定前瞻收益统计 (兼容历史缓存结果) */}
|
||||
{result && !result.error && result.stats && result.stats.mode === 'full' && result.stats.full_kind !== 'candidate_execution' && (
|
||||
<motion.div
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
// @vitest-environment jsdom
|
||||
import { act } from 'react'
|
||||
import { createRoot, type Root } from 'react-dom/client'
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
|
||||
import type { BacktestEvidence } from '@/lib/api'
|
||||
import { BacktestEvidencePanel } from './BacktestEvidencePanel'
|
||||
|
||||
let host: HTMLDivElement
|
||||
let root: Root
|
||||
const originalCreateUrl = Object.getOwnPropertyDescriptor(URL, 'createObjectURL')
|
||||
const originalRevokeUrl = Object.getOwnPropertyDescriptor(URL, 'revokeObjectURL')
|
||||
|
||||
beforeEach(() => {
|
||||
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true })
|
||||
host = document.createElement('div')
|
||||
document.body.append(host)
|
||||
root = createRoot(host)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await act(async () => root.unmount())
|
||||
host.remove()
|
||||
vi.restoreAllMocks()
|
||||
for (const [key, descriptor] of [['createObjectURL', originalCreateUrl], ['revokeObjectURL', originalRevokeUrl]] as const) {
|
||||
if (descriptor) Object.defineProperty(URL, key, descriptor)
|
||||
else Reflect.deleteProperty(URL, key)
|
||||
}
|
||||
})
|
||||
|
||||
function evidence(status: BacktestEvidence['lookahead']['status'] = 'passed'): BacktestEvidence {
|
||||
return {
|
||||
schema_version: 1, run_id: '012345abcd', created_at: '2026-11-01T02:00:00Z',
|
||||
runtime: { application: '0.3.25', python: '3.11', polars: '1', numpy: '2', code_sha256: 'e'.repeat(64) },
|
||||
config: {}, config_sha256: 'c'.repeat(64),
|
||||
strategy: { id: 'test', backend: 'matrix_native', sha256: 'b'.repeat(64), changed_during_run: false },
|
||||
data: { sha256: 'a'.repeat(64), asset_type: 'stock', origin: 'local_normalized_snapshot', daily_price_basis: 'qfq', datasets: {} },
|
||||
reproducibility_sha256: 'd'.repeat(64),
|
||||
lookahead: { status, sampled: true, differences: 0, checks: [], reports: [] },
|
||||
persistence: { status: 'saved' }, limitations: [],
|
||||
}
|
||||
}
|
||||
|
||||
it('历史结果缺少记录时明确提示重新回测,不展示检查通过', async () => {
|
||||
await act(async () => root.render(<BacktestEvidencePanel />))
|
||||
expect(host.textContent).toContain('重新回测后自动生成')
|
||||
expect(host.textContent).not.toContain('抽样未发现差异')
|
||||
})
|
||||
|
||||
it('通过状态注明抽样范围,展示独立版本并导出完整核查报告', async () => {
|
||||
const value = evidence()
|
||||
const createUrl = vi.fn((_blob: Blob) => 'blob:test')
|
||||
const revoke = vi.fn()
|
||||
Object.defineProperty(URL, 'createObjectURL', { value: createUrl, configurable: true })
|
||||
Object.defineProperty(URL, 'revokeObjectURL', { value: revoke, configurable: true })
|
||||
let filename = ''
|
||||
vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function (this: HTMLAnchorElement) { filename = this.download })
|
||||
await act(async () => root.render(<BacktestEvidencePanel evidence={value} />))
|
||||
expect(host.textContent).toContain('抽样未发现差异')
|
||||
expect(host.textContent).not.toContain('保证没有未来数据')
|
||||
expect(host.textContent).toContain('数据版本')
|
||||
expect(host.textContent).toContain('a'.repeat(20))
|
||||
expect(host.textContent).toContain('参数版本')
|
||||
expect(host.querySelector('details')?.open).toBe(false)
|
||||
await act(async () => host.querySelector('button')!.click())
|
||||
expect(filename).toBe('backtest-evidence-012345abcd.json')
|
||||
expect(createUrl).toHaveBeenCalledOnce()
|
||||
expect(createUrl.mock.calls[0][0]).toBeInstanceOf(Blob)
|
||||
expect(revoke).toHaveBeenCalledWith('blob:test')
|
||||
})
|
||||
|
||||
it('发现差异时自动展开具体时间、标的、字段与重算值', async () => {
|
||||
const value = evidence('failed')
|
||||
value.lookahead.differences = 1
|
||||
value.lookahead.checks = [{
|
||||
kind: 'truncate', cutoff: '2024-01-05', status: 'completed', compared_values: 10, differences: 1,
|
||||
examples: [{ date: '2024-01-05', symbol: '600000.SH', field: 'score', cutoff: '2024-01-05', baseline: 30, recomputed: 10 }],
|
||||
}]
|
||||
await act(async () => root.render(<BacktestEvidencePanel evidence={value} />))
|
||||
expect(host.querySelector('details')?.open).toBe(true)
|
||||
expect(host.textContent).toContain('检测到历史输出变化')
|
||||
expect(host.textContent).toContain('600000.SH')
|
||||
expect(host.textContent).toContain('评分')
|
||||
expect(host.textContent).toContain('30 → 10')
|
||||
})
|
||||
|
||||
it('检查错误与保存失败保持可见,并允许导出当前内存中的报告', async () => {
|
||||
const value = evidence('error')
|
||||
value.execution_timing = { status: 'failed', compared_values: 1, differences: 1, examples: [] }
|
||||
value.persistence = { status: 'error' }
|
||||
value.lookahead.checks = [{ kind: 'truncate', cutoff: '2024-01-05', status: 'error', differences: 0, compared_values: 0, examples: [], error: '重算未完成(ValueError)' }]
|
||||
await act(async () => root.render(<BacktestEvidencePanel evidence={value} />))
|
||||
expect(host.textContent).toContain('检查未完成')
|
||||
expect(host.textContent).toContain('版本记录未能保存')
|
||||
expect(host.textContent).toContain('信号与成交时间关系异常')
|
||||
expect(host.textContent).toContain('重算未完成(ValueError)')
|
||||
expect(host.querySelector('button')?.disabled).toBe(false)
|
||||
})
|
||||
@@ -0,0 +1,99 @@
|
||||
import { Download, ScanSearch } from 'lucide-react'
|
||||
import type { BacktestEvidence } from '@/lib/api'
|
||||
|
||||
const STATUS_LABELS: Record<BacktestEvidence['lookahead']['status'], string> = {
|
||||
passed: '抽样未发现差异', failed: '检测到历史输出变化', partial: '部分检查未完成',
|
||||
error: '检查未完成', insufficient: '数据不足以检查', cancelled: '检查已取消',
|
||||
}
|
||||
const DATA_LABELS: Record<string, string> = {
|
||||
daily_base: '日线基础数据', daily_matrix: '日线行情与输入字段',
|
||||
instruments: '标的信息', historical_shares: '历史股本',
|
||||
financial_announcements: '财务公告', regime_history: '市场环境', benchmark: '同期基准',
|
||||
}
|
||||
const FIELD_LABELS: Record<string, string> = {
|
||||
entry: '买入信号', exit: '卖出信号', score: '评分',
|
||||
_audit_candidate: '候选标的', _audit_entry: '买入信号', _audit_exit: '卖出信号',
|
||||
signal_presence: '信号出现时间', entry_signal_code: '买入触发器', exit_signal_code: '卖出触发器',
|
||||
}
|
||||
const CHECK_LABELS: Record<string, string> = { truncate: '截断未来', perturb: '扰动未来', repeat: '基线复算' }
|
||||
|
||||
function exportEvidence(evidence: BacktestEvidence) {
|
||||
const blob = new Blob([JSON.stringify(evidence, null, 2)], { type: 'application/json;charset=utf-8' })
|
||||
const url = URL.createObjectURL(blob)
|
||||
const anchor = document.createElement('a')
|
||||
anchor.href = url
|
||||
anchor.download = `backtest-evidence-${evidence.run_id}.json`
|
||||
anchor.click()
|
||||
URL.revokeObjectURL(url)
|
||||
}
|
||||
|
||||
export function BacktestEvidencePanel({ evidence }: { evidence?: BacktestEvidence | null }) {
|
||||
if (!evidence) {
|
||||
return <p className="text-xs text-muted">此结果没有检查与版本记录,重新回测后自动生成。</p>
|
||||
}
|
||||
const { lookahead } = evidence
|
||||
const completed = lookahead.checks.filter(check => check.status === 'completed').length
|
||||
const examples = lookahead.checks.flatMap(check => check.examples.map(example => ({ ...example, kind: check.kind }))).slice(0, 15)
|
||||
const failed = lookahead.status === 'failed'
|
||||
const datasets = Object.entries(evidence.data.datasets)
|
||||
const color = failed ? 'text-danger' : lookahead.status === 'passed' ? 'text-foreground' : 'text-warning'
|
||||
|
||||
return (
|
||||
<section aria-label="回测检查与版本记录" className="min-w-0 rounded-card border border-border bg-surface px-4 py-3">
|
||||
<div className="flex flex-wrap items-center gap-x-3 gap-y-2">
|
||||
<div className="flex min-w-0 flex-1 flex-wrap items-center gap-2 text-xs">
|
||||
<ScanSearch className="h-4 w-4 shrink-0 text-secondary" aria-hidden="true" />
|
||||
<span className="font-medium text-foreground">未来数据检查</span>
|
||||
<span className={color}>{STATUS_LABELS[lookahead.status]}</span>
|
||||
</div>
|
||||
<button type="button" onClick={() => exportEvidence(evidence)} className="inline-flex items-center gap-1 rounded-btn border border-border px-2 py-1 text-xs text-secondary hover:text-accent">
|
||||
<Download className="h-3 w-3" aria-hidden="true" />导出核查报告
|
||||
</button>
|
||||
</div>
|
||||
{failed && <p className="mt-2 text-xs leading-5 text-danger">改变未来数据后,部分历史输出发生变化。请结合下方差异排查策略或指标。</p>}
|
||||
{evidence.execution_timing?.status === 'failed' && (
|
||||
<p role="status" className="mt-2 text-xs leading-5 text-danger">信号与成交时间关系异常,共 {evidence.execution_timing.differences} 处。请查看导出报告中的时间核对记录。</p>
|
||||
)}
|
||||
{evidence.persistence.status === 'error' && <p role="status" className="mt-2 text-xs text-warning">版本记录未能保存,请导出本次报告。</p>}
|
||||
{evidence.strategy.changed_during_run && <p role="status" className="mt-2 text-xs text-warning">定义在回测期间发生变化,请重新回测确认。</p>}
|
||||
<details className="mt-2 text-xs text-secondary" open={failed || undefined}>
|
||||
<summary className="cursor-pointer py-1">检查与版本详情 · 完成 {completed}/{lookahead.checks.length} 项</summary>
|
||||
<div className="mt-3 space-y-3">
|
||||
<p className="leading-5">固定参数与预热起点,抽样截断、扰动未来数据并重算历史输出。检查结果仅覆盖本次样本和已加载的数据。</p>
|
||||
<dl className="grid grid-cols-1 gap-x-5 gap-y-2 sm:grid-cols-2">
|
||||
{[
|
||||
['数据版本', evidence.data.sha256], ['策略版本', evidence.strategy.sha256],
|
||||
['参数版本', evidence.config_sha256], ['回测编号', evidence.run_id],
|
||||
].map(([label, value]) => (
|
||||
<div key={label} className="min-w-0"><dt className="text-muted">{label}</dt><dd className="mt-0.5 break-all font-mono text-foreground" title={value}>{value.slice(0, 20)}</dd></div>
|
||||
))}
|
||||
</dl>
|
||||
<p>程序版本 {evidence.runtime.application} · 记录时间 {new Date(evidence.created_at).toLocaleString('zh-CN')}</p>
|
||||
{evidence.execution_timing && (
|
||||
<p>信号与成交时间:{evidence.execution_timing.status === 'passed' ? `已核对 ${evidence.execution_timing.compared_values} 项` : evidence.execution_timing.status === 'failed' ? '存在异常' : '暂无可核对的已完成交易记录'}</p>
|
||||
)}
|
||||
{lookahead.checks.some(check => check.status === 'error') && (
|
||||
<ul className="space-y-1 text-warning">{lookahead.checks.filter(check => check.status === 'error').map((check, i) => <li key={i}>{check.cutoff} · {CHECK_LABELS[check.kind] ?? check.kind} · {check.error}</li>)}</ul>
|
||||
)}
|
||||
{examples.length > 0 && (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full whitespace-nowrap text-left text-xs">
|
||||
<caption className="pb-2 text-left text-muted">差异示例(最多展示 15 条,完整抽样记录见导出报告)</caption>
|
||||
<thead><tr className="border-b border-border text-muted">{['历史时间 / 标的', '字段', '原值 → 重算值', '检查截点'].map(label => <th key={label} className="px-2 py-2 font-medium">{label}</th>)}</tr></thead>
|
||||
<tbody>{examples.map((row, i) => <tr key={i} className="border-b border-border/50"><td className="px-2 py-2">{row.date}<br />{row.symbol}</td><td className="px-2 py-2">{FIELD_LABELS[row.field] ?? row.field}</td><td className="px-2 py-2 font-mono">{String(row.baseline ?? '空')} → {String(row.recomputed ?? '空')}</td><td className="px-2 py-2">{row.cutoff}<br />{CHECK_LABELS[row.kind] ?? row.kind}</td></tr>)}</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
<div className="space-y-1">
|
||||
<p className="font-medium text-foreground">本次数据范围</p>
|
||||
{datasets.slice(0, 20).map(([name, data]) => (
|
||||
<p key={name} className="break-words">{DATA_LABELS[name] ?? (name.startsWith('minute_') ? '分钟数据 ' + name.split('/').slice(1).join(' · ') : name)} · {data.rows.toLocaleString()} 行{data.start && ` · ${data.start} ~ ${data.end}`}</p>
|
||||
))}
|
||||
{datasets.length > 20 && <p className="text-muted">共 {datasets.length} 批数据,完整范围见导出报告。</p>}
|
||||
</div>
|
||||
<p className="leading-5 text-muted">版本记录保存内容指纹与有效参数。它不备份全量行情,也不验证供应商历史修订、历史股票池和静态维表是否符合当时可用的信息。</p>
|
||||
</div>
|
||||
</details>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user