feat(backtest): 增加未来数据检查与数据版本追溯

Signed-off-by: shy3130 <shy3130@qq.com>
This commit is contained in:
shy3130
2026-11-01 17:20:55 +08:00
parent ef6980d996
commit 316ae2954a
15 changed files with 1512 additions and 50 deletions
+14
View File
@@ -468,6 +468,20 @@ def _stamp_daily_data(result, revisions=None) -> dict:
return {**payload, "daily_data_revisions": revisions if revisions is not None else _daily_data_revisions()}
@router.get("/strategy/runs/{run_id}/evidence")
def strategy_run_evidence(run_id: str):
from app.backtest.evidence import load_evidence
try:
return load_evidence(settings.data_dir, run_id)
except FileNotFoundError as exc:
raise HTTPException(404, detail="该回测没有版本记录,旧结果可重新回测生成") from exc
except ValueError as exc:
raise HTTPException(400, detail=str(exc)) from exc
except OSError as exc:
raise HTTPException(503, detail="暂时无法读取回测版本记录") from exc
class _BacktestJob:
"""单个回测任务的状态, 存模块级供重连使用。"""
__slots__ = ("key", "cancel_event", "progress", "result", "error", "done", "finish_ts", "daily_data_revisions")
+61 -24
View File
@@ -387,6 +387,7 @@ class BacktestEngine:
def __init__(self, repo: KlineRepository) -> None:
self.repo = repo
self._cache = PanelCache()
self._evidence = None
# ── 数据加载 ──────────────────────────────────────
@@ -484,14 +485,10 @@ class BacktestEngine:
end: date,
feature_plan,
asset_type: str = "stock",
*,
evidence=None,
) -> pl.DataFrame:
"""按解析后的依赖加载窄基础列并计算回测所需特征。"""
from app.indicators.pipeline import (
compute_indicators,
compute_limit_signals,
compute_signals,
)
df = self.load_panel(
symbols,
start,
@@ -503,7 +500,6 @@ class BacktestEngine:
return df
from app.backtest.fundamentals import (
attach_fundamental_factors,
load_fundamental_snapshot,
)
@@ -511,25 +507,59 @@ class BacktestEngine:
getattr(feature_plan, "fundamental_columns", frozenset())
or frozenset()
)
if fundamental_names:
# 财务因子列不落 enriched 存储, 在加载口按公告日门控并入。
df = attach_fundamental_factors(
df,
load_fundamental_snapshot(
self.repo.store.data_dir if self.repo is not None else None
),
fundamental_names,
)
financials = load_fundamental_snapshot(
self.repo.store.data_dir if self.repo is not None else None,
) if fundamental_names else None
instruments = (
self.repo.get_instruments_asset(asset_type)
if self.repo is not None
else pl.DataFrame()
)
historical_shares = (
self.repo.get_historical_shares()
if asset_type == "stock" and self.repo is not None and not instruments.is_empty()
else None
)
if evidence is not None:
from app.factors.ext_factors import attach_ext_columns
# 扩展时序值也要固定在本次原始快照内;检查期间不重新读取磁盘。
df = attach_ext_columns(df, include_snapshot=False, data_dir=self.repo.store.data_dir)
def prepare(raw):
return self._prepare_panel_features(
raw, feature_plan, instruments, historical_shares, financials,
attach_extensions=evidence is None,
)
if evidence is not None:
evidence.capture_panel(df, prepare, {
"instruments": instruments, "historical_shares": historical_shares,
"financial_announcements": financials,
})
return prepare(df)
@staticmethod
def _prepare_panel_features(
df, feature_plan, instruments, historical_shares, financials, *, attach_extensions=True,
):
from app.backtest.fundamentals import attach_fundamental_factors
from app.indicators.pipeline import (
compute_indicators,
compute_limit_signals,
compute_signals,
)
fundamental_names = sorted(getattr(feature_plan, "fundamental_columns", ()) or ())
if fundamental_names:
df = attach_fundamental_factors(df, financials, fundamental_names)
matrix_native = feature_plan.execution_backend == "matrix_native"
if not matrix_native:
df = compute_indicators(df, needed=set(feature_plan.indicator_columns))
df = compute_signals(df, needed=set(feature_plan.signal_columns))
df = compute_signals(
df, needed=set(feature_plan.signal_columns),
**({"attach_extensions": False} if not attach_extensions else {}),
)
if not instruments.is_empty():
df = compute_limit_signals(
df,
@@ -537,11 +567,7 @@ class BacktestEngine:
needed={"signal_limit_up", "signal_limit_down"}
if matrix_native
else set(feature_plan.signal_columns),
historical_shares=(
self.repo.get_historical_shares()
if asset_type == "stock" and self.repo is not None
else None
),
historical_shares=historical_shares,
)
join_cols = ["symbol"] if "symbol" in instruments.columns else []
join_cols.extend(
@@ -595,6 +621,8 @@ class BacktestEngine:
parquet_root = self.repo.store.data_dir / enriched_dirname(asset_type)
instruments = self.repo.get_instruments_asset(asset_type)
if self._evidence is not None:
self._evidence.record_frame("instruments", instruments)
field_columns = (
set(feature_plan.base_columns)
| set(feature_plan.instrument_columns)
@@ -673,6 +701,7 @@ class BacktestEngine:
market,
self.repo.store.data_dir if self.repo is not None else None,
fundamental_names,
evidence=self._evidence,
)
if ext_wanted:
from app.factors.ext_factors import attach_matrix_ext_fields
@@ -988,7 +1017,9 @@ class BacktestEngine:
dates = {matrix.timestamp_labels[int(t)][:10] for t in trigger_times}
symbols = {matrix.symbols[int(a)] for a in trigger_assets}
if dates and symbols:
loaded = self._load_minute_for_fills(self.repo, list(symbols), dates, config.asset_type)
loaded = self._load_minute_for_fills(
self.repo, list(symbols), dates, config.asset_type, evidence=self._evidence,
)
minute_cache = {key: value for key, value in loaded.items() if value is not None and len(value) > 0}
def _count(key: str) -> None:
@@ -1398,6 +1429,7 @@ class BacktestEngine:
if _trigger_dates and _trigger_symbols:
_loaded = self._load_minute_for_fills(
self.repo, list(_trigger_symbols), _trigger_dates, "stock",
evidence=self._evidence,
)
for _key, _marr in _loaded.items():
if _marr is not None and len(_marr) > 0:
@@ -1848,6 +1880,7 @@ class BacktestEngine:
symbols: list[str],
dates_needed: set,
asset_type: str,
*, evidence=None,
) -> dict:
"""按触发日加载分钟K, 返回 {(symbol, date_str): float64 2D ndarray}。
@@ -1879,6 +1912,8 @@ class BacktestEngine:
continue
if df.is_empty():
continue
if evidence is not None:
evidence.record_frame(f"minute_fill/{asset_type}/{batch[0]}/{batch[-1]}", df)
# 按 (symbol, 日期) 分组, 每组转紧凑 float64 数组存入 cache
df = df.sort(["symbol", "datetime"]).with_columns(
pl.col("datetime").dt.strftime("%Y-%m-%d").alias("_d_str")
@@ -1985,6 +2020,7 @@ class BacktestEngine:
if trigger_dates and trigger_symbols:
loaded = self._load_minute_for_fills(
self.repo, list(trigger_symbols), trigger_dates, config.asset_type,
evidence=self._evidence,
)
minute_cache = {key: value for key, value in loaded.items() if value is not None and len(value) > 0}
@@ -2601,6 +2637,7 @@ class BacktestEngine:
) else "stock"
loaded = self._load_minute_for_fills(
self.repo, list(trigger_symbols), trigger_dates, asset_type,
evidence=self._evidence,
)
for key, marr in loaded.items():
if marr is not None and len(marr) > 0:
+538
View File
@@ -0,0 +1,538 @@
"""回测输入指纹及有限次时间因果检查。只在正式回测 worker 中启用。
检查固定本次输入、参数和预热起点,比较截断/扰动未来后的历史输出;
它不能证明上游供应商数据本身满足 point-in-time,也不替代样本外验证。
"""
from __future__ import annotations
import hashlib
import io
import json
import os
import re
import sys
from collections.abc import Callable
from dataclasses import replace
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import numpy as np
import polars as pl
from app.backtest.matrix import MarketDataMatrix, SignalMatrix, slice_market_data_matrix
_SAMPLES = 3
_EXAMPLES = 5
_RUN_ID = re.compile(r"[0-9a-f]{10,32}\Z")
_MAX_MANIFEST_BYTES = 4 * 1024 * 1024
def json_digest(value: Any) -> str:
return hashlib.sha256(
json.dumps(value, sort_keys=True, ensure_ascii=False, default=str, allow_nan=False).encode()
).hexdigest()
def _array_digest(array: np.ndarray) -> str:
digest = hashlib.sha256()
digest.update(str(array.dtype).encode())
digest.update(str(array.shape).encode())
# memmap 和非连续视图分块处理,避免为整个回测矩阵再复制一份。
for start in range(0, len(array), 256):
block = np.ascontiguousarray(array[start:start + 256])
digest.update(memoryview(block).cast("B"))
return digest.hexdigest()
def fingerprint_matrix(market: MarketDataMatrix) -> dict:
fields = {
name: _array_digest(getattr(market, name))
for name in (
"timestamps", "session_ids", "open", "high", "low", "close", "volume",
"tradable", "limit_up_locked", "limit_down_locked",
)
}
fields.update({f"field:{key}": _array_digest(value) for key, value in sorted(market.fields.items())})
identity = {"timestamps": market.timestamp_labels, "symbols": market.symbols, "names": market.names}
return {
"sha256": json_digest({"identity": identity, "fields": fields}),
"fields": fields,
"rows": market.shape[0] * market.shape[1],
"symbols": len(market.symbols),
"start": market.timestamp_labels[0] if market.timestamp_labels else None,
"end": market.timestamp_labels[-1] if market.timestamp_labels else None,
}
def fingerprint_frame(frame: pl.DataFrame) -> dict:
keys = [name for name in ("symbol", "date", "datetime") if name in frame.columns]
# 公告/股本表同一标的有多行;补齐排序键,避免物理行序变化被误判为内容变化。
frame = frame.sort(keys + sorted(set(frame.columns) - set(keys)))
fields = {}
for name in sorted(frame.columns):
digest = hashlib.sha256(str(frame.schema[name]).encode())
for block in frame.select(name).iter_slices(65536):
buffer = io.BytesIO()
block.rechunk().write_ipc(buffer)
digest.update(buffer.getbuffer())
fields[name] = digest.hexdigest()
time_col = next((col for col in ("datetime", "date") if col in frame.columns), None)
return {
"sha256": json_digest({"rows": frame.height, "fields": fields}),
"fields": fields,
"rows": frame.height,
"symbols": frame["symbol"].n_unique() if "symbol" in frame.columns else None,
"start": str(frame[time_col].min()) if time_col and frame.height else None,
"end": str(frame[time_col].max()) if time_col and frame.height else None,
}
def _cutoffs(values: list) -> list:
if not values:
return []
indices = np.linspace(0, len(values) - 1, min(_SAMPLES, len(values)), dtype=int)
return [values[int(i)] for i in indices]
def _scalar(value):
if isinstance(value, np.generic):
value = value.item()
if isinstance(value, float) and not np.isfinite(value):
return None
return value
def _different(left: np.ndarray, right: np.ndarray) -> np.ndarray:
if left.shape != right.shape:
raise ValueError("重算输出形状与基线不一致")
if left.dtype.kind in "iufb" and right.dtype.kind in "iufb":
return ~np.isclose(left, right, rtol=1e-6, atol=1e-7, equal_nan=True)
return ~((left == right) | ((left == None) & (right == None))) # noqa: E711
def _report(checks: list[dict]) -> dict:
differences = sum(c.get("differences", 0) for c in checks)
status = (
"error" if any(c["kind"] == "repeat" and c["status"] == "error" for c in checks)
else "failed" if differences else "error" if any(c["status"] == "error" for c in checks)
else "passed" if any(c.get("compared_values", 0) > 0 for c in checks) else "insufficient"
)
return {"status": status, "checks": checks, "differences": differences}
def _check(kind: str, cutoff, compare: Callable[[], dict]) -> dict:
try:
return {"kind": kind, "cutoff": str(cutoff), "status": "completed", **compare()}
except Exception as exc: # noqa: BLE001 — 用户策略异常必须转为可见的检查失败。
return {
"kind": kind, "cutoff": str(cutoff), "status": "error",
"error": f"重算未完成({type(exc).__name__})",
"differences": 0, "compared_values": 0, "examples": [],
}
def _perturb_matrix(market: MarketDataMatrix, cutoff: int) -> MarketDataMatrix:
def change(array):
if array.ndim != 2 or array.dtype.kind not in "fiu":
return array
values = array.copy()
scale = 1.5 + np.arange(len(values) - cutoff - 1)[:, None] * 0.03
future = values[cutoff + 1:]
values[cutoff + 1:] = (future * scale + (future == 0)).astype(array.dtype)
values.flags.writeable = False
return values
return replace(
market,
**{name: change(getattr(market, name)) for name in ("open", "high", "low", "close", "volume")},
fields={name: change(value) for name, value in market.fields.items()},
_valid_bars=None,
)
def audit_matrix(
market: MarketDataMatrix,
baseline: SignalMatrix,
evaluate: Callable[[MarketDataMatrix], SignalMatrix],
*, start: str, end: str, cancel_event=None,
) -> dict:
candidates = [
i for i, label in enumerate(market.timestamp_labels[:-1])
if start <= label[:10] <= end
]
checks = []
def compare(kind, cutoff):
label = market.timestamp_labels[cutoff]
if kind == "repeat":
data = market
elif kind == "truncate":
data = slice_market_data_matrix(market, 0, cutoff + 1)
else:
data = _perturb_matrix(market, cutoff)
actual = evaluate(data)
if actual.shape != data.shape:
raise ValueError("重算信号形状不匹配")
rows = np.array([start <= t[:10] <= label[:10] for t in market.timestamp_labels[:cutoff + 1]])
count, compared, examples = 0, 0, []
for name in ("entry", "exit", "score", "entry_signal_code", "exit_signal_code"):
expected = getattr(baseline, name)[:cutoff + 1]
observed = getattr(actual, name)[:cutoff + 1]
diff = _different(expected, observed) & rows[:, None]
count += int(diff.sum())
compared += int(rows.sum()) * market.shape[1]
for t, a in np.argwhere(diff)[:max(0, _EXAMPLES - len(examples))]:
examples.append({
"date": market.timestamp_labels[t], "symbol": market.symbols[a],
"field": name, "baseline": _scalar(expected[t, a]),
"recomputed": _scalar(observed[t, a]), "cutoff": label,
})
return {"differences": count, "compared_values": compared, "examples": examples}
for cutoff in _cutoffs(candidates):
label = market.timestamp_labels[cutoff]
for kind in ("truncate", "perturb"):
if cancel_event is not None and cancel_event.is_set():
return {**_report(checks), "status": "cancelled"}
checks.append(_check(kind, label, lambda kind=kind, cutoff=cutoff: compare(kind, cutoff)))
if any(check["differences"] for check in checks):
# 先排除非确定性/带状态策略,避免把随机变化直接解释为未来数据依赖。
kind = "repeat"
cutoff = max(i for i, label in enumerate(market.timestamp_labels) if start <= label[:10] <= end)
label = market.timestamp_labels[cutoff]
control = _check(kind, label, lambda: compare("repeat", cutoff))
if control["differences"]:
control.update(status="error", error="相同输入重算结果不一致,无法归因于未来数据")
checks.append(control)
return _report(checks)
def _perturb_frame(frame: pl.DataFrame, time_col: str, cutoff) -> pl.DataFrame:
future = pl.col(time_col) > cutoff
return frame.with_columns([
pl.when(future).then(
pl.col(name) * (1.5 + pl.col(time_col).rank("dense") * 0.03) + (pl.col(name) == 0)
)
.otherwise(pl.col(name)).cast(dtype).alias(name)
for name, dtype in frame.schema.items()
if dtype.is_numeric() and name != time_col
])
def _compare_frames(baseline, actual, time_col, start, cutoff):
keys = ["symbol", time_col]
bounds = (pl.col(time_col) >= start) & (pl.col(time_col) <= cutoff)
left, right = baseline.filter(bounds), actual.filter(bounds)
if left.select(keys).is_duplicated().any() or right.select(keys).is_duplicated().any():
raise ValueError("检查输出包含重复时间和标的")
names = [name for name, dtype in left.schema.items() if name not in keys and (dtype.is_numeric() or dtype == pl.Boolean)]
if any(name not in right.columns for name in names):
raise ValueError("重算输出缺少字段")
# full join 能发现消失或新增的历史信号,而不只检查两边都存在的行。
joined = (
left.select(*keys, *names).with_columns(pl.lit(True).alias("_present"))
.join(
right.select(*keys, *names).with_columns(pl.lit(True).alias("_present")),
on=keys, how="full", coalesce=True, suffix="__audit",
).sort(keys)
)
count, examples = 0, []
for name in ["_present", *names]:
expected, observed = joined[name].to_numpy(), joined[f"{name}__audit"].to_numpy()
diff = _different(expected, observed)
count += int(diff.sum())
for row in np.flatnonzero(diff)[:max(0, _EXAMPLES - len(examples))]:
examples.append({
"date": str(joined[time_col][int(row)]), "symbol": joined["symbol"][int(row)],
"field": "signal_presence" if name == "_present" else name,
"baseline": _scalar(expected[row]), "recomputed": _scalar(observed[row]),
"cutoff": str(cutoff),
})
return {"differences": count, "compared_values": joined.height * (len(names) + 1), "examples": examples}
def audit_frame(
raw: pl.DataFrame, baseline: pl.DataFrame, evaluate: Callable[[pl.DataFrame], pl.DataFrame],
*, start, end, time_col: str = "date", cancel_event=None,
) -> dict:
times = raw[time_col].unique().sort().to_list()
if times and isinstance(start, str) and not isinstance(times[0], str):
start = type(times[0]).fromisoformat(start)
end = type(times[0]).fromisoformat(end)
candidates = [t for t in times[:-1] if start <= t <= end]
checks = []
def compare(kind, cutoff):
if kind == "repeat":
variant = raw
elif kind == "truncate":
variant = raw.filter(pl.col(time_col) <= cutoff)
else:
variant = _perturb_frame(raw, time_col, cutoff)
return _compare_frames(baseline, evaluate(variant), time_col, start, cutoff)
for cutoff in _cutoffs(candidates):
for kind in ("truncate", "perturb"):
if cancel_event is not None and cancel_event.is_set():
return {**_report(checks), "status": "cancelled"}
checks.append(_check(kind, cutoff, lambda kind=kind, cutoff=cutoff: compare(kind, cutoff)))
if any(check["differences"] for check in checks):
kind, cutoff = "repeat", end
control = _check(kind, cutoff, lambda: compare("repeat", cutoff))
if control["differences"]:
control.update(status="error", error="相同输入重算结果不一致,无法归因于未来数据")
checks.append(control)
return _report(checks)
def _manifest_path(data_dir: Path, run_id: str) -> Path:
if not _RUN_ID.fullmatch(run_id):
raise ValueError("回测编号无效")
root = Path(data_dir).resolve()
path = root / "backtest_runs" / f"{run_id}.json"
if path.resolve() != path:
raise ValueError("回测记录目录无效")
return path
def save_evidence(data_dir: Path, evidence: dict) -> None:
from app.services.fs_utils import atomic_write_text
path = _manifest_path(data_dir, evidence["run_id"])
path.parent.mkdir(parents=True, exist_ok=True)
content = json.dumps(evidence, ensure_ascii=False, sort_keys=True, allow_nan=False, default=str)
if len(content.encode()) > _MAX_MANIFEST_BYTES:
raise ValueError("回测版本记录超过大小限制")
# 先占用同编号写锁,防止两个 worker 的检查与原子替换之间发生覆盖。
lock_path = path.with_suffix(".lock")
descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
os.close(descriptor)
try:
if path.exists():
raise FileExistsError("回测版本记录已存在")
atomic_write_text(path, content)
finally:
lock_path.unlink(missing_ok=True)
def load_evidence(data_dir: Path, run_id: str) -> dict:
path = _manifest_path(data_dir, run_id)
if path.stat().st_size > _MAX_MANIFEST_BYTES:
raise ValueError("回测版本记录超过大小限制")
value = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(value, dict) or value.get("schema_version") != 1 or value.get("run_id") != run_id:
raise ValueError("回测版本记录无效")
return value
def audit_trade_timing(trades: list[dict], config: dict, backend: str) -> dict:
"""检查已记录的信号日与成交日,不把期末估值记录当成次日信号成交。"""
differences, compared, examples = 0, 0, []
for trade in trades:
for side in ("entry", "exit"):
signal = trade.get(f"{side}_signal_date")
fill = trade.get(f"{side}_date")
if not signal or not fill:
continue
signal, fill = str(signal)[:10], str(fill)[:10]
next_open = config.get(f"{side}_fill") == "open_t+1"
if side == "entry" and backend == "minute_filter":
next_open = False # 分钟策略按触发分钟入场,忽略日线 entry_fill。
if side == "exit" and trade.get("exit_reason") != "signal":
next_open = False
compared += 1
if signal > fill or (next_open and signal == fill):
differences += 1
if len(examples) < _EXAMPLES:
examples.append({
"symbol": trade["symbol"], "side": side,
"signal_date": signal, "fill_date": fill,
})
return {
"status": "failed" if differences else "passed" if compared else "insufficient",
"compared_values": compared, "differences": differences, "examples": examples,
}
def definition_versions(data_dir: Path, strategy) -> dict:
"""只读取策略及定义目录;绝不扫描偏好、密钥或环境变量。"""
files = {}
if strategy.file_path is not None:
files["strategy"] = _file_digest(Path(strategy.file_path))
for directory in ("custom_factors", "custom_signals", "position_policies"):
root = Path(data_dir) / "user_data" / directory
for path in sorted(root.glob("*")):
if path.is_file() and path.suffix in {".json", ".py"}:
files[f"{directory}/{path.name}"] = _file_digest(path)
return {"sha256": json_digest(files), "files": files}
def _file_digest(path: Path) -> str | None:
try:
return hashlib.sha256(path.read_bytes()).hexdigest()
except OSError:
return None
def runtime_version() -> dict:
from app import __version__
root = Path(__file__).resolve().parents[1]
paths = [
"backtest/evidence.py", "backtest/engine.py", "backtest/strategy.py",
"backtest/matrix.py", "backtest/fundamentals.py", "backtest/minute_replay.py",
"strategy/engine.py", "strategy/scoring.py", "indicators/pipeline.py",
]
files = {
name: _file_digest(root / name) # 冻结发行版可能只携带字节码。
for name in paths
}
return {
"application": __version__, "python": sys.version.split()[0],
"polars": pl.__version__, "numpy": np.__version__,
"code_sha256": json_digest(files), "code_files": files,
}
class EvidenceRecorder:
"""一次回测专属的观察器,不进入共享缓存,也不保存全量行情副本。"""
def __init__(self, data_dir: Path, config, strategy, strategy_engine, cancel_event=None):
self.data_dir = data_dir
self.config = config
self.strategy = strategy
self.cancel_event = cancel_event
self.datasets: dict[str, dict] = {}
self.outputs: dict[str, dict] = {}
self.reports: list[dict] = []
self.raw_panel: pl.DataFrame | None = None
self.prepare_panel: Callable | None = None
self.definitions = definition_versions(data_dir, strategy)
self.runtime = runtime_version()
self.children = []
if strategy.composite is not None:
for child in strategy.composite.children:
child_def = strategy_engine.get(child.strategy_id)
self.children.append({
"id": child.strategy_id, "weight": child.weight,
"definitions": definition_versions(data_dir, child_def),
})
def record_frame(self, name: str, frame: pl.DataFrame) -> None:
self.datasets[name] = fingerprint_frame(frame)
def capture_panel(self, raw, prepare, auxiliaries) -> None:
self.raw_panel, self.prepare_panel = raw, prepare
self.record_frame("daily_base", raw)
for name, frame in auxiliaries.items():
if frame is not None:
self.record_frame(name, frame)
def check_matrix(self, market, signals, evaluate) -> None:
self.datasets["daily_matrix"] = fingerprint_matrix(market)
self.reports.append({
"scope": "daily_signals",
**audit_matrix(
market, signals, evaluate, start=str(self.config.start), end=str(self.config.end),
cancel_event=self.cancel_event,
),
})
def check_panel(self, baseline, evaluate) -> None:
self.outputs["daily_features_and_signals"] = fingerprint_frame(baseline)
if self.raw_panel is None or self.prepare_panel is None:
self.reports.append({"scope": "daily_signals", "status": "error", "checks": [], "differences": 0})
return
self.reports.append({
"scope": "daily_features_and_signals",
**audit_frame(
self.raw_panel, baseline, lambda raw: evaluate(self.prepare_panel(raw)),
start=self.config.start, end=self.config.end, cancel_event=self.cancel_event,
),
})
# 检查完成后释放对大面板及准备上下文的引用。
self.raw_panel, self.prepare_panel = None, None
def check_minute(self, history, result, evaluate) -> None:
dtype = history.schema["datetime"]
def decisions(value):
rows = []
for row in value.rows:
timestamp = row.get("last_datetime")
if timestamp is None:
raise ValueError("分钟信号缺少触发时间,无法检查")
if isinstance(timestamp, str):
timestamp = datetime.fromisoformat(timestamp)
rows.append({
"symbol": str(row["symbol"]), "datetime": timestamp,
"close": row.get("close"), "score": value.scores.get(str(row["symbol"]), 0.0),
})
return pl.DataFrame(rows, schema={
"symbol": pl.Utf8, "datetime": dtype, "close": pl.Float64, "score": pl.Float64,
})
try:
report = audit_frame(
history, decisions(result), lambda variant: decisions(evaluate(variant)),
start=history["datetime"].min(), end=history["datetime"].max(),
time_col="datetime", cancel_event=self.cancel_event,
)
except (ValueError, TypeError, pl.exceptions.PolarsError):
report = {"status": "error", "checks": [], "differences": 0}
self.reports.append({"scope": "minute_signals", **report})
@staticmethod
def minute_sample_days(days):
return set(_cutoffs(days))
def finish(self, result, effective_config: dict) -> dict:
checks = [check for report in self.reports for check in report["checks"]]
summary = _report(checks)
if any(r["status"] == "cancelled" for r in self.reports):
summary["status"] = "cancelled"
elif any(r["status"] == "error" for r in self.reports) and summary["status"] != "failed":
summary["status"] = "error"
elif any(r["status"] == "insufficient" for r in self.reports) and summary["status"] == "passed":
summary["status"] = "partial"
definitions_after = definition_versions(self.data_dir, self.strategy)
data_digest = json_digest({key: value["sha256"] for key, value in sorted(self.datasets.items())})
config_digest = json_digest(effective_config)
strategy = {
"id": self.config.strategy_id, "backend": self.strategy.execution_backend,
"meta": self.strategy.meta, "definitions": self.definitions, "children": self.children,
"changed_during_run": definitions_after != self.definitions,
}
strategy["sha256"] = json_digest(strategy)
runtime = self.runtime
return {
"schema_version": 1, "run_id": result.run_id,
"created_at": datetime.now(UTC).isoformat(),
"runtime": runtime,
"config": effective_config, "config_sha256": config_digest,
"strategy": strategy,
"data": {
"sha256": data_digest, "origin": "local_normalized_snapshot",
"asset_type": self.config.asset_type, "daily_price_basis": "qfq",
"datasets": self.datasets,
},
"reproducibility_sha256": json_digest({
"data": data_digest, "strategy": strategy["sha256"], "config": config_digest, "runtime": runtime,
}),
"lookahead": {**summary, "reports": self.reports, "sampled": True},
"execution_timing": audit_trade_timing(result.trades, effective_config, self.strategy.execution_backend),
"outputs": self.outputs,
"result_summary": {
key: _scalar(result.stats.get(key)) for key in (
"total_return", "annual_return", "max_drawdown", "sharpe", "win_rate", "n_trades",
)
},
"limitations": [
"按交易时间抽样检查本次加载的输入、特征和信号,不是全时点无未来数据证明。",
"供应商历史修订、历史股票池及静态维表的时点正确性需另行核实。",
"记录内容指纹和有效参数,不归档全量行情;原始数据被覆盖后不能仅凭指纹还原。",
],
}
+3 -1
View File
@@ -201,7 +201,7 @@ def build_fundamental_matrices(
return result
def attach_matrix_fundamental_fields(market: Any, data_dir: Path | None, names: Any) -> Any:
def attach_matrix_fundamental_fields(market: Any, data_dir: Path | None, names: Any, *, evidence=None) -> Any:
"""把财务因子作为 matrix fields 附加到 (frozen) MarketDataMatrix 副本。"""
import dataclasses
@@ -209,6 +209,8 @@ def attach_matrix_fundamental_fields(market: Any, data_dir: Path | None, names:
if not requested:
return market
snapshot = load_fundamental_snapshot(data_dir)
if evidence is not None and snapshot is not None:
evidence.record_frame("financial_announcements", snapshot)
extra = build_fundamental_matrices(market, snapshot, requested)
if not extra:
return market
+14 -1
View File
@@ -13,7 +13,7 @@ from __future__ import annotations
import time
from collections.abc import Callable
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from datetime import date, timedelta
import polars as pl
@@ -135,6 +135,7 @@ class MinuteSignalReplayer:
symbols: list[str] | None = None,
progress_cb: Callable[[dict], None] | None = None,
cancel_event=None,
evidence=None,
) -> MinuteReplayResult:
t0 = time.perf_counter()
result = MinuteReplayResult()
@@ -158,6 +159,7 @@ class MinuteSignalReplayer:
replay_days = [day for day in panel_dates if start <= day <= end]
result.skipped_days = [day for day in replay_days if day not in minute_day_set]
total = len(minute_days)
audit_days = evidence.minute_sample_days(minute_days) if evidence is not None else set()
# 逐标的的 T-1 原始收盘/复权收盘查表 (涨停价与复权折算用)。
prev_raw_close: dict[str, float] = {}
@@ -179,6 +181,8 @@ class MinuteSignalReplayer:
if history.is_empty():
result.skipped_days.append(day)
continue
if evidence is not None:
evidence.record_frame(f"minute_replay/{day}", history)
# 日线窗口: 截至 T-1 的完成态日K (index of last panel date < day)。
prior = [d for d in panel_dates if d < day]
@@ -223,6 +227,15 @@ class MinuteSignalReplayer:
result.skipped_days.append(day)
continue
if evidence is not None and day in audit_days:
evidence.check_minute(
history, run_result,
lambda variant, context=context: self.strategy_engine.run(
strategy.meta.get("id", ""), replace(context, history=variant),
pool, params, overrides,
),
)
result.replayed_days += 1
result.strategy_matches += len(run_result.rows)
for row in run_result.rows:
+120 -21
View File
@@ -680,6 +680,7 @@ class StrategyBacktestResult:
factor_attribution: dict | None = None
elapsed_ms: float = 0.0
error: str | None = None
evidence: dict | None = None
@dataclass(frozen=True)
@@ -1259,6 +1260,40 @@ class StrategyBacktestService:
sim_encoding=sim_encoding,
)
def run_with_evidence(self, config, progress_cb=None, cancel_event=None) -> StrategyBacktestResult:
"""正式交互回测启用检查和留档;优化器继续复用轻量 run。"""
from app.backtest.evidence import EvidenceRecorder, save_evidence
started = time.perf_counter()
try:
strategy = self.strategy_engine.get(config.strategy_id)
except ValueError:
return self.run(config, progress_cb, cancel_event)
data_dir = self.engine.repo.store.data_dir
recorder = EvidenceRecorder(data_dir, config, strategy, self.strategy_engine, cancel_event)
# 每个正式回测 worker 拥有独立引擎;仅在本次运行内观察成交用分钟数据。
previous = getattr(self.engine, "_evidence", None)
self.engine._evidence = recorder
try:
result = self.run(config, progress_cb, cancel_event, evidence=recorder)
finally:
self.engine._evidence = previous
if result.error == "cancelled":
return result
effective = {
**result.config,
"params": self._normalize_params(config.params or {}, strategy),
}
result.evidence = recorder.finish(result, effective)
try:
# 保存状态是结果的一部分;写入失败时不会宣称记录已经落盘。
result.evidence["persistence"] = {"status": "saved"}
save_evidence(data_dir, result.evidence)
except (OSError, ValueError):
result.evidence["persistence"] = {"status": "error", "error": "版本记录未能保存,请导出本次报告"}
result.elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
return result
def run(
self,
config: StrategyBacktestConfig,
@@ -1267,6 +1302,7 @@ class StrategyBacktestService:
prepared: PreparedMatrixBacktest | None = None,
result_policy: BacktestResultPolicy | None = None,
prepared_panel: PreparedPanelBacktest | None = None,
evidence=None,
) -> StrategyBacktestResult:
t0 = time.perf_counter()
run_id = uuid.uuid4().hex[:10]
@@ -1372,6 +1408,7 @@ class StrategyBacktestService:
result_policy=result_policy,
run_id=run_id,
t0=t0,
evidence=evidence,
)
try:
@@ -1528,6 +1565,7 @@ class StrategyBacktestService:
load_end,
feature_plan,
asset_type=config.asset_type,
**({"evidence": evidence} if evidence is not None else {}),
)
except (ValueError, pl.exceptions.PolarsError) as e:
return _err(f"回测特征准备失败: {e}")
@@ -1594,6 +1632,7 @@ class StrategyBacktestService:
getattr(getattr(self.engine.repo, "store", None), "data_dir", None),
required_start=config.start,
required_end=config.end,
evidence=evidence,
)
except ValueError as e:
return _err(str(e))
@@ -1638,6 +1677,15 @@ class StrategyBacktestService:
except ValueError as e:
return _err(str(e))
if evidence is not None:
evidence.check_matrix(
market_data, signal_matrix,
lambda data: self._generate_composite_signal_matrix(
composite_children_resolved, data, merge_mode, min_confirm,
composite_max_hold, {},
),
)
sim_market_data = slice_market_data_matrix(market_data, start_id, stop_id)
sim_signal_matrix = slice_signal_matrix(signal_matrix, start_id, stop_id)
sim_signal_matrix = apply_time_masks(
@@ -1698,6 +1746,7 @@ class StrategyBacktestService:
getattr(getattr(self.engine.repo, "store", None), "data_dir", None),
required_start=config.start,
required_end=config.end,
evidence=evidence,
)
except ValueError as e:
return _err(str(e))
@@ -1756,6 +1805,14 @@ class StrategyBacktestService:
except (TypeError, ValueError) as e:
return _err(f"矩阵策略信号计算失败: {e}")
if evidence is not None:
evidence.check_matrix(
market_data, signal_matrix,
lambda data: MatrixStrategyPipeline().run(
s.matrix_strategy, data, params, pipeline_config,
),
)
sim_market_data = slice_market_data_matrix(market_data, start_id, stop_id)
sim_signal_matrix = slice_signal_matrix(signal_matrix, start_id, stop_id)
sim_signal_matrix = apply_time_masks(
@@ -1810,31 +1867,25 @@ class StrategyBacktestService:
else:
if panel is None or formal_range is None:
return _err("非矩阵策略不能使用共享基础矩阵")
# basic_filter 只影响买入候选,不能删除持仓估值和卖出所需行情。
basic_mask = pl.Series("_basic", [True] * len(panel), dtype=pl.Boolean)
if basic_filter and basic_filter.get("enabled", True):
expr = StrategyEngine._basic_filter_expr(panel, basic_filter)
if expr is not None:
try:
basic_mask = panel.select(expr.alias("_basic"))["_basic"].fill_null(False).cast(pl.Boolean)
except Exception as e: # noqa: BLE001
logger.warning("basic_filter mask failed: %s", e)
return _err(f"基础过滤计算失败: {e}")
candidate_filter_mask = self._build_candidate_filter_mask(panel, s, params)
candidate_mask = basic_mask & candidate_filter_mask
panel = self._apply_score(panel, s, overrides, universe_mask=candidate_mask, factor_snapshot=factor_snapshot)
try:
panel, candidate_mask, entry_mask, raw_exit_mask = self._evaluate_panel_decisions(
panel, s, params, overrides, basic_filter, entry_signals, exit_signals,
factor_snapshot=factor_snapshot,
)
except ValueError as exc:
return _err(str(exc))
if evidence is not None:
evidence.check_panel(
self._decision_frame(panel, candidate_mask, entry_mask, raw_exit_mask),
lambda data: self._decision_frame(*self._evaluate_panel_decisions(
data, s, params, overrides, basic_filter, entry_signals, exit_signals,
)),
)
formal_candidate_mask = candidate_mask & formal_range
entry_mask = self._build_entry_mask_from_candidate(panel, candidate_mask, s, entry_signals)
entry_mask = entry_mask & formal_range
# 叠加条件: 在裁剪前的完整 panel (ext_*/因子列已在, polars 路径经
# compute_signals 附带 ext 列) 上评估, 只 AND 进 entry — exit 掩码
# 独立构建, 已持仓卖出不受影响 (与 basic_filter 同口径)。
if overlay_conditions:
overlay_mask, overlay_err = self._overlay_panel_mask(panel, overlay_conditions)
if overlay_err is not None:
return _err(overlay_err)
entry_mask = entry_mask & overlay_mask
if config.regime_filter:
date_values = panel.get_column("date").unique().sort().to_list()
date_labels = tuple(str(value)[:10] for value in date_values)
@@ -1845,6 +1896,7 @@ class StrategyBacktestService:
getattr(getattr(self.engine.repo, "store", None), "data_dir", None),
required_start=config.start,
required_end=config.end,
evidence=evidence,
)
except ValueError as e:
return _err(str(e))
@@ -1856,7 +1908,6 @@ class StrategyBacktestService:
regime_row_mask = panel.get_column("date").is_in(allowed_dates).fill_null(False)
formal_candidate_mask = formal_candidate_mask & regime_row_mask
entry_mask = entry_mask & regime_row_mask
raw_exit_mask = self._build_signal_mask(panel, exit_signals, "_exit")
exit_range = self._date_range_mask(panel, config.start, load_end) if config.mode == "full" else formal_range
exit_mask = raw_exit_mask & exit_range
timing_ms["signals_score"] = round((time.perf_counter() - t_signal) * 1000, 1)
@@ -2076,6 +2127,7 @@ class StrategyBacktestService:
result_policy: BacktestResultPolicy,
run_id: str,
t0: float,
evidence=None,
) -> StrategyBacktestResult:
def _err(msg: str) -> StrategyBacktestResult:
return StrategyBacktestResult(
@@ -2117,12 +2169,15 @@ class StrategyBacktestService:
load_end,
feature_plan,
asset_type="stock",
**({"evidence": evidence} if evidence is not None else {}),
)
except (ValueError, OSError, pl.exceptions.PolarsError) as e:
return _err(f"回测特征准备失败: {e}")
timing_ms["load_panel"] = round((time.perf_counter() - t_load) * 1000, 1)
if panel.is_empty():
return _err("无日线数据, 请检查日期范围或先运行盘后管道")
if evidence is not None:
evidence.record_frame("minute_daily_features", panel)
replayer = MinuteSignalReplayer(self.engine, self.strategy_engine)
replay = replayer.replay(
@@ -2135,6 +2190,7 @@ class StrategyBacktestService:
symbols=config.symbols,
progress_cb=progress_cb,
cancel_event=cancel_event,
**({"evidence": evidence} if evidence is not None else {}),
)
timing_ms["minute_replay"] = replay.elapsed_ms
if cancel_event is not None and cancel_event.is_set():
@@ -2516,6 +2572,7 @@ class StrategyBacktestService:
*,
required_start: date | None = None,
required_end: date | None = None,
evidence=None,
) -> np.ndarray | None:
"""构造逐日 T-1 regime mask, 保留历史静态入口兼容调用方。"""
if not regime_filter:
@@ -2531,6 +2588,8 @@ class StrategyBacktestService:
from app.services import regime_builder
regime_df = regime_builder.load_regime_history(data_dir)
if evidence is not None:
evidence.record_frame("regime_history", regime_df)
regime_by_date = {
row["date"]: {
"state": row.get("state", ""),
@@ -2547,6 +2606,40 @@ class StrategyBacktestService:
required_end=required_end,
)
def _evaluate_panel_decisions(
self, panel, strategy, params, overrides, basic_filter, entry_signals, exit_signals,
*, factor_snapshot=None,
):
"""生产回测与时间切片检查共用信号管线,日期/成交延迟在其后应用。"""
basic_mask = pl.Series("_basic", [True] * len(panel), dtype=pl.Boolean)
if basic_filter and basic_filter.get("enabled", True):
expr = StrategyEngine._basic_filter_expr(panel, basic_filter)
if expr is not None:
basic_mask = panel.select(expr.alias("_basic"))["_basic"].fill_null(False).cast(pl.Boolean)
candidate = basic_mask & self._build_candidate_filter_mask(panel, strategy, params)
panel = self._apply_score(
panel, strategy, overrides, universe_mask=candidate, factor_snapshot=factor_snapshot,
)
entry = self._build_entry_mask_from_candidate(panel, candidate, strategy, entry_signals)
overlay = StrategyEngine._effective_overlay(overrides)
if overlay:
mask, error = self._overlay_panel_mask(panel, overlay)
if error is not None:
raise ValueError(error)
entry = entry & mask
exits = self._build_signal_mask(panel, exit_signals, "_exit")
return panel, candidate, entry, exits
@staticmethod
def _decision_frame(panel, candidate, entry, exits):
names = [
name for name, dtype in panel.schema.items()
if name in {"symbol", "date"} or dtype.is_numeric() or dtype == pl.Boolean
]
return panel.select(names).with_columns(
candidate.rename("_audit_candidate"), entry.rename("_audit_entry"), exits.rename("_audit_exit"),
)
def _build_candidate_filter_mask(
self,
panel: pl.DataFrame,
@@ -2741,6 +2834,10 @@ class StrategyBacktestService:
logger.warning("load benchmark %s failed: %s", BENCHMARK_SYMBOL, e)
return []
evidence = getattr(self.engine, "_evidence", None)
if evidence is not None:
evidence.record_frame("benchmark", df)
if df.is_empty() or "close" not in df.columns:
return []
@@ -2918,6 +3015,8 @@ class StrategyBacktestService:
"initial_capital": c.initial_capital,
"position_sizing": c.position_sizing,
"position_policy": c.position_policy,
"position_policy_params": c.position_policy_params,
"asset_type": c.asset_type,
"mode": c.mode,
"holding_days": c.holding_days,
"minute_fill": c.minute_fill,
+1 -1
View File
@@ -235,7 +235,7 @@ def _worker_entry(task: dict[str, Any], event_queue, cancel_event) -> None:
kind = task["kind"]
if kind == "backtest":
config = _decode_backtest_config(task["config"])
result = asdict(service.run(config, _progress, cancel_event))
result = asdict(service.run_with_evidence(config, _progress, cancel_event))
elif kind == "robust_optimize":
from app.backtest.robust_optimizer import RobustOptimizeService
+5 -2
View File
@@ -654,7 +654,9 @@ def _needed_references_limit_prices(needed: set[str]) -> bool:
return True
return False
def compute_signals(df: pl.DataFrame, needed: set[str] | None = None) -> pl.DataFrame:
def compute_signals(
df: pl.DataFrame, needed: set[str] | None = None, *, attach_extensions: bool = True,
) -> pl.DataFrame:
"""注入自定义信号布尔列 (含迁移自内置的 signal_* 定义)。
输入必须包含 compute_indicators() 产出的指标列。
@@ -667,7 +669,8 @@ def compute_signals(df: pl.DataFrame, needed: set[str] | None = None) -> pl.Data
# 都按列存在性解析。历史多日帧仅注入时序模式 —— 快照代表"最新值",
# 历史回看注入会引入未来数据 (CONTRIBUTING §5.3)。
from app.factors import ext_factors
df = ext_factors.attach_ext_columns(df, include_snapshot=False)
if attach_extensions:
df = ext_factors.attach_ext_columns(df, include_snapshot=False)
# 条件引用的注册表因子列先复用评分物化管线补算 (虚拟/自定义/复合均可)。
from app.strategy import custom_signals
exprs = _get_custom_signal_exprs()
+447
View File
@@ -0,0 +1,447 @@
from __future__ import annotations
from datetime import date, timedelta
import numpy as np
import polars as pl
import pytest
from app.backtest.evidence import (
audit_frame,
audit_matrix,
audit_trade_timing,
fingerprint_frame,
load_evidence,
save_evidence,
)
from app.backtest.matrix import build_market_data_matrix, make_signal_matrix, rolling_mean
@pytest.fixture
def service_factory(tmp_path, monkeypatch):
from app.backtest.engine import BacktestEngine
from app.backtest.strategy import StrategyBacktestService
from app.config import settings
from app.strategy.engine import StrategyEngine
from app.tickflow.repository import DataStore, KlineRepository
from tests.signal_seeds import install_pipeline_caches
monkeypatch.setattr(settings, "data_dir", tmp_path)
install_pipeline_caches(monkeypatch)
frame = _panel().with_columns(
pl.col("close").alias("raw_close"), pl.col("high").alias("raw_high"),
pl.col("low").alias("raw_low"), (pl.col("close") * pl.col("volume")).alias("amount"),
pl.lit(1.0).alias("turnover_rate"),
pl.lit(0).alias("consecutive_limit_ups"), pl.lit(0).alias("consecutive_limit_downs"),
)
for part in frame.partition_by("date"):
path = tmp_path / "kline_daily_enriched" / f"date={part['date'][0]}" / "part.parquet"
path.parent.mkdir(parents=True)
part.write_parquet(path)
instruments = tmp_path / "instruments" / "part.parquet"
instruments.parent.mkdir()
pl.DataFrame({
"symbol": ["600000.SH", "000001.SZ"], "name": ["A", "B"],
"float_shares": [1e8, 2e8], "total_shares": [2e8, 4e8],
}).write_parquet(instruments)
strategies = tmp_path / "strategies" / "custom"
strategies.mkdir(parents=True)
store = DataStore(tmp_path)
repo = KlineRepository(store)
def build(backend="matrix_native", leak=False):
if backend == "matrix_native":
source = '''
import numpy as np
from app.backtest.matrix import make_signal_matrix
META = {"id": "audit_test", "name": "audit", "params": [{"id": "offset", "type": "int", "default": 0}], "scoring": {}}
EXECUTION_BACKEND = "matrix_native"
class Strategy:
def required_fields(self):
return frozenset({"open", "high", "low", "close", "volume"})
def required_warmup_bars(self, params):
return 1
def compute_signals(self, market, params):
score = SCORE + params.get("offset", 0)
return make_signal_matrix(market.shape, entry=np.ones(market.shape), score=score)
MATRIX_STRATEGY = Strategy()
'''.replace("SCORE", "np.broadcast_to(market.close[-1], market.shape)" if leak else "market.close")
else:
source = '''
import polars as pl
META = {"id": "audit_test", "name": "audit", "scoring": {}, "params": []}
EXECUTION_BACKEND = "polars_expr"
def filter(df, params):
return EXPR
'''.replace("EXPR", 'pl.col("close") > pl.col("close").mean().over("symbol")' if leak else 'pl.col("close") > 0')
(strategies / "audit_test.py").write_text(source, encoding="utf-8")
return StrategyBacktestService(BacktestEngine(repo), StrategyEngine(strategy_dirs=[strategies]))
yield build
store.db.close()
def _config(**kwargs):
from app.backtest.strategy import StrategyBacktestConfig
return StrategyBacktestConfig(
strategy_id="audit_test", symbols=None, start=date(2024, 1, 4), end=date(2024, 1, 12),
overrides={"basic_filter": {"enabled": False}}, **kwargs,
)
def _panel() -> pl.DataFrame:
return pl.DataFrame([
{
"symbol": symbol,
"date": date(2024, 1, 1) + timedelta(days=i),
"open": float(i + offset), "high": float(i + offset + 1),
"low": float(i + offset - 1), "close": float(i + offset),
"volume": float(100 + i),
}
for symbol, offset in [("600000.SH", 10), ("000001.SZ", 20)]
for i in range(12)
]).sort(["symbol", "date"])
def test_matrix_causal_rolling_passes_without_modifying_inputs():
market = build_market_data_matrix(_panel())
original = market.close.copy()
def evaluate(data):
score = rolling_mean(data.close, 3)
return make_signal_matrix(data.shape, entry=data.close > score, score=np.nan_to_num(score))
report = audit_matrix(market, evaluate(market), evaluate, start="2024-01-04", end="2024-01-12")
assert report["status"] == "passed"
assert {c["kind"] for c in report["checks"]} == {"truncate", "perturb"}
assert len(report["checks"]) == 6
assert all(c["compared_values"] > 0 for c in report["checks"])
np.testing.assert_equal(market.close, original)
@pytest.mark.parametrize("leak", ["next_bar", "global_mean", "last_bar"])
def test_matrix_detects_future_dependency_with_location(leak):
market = build_market_data_matrix(_panel())
def evaluate(data):
if leak == "next_bar":
score = np.roll(data.close, -1, axis=0)
elif leak == "global_mean":
score = np.broadcast_to(np.mean(data.close, axis=0), data.shape)
else:
score = np.broadcast_to(data.close[-1], data.shape)
return make_signal_matrix(data.shape, entry=np.ones(data.shape), score=score)
report = audit_matrix(market, evaluate(market), evaluate, start="2024-01-04", end="2024-01-12")
assert report["status"] == "failed"
assert report["differences"] > 0
example = next(c for c in report["checks"] if c["examples"])["examples"][0]
assert example["field"] == "score"
assert example["symbol"] in market.symbols
assert example["date"] <= example["cutoff"]
def test_frame_recomputes_features_on_prefix_and_perturbed_future():
raw = _panel()
def evaluate(frame):
return frame.with_columns(
pl.col("close").shift(-1).over("symbol").alias("future_price"),
pl.lit(True).alias("entry"),
).select("symbol", "date", "future_price", "entry")
report = audit_frame(raw, evaluate(raw), evaluate, start="2024-01-04", end="2024-01-12")
assert report["status"] == "failed"
assert any(e["field"] == "future_price" for c in report["checks"] for e in c["examples"])
assert raw.equals(_panel())
def test_audit_insufficient_or_failed_recompute_never_passes():
market = build_market_data_matrix(_panel().filter(pl.col("date") == date(2024, 1, 1)))
signals = make_signal_matrix(market.shape, entry=np.ones(market.shape))
report = audit_matrix(market, signals, lambda _: signals, start="2024-01-01", end="2024-01-01")
assert report["status"] == "insufficient"
market = build_market_data_matrix(_panel())
def broken(_):
raise ValueError("bad prefix")
report = audit_matrix(market, make_signal_matrix(market.shape), broken, start="2024-01-04", end="2024-01-12")
assert report["status"] == "error"
assert all(c["status"] == "error" for c in report["checks"])
def test_fingerprint_stable_across_row_and_column_order_but_changes_with_values():
frame = _panel()
first = fingerprint_frame(frame)
assert first["sha256"] == fingerprint_frame(frame.reverse().select(reversed(frame.columns)))["sha256"]
changed = frame.with_columns((pl.col("volume") + 1).alias("volume"))
second = fingerprint_frame(changed)
assert first["sha256"] != second["sha256"]
assert first["fields"]["close"] == second["fields"]["close"]
assert first["fields"]["volume"] != second["fields"]["volume"]
def test_announcement_fingerprint_ignores_physical_order_of_same_symbol_rows():
frame = pl.DataFrame({
"symbol": ["600000.SH", "600000.SH"],
"announce_date": ["2024-01-04", "2024-01-05"], "value": [1.0, 2.0],
})
assert fingerprint_frame(frame)["sha256"] == fingerprint_frame(frame.reverse())["sha256"]
def test_manifest_persists_and_rejects_unsafe_ids_and_overwrites(tmp_path):
evidence = {"schema_version": 1, "run_id": "012345abcd", "data": {"sha256": "a" * 64}}
save_evidence(tmp_path, evidence)
assert load_evidence(tmp_path, "012345abcd") == evidence
with pytest.raises(FileExistsError):
save_evidence(tmp_path, evidence)
with pytest.raises(ValueError):
load_evidence(tmp_path, "../../secrets")
with pytest.raises(FileNotFoundError):
load_evidence(tmp_path, "0000000000")
@pytest.mark.parametrize("backend", ["matrix_native", "polars_expr"])
@pytest.mark.parametrize("leak", [False, True])
def test_real_service_checks_and_persists_exact_result_inputs(service_factory, tmp_path, backend, leak):
service = service_factory(backend, leak)
result = service.run_with_evidence(_config())
assert result.error is None
assert result.evidence["lookahead"]["status"] == ("failed" if leak else "passed")
assert result.evidence["persistence"]["status"] == "saved"
assert load_evidence(tmp_path, result.run_id) == result.evidence
assert result.evidence["data"]["datasets"]
assert result.evidence["config"]["asset_type"] == "stock"
assert service.engine._evidence is None
assert result.evidence["execution_timing"]["status"] == "passed"
def test_versions_separate_data_parameters_and_strategy_changes(service_factory, tmp_path):
service = service_factory()
first = service.run_with_evidence(_config()).evidence
same = service.run_with_evidence(_config()).evidence
assert first["reproducibility_sha256"] == same["reproducibility_sha256"]
params = service.run_with_evidence(_config(params={"offset": 2})).evidence
assert params["data"]["sha256"] == first["data"]["sha256"]
assert params["config_sha256"] != first["config_sha256"]
assert params["strategy"]["sha256"] == first["strategy"]["sha256"]
from app.enriched_generation import bump_enriched_generation
path = tmp_path / "kline_daily_enriched" / "date=2024-01-08" / "part.parquet"
pl.read_parquet(path).with_columns((pl.col("volume") + 5).alias("volume")).write_parquet(path)
bump_enriched_generation(tmp_path, "stock")
data = service.run_with_evidence(_config()).evidence
assert data["data"]["sha256"] != first["data"]["sha256"]
assert data["config_sha256"] == first["config_sha256"]
assert data["strategy"]["sha256"] == first["strategy"]["sha256"]
assert load_evidence(tmp_path, first["run_id"]) == first
changed_service = service_factory(leak=True)
strategy = changed_service.run_with_evidence(_config()).evidence
assert strategy["data"]["sha256"] == data["data"]["sha256"]
assert strategy["strategy"]["sha256"] != data["strategy"]["sha256"]
def test_unwritable_manifest_is_visible_and_normal_trials_do_not_audit(service_factory, monkeypatch):
service = service_factory()
assert service.run(_config()).evidence is None
def denied(*_args):
raise PermissionError("synthetic")
monkeypatch.setattr("app.backtest.evidence.save_evidence", denied)
result = service.run_with_evidence(_config())
assert result.error is None
assert result.evidence["persistence"]["status"] == "error"
assert service.engine._evidence is None
def test_nondeterministic_strategy_is_inconclusive_not_a_future_leak_verdict():
market = build_market_data_matrix(_panel())
calls = 0
def evaluate(data):
nonlocal calls
calls += 1
return make_signal_matrix(data.shape, score=np.full(data.shape, calls))
result = audit_matrix(market, evaluate(market), evaluate, start="2024-01-04", end="2024-01-12")
assert result["status"] == "error"
assert result["checks"][-1]["kind"] == "repeat"
def test_evidence_api_reads_saved_manifest_and_handles_missing_invalid_and_corrupt(tmp_path, monkeypatch):
from fastapi import FastAPI
from fastapi.testclient import TestClient
from app.api.backtest import router
from app.config import settings
monkeypatch.setattr(settings, "data_dir", tmp_path)
app = FastAPI()
app.include_router(router)
client = TestClient(app)
evidence = {"schema_version": 1, "run_id": "012345abcd"}
save_evidence(tmp_path, evidence)
assert client.get("/api/backtest/strategy/runs/012345abcd/evidence").json() == evidence
assert client.get("/api/backtest/strategy/runs/0000000000/evidence").status_code == 404
assert client.get("/api/backtest/strategy/runs/invalid/evidence").status_code == 400
(tmp_path / "backtest_runs" / "012345abcd.json").write_text("{", encoding="utf-8")
assert client.get("/api/backtest/strategy/runs/012345abcd/evidence").status_code == 400
def test_real_post_api_returns_same_persisted_report_as_worker(service_factory, tmp_path):
from fastapi import FastAPI
from fastapi.testclient import TestClient
from app.api.backtest import router
service = service_factory()
app = FastAPI()
app.state.repo = service.engine.repo
app.state.strategy_engine = service.strategy_engine
app.include_router(router)
client = TestClient(app)
response = client.post("/api/backtest/strategy/run", json={
"strategy_id": "audit_test", "start": "2024-01-04", "end": "2024-01-12",
"overrides": {"basic_filter": {"enabled": False}},
})
assert response.status_code == 200
result = response.json()
assert result["error"] is None
assert result["evidence"]["lookahead"]["status"] == "passed"
assert load_evidence(tmp_path, result["run_id"]) == result["evidence"]
assert client.get(f"/api/backtest/strategy/runs/{result['run_id']}/evidence").json() == result["evidence"]
def test_panel_audit_freezes_external_timeseries_values(service_factory, monkeypatch):
from app.factors import ext_factors
service = service_factory("polars_expr")
calls = 0
def attach(frame, **_kwargs):
nonlocal calls
calls += 1
return frame.with_columns(pl.lit(float(calls)).alias("ext_test_value"))
monkeypatch.setattr(ext_factors, "attach_ext_columns", attach)
result = service.run_with_evidence(_config())
assert result.error is None
assert result.evidence["lookahead"]["status"] == "passed"
assert calls == 1
assert "ext_test_value" in result.evidence["data"]["datasets"]["daily_base"]["fields"]
def test_report_path_rejects_symlink_even_inside_data_directory(tmp_path):
target = tmp_path / "other"
target.mkdir()
(tmp_path / "backtest_runs").symlink_to(target, target_is_directory=True)
with pytest.raises(ValueError, match="目录无效"):
save_evidence(tmp_path, {"schema_version": 1, "run_id": "012345abcd"})
def test_concurrent_manifest_writes_never_replace_an_existing_version(tmp_path):
from concurrent.futures import ThreadPoolExecutor
def write(value):
try:
save_evidence(tmp_path, {"schema_version": 1, "run_id": "012345abcd", "value": value})
return value
except FileExistsError:
return None
with ThreadPoolExecutor(max_workers=4) as pool:
written = [value for value in pool.map(write, range(8)) if value is not None]
assert len(written) == 1
assert load_evidence(tmp_path, "012345abcd")["value"] == written[0]
def test_trade_timing_checks_future_dates_and_next_open_but_respects_minute_entries():
trade = {"symbol": "600000.SH", "entry_signal_date": "2024-01-05", "entry_date": "2024-01-04"}
assert audit_trade_timing([trade], {}, "matrix_native")["status"] == "failed"
trade["entry_date"] = "2024-01-05"
assert audit_trade_timing([trade], {"entry_fill": "open_t+1"}, "matrix_native")["status"] == "failed"
assert audit_trade_timing([trade], {"entry_fill": "close_t"}, "matrix_native")["status"] == "passed"
assert audit_trade_timing([trade], {"entry_fill": "open_t+1"}, "minute_filter")["status"] == "passed"
trade.update(exit_signal_date="2024-01-08", exit_date="2024-01-08", exit_reason="signal")
assert audit_trade_timing([trade], {"exit_fill": "open_t+1"}, "matrix_native")["status"] == "failed"
trade["exit_reason"] = "forced_close"
assert audit_trade_timing([trade], {"exit_fill": "open_t+1"}, "matrix_native")["status"] == "passed"
def test_unknown_strategy_keeps_legacy_error_contract(service_factory):
from dataclasses import replace
result = service_factory().run_with_evidence(replace(_config(), strategy_id="missing"))
assert result.error
assert result.evidence is None
def test_empty_minute_signal_comparisons_are_insufficient_instead_of_passed():
raw = _panel()
empty = raw.head(0)
result = audit_frame(raw, empty, lambda _: empty, start=date(2024, 1, 4), end=date(2024, 1, 12))
assert result["status"] == "insufficient"
def test_perturbation_also_exercises_zero_valued_future_fields():
market = build_market_data_matrix(_panel().with_columns(pl.lit(0.0).alias("volume")))
def evaluate(data):
return make_signal_matrix(data.shape, score=np.broadcast_to(data.volume.mean(axis=0), data.shape))
result = audit_matrix(market, evaluate(market), evaluate, start="2024-01-04", end="2024-01-12")
assert result["status"] == "failed"
assert any(check["kind"] == "perturb" and check["differences"] > 0 for check in result["checks"])
def test_cancelled_checks_are_not_reported_as_passed():
import threading
event = threading.Event()
event.set()
market = build_market_data_matrix(_panel())
result = audit_matrix(market, make_signal_matrix(market.shape), lambda _: None,
start="2024-01-04", end="2024-01-12", cancel_event=event)
assert result["status"] == "cancelled"
@pytest.mark.parametrize("leak", [False, True])
def test_minute_replay_checks_intraday_future_and_tracks_minute_inputs(tmp_path, monkeypatch, leak):
from types import SimpleNamespace
from app.config import settings
from tests.backtest.test_minute_backtest import (
TEST_STRATEGY_SOURCE,
_daily_panel,
_make_service,
_minute_frame,
_trading_days,
)
from tests.backtest.test_minute_backtest import (
_config as minute_config,
)
monkeypatch.setattr(settings, "data_dir", tmp_path)
days = _trading_days(10)
frame = _daily_panel(days, ["600000.SH"])
minutes = {
day: _minute_frame(day, [("600000.SH", f"09:{31 + i}", 10.7 + i * 0.01) for i in range(6)])
for day in days[5:9]
}
service = _make_service(tmp_path, frame, minutes)
service.engine.repo.store = SimpleNamespace(data_dir=tmp_path)
if leak:
source = TEST_STRATEGY_SOURCE.replace(
'joined = df.join(prev, on="symbol", how="inner")',
'df = df.with_columns(pl.col("close").shift(-1).over("symbol").alias("close"))\n joined = df.join(prev, on="symbol", how="inner")',
)
(tmp_path / "strategies" / "test_minute_ping.py").write_text(source, encoding="utf-8")
service.strategy_engine.reload()
result = service.run_with_evidence(minute_config(days[5], days[8]))
assert result.error is None
assert result.evidence["lookahead"]["status"] == ("failed" if leak else "passed")
assert len([key for key in result.evidence["data"]["datasets"] if key.startswith("minute_replay/")]) == 4
@@ -161,6 +161,10 @@ def test_spawn_worker_returns_compact_result_and_memory_metrics(tmp_path):
assert result["error"] is None
assert result["stats"]["execution_backend"] == "matrix_native"
from app.backtest.evidence import load_evidence
assert result["evidence"]["lookahead"]["status"] == "passed"
assert load_evidence(data_dir, result["run_id"]) == result["evidence"]
worker = result["stats"]["worker"]
assert worker["peak_rss_bytes"] > 0
assert worker["serialized_result_bytes"] > 0
+45
View File
@@ -0,0 +1,45 @@
# 回测未来数据检查与版本记录
从本功能起,通过策略回测页面、`POST /api/backtest/strategy/run` 或 SSE 发起的正式回测,会自动生成核查报告。日线表达式、历史 Python 策略、矩阵策略、叠加策略及分钟策略复用各自的生产信号管线;优化器的每个参数试算保持轻量,需将选中的参数载入正式回测生成报告。
## 时间检查
- 固定本次数据快照、策略参数、标的集合和预热起点,最多选择 3 个实际交易时点;分钟策略选择最多 3 个交易日,每日最多 3 个分钟截点。
- 分别删除截点后的行情、扰动截点后的数值,重算后比较截点及之前的历史特征、候选、买卖信号和评分。矩阵策略检查其公开的信号输出;分钟策略检查触发时间、触发价及评分。
- 若发现变化,再用完全相同的输入复算基线,区分未来数据依赖与非确定性/带状态策略。
- 浮点比较容差为 `rtol=1e-6, atol=1e-7`,空值与空值相等。报告记录差异数量,并为每次重算保留最多 5 条日期、标的、字段和前后数值示例。
- 状态分别为抽样未发现差异、检测到变化、数据不足、检查错误、部分未完成和取消。缺字段、重算异常、无可切分时间点都不会记为通过。发现变化只标记结果供排查,不擅自修改用户策略。
日线表达式路径从基础数据重新计算指标与信号,不是切割已经算好的信号。公告日门控、交易日、股票 T+1、费用、滑点和涨跌停成交约束继续由原有管线执行。
同时单独核对已完成交易的信号日和成交日:信号不能晚于成交,日线的「次日开盘」信号成交必须晚于信号日;分钟触发入场及期末估值按原有语义处理。时间核对不修改成交结果,异常在页面和报告中单独标出。扩展时序数据在准备时固定为快照,检查重算不会再次读取更新后的扩展数据。
这不是全时点无泄漏证明。检查不能证明供应商数据、复权历史修订、历史股票池、静态维表及外部 Python 导入满足 point-in-time;未触发的逻辑也可能未被覆盖。应结合公告时点验证、滚动样本外验证及模拟盘结果判断。
## 版本追溯
报告分别记录:
- 本次实际加载的本地标准化数据内容 SHA-256、字段指纹、行数和时间范围,区分股票/ETF。包括预热和全量模式退出尾部;分钟回放和分钟精确成交按实际读取批次记录。
- 标的信息、历史股本、财务公告、市场环境和同期基准等实际使用的辅助数据。
- 策略文件、叠加子策略及本地信号/因子/仓位定义目录的内容指纹。定义目录指纹覆盖整个注册目录,未使用定义的修改也会改变该部分指纹。
- 包含默认参数的有效运行配置、独立参数指纹、应用/核心代码/Python/Polars/NumPy 版本及组合指纹。冻结发行版缺少源码文件时,对应代码文件指纹为 `null`,应用版本仍保留。
- 检查报告、核心回测统计及策略定义在运行期间是否发生变化。
数据来源标注为本次读取的本地标准化快照,**不把当前配置的数据源冒充为历史入库来源**。不读取密钥、环境变量或完整偏好配置。
每次正式回测的报告保存在运行数据目录的 `backtest_runs/<run_id>.json`。按回测编号读取:
```text
GET /api/backtest/strategy/runs/{run_id}/evidence
```
策略回测结果的 `evidence` 字段包含同一份报告,页面可展开查看及导出 JSON;CSV 概要携带回测编号及三个版本指纹。保存失败会明确标记,仍可导出内存中的报告。旧结果缺少 `evidence` 时正常显示,重新回测后才生成记录。
**内容指纹不是全量行情备份。** 它用于确认两次运行的数据、策略或参数是否变化、定位变化字段。原始行情被覆盖后,不能仅凭指纹恢复旧数据或自动重放旧回测。
## 缓存与兼容边界
版本与检查仅在独立回测 worker 内收集。检查重用本次不可变输入,变体不写行情仓库、Parquet、共享矩阵缓存或策略结果缓存;报告不进入缓存键。SSE 重连复用原结果时携带原报告,不重新盖章。原有日K修订失效检查保持生效。
未修改行情存储 schema、数据源协议或历史配置。核查报告按版本 1 独立落盘,读写拒绝非法回测编号、符号链接越界和损坏记录;原有用户数据不迁移、不覆盖。
+54
View File
@@ -1893,6 +1893,56 @@ export interface StrategyBacktestTrade {
exit_signal_id?: string | null
}
export interface BacktestEvidenceCheck {
kind: string
cutoff: string
status: 'completed' | 'error'
differences: number
compared_values: number
error?: string
examples: {
date: string; symbol: string; field: string; cutoff: string
baseline: number | string | boolean | null
recomputed: number | string | boolean | null
}[]
}
export interface BacktestEvidence {
schema_version: number
run_id: string
created_at: string
runtime: { application: string; python: string; polars: string; numpy: string; code_sha256: string }
config: Record<string, unknown>
config_sha256: string
strategy: { id: string; backend: string; sha256: string; changed_during_run: boolean }
data: {
sha256: string
origin: string
asset_type: string
daily_price_basis: string
datasets: Record<string, {
sha256: string; fields: Record<string, string>; rows: number; symbols: number | null
start: string | null; end: string | null
}>
}
reproducibility_sha256: string
lookahead: {
status: 'passed' | 'failed' | 'partial' | 'error' | 'insufficient' | 'cancelled'
sampled: boolean
differences: number
checks: BacktestEvidenceCheck[]
reports: { scope: string; status: string; checks: BacktestEvidenceCheck[]; differences: number }[]
}
persistence: { status: 'saved' | 'error'; error?: string }
execution_timing?: {
status: 'passed' | 'failed' | 'insufficient'
compared_values: number
differences: number
examples: { symbol: string; side: string; signal_date: string; fill_date: string }[]
}
limitations: string[]
}
export interface StrategyBacktestResult {
run_id: string
config: Record<string, any>
@@ -1936,6 +1986,7 @@ export interface StrategyBacktestResult {
}
elapsed_ms: number
error: string | null
evidence?: BacktestEvidence | null
}
// ===== Settings =====
@@ -3744,6 +3795,9 @@ export const api = {
method: 'DELETE',
}),
strategyBacktestEvidence: (runId: string) =>
request<BacktestEvidence>(`/api/backtest/strategy/runs/${encodeURIComponent(runId)}/evidence`),
strategyBacktestRun: (payload: {
strategy_id: string
symbols?: string[] | null
@@ -42,6 +42,7 @@ import { ConditionEditor } from '@/components/signals/ConditionEditor'
import { WatchlistGroupMenu } from '@/components/WatchlistAddMenu'
import { ScoringEditor } from '@/components/ScoringEditor'
import { strategyResultCandidate } from './researchCandidates'
import { BacktestEvidencePanel } from './components/BacktestEvidencePanel'
import { buildDailyNavItems, buildTradeKeyMap, buildTradesNavItems, type TradeNavItem } from './tradeNav'
const formatDate = (date: Date) => date.toISOString().slice(0, 10)
@@ -1345,6 +1346,13 @@ export function StrategyBacktest({ loadCandidate, onLoadConsumed }: {
lines.push(`策略名称,${name}`)
if (result.strategy_info?.id) lines.push(`策略ID,${result.strategy_info.id}`)
lines.push(`回测区间,${start} ~ ${end}`)
if (result.evidence) {
lines.push(`回测编号,${result.run_id}`)
lines.push(`数据版本,${result.evidence.data.sha256}`)
lines.push(`策略版本,${result.evidence.strategy.sha256}`)
lines.push(`参数版本,${result.evidence.config_sha256}`)
lines.push(`时间检查状态,${result.evidence.lookahead.status}`)
}
lines.push(`净值曲线天数,${result.equity_curve?.length ?? 0}`)
lines.push(`完成交易数,${result.trades?.length ?? 0}`)
lines.push(`总收益,${pct(strategyReturn)}`)
@@ -2350,6 +2358,8 @@ export function StrategyBacktest({ loadCandidate, onLoadConsumed }: {
</motion.div>
)}
{result && <BacktestEvidencePanel key={result.run_id} evidence={result.evidence} />}
{/* 旧全量模拟结果: 固定前瞻收益统计 (兼容历史缓存结果) */}
{result && !result.error && result.stats && result.stats.mode === 'full' && result.stats.full_kind !== 'candidate_execution' && (
<motion.div
@@ -0,0 +1,97 @@
// @vitest-environment jsdom
import { act } from 'react'
import { createRoot, type Root } from 'react-dom/client'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import type { BacktestEvidence } from '@/lib/api'
import { BacktestEvidencePanel } from './BacktestEvidencePanel'
let host: HTMLDivElement
let root: Root
const originalCreateUrl = Object.getOwnPropertyDescriptor(URL, 'createObjectURL')
const originalRevokeUrl = Object.getOwnPropertyDescriptor(URL, 'revokeObjectURL')
beforeEach(() => {
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true })
host = document.createElement('div')
document.body.append(host)
root = createRoot(host)
})
afterEach(async () => {
await act(async () => root.unmount())
host.remove()
vi.restoreAllMocks()
for (const [key, descriptor] of [['createObjectURL', originalCreateUrl], ['revokeObjectURL', originalRevokeUrl]] as const) {
if (descriptor) Object.defineProperty(URL, key, descriptor)
else Reflect.deleteProperty(URL, key)
}
})
function evidence(status: BacktestEvidence['lookahead']['status'] = 'passed'): BacktestEvidence {
return {
schema_version: 1, run_id: '012345abcd', created_at: '2026-11-01T02:00:00Z',
runtime: { application: '0.3.25', python: '3.11', polars: '1', numpy: '2', code_sha256: 'e'.repeat(64) },
config: {}, config_sha256: 'c'.repeat(64),
strategy: { id: 'test', backend: 'matrix_native', sha256: 'b'.repeat(64), changed_during_run: false },
data: { sha256: 'a'.repeat(64), asset_type: 'stock', origin: 'local_normalized_snapshot', daily_price_basis: 'qfq', datasets: {} },
reproducibility_sha256: 'd'.repeat(64),
lookahead: { status, sampled: true, differences: 0, checks: [], reports: [] },
persistence: { status: 'saved' }, limitations: [],
}
}
it('历史结果缺少记录时明确提示重新回测,不展示检查通过', async () => {
await act(async () => root.render(<BacktestEvidencePanel />))
expect(host.textContent).toContain('重新回测后自动生成')
expect(host.textContent).not.toContain('抽样未发现差异')
})
it('通过状态注明抽样范围,展示独立版本并导出完整核查报告', async () => {
const value = evidence()
const createUrl = vi.fn((_blob: Blob) => 'blob:test')
const revoke = vi.fn()
Object.defineProperty(URL, 'createObjectURL', { value: createUrl, configurable: true })
Object.defineProperty(URL, 'revokeObjectURL', { value: revoke, configurable: true })
let filename = ''
vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function (this: HTMLAnchorElement) { filename = this.download })
await act(async () => root.render(<BacktestEvidencePanel evidence={value} />))
expect(host.textContent).toContain('抽样未发现差异')
expect(host.textContent).not.toContain('保证没有未来数据')
expect(host.textContent).toContain('数据版本')
expect(host.textContent).toContain('a'.repeat(20))
expect(host.textContent).toContain('参数版本')
expect(host.querySelector('details')?.open).toBe(false)
await act(async () => host.querySelector('button')!.click())
expect(filename).toBe('backtest-evidence-012345abcd.json')
expect(createUrl).toHaveBeenCalledOnce()
expect(createUrl.mock.calls[0][0]).toBeInstanceOf(Blob)
expect(revoke).toHaveBeenCalledWith('blob:test')
})
it('发现差异时自动展开具体时间、标的、字段与重算值', async () => {
const value = evidence('failed')
value.lookahead.differences = 1
value.lookahead.checks = [{
kind: 'truncate', cutoff: '2024-01-05', status: 'completed', compared_values: 10, differences: 1,
examples: [{ date: '2024-01-05', symbol: '600000.SH', field: 'score', cutoff: '2024-01-05', baseline: 30, recomputed: 10 }],
}]
await act(async () => root.render(<BacktestEvidencePanel evidence={value} />))
expect(host.querySelector('details')?.open).toBe(true)
expect(host.textContent).toContain('检测到历史输出变化')
expect(host.textContent).toContain('600000.SH')
expect(host.textContent).toContain('评分')
expect(host.textContent).toContain('30 → 10')
})
it('检查错误与保存失败保持可见,并允许导出当前内存中的报告', async () => {
const value = evidence('error')
value.execution_timing = { status: 'failed', compared_values: 1, differences: 1, examples: [] }
value.persistence = { status: 'error' }
value.lookahead.checks = [{ kind: 'truncate', cutoff: '2024-01-05', status: 'error', differences: 0, compared_values: 0, examples: [], error: '重算未完成(ValueError)' }]
await act(async () => root.render(<BacktestEvidencePanel evidence={value} />))
expect(host.textContent).toContain('检查未完成')
expect(host.textContent).toContain('版本记录未能保存')
expect(host.textContent).toContain('信号与成交时间关系异常')
expect(host.textContent).toContain('重算未完成(ValueError)')
expect(host.querySelector('button')?.disabled).toBe(false)
})
@@ -0,0 +1,99 @@
import { Download, ScanSearch } from 'lucide-react'
import type { BacktestEvidence } from '@/lib/api'
const STATUS_LABELS: Record<BacktestEvidence['lookahead']['status'], string> = {
passed: '抽样未发现差异', failed: '检测到历史输出变化', partial: '部分检查未完成',
error: '检查未完成', insufficient: '数据不足以检查', cancelled: '检查已取消',
}
const DATA_LABELS: Record<string, string> = {
daily_base: '日线基础数据', daily_matrix: '日线行情与输入字段',
instruments: '标的信息', historical_shares: '历史股本',
financial_announcements: '财务公告', regime_history: '市场环境', benchmark: '同期基准',
}
const FIELD_LABELS: Record<string, string> = {
entry: '买入信号', exit: '卖出信号', score: '评分',
_audit_candidate: '候选标的', _audit_entry: '买入信号', _audit_exit: '卖出信号',
signal_presence: '信号出现时间', entry_signal_code: '买入触发器', exit_signal_code: '卖出触发器',
}
const CHECK_LABELS: Record<string, string> = { truncate: '截断未来', perturb: '扰动未来', repeat: '基线复算' }
function exportEvidence(evidence: BacktestEvidence) {
const blob = new Blob([JSON.stringify(evidence, null, 2)], { type: 'application/json;charset=utf-8' })
const url = URL.createObjectURL(blob)
const anchor = document.createElement('a')
anchor.href = url
anchor.download = `backtest-evidence-${evidence.run_id}.json`
anchor.click()
URL.revokeObjectURL(url)
}
export function BacktestEvidencePanel({ evidence }: { evidence?: BacktestEvidence | null }) {
if (!evidence) {
return <p className="text-xs text-muted">此结果没有检查与版本记录,重新回测后自动生成。</p>
}
const { lookahead } = evidence
const completed = lookahead.checks.filter(check => check.status === 'completed').length
const examples = lookahead.checks.flatMap(check => check.examples.map(example => ({ ...example, kind: check.kind }))).slice(0, 15)
const failed = lookahead.status === 'failed'
const datasets = Object.entries(evidence.data.datasets)
const color = failed ? 'text-danger' : lookahead.status === 'passed' ? 'text-foreground' : 'text-warning'
return (
<section aria-label="回测检查与版本记录" className="min-w-0 rounded-card border border-border bg-surface px-4 py-3">
<div className="flex flex-wrap items-center gap-x-3 gap-y-2">
<div className="flex min-w-0 flex-1 flex-wrap items-center gap-2 text-xs">
<ScanSearch className="h-4 w-4 shrink-0 text-secondary" aria-hidden="true" />
<span className="font-medium text-foreground">未来数据检查</span>
<span className={color}>{STATUS_LABELS[lookahead.status]}</span>
</div>
<button type="button" onClick={() => exportEvidence(evidence)} className="inline-flex items-center gap-1 rounded-btn border border-border px-2 py-1 text-xs text-secondary hover:text-accent">
<Download className="h-3 w-3" aria-hidden="true" />导出核查报告
</button>
</div>
{failed && <p className="mt-2 text-xs leading-5 text-danger">改变未来数据后,部分历史输出发生变化。请结合下方差异排查策略或指标。</p>}
{evidence.execution_timing?.status === 'failed' && (
<p role="status" className="mt-2 text-xs leading-5 text-danger">信号与成交时间关系异常,共 {evidence.execution_timing.differences} 处。请查看导出报告中的时间核对记录。</p>
)}
{evidence.persistence.status === 'error' && <p role="status" className="mt-2 text-xs text-warning">版本记录未能保存,请导出本次报告。</p>}
{evidence.strategy.changed_during_run && <p role="status" className="mt-2 text-xs text-warning">定义在回测期间发生变化,请重新回测确认。</p>}
<details className="mt-2 text-xs text-secondary" open={failed || undefined}>
<summary className="cursor-pointer py-1">检查与版本详情 · 完成 {completed}/{lookahead.checks.length} 项</summary>
<div className="mt-3 space-y-3">
<p className="leading-5">固定参数与预热起点,抽样截断、扰动未来数据并重算历史输出。检查结果仅覆盖本次样本和已加载的数据。</p>
<dl className="grid grid-cols-1 gap-x-5 gap-y-2 sm:grid-cols-2">
{[
['数据版本', evidence.data.sha256], ['策略版本', evidence.strategy.sha256],
['参数版本', evidence.config_sha256], ['回测编号', evidence.run_id],
].map(([label, value]) => (
<div key={label} className="min-w-0"><dt className="text-muted">{label}</dt><dd className="mt-0.5 break-all font-mono text-foreground" title={value}>{value.slice(0, 20)}</dd></div>
))}
</dl>
<p>程序版本 {evidence.runtime.application} · 记录时间 {new Date(evidence.created_at).toLocaleString('zh-CN')}</p>
{evidence.execution_timing && (
<p>信号与成交时间:{evidence.execution_timing.status === 'passed' ? `已核对 ${evidence.execution_timing.compared_values} 项` : evidence.execution_timing.status === 'failed' ? '存在异常' : '暂无可核对的已完成交易记录'}</p>
)}
{lookahead.checks.some(check => check.status === 'error') && (
<ul className="space-y-1 text-warning">{lookahead.checks.filter(check => check.status === 'error').map((check, i) => <li key={i}>{check.cutoff} · {CHECK_LABELS[check.kind] ?? check.kind} · {check.error}</li>)}</ul>
)}
{examples.length > 0 && (
<div className="overflow-x-auto">
<table className="w-full whitespace-nowrap text-left text-xs">
<caption className="pb-2 text-left text-muted">差异示例(最多展示 15 条,完整抽样记录见导出报告)</caption>
<thead><tr className="border-b border-border text-muted">{['历史时间 / 标的', '字段', '原值 → 重算值', '检查截点'].map(label => <th key={label} className="px-2 py-2 font-medium">{label}</th>)}</tr></thead>
<tbody>{examples.map((row, i) => <tr key={i} className="border-b border-border/50"><td className="px-2 py-2">{row.date}<br />{row.symbol}</td><td className="px-2 py-2">{FIELD_LABELS[row.field] ?? row.field}</td><td className="px-2 py-2 font-mono">{String(row.baseline ?? '空')} → {String(row.recomputed ?? '空')}</td><td className="px-2 py-2">{row.cutoff}<br />{CHECK_LABELS[row.kind] ?? row.kind}</td></tr>)}</tbody>
</table>
</div>
)}
<div className="space-y-1">
<p className="font-medium text-foreground">本次数据范围</p>
{datasets.slice(0, 20).map(([name, data]) => (
<p key={name} className="break-words">{DATA_LABELS[name] ?? (name.startsWith('minute_') ? '分钟数据 ' + name.split('/').slice(1).join(' · ') : name)} · {data.rows.toLocaleString()} 行{data.start && ` · ${data.start} ~ ${data.end}`}</p>
))}
{datasets.length > 20 && <p className="text-muted">共 {datasets.length} 批数据,完整范围见导出报告。</p>}
</div>
<p className="leading-5 text-muted">版本记录保存内容指纹与有效参数。它不备份全量行情,也不验证供应商历史修订、历史股票池和静态维表是否符合当时可用的信息。</p>
</div>
</details>
</section>
)
}