"""Anti-look-ahead tests for `ExecutionPrice.custom(...)` with `signal_delay=0`. This is the configuration of `examples/21_fill_at_computed_level.py`, and the one with the most room for leakage in the whole engine: the fill price is read from a strategy signal, the order acts on the same bar it was computed on, and the level itself comes from a higher timeframe. Three chances for a bar to be priced with information it could not have had. Two independent methods, because a single one can pass for the wrong reason: * **future perturbation** — corrupt every bar after K, re-run, and require the equity of bars 0..K to be *bit-identical*. Any decision that read a future bar moves the prefix. * **the engine's own detector** — `detect_lookahead`, which compares the trades of truncated runs against the full run. Both carry an anti-vacuity guard. A look-ahead test that compares nothing passes just as loudly as one that compares everything, which is exactly how the built-in detector reports PASS when its splits fall outside the data. """ import os import tempfile import pytest import manifoldbt as bt np = pytest.importorskip("numpy") pd = pytest.importorskip("pandas") from manifoldbt.helpers import ExecutionPrice, Interval, Slippage # noqa: E402 # Three days of 1-minute bars: enough for the hourly SMA to have a history, # small enough not to weigh on the suite. N_BARS = 3 * 1440 SPLIT = 2 * 1440 # perturb everything after this bar def _mean_reverting_bars(seed=7): """The construction of example 21, at a size a test can afford.""" rng = np.random.default_rng(seed) steps = rng.normal(0.0, 0.0010, N_BARS) level = np.cumsum(steps) * 0.85 px = 100.0 * np.exp(level - np.linspace(0, level[-1], N_BARS)) o = px c = np.roll(px, -1) c[-1] = px[-1] amp = np.abs(rng.normal(0.0, 0.0012, N_BARS)) return pd.DataFrame({ "timestamp": pd.date_range("2024-01-01", periods=N_BARS, freq="1min", tz="UTC"), "open": o, "high": np.maximum(o, c) * (1 + amp), "low": np.minimum(o, c) * (1 - amp), "close": c, "volume": rng.uniform(1_000, 5_000, N_BARS), }) def _band_strategy(): """Short the upper band, cover the lower one, filling ON the band.""" from manifoldbt.indicators import close, high, low, open as open_px, sma h1 = bt.tf("1h") band_up = sma(h1.close, 8) * 1.004 band_dn = sma(h1.close, 8) * 0.997 touch_up = high >= band_up touch_dn = low <= band_dn target = bt.when(touch_dn, 0.0, bt.when(touch_up, -1.0)) exec_level = bt.when( touch_dn, bt.when(open_px <= band_dn, open_px, band_dn), bt.when(touch_up, bt.when(open_px >= band_up, open_px, band_up), close), ) return ( bt.Strategy.create("band_touch_short") .signal("position", target) .signal("exec_level", exec_level) .size(target) .stop_loss(pct=25.0) ) def _store(frame, tmp_path, tag): root = os.path.join(str(tmp_path), tag) return bt.import_dataframe( frame, symbol="SYNTH", symbol_id=1, interval="1m", data_root=os.path.join(root, "data"), metadata_db=os.path.join(root, "meta.sqlite"), ) def _config(frame): """A time range bounded by the DATA, not by the epoch. `time_range_start=0` would stretch the range over 54 years, which is what makes the built-in detector split outside the data and pass vacuously. """ ts = pd.DatetimeIndex(frame["timestamp"]) return bt.BacktestConfig( universe=[1], time_range_start=int(ts[0].value), time_range_end=int(ts[-1].value) + 60_000_000_000, bar_interval=Interval.minutes(1), initial_capital=10_000, execution=bt.ExecutionConfig( signal_delay=0, execution_price=ExecutionPrice.custom("exec_level"), max_position_pct=0.4, allow_short=True, position_sizing_mode="FractionOfEquity", ), fees=bt.FeeConfig.binance_perps(), slippage=Slippage.fixed_bps(2), warmup_bars=60 * 4, extra_timeframes={"1h": Interval.hours(1)}, ) def _equity(result): return np.array([float(x) for x in result.equity_curve]) def test_future_bars_cannot_move_the_past(tmp_path): """The decisive test: corrupt the future, the past must not budge.""" frame = _mean_reverting_bars() strategy = _band_strategy() config = _config(frame) reference = _equity(bt.run(strategy, _config(frame), _store(frame, tmp_path, "ref"))) # Same multiplicative factor on all four price columns, so the bars stay # valid (high >= max(open, close), low <= min(open, close)). Small enough # that the short strategy survives it: a 3x future turns the equity # negative and the run refuses, which would prove nothing. rng = np.random.default_rng(1234) corrupted = frame.copy() tail = slice(SPLIT + 1, None) factor = 1.0 + rng.uniform(-0.005, 0.005, N_BARS - SPLIT - 1) for col in ("open", "high", "low", "close"): corrupted.loc[corrupted.index[tail], col] = corrupted[col].to_numpy()[tail] * factor perturbed = _equity(bt.run(strategy, config, _store(corrupted, tmp_path, "pert"))) # Anti-vacuity: if the corruption changed nothing at all, an identical # prefix would be meaningless. assert abs(reference[-1] - perturbed[-1]) > 1e-6, ( "the perturbation left the future untouched; the test would be vacuous" ) n = min(len(reference), len(perturbed), SPLIT + 1) assert n > 1000, f"only {n} bars compared, too few to conclude" delta = np.abs(reference[:n] - perturbed[:n]) first = int(np.argmax(delta > 0)) if delta.max() > 0 else -1 assert delta.max() == 0.0, ( f"future data leaked into the past: bar {first} differs by {delta.max():.3e}" ) def test_builtin_detector_agrees_and_is_not_vacuous(tmp_path): """The engine's own detector, plus a check that it compared something. `trades=0, mismatched=0` is reported as PASS. Asserting only on `.passed` would accept that empty verdict. """ from manifoldbt.diagnostics import detect_lookahead frame = _mean_reverting_bars() result = detect_lookahead( _band_strategy(), _config(frame), _store(frame, tmp_path, "det"), mode="all" ) assert result.passed, f"look-ahead reported: {result}" compared = sum(r.total_trades_overlap for r in result.reports) assert compared > 0, ( f"the detector compared no trade at all, its PASS is empty: {result.reports}" ) def test_every_fill_lands_on_a_level_known_before_the_bar(tmp_path): """No fill may be priced at its own bar's close. A fill at the close is only knowable once the bar is over. It is also what the example would produce if `custom(...)` silently fell back to AtClose, which would make the whole feature a no-op. """ frame = _mean_reverting_bars() result = bt.run(_band_strategy(), _config(frame), _store(frame, tmp_path, "fills")) trades = result.trades_df() assert len(trades) > 20, f"only {len(trades)} trades, too few to conclude" bars = frame.set_index(pd.DatetimeIndex(frame["timestamp"])) at = bars.reindex(pd.DatetimeIndex(pd.to_datetime(trades["execution_timestamp"], utc=True))) intended = trades["intended_price"].to_numpy() on_close = np.isclose(intended, at["close"].to_numpy(), rtol=0, atol=1e-12) assert not on_close.any(), ( f"{int(on_close.sum())} fill(s) landed on their bar's close, " "which is AtClose behaviour, not a computed level" ) def _leaking_strategy(): """The clean strategy, with one deliberate leak: the entry reads ahead.""" from manifoldbt.indicators import close, low, open as open_px, sma h1 = bt.tf("1h") base = sma(h1.close, 8) band_up = base * 1.004 band_dn = base * 0.997 touch_up = close.lead(5) >= band_up # <- the leak touch_dn = low <= band_dn target = bt.when(touch_dn, 0.0, bt.when(touch_up, -1.0)) exec_level = bt.when( touch_dn, bt.when(open_px <= band_dn, open_px, band_dn), bt.when(touch_up, bt.when(open_px >= band_up, open_px, band_up), close), ) return ( bt.Strategy.create("leaky") .signal("position", target) .signal("exec_level", exec_level) .size(target) .stop_loss(pct=25.0) ) def test_the_perturbation_method_catches_a_real_leak(tmp_path): """A look-ahead test that cannot fail proves nothing. Same data, same perturbation, same comparison as :func:`test_future_bars_cannot_move_the_past` -- only the strategy reads five bars ahead. The prefix MUST diverge, or the method above is blind and its PASS is worthless. """ frame = _mean_reverting_bars() strategy = _leaking_strategy() config = _config(frame) reference = _equity(bt.run(strategy, config, _store(frame, tmp_path, "leak_ref"))) rng = np.random.default_rng(1234) corrupted = frame.copy() tail = slice(SPLIT + 1, None) factor = 1.0 + rng.uniform(-0.005, 0.005, N_BARS - SPLIT - 1) for col in ("open", "high", "low", "close"): corrupted.loc[corrupted.index[tail], col] = corrupted[col].to_numpy()[tail] * factor perturbed = _equity(bt.run(strategy, config, _store(corrupted, tmp_path, "leak_pert"))) n = min(len(reference), len(perturbed), SPLIT + 1) delta = np.abs(reference[:n] - perturbed[:n]) assert delta.max() > 0.0, ( "a strategy reading 5 bars ahead went undetected: the perturbation " "method is blind and every PASS in this file is meaningless" ) # The divergence must sit just before the split, where the lead reaches # into the corrupted tail -- not somewhere unrelated. first = int(np.argmax(delta > 0)) assert SPLIT - 60 <= first <= SPLIT, ( f"divergence at bar {first}, expected it near the split at {SPLIT}" ) def test_detector_splits_on_the_data_not_on_the_configured_range(tmp_path): """A config starting at the epoch must not empty the detector. `time_range_start=0` is what `examples/21_fill_at_computed_level.py` writes, and it stretches the period over five decades: both split points used to land before the first bar, so both truncated runs saw no data and the detector announced PASS having compared nothing. """ from manifoldbt.diagnostics import detect_lookahead frame = _mean_reverting_bars() config = _config(frame) config.time_range_start = 0 # the epoch, as the example does result = detect_lookahead( _band_strategy(), config, _store(frame, tmp_path, "epoch"), mode="all" ) compared = sum(r.total_trades_overlap for r in result.reports) assert compared > 0, ( "the detector compared no trade: its splits fell outside the data again" ) assert result.passed, f"look-ahead reported: {result}"