manifoldbt/examples/25_lookahead_trap.py

"""Look-ahead — the leak the detector cannot see.

The classic beginner's leak, and the one that survives review: computing a
statistic over the *whole* dataset in the notebook, then using it as a strategy
parameter. Here it is the mean price of the entire asset, used from bar 0 —
where 749 of the 750 days it summarises are still in the future.

**There IS a leak in this strategy. It is deliberate.** The question the
example answers is not whether the leak exists, but which audit methods can
see it — and two of the three cannot. Read every "sees nothing" below as a
statement about the *method*, never as a clean bill of health for the
strategy.

What this example shows, in order:

  1. the leak triples the return and doubles the Sharpe;
  2. `detect_lookahead` sees nothing;
  3. perturbing every future bar sees nothing either;
  4. the one method that catches it: re-derive the parameter on truncated data.

**Why the first two are blind.** Both re-run the *same strategy* on shorter or
altered data. The mean is not recomputed by the engine: it is a number baked
into the strategy at research time. Re-running cannot see it, because the leak
already happened, in the notebook, before the backtest existed.

That is not a bug to fix in the detector, it is the shape of the problem. No
re-run method can audit a constant. The only defence is to treat every
parameter derived from data as part of the pipeline, and re-derive it on
whatever window you are testing.

Self-contained: generates its own synthetic data in a temp store.

Demonstrates:
  - a parameter computed over the whole dataset, used from bar 0
  - two audit methods that do not see it, and why
  - the one that does: re-deriving the parameter per window

Data: synthetic (seed 11) — generated by this file, reproducible.
The fixture DETERMINES the outcome: see examples/README.md.

Usage:
    python examples/25_lookahead_trap.py
"""
import os
import tempfile

import numpy as np
import pandas as pd

import manifoldbt as mbt
from manifoldbt.indicators import close
from manifoldbt.helpers import Interval, Slippage

# -- A mean-reverting series: exactly where knowing the mean is gold ----------
N_DAYS = 750
rng = np.random.default_rng(11)
level = np.cumsum(rng.normal(0.0, 0.018, N_DAYS))
px = 100.0 * np.exp(level - np.linspace(0, level[-1], N_DAYS))
o = px
c = np.roll(px, -1)
c[-1] = px[-1]
amp = np.abs(rng.normal(0.0, 0.004, N_DAYS))
frame = pd.DataFrame({
    "timestamp": pd.date_range("2022-01-01", periods=N_DAYS, freq="1D", tz="UTC"),
    "open": o,
    "high": np.maximum(o, c) * (1 + amp),
    "low": np.minimum(o, c) * (1 - amp),
    "close": c,
    "volume": rng.uniform(1_000, 5_000, N_DAYS),
})


def store_of(df, tag):
    root = tempfile.mkdtemp(prefix=f"mbt_ex25_{tag}_")
    return mbt.import_dataframe(
        df, symbol="SYNTH", symbol_id=1, interval="1d",
        data_root=os.path.join(root, "data"),
        metadata_db=os.path.join(root, "meta.sqlite"),
    )


def config_of(df):
    ts = pd.DatetimeIndex(df["timestamp"])
    return mbt.BacktestConfig(
        universe=[1],
        time_range_start=int(ts[0].value),
        time_range_end=int(ts[-1].value) + 86_400_000_000_000,
        bar_interval=Interval.days(1),
        initial_capital=10_000,
        execution=mbt.ExecutionConfig(
            signal_delay=1, max_position_pct=1.0,
            allow_short=True, position_sizing_mode="FractionOfEquity",
        ),
        slippage=Slippage.fixed_bps(0),
        warmup_bars=0,
    )


def leaky(mean_price):
    """THE LEAK: `mean_price` comes from `df.close.mean()` over everything."""
    return (
        mbt.Strategy.create("global_mean_leak")
        .signal("edge", close)
        .size(mbt.when(close < mean_price, 1.0, -1.0))
        .describe("Long below the global mean, short above")
    )


def causal(window):
    """The honest twin: a rolling mean knows only the past."""
    return (
        mbt.Strategy.create("rolling_mean")
        .signal("edge", close)
        .size(mbt.when(close < close.rolling_mean(window), 1.0, -1.0))
        .describe("Long below the rolling mean")
    )


def equity(result):
    return np.array([float(x) for x in result.equity_curve])


# The two words this example turns on. "SEES NOTHING" describes the METHOD,
# never the strategy: the leak below is there whatever any audit reports.
BLIND, CAUGHT = "SEES NOTHING", "CATCHES THE LEAK"


if __name__ == "__main__":
    global_mean = float(frame["close"].mean())
    store = store_of(frame, "full")

    # -- 1. The seduction -----------------------------------------------------
    leaked = mbt.run(leaky(global_mean), config_of(frame), store)
    honest = mbt.run(causal(60), config_of(frame), store)

    print(f"Global mean over all {N_DAYS} days: {global_mean:.4f}")
    print("(at bar 0, 749 of those days have not happened yet)")
    print("\nThis strategy IS leaking. Below: which audits notice.\n")
    for label, r in (("with the leak ", leaked), ("rolling mean  ", honest)):
        m = r.metrics
        print(f"  {label} return {100 * m['total_return']:+8.2f}%   "
              f"sharpe {m['sharpe']:5.2f}   trades {r.trade_count}")

    # -- 2. The detector -------------------------------------------------------
    from manifoldbt.diagnostics import detect_lookahead

    report = detect_lookahead(leaky(global_mean), config_of(frame), store, mode="all")
    compared = sum(r.total_trades_overlap for r in report.reports)
    verdict = BLIND if report.passed else CAUGHT
    print(f"\n  [1] detect_lookahead .......... {verdict}")
    print(f"      ({compared} trades compared, so the verdict is not empty)")

    # -- 3. Perturbing the future ---------------------------------------------
    split = 500
    reference = equity(mbt.run(leaky(global_mean), config_of(frame), store))
    corrupted = frame.copy()
    tail = slice(split + 1, None)
    factor = 1.0 + np.random.default_rng(99).uniform(-0.05, 0.05, N_DAYS - split - 1)
    for col in ("open", "high", "low", "close"):
        corrupted.loc[corrupted.index[tail], col] = corrupted[col].to_numpy()[tail] * factor
    perturbed = equity(mbt.run(leaky(global_mean), config_of(frame),
                               store_of(corrupted, "pert")))
    n = min(len(reference), len(perturbed), split + 1)
    drift = float(np.abs(reference[:n] - perturbed[:n]).max())
    verdict = CAUGHT if drift > 0 else BLIND
    print(f"  [2] future perturbation ....... {verdict}")
    print(f"      (past moved by {drift:.3e} while the future moved by "
          f"{abs(reference[-1] - perturbed[-1]):.0f})")

    # -- 4. The method that works ---------------------------------------------
    # Treat the mean as what it is: a pipeline step, not a constant. A
    # researcher standing at bar 500 only has the first 500 bars.
    truncated = frame.iloc[:split + 1]
    honest_mean = float(truncated["close"].mean())
    with_future = equity(mbt.run(leaky(global_mean), config_of(truncated),
                                 store_of(truncated, "t1")))
    with_past = equity(mbt.run(leaky(honest_mean), config_of(truncated),
                               store_of(truncated, "t2")))
    m = min(len(with_future), len(with_past))
    gap = float(np.abs(with_future[:m] - with_past[:m]).max())
    verdict = CAUGHT if gap > 0 else BLIND
    print(f"  [3] re-deriving the parameter . {verdict}")
    print(f"      mean from the whole set : {global_mean:.4f} -> "
          f"final equity {with_future[-1]:,.0f}")
    print(f"      mean from the past only : {honest_mean:.4f} -> "
          f"final equity {with_past[-1]:,.0f}")
    print(f"      same window, same bars, equity differs by {gap:.0f}")
    print("\n  Verdict: the leak is real, and only [3] found it. A")
    print("  'SEES NOTHING' is a statement about the method, not about")
    print("  the strategy.")