From 8ebd171f71a878355a48137c2154f80fc8146746 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 12 Oct 2026 23:00:48 +0000 Subject: [PATCH] release: v0.27.1 --- README.md | 8 +- docs/strategy-authoring.md | 13 ++- examples/07_walk_forward.py | 5 +- examples/08_sweep_2d_heatmap.py | 9 +- examples/25_lookahead_trap.py | 27 +++-- pyproject.toml | 2 +- python/manifoldbt/__init__.py | 42 +++++++ python/manifoldbt/config.py | 2 + python/manifoldbt/diagnostics.py | 55 +++++++-- python/manifoldbt/strategy.py | 37 ++++-- python/tests/test_diagnostics_exo.py | 76 +++++++++++++ .../test_garde_configuration_execution.py | 31 +++++ python/tests/test_lookahead_sans_trade.py | 79 +++++++++++++ python/tests/test_param_sans_valeur.py | 81 +++++++++++++ python/tests/test_walk_forward_exo_sources.py | 79 +++++++++++++ python/tests/test_walk_forward_resolution.py | 106 ++++++++++++++++++ 16 files changed, 610 insertions(+), 42 deletions(-) create mode 100644 python/tests/test_diagnostics_exo.py create mode 100644 python/tests/test_lookahead_sans_trade.py create mode 100644 python/tests/test_param_sans_valeur.py create mode 100644 python/tests/test_walk_forward_exo_sources.py create mode 100644 python/tests/test_walk_forward_resolution.py diff --git a/README.md b/README.md index 7aa47ae..9a03e0d 100644 --- a/README.md +++ b/README.md @@ -259,9 +259,11 @@ precomputed with future bars and imported as data). Bar T sees the same T+1 in every window, so the shorter run trades exactly like the full one and the re-runs report PASS, next to a Sharpe in the hundreds. No choice of split changes that. The detector therefore also walks the strategy's expressions -and fails any explicit `lead()`, naming the signal; for an implicit one, the -check that works is to perturb the bars after some K and require the equity -up to K to be bit-identical, which `examples/25_lookahead_trap.py` does. +and fails any explicit `lead()`, naming the signal. For an implicit one, the +check belongs to the pipeline that built the column: perturb the raw data +after some bar K, rebuild the column from it, and require the equity up to K +to be bit-identical. Perturbing the bars alone leaves a precomputed column +untouched and sees nothing, as `examples/25_lookahead_trap.py` shows. A parameter derived from the data *before* the backtest is a different question: a threshold computed over the whole history in a notebook and then diff --git a/docs/strategy-authoring.md b/docs/strategy-authoring.md index 39901eb..db9a10e 100644 --- a/docs/strategy-authoring.md +++ b/docs/strategy-authoring.md @@ -3618,9 +3618,16 @@ store = mbt.DataStore(data_root="data", metadata_db="...", dataset="bars_1m") # Look-ahead bias detection: a static walk for every non-causal operator, # then two re-runs over shorter windows. The two halves catch different # things. The re-runs cannot see a fixed read-ahead (close.lead(1) trades -# the same in every window); the static walk names it. The static walk -# only knows the operators, so it cannot see a leak that comes from the -# data itself; the re-runs can. +# the same in every window); the static walk names it. The re-runs catch a +# strategy whose past trades change with how much data the run is given. +# NEITHER half sees a leak carried by the data itself: a column computed +# with future bars and imported, or a constant fitted on the whole history, +# reads the same in every window and passes both. That one is caught +# upstream, by re-deriving every data-dependent input on the window under +# test (examples/25_lookahead_trap.py). +# +# A re-run with nothing to compare (a strategy that never trades, a cut +# run that fails) reports INCONCLUSIVE, not PASS. # # The static walk reads a registry in which every DSL operator is # classified causal or not, so a new operator cannot be added without diff --git a/examples/07_walk_forward.py b/examples/07_walk_forward.py index 09f04e0..5c008da 100644 --- a/examples/07_walk_forward.py +++ b/examples/07_walk_forward.py @@ -19,8 +19,9 @@ from _bootstrap import open_store, plots_available # -- Strategy with tunable parameters ---------------------------------------- # The sweep engine substitutes each grid value into the param() references at -# runtime. The period MUST be param("..."): a hardcoded int compiles every -# combo to the same strategy, so the sweep becomes a silent no-op. +# runtime. The period MUST be param("..."): with a hardcoded int no expression +# reads the grid key, every combo would be the same strategy, and the engine +# refuses the run with a StrategyError naming the undeclared parameter. fast = ema(close, mbt.param("fast", default=12)) slow = ema(close, mbt.param("slow", default=26)) diff --git a/examples/08_sweep_2d_heatmap.py b/examples/08_sweep_2d_heatmap.py index 3d33084..603469b 100644 --- a/examples/08_sweep_2d_heatmap.py +++ b/examples/08_sweep_2d_heatmap.py @@ -2,7 +2,7 @@ Demonstrates: - param() in indicator periods (engine re-compiles per combo) - - run_sweep() for Cartesian grid search + - run_sweep_lite() for Cartesian grid search - Heatmap visualization with mbt.plot.heatmap_2d() Data: shared store — real market data from `data/` (see examples/README.md) @@ -18,8 +18,11 @@ from manifoldbt.helpers import time_range, Slippage, Interval from _bootstrap import open_store, plots_available # -- Strategy (single definition, param() in periods) ------------------------ -fast = ema(close, mbt.param("fast")) -slow = ema(close, mbt.param("slow")) +# The sweep sets both periods for every combo. The defaults are what a plain +# mbt.run(strategy, ...) or a diagnostic uses: without one, a period has no +# value outside the sweep. +fast = ema(close, mbt.param("fast", default=20)) +slow = ema(close, mbt.param("slow", default=100)) signal = mbt.when(fast > slow, 0.25, mbt.when(fast < slow, -0.25, 0.0)) diff --git a/examples/25_lookahead_trap.py b/examples/25_lookahead_trap.py index 489894b..e8fc16e 100644 --- a/examples/25_lookahead_trap.py +++ b/examples/25_lookahead_trap.py @@ -1,4 +1,4 @@ -"""Look-ahead — the leak no re-run method can see, and the one that does. +"""Look-ahead — the leak no re-run method can see, and how to backtest without it. The classic beginner's leak, and the one that survives review: computing a statistic over the *whole* dataset in the notebook, then using it as a strategy @@ -16,7 +16,9 @@ What this example shows, in order: 1. the leak triples the return and doubles the Sharpe; 2. `detect_lookahead` sees nothing; 3. perturbing every future bar sees nothing either; - 4. the one method that catches it: re-derive the parameter on truncated data. + 4. what to do instead: re-derive the parameter on the window under test. + That is not a detector (two different constants give two different + curves by construction); it is the backtest you can report. **Why the first two are blind.** Both re-run the *same strategy* on shorter or altered data. The mean is not recomputed by the engine: it is a number baked @@ -33,7 +35,7 @@ Self-contained: generates its own synthetic data in a temp store. Demonstrates: - a parameter computed over the whole dataset, used from bar 0 - two audit methods that do not see it, and why - - the one that does: re-deriving the parameter per window + - the method that avoids it: re-deriving the parameter per window Data: synthetic (seed 11) — generated by this file, reproducible. The fixture DETERMINES the outcome: see examples/README.md. @@ -157,7 +159,7 @@ if __name__ == "__main__": except LicenseError: # A Pro diagnostic. The file's argument does not need it to run: this # is the audit the leak is INVISIBLE to (run under Pro, it reports - # SEES NOTHING here), and the audit that does catch it, [3], is free. + # SEES NOTHING here), and the method that avoids it, [3], is free. print("\n [1] detect_lookahead .......... skipped (Pro diagnostic)") # -- 3. Perturbing the future --------------------------------------------- @@ -186,15 +188,16 @@ if __name__ == "__main__": store_of(truncated, "t1"))) with_past = equity(mbt.run(leaky(honest_mean), config_of(truncated), store_of(truncated, "t2"))) - m = min(len(with_future), len(with_past)) - gap = float(np.abs(with_future[:m] - with_past[:m]).max()) - verdict = CAUGHT if gap > 0 else BLIND - print(f" [3] re-deriving the parameter . {verdict}") + # Not a detector: the two runs differ by one constant, so their curves + # differ whenever the constants do, leak or not. What [3] gives is the + # backtest that only uses what bar 500 knew: the one you can report. + print(" [3] re-deriving the parameter . the honest backtest") print(f" mean from the whole set : {global_mean:.4f} -> " f"final equity {with_future[-1]:,.0f}") print(f" mean from the past only : {honest_mean:.4f} -> " f"final equity {with_past[-1]:,.0f}") - print(f" same window, same bars, equity differs by {gap:.0f}") - print("\n Verdict: the leak is real, and only [3] found it. A") - print(" 'SEES NOTHING' is a statement about the method, not about") - print(" the strategy.") + print(" (a different constant gives a different curve by construction:") + print(" the second line is the number to report, not a test verdict)") + print("\n Verdict: the leak is real, and no audit of the finished") + print(" strategy can see it. A 'SEES NOTHING' is a statement about the") + print(" method, not about the strategy: the defence is upstream, [3].") diff --git a/pyproject.toml b/pyproject.toml index 7173950..286f03a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [project] name = "manifoldbt" -version = "0.27.0" +version = "0.27.1" description = "Rust-powered backtesting engine for quantitative research" readme = "README.md" requires-python = ">=3.9" diff --git a/python/manifoldbt/__init__.py b/python/manifoldbt/__init__.py index ac2d08a..2c7e14d 100644 --- a/python/manifoldbt/__init__.py +++ b/python/manifoldbt/__init__.py @@ -545,6 +545,8 @@ def _reject_intrabar_price_without_delay(cfg: BacktestConfig) -> None: Measured on a driftless random walk, "long when close > open" returns +1913% under `AtOpen`, +361% under `AtVwap`, `MidPrice` and `custom("low")`, against +5% under the defaults. The engine said nothing. + `fill_model.intra_bar_price` (`TypicalPrice`, `OhlcAverage`) is the same + average reached by another key, and is refused the same way. **`custom()` is exempt, and deliberately.** There the level is one the DSL computed, and the fill only books if the bar @@ -556,6 +558,23 @@ def _reject_intrabar_price_without_delay(cfg: BacktestConfig) -> None: execution = getattr(cfg, "execution", None) if execution is None or getattr(execution, "signal_delay", 1) != 0: return + # The same question under a second key: `fill_model.intra_bar_price` + # averages the execution bar ((H+L+C)/3, (O+H+L+C)/4) whatever + # `execution_price` says, and at delay zero that bar is the signal's own. + # Read first, because the default price (the close) returns early below, + # and that is the very combination that slipped through: Sharpe 37 and 65 + # on a driftless walk, against 1.3 at the close. + fill = getattr(execution, "fill_model", None) + intra = fill.get("intra_bar_price", "SinglePoint") if isinstance(fill, dict) else "SinglePoint" + if intra not in (None, "SinglePoint"): + raise ConfigError( + f'fill_model["intra_bar_price"]={intra!r} needs signal_delay >= 1: ' + "with signal_delay=0 the fill is an average over the same bar whose " + "close produced the signal, prices the decision came after. Set " + "ExecutionConfig(signal_delay=1) to decide on one bar and execute on " + 'the next, or drop intra_bar_price ("SinglePoint", the default) to ' + "fill at the very close the signal was computed on." + ) price = getattr(execution, "execution_price", "AtClose") # `ExecutionPrice.custom(name)` serialises as {"Custom": name}. label = next(iter(price)) if isinstance(price, dict) else price @@ -1947,6 +1966,28 @@ def _by_day_arg(by_day) -> Optional[bool]: raise ValueError(f'by_day takes "auto", True or False, got {by_day!r}') +def _reject_unset_params(strategy) -> None: + """A plain run needs a value for every param() its expressions read. + + Without a default, a param() is set by a sweep and by nothing else. A run + read it as an empty series: an EMA of no period, no signal, no trade, and + no error (example 08's strategy run on its own made 0 trades, and + detect_lookahead answered PASS on it). Sweeps never come through here. + """ + unset = getattr(strategy, "_unset_params", lambda: [])() + if not unset: + return + names = ", ".join(repr(n) for n in unset) + first = unset[0] + raise StrategyError( + f"strategy {strategy.name!r}: param({names}) has no value in a plain run. " + "A param() without a default is set by a sweep; run() would read it as " + f"empty and the strategy would make no trade. Give it one, " + f"param({first!r}, default=...), or declare it with " + f"Strategy.param({first!r}, default=...), or sweep it (run_sweep_lite)." + ) + + def run( strategy: Strategy, config: BacktestConfig, @@ -1988,6 +2029,7 @@ def run( graphics card, with the same result; a build or a machine without CUDA prepares them on the CPU and says so in a warning. """ + _reject_unset_params(strategy) if _quotes(strategy): return _run_quotes(strategy, config, store, by_day=by_day, device=device) if not (isinstance(by_day, str) and by_day == "auto") or device != "cpu": diff --git a/python/manifoldbt/config.py b/python/manifoldbt/config.py index 30a82c9..eee7915 100644 --- a/python/manifoldbt/config.py +++ b/python/manifoldbt/config.py @@ -513,6 +513,8 @@ class ExecutionConfig: """Fill model configuration. None = Rust defaults (atomic fill, single point). Example: {"max_participation_rate": 0.1, "intra_bar_price": "TypicalPrice"} intra_bar_price options: "SinglePoint", "TypicalPrice", "OhlcAverage" + (the two averages need signal_delay >= 1: at delay 0 they would average + the bar whose close produced the signal, and run() refuses them) passive_fill options: "touch" (default: a maker fill books when the bar reaches its level) or "traverse" (the bar must trade through the level; conservative, and the run stays on the general loop). The result's diff --git a/python/manifoldbt/diagnostics.py b/python/manifoldbt/diagnostics.py index 9009a37..85d3f02 100644 --- a/python/manifoldbt/diagnostics.py +++ b/python/manifoldbt/diagnostics.py @@ -41,9 +41,18 @@ class LookaheadReport: mismatched: int method: str = "" details: List[Dict[str, Any]] = field(default_factory=list) + inconclusive: Optional[str] = None + """Why a re-run sub-test compared nothing (a strategy that never trades, a + truncated run that failed). Such a report is not a pass: ``passed`` is + False, and it prints INCONCLUSIVE rather than PASS or FAIL.""" def assert_clean(self) -> None: - """Raise AssertionError if look-ahead bias was detected.""" + """Raise AssertionError if look-ahead bias was detected, or if the + sub-test could not look for it.""" + if self.inconclusive: + raise AssertionError( + f"Look-ahead check inconclusive ({self.method}): {self.inconclusive}" + ) if not self.passed: if self.method == "static": msg = ( @@ -63,6 +72,8 @@ class LookaheadReport: raise AssertionError(msg) def __str__(self) -> str: + if self.inconclusive: + return f" [{self.method}] INCONCLUSIVE ({self.inconclusive})" status = "PASS" if self.passed else "FAIL" if self.method == "static": # "non-causal", pas "reads ahead" : le compte couvre maintenant @@ -102,7 +113,12 @@ class DiagnosticsResult: r.assert_clean() def __str__(self) -> str: - status = "PASS" if self.passed else "FAIL" + if self.passed: + status = "PASS" + elif all(r.passed or r.inconclusive for r in self.reports): + status = "INCONCLUSIVE" + else: + status = "FAIL" parts = [f"Lookahead diagnostics: {status}"] for r in self.reports: parts.append(str(r)) @@ -135,15 +151,25 @@ def detect_lookahead( classified in a registry the engine checks exhaustively, so a new operator cannot be added without answering the question. * **extension** -- re-run on the first 2/3 of the data and compare the - trades with the full run. Catches look-ahead that depends on how much - data the run was given (a statistic over the whole series, a - normalisation by the last value). + trades with the full run. Catches a strategy whose past trades change + with how much data the run was given: in this engine that takes a + non-causal operator over the whole column, which the static test + already names. A statistic computed outside the engine (a z-score or + a normalisation fitted on the whole history, handed in as a constant + or a column) reads the same in every window and is NOT caught: see + the warning and the note below. * **truncation** -- the same comparison, split at 1/3. The re-runs load the data once and slice it (no redundant I/O). The verdict is the conjunction: ``.passed`` is False as soon as one sub-test fails. + A re-run sub-test that has nothing to compare reports INCONCLUSIVE, not + PASS, and ``.passed`` is False: a strategy that never trades (a + ``param()`` left without a value is enough), a truncated run that fails, + or no trade before the split. A cut run with no trade while the full run + traded before the split is a FAIL: the same bars gave two answers. + .. warning:: **What the re-runs cannot see.** A leak that reads a *fixed* number of bars ahead (``close.lead(1)``, or a column precomputed with future bars @@ -158,9 +184,12 @@ def detect_lookahead( clean bill of health. The ``static`` sub-test is the one that names an explicit ``lead()``. - For an implicit one (a column built outside the engine) the check that - works is to perturb the bars after some bar K and require the equity - up to K to be bit-identical: ``examples/25_lookahead_trap.py`` does it. + For an implicit one (a column built outside the engine) the check + belongs to the pipeline that built it: perturb the raw data after some + bar K, REBUILD the column from it, and require the equity up to K to be + bit-identical. Perturbing the bars alone leaves a precomputed column + untouched, so it sees nothing: ``examples/25_lookahead_trap.py`` shows + exactly that. .. note:: **Scope.** The re-runs re-run the *same strategy* over a different @@ -227,6 +256,7 @@ def detect_lookahead( mismatched=r["mismatched"], method=r["method"], details=r["details"], + inconclusive=r.get("inconclusive"), ) for r in raw ] @@ -746,9 +776,14 @@ def check_exposure_stability( Sub-tests: * **extension** — compare full period vs first 2/3. - Catches global normalization (e.g. zscore over entire series). * **truncation** — compare full period vs first 1/3. - Catches rolling window that peeks ahead. + + Both catch sizing whose past values change with how much data the run is + given, which in this engine takes a non-causal operator over the whole + column. Neither catches a fixed read-ahead (a window shifted by ``lead`` + reads the same bars in every window), nor a statistic fitted outside the + engine on the whole history and handed in as a constant or a column: see + :func:`detect_lookahead` for what each method can and cannot see. Args: strategy: Strategy definition. diff --git a/python/manifoldbt/strategy.py b/python/manifoldbt/strategy.py index b089862..c89e699 100644 --- a/python/manifoldbt/strategy.py +++ b/python/manifoldbt/strategy.py @@ -637,6 +637,34 @@ class Strategy: """Order config dict (stop-loss, take-profit, trailing), or None.""" return self._orders + def _expression_params(self) -> "_Declarations": + """The param() references this strategy's expressions read.""" + declared = _Declarations() + for expr in self.signals.values(): + _collect_params(expr, declared) + _collect_params(self.position_sizing, declared) + for q in self._quotes: + for key in ("price", "size", "enabled"): + if key in q: + _collect_params(q[key], declared) + return declared + + def _unset_params(self) -> List[str]: + """Parameters an expression reads that have no value outside a sweep. + + A param() without a default serialises with a null default. A sweep + sets every name it sweeps, but a plain run reads the null, which the + engine evaluated as an empty series: no signal, no trade, no error. A + name only declared with ``.param()`` and read by no expression is not + reported: nothing reads it. + """ + specs = self.to_json_dict()["parameters"] + return sorted( + name + for name in self._expression_params().metas + if specs.get(name, {}).get("default") == "Null" + ) + def to_json_dict(self) -> dict: """Serialize to a dict matching Rust ``StrategyDef`` serde format.""" # `signals` is a plain public dict, so a caller can drop anything into it @@ -649,14 +677,7 @@ class Strategy: # Auto-collect params from this strategy's own expressions: the # default of a param() is the one written in the expression that uses # it, never one another strategy of the process declared. - declared = _Declarations() - for expr in self.signals.values(): - _collect_params(expr, declared) - _collect_params(self.position_sizing, declared) - for q in self._quotes: - for key in ("price", "size", "enabled"): - if key in q: - _collect_params(q[key], declared) + declared = self._expression_params() # Merge: explicit .param() calls override auto-collected. One without # a default keeps the default the expressions give (like a param() diff --git a/python/tests/test_diagnostics_exo.py b/python/tests/test_diagnostics_exo.py new file mode 100644 index 0000000..29eee49 --- /dev/null +++ b/python/tests/test_diagnostics_exo.py @@ -0,0 +1,76 @@ +"""Les diagnostics doivent accepter les series EXOGENES, comme `run`. + +`detect_lookahead` relancait la strategie avec des tables exo vides, et +`AlignedData` (sur lequel `check_exposure_stability` relance ses runs) n'en +portait aucune : une strategie lisant `exo..` levait +"unknown input column" dans les deux, alors que `mbt.run` la faisait tourner. +""" +import os + +import pytest + +import manifoldbt as bt + +# Safety diagnostics are a Pro feature. +pytestmark = pytest.mark.skipif( + bt.license_info()[0] != "Pro", + reason="requires Pro: skipped on a Community wheel", +) + +pd = pytest.importorskip("pandas") +np = pytest.importorskip("numpy") + + +def _monte(tmp_path): + """Serie horaire reguliere, plus une moyenne posee en serie exogene.""" + n = 2400 + idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp") + pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0) + px = 100.0 + np.cumsum(pas) * 0.05 + df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996, + "close": px, "volume": np.full(n, 1000.0)}, index=idx) + racine = str(tmp_path / "data") + meta = str(tmp_path / "meta.sqlite") + os.makedirs(racine, exist_ok=True) + store = bt.import_dataframe(df.reset_index(), symbol="ZDEX", symbol_id=1, + interval="1h", asset_class="equity", + exchange="TEST", data_root=racine, metadata_db=meta) + # Moyenne CAUSALE (fenetre glissante sur le passe) : aucune fuite. + moy = pd.Series(px).rolling(24).mean().to_numpy() + bt.register_exo("moyenne", pd.DataFrame({"timestamp": idx, "sma": moy}), + store=store, data_root=racine, timeframe="1h") + tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11") + cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1, + initial_capital=1000.0, provider="TEST", + bar_interval=bt.Interval.hours(1), symbol_names={"ZDEX": 1}) + cfg.warmup_bars = 0 + cfg.exo_data = ["moyenne"] + return store, cfg + + +def _strategie(): + from manifoldbt.indicators import close, col + m = col("exo.moyenne.sma") + return (bt.Strategy.create("s") + .signal("m", m) + .size(bt.when(close > m, 1.0, 0.0))) + + +def test_la_strategie_a_exo_trade_dans_run(tmp_path): + store, cfg = _monte(tmp_path) + r = bt.run(_strategie(), cfg, store) + assert r.trade_count > 0 + + +def test_detect_lookahead_accepte_une_serie_exogene(tmp_path): + store, cfg = _monte(tmp_path) + rapport = bt.diagnostics.detect_lookahead(_strategie(), cfg, store, mode="all") + # Une moyenne glissante sur le passe ne fuit pas : les re-runs tronques + # doivent retrouver les trades du run complet. + assert rapport.passed, rapport + + +def test_check_exposure_stability_accepte_une_serie_exogene(tmp_path): + store, cfg = _monte(tmp_path) + rapport = bt.diagnostics.check_exposure_stability(_strategie(), cfg, store) + assert rapport.passed, rapport diff --git a/python/tests/test_garde_configuration_execution.py b/python/tests/test_garde_configuration_execution.py index f0d3a26..85b887c 100644 --- a/python/tests/test_garde_configuration_execution.py +++ b/python/tests/test_garde_configuration_execution.py @@ -149,6 +149,37 @@ def test_le_message_nomme_le_prix_et_la_sortie(store): assert "AtClose" in message +@pytest.mark.parametrize("moyenne", ["TypicalPrice", "OhlcAverage"]) +def test_une_moyenne_intrabarre_sans_delai_est_refusee(store, moyenne): + """`fill_model.intra_bar_price` averages the execution bar whatever the + price says: at delay 0, with the default close, it filled at (H+L+C)/3 or + (O+H+L+C)/4 of the signal's own bar (Sharpe 37 and 65 on a driftless + walk, `detect_lookahead` PASS). Refused like `AtVwap`.""" + with pytest.raises(ConfigError, match="needs signal_delay >= 1") as exc: + bt.run(_reads_its_own_bar(), + _config(signal_delay=0, fill_model={"intra_bar_price": moyenne}), + store) + assert moyenne in str(exc.value) + assert "signal_delay=1" in str(exc.value) + + +@pytest.mark.parametrize("moyenne", ["TypicalPrice", "OhlcAverage"]) +def test_une_moyenne_intrabarre_avec_delai_reste_permise(store, moyenne): + """With a delay the averaged bar follows the signal: nothing to refuse.""" + result = bt.run(_reads_its_own_bar(), + _config(signal_delay=1, fill_model={"intra_bar_price": moyenne}), + store) + assert abs(result.metrics["total_return"]) < 0.5 + + +def test_single_point_sans_delai_reste_permis(store): + """`SinglePoint` is the default: the price named by `execution_price`.""" + result = bt.run(_reads_its_own_bar(), + _config(signal_delay=0, fill_model={"intra_bar_price": "SinglePoint"}), + store) + assert abs(result.metrics["total_return"]) < 0.5 + + @pytest.mark.parametrize("price", ["AtClose", "NextBarClose"]) def test_la_cloture_reste_permise_sans_delai(store, price): """These two fill at the close that produced the signal: nothing to refuse. diff --git a/python/tests/test_lookahead_sans_trade.py b/python/tests/test_lookahead_sans_trade.py new file mode 100644 index 0000000..79f127d --- /dev/null +++ b/python/tests/test_lookahead_sans_trade.py @@ -0,0 +1,79 @@ +"""Un sous-test de re-run qui n'a rien compare ne rend pas PASS. + +detect_lookahead rendait PASS sur une strategie qui ne trade jamais (un +param() sans valeur suffit), ou quand le run tronque echouait : un verdict +vide lu comme un feu vert. Il rend maintenant INCONCLUSIVE, `.passed` vaut +False et `assert_clean()` leve, en disant pourquoi. +""" +import os + +import pytest + +import manifoldbt as bt + +# Safety diagnostics are a Pro feature. +pytestmark = pytest.mark.skipif( + bt.license_info()[0] != "Pro", + reason="requires Pro: skipped on a Community wheel", +) + +pd = pytest.importorskip("pandas") +np = pytest.importorskip("numpy") + + +def _monte(tmp_path): + n = 2400 + idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp") + pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0) + px = 100.0 + np.cumsum(pas) * 0.05 + df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996, + "close": px, "volume": np.full(n, 1000.0)}, index=idx) + racine = str(tmp_path / "data") + os.makedirs(racine, exist_ok=True) + store = bt.import_dataframe(df.reset_index(), symbol="ZLSV", symbol_id=1, + interval="1h", asset_class="equity", + exchange="TEST", data_root=racine, + metadata_db=str(tmp_path / "meta.sqlite")) + tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11") + cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1, + initial_capital=1000.0, provider="TEST", + bar_interval=bt.Interval.hours(1), symbol_names={"ZLSV": 1}) + cfg.warmup_bars = 0 + return store, cfg + + +def _jamais(): + from manifoldbt.indicators import close + return (bt.Strategy.create("jamais") + .signal("c", close) + .size(bt.lit(0.0))) + + +def _croise(): + from manifoldbt.indicators import close, sma + m = sma(close, 24) + return bt.Strategy.create("croise").signal("m", m).size(bt.when(close > m, 1.0, 0.0)) + + +def test_une_strategie_qui_ne_trade_pas_est_inconclusive(tmp_path): + store, cfg = _monte(tmp_path) + rapport = bt.diagnostics.detect_lookahead(_jamais(), cfg, store, mode="all") + assert not rapport.passed + re_runs = [r for r in rapport.reports if r.method != "static"] + assert {r.method for r in re_runs} == {"extension", "truncation"} + for r in re_runs: + assert r.inconclusive and "made no trade" in r.inconclusive + assert "INCONCLUSIVE" in str(r) + statique = [r for r in rapport.reports if r.method == "static"] + assert statique and statique[0].passed + assert str(rapport).startswith("Lookahead diagnostics: INCONCLUSIVE") + with pytest.raises(AssertionError, match="inconclusive"): + rapport.assert_clean() + + +def test_une_strategie_causale_qui_trade_reste_pass(tmp_path): + store, cfg = _monte(tmp_path) + rapport = bt.diagnostics.detect_lookahead(_croise(), cfg, store, mode="all") + assert rapport.passed, rapport + assert all(r.inconclusive is None for r in rapport.reports) + assert str(rapport).startswith("Lookahead diagnostics: PASS") diff --git a/python/tests/test_param_sans_valeur.py b/python/tests/test_param_sans_valeur.py new file mode 100644 index 0000000..f7e3cb8 --- /dev/null +++ b/python/tests/test_param_sans_valeur.py @@ -0,0 +1,81 @@ +"""run() refuse un param() lu sans valeur, au lieu de ne rien trader. + +Un param() sans defaut n'a de valeur que dans un balayage. Un run seul le +lisait comme une serie vide : EMA sans periode, aucun signal, 0 trade, aucune +erreur (la strategie de l'exemple 08 lancee seule). Les balayages, eux, +fixent chaque nom balaye et ne passent pas par cette garde. +""" +import os + +import pytest + +import manifoldbt as bt +from manifoldbt.exceptions import StrategyError +from manifoldbt.indicators import close, ema + +pd = pytest.importorskip("pandas") +np = pytest.importorskip("numpy") + + +@pytest.fixture(scope="module") +def monte(tmp_path_factory): + tmp = tmp_path_factory.mktemp("param") + n = 2400 + idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp") + pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0) + px = 100.0 + np.cumsum(pas) * 0.05 + df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996, + "close": px, "volume": np.full(n, 1000.0)}, index=idx) + racine = str(tmp / "data") + os.makedirs(racine, exist_ok=True) + store = bt.import_dataframe(df.reset_index(), symbol="ZPSV", symbol_id=1, + interval="1h", asset_class="equity", + exchange="TEST", data_root=racine, + metadata_db=str(tmp / "meta.sqlite")) + tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11") + cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1, + initial_capital=1000.0, provider="TEST", + bar_interval=bt.Interval.hours(1), symbol_names={"ZPSV": 1}) + return store, cfg + + +def _croisement(fast, slow): + f, s = ema(close, fast), ema(close, slow) + return (bt.Strategy.create("croisement").signal("f", f).signal("s", s) + .size(bt.when(f > s, 0.25, bt.when(f < s, -0.25, 0.0)))) + + +def test_un_param_lu_sans_valeur_est_refuse_et_nomme(monte): + store, cfg = monte + with pytest.raises(StrategyError, match="has no value in a plain run") as exc: + bt.run(_croisement(bt.param("fast"), bt.param("slow")), cfg, store) + assert "'fast'" in str(exc.value) and "'slow'" in str(exc.value) + + +def test_un_param_avec_defaut_tourne(monte): + store, cfg = monte + r = bt.run(_croisement(bt.param("fast", default=12), bt.param("slow", default=26)), + cfg, store) + assert r.trade_count > 0 + + +def test_une_declaration_strategy_param_donne_la_valeur(monte): + store, cfg = monte + s = (_croisement(bt.param("fast"), bt.param("slow")) + .param("fast", default=12).param("slow", default=26)) + assert bt.run(s, cfg, store).trade_count > 0 + + +def test_un_param_declare_mais_lu_par_rien_ne_bloque_pas(monte): + store, cfg = monte + s = _croisement(12, 26).param("seuil") + assert bt.run(s, cfg, store).trade_count > 0 + + +def test_un_balayage_fixe_les_valeurs_et_passe(monte): + store, cfg = monte + if bt.license_info()[0] != "Pro": + pytest.skip("requires Pro beyond the Community combination limit") + b = bt.run_sweep_lite(_croisement(bt.param("fast"), bt.param("slow")), + {"fast": [8, 12], "slow": [26]}, cfg, store) + assert len(b) == 2 diff --git a/python/tests/test_walk_forward_exo_sources.py b/python/tests/test_walk_forward_exo_sources.py new file mode 100644 index 0000000..78f6ef4 --- /dev/null +++ b/python/tests/test_walk_forward_exo_sources.py @@ -0,0 +1,79 @@ +"""Le walk-forward doit accepter les sources exogenes PAR SYMBOLE (`exo_sources`). + +Une strategie lisant `exo.` d'une source par symbole tournait dans +`run`, mais le walk-forward passait des sources vides a ses trois runs +(selection, run d'apprentissage, run hors echantillon) et levait +"unknown input column". +""" +import os + +import pytest + +import manifoldbt as bt + +pytestmark = pytest.mark.skipif( + bt.license_info()[0] != "Pro", + reason="requires Pro: skipped on a Community wheel", +) + +pd = pytest.importorskip("pandas") +np = pytest.importorskip("numpy") + + +def _monte(tmp_path): + """Le titre trade, une seconde source porte une moyenne causale du titre.""" + n = 2400 + idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp") + pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0) + px = 100.0 + np.cumsum(pas) * 0.05 + aux = pd.Series(px).rolling(24, min_periods=1).mean().to_numpy() + + def barres(p): + return pd.DataFrame({"open": p, "high": p * 1.004, "low": p * 0.996, + "close": p, "volume": np.full(n, 1000.0)}, + index=idx).reset_index() + + racine = str(tmp_path / "data") + meta = str(tmp_path / "meta.sqlite") + os.makedirs(racine, exist_ok=True) + store = bt.import_dataframe(barres(px), symbol="ZESM", symbol_id=1, interval="1h", + asset_class="equity", exchange="TEST", + data_root=racine, metadata_db=meta) + bt.import_dataframe(barres(aux), symbol="ZESA", symbol_id=2, interval="1h", + asset_class="equity", exchange="AUX", + data_root=racine, metadata_db=meta) + tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11") + cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1, + initial_capital=1000.0, provider="TEST", + bar_interval=bt.Interval.hours(1), symbol_names={"ZESM": 1}) + cfg.warmup_bars = 0 + cfg.exo_sources = {1: ("AUX", "ZESA")} + return store, cfg + + +def _strategie(): + from manifoldbt.indicators import close, col + m = col("exo.close") + bande = m * (bt.lit(1.0) - bt.param("dev", default=0.002)) + return (bt.Strategy.create("s") + .signal("b", bande) + .size(bt.when(close < bande, 1.0, bt.when(close > m, 0.0, bt.hold())))) + + +WF = {"method": "Anchored", "n_splits": 3, "train_ratio": 0.5, + "optimize_metric": "sharpe", "param_grid": {"dev": [0.002, 0.005, 0.01]}} + + +def test_la_strategie_trade_dans_run(tmp_path): + store, cfg = _monte(tmp_path) + assert bt.run(_strategie(), cfg, store).trade_count > 0 + + +def test_walk_forward_accepte_une_source_par_symbole(tmp_path): + store, cfg = _monte(tmp_path) + r = bt.run_walk_forward(_strategie(), WF, cfg, store) + assert r["n_folds"] == 3 + # La source est lue dans les trois runs : la selection (metriques IS) et + # les deux courbes. Sans elle, aucun pli ne trade. + assert any(f["oos_metrics"]["total_return"] != 0.0 for f in r["folds"]) + assert any(len(set(f["is_equity"])) > 1 for f in r["folds"]) diff --git a/python/tests/test_walk_forward_resolution.py b/python/tests/test_walk_forward_resolution.py new file mode 100644 index 0000000..c6e0959 --- /dev/null +++ b/python/tests/test_walk_forward_resolution.py @@ -0,0 +1,106 @@ +"""Walk-forward sur un signal plus grossier que le store. + +Un signal 12h sur des barres 1h (mode hybride) rendait, en sortie +automatique, une courbe OOS sous-echantillonnee au pas du signal. Sa longueur +ne valait plus celle des barres natives, et le walk-forward retombait sur les +metriques du run ENTIER, chauffe plate comprise : Sharpe hors echantillon +dilue, WFE faux, sans un avertissement. Avec `output_resolution` a 1h, le meme +walk-forward donnait le bon chiffre. + +Le contrat : les metriques hors echantillon ne dependent pas de la resolution +de sortie, et la courbe rendue suit celle-ci, avec autant d'horodatages que de +points. +""" +import os + +import pytest + +import manifoldbt as bt + +pytestmark = pytest.mark.skipif( + bt.license_info()[0] != "Pro", + reason="requires Pro: skipped on a Community wheel", +) + +pd = pytest.importorskip("pandas") +np = pytest.importorskip("numpy") + +HEURE_NS = 3_600 * 1_000_000_000 + + +def _magasin_horaire(tmp_path): + """100 jours de barres horaires, prix cyclique pour garantir des trades.""" + n = 2400 + idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp") + pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0) + px = 100.0 + np.cumsum(pas) * 0.05 + df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996, + "close": px, "volume": np.full(n, 1000.0)}, index=idx) + racine = str(tmp_path / "data") + meta = str(tmp_path / "meta.sqlite") + os.makedirs(racine, exist_ok=True) + store = bt.import_dataframe(df.reset_index(), symbol="ZWFR", symbol_id=1, + interval="1h", asset_class="equity", + exchange="TEST", data_root=racine, metadata_db=meta) + return store + + +def _config(signal): + tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11") + cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1, + initial_capital=1000.0, provider="TEST", + bar_interval=signal, symbol_names={"ZWFR": 1}) + cfg.warmup_bars = 0 + return cfg + + +def _strategie(): + from manifoldbt.indicators import close, sma + m = sma(close, bt.param("n")) + return (bt.Strategy.create("s") + .signal("m", m) + .size(bt.when(close > m, 1.0, 0.0))) + + +WF = {"geometry": "anchored", "n_splits": 2, "train_ratio": 0.6, + "optimize_metric": "total_return", "param_grid": {"n": [4, 8]}} + + +def test_les_metriques_oos_ne_dependent_pas_de_la_sortie(tmp_path): + store = _magasin_horaire(tmp_path) + auto = bt.run_walk_forward(_strategie(), WF, _config(bt.Interval.hours(12)), store) + cfg_native = _config(bt.Interval.hours(12)) + cfg_native.output_resolution = bt.Interval.hours(1) + native = bt.run_walk_forward(_strategie(), WF, cfg_native, store) + + assert auto["n_folds"] == native["n_folds"] == 2 + for fa, fn in zip(auto["folds"], native["folds"]): + assert fa["best_params"] == fn["best_params"] + for k in ("sharpe", "cagr", "total_return", "volatility", "max_drawdown"): + assert fa["oos_metrics"][k] == pytest.approx(fn["oos_metrics"][k], rel=1e-12), k + assert fa["oos_metrics"]["sharpe"] != 0.0, "le pli doit trader" + if fn["wfe"] is None: + assert fa["wfe"] is None + else: + assert fa["wfe"] == pytest.approx(fn["wfe"], rel=1e-12) + + +def test_la_courbe_oos_suit_la_resolution_de_sortie(tmp_path): + store = _magasin_horaire(tmp_path) + auto = bt.run_walk_forward(_strategie(), WF, _config(bt.Interval.hours(12)), store) + cfg_native = _config(bt.Interval.hours(12)) + cfg_native.output_resolution = bt.Interval.hours(1) + native = bt.run_walk_forward(_strategie(), WF, cfg_native, store) + for fa, fn in zip(auto["folds"], native["folds"]): + eq, ts = fa["oos_equity"], fa["oos_timestamps"] + assert len(eq) == len(ts) > 0 + assert ts[0] >= fa["test_range"]["start"] + assert ts[-1] < fa["test_range"]["end"] + # Un point par seau de sortie, au plus 12h d'ecart... + pas = np.diff(np.asarray(ts)) + assert pas.max() == 12 * HEURE_NS and pas.min() > 0 + # ... et chaque point est celui de la courbe native a cet instant. + natif = dict(zip(fn["oos_timestamps"], fn["oos_equity"])) + assert len(natif) == len(fn["oos_timestamps"]) > len(ts) + for t, e in zip(ts, eq): + assert natif[t] == e