From b47b07e8e6ef315b936baac8c23cfb2c68bfd98d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 21 Aug 2026 01:48:35 +0000 Subject: [PATCH] release: v0.18.0 --- README.md | 79 +++++++-- pyproject.toml | 2 +- python/manifoldbt/__init__.py | 57 ++++++- python/manifoldbt/_cuda_libs.py | 72 ++++++++ python/manifoldbt/_reprs.py | 15 +- python/manifoldbt/plot/research.py | 183 +++++++++++++++------ python/tests/test_sweep_validation.py | 2 +- python/tests/test_walk_forward_exo.py | 87 ++++++++++ python/tests/test_walk_forward_geometry.py | 128 ++++++++++++++ 9 files changed, 544 insertions(+), 81 deletions(-) create mode 100644 python/manifoldbt/_cuda_libs.py create mode 100644 python/tests/test_walk_forward_exo.py create mode 100644 python/tests/test_walk_forward_geometry.py diff --git a/README.md b/README.md index 8af6ee1..1cc7dfe 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,10 @@ sequential fill simulation with realistic fees, slippage, funding and look-ahead ## Why ManifoldBT -- **Fast** — 500K bars in ~13 ms. 353x faster than vectorbt, ~3,500x faster than backtrader. -- **Expressive** — fluent DSL with 30+ indicators, conditional logic, cross-asset references -- **Rigorous** — Monte Carlo, walk-forward, parameter sweeps, lookahead detection, exposure diagnostics -- **Portable** — `pip install`, no Rust toolchain needed. Works on Python 3.9+. +- **Fast**: 10M bars in 317 ms. 78x faster than vectorbt, 308x once you also want drawdown and Sharpe, ~3,500x faster than backtrader. [Measured in public CI](#performance), every run linked. +- **Expressive**: fluent DSL with 30+ indicators, conditional logic, cross-asset references +- **Rigorous**: Monte Carlo, walk-forward, parameter sweeps, lookahead detection, exposure diagnostics +- **Portable**: `pip install`, no Rust toolchain needed. Works on Python 3.9+. ## Installation @@ -37,12 +37,19 @@ sequential fill simulation with realistic fees, slippage, funding and look-ahead pip install manifoldbt # engine only: backtests, sweeps, metrics pip install manifoldbt[plot] # + interactive charts and native windows (show=True) pip install manifoldbt[all] # everything: plots, windows, PNG export, pandas/polars +pip install manifoldbt[gpu] # + NVIDIA runtime compiler, for device="cuda" (Pro) ``` The base install stays light (no browser, no GUI) for scripts, servers and CI. `[plot]` adds plotly and a native window backend; `[all]` also pulls kaleido for static PNG/SVG export (which bundles a headless Chromium). +The Linux and Windows x86_64 wheels already carry the CUDA kernels, so `[gpu]` +only adds the NVIDIA runtime compiler (~180 MB) that compiles them on your +machine. Skip it if you already have a CUDA toolkit installed. An NVIDIA driver +is required, and GPU acceleration is a Pro feature; everything else runs at full +speed on the CPU. + ## Quick Start ```python @@ -83,17 +90,17 @@ print(result.summary()) ## Loading data -Bring your own data, or pull it from a built-in connector — both return a +Bring your own data, or pull it from a built-in connector. Both return a `DataStore` ready for `mbt.run(...)`. -**CSV** — free on all tiers, auto-detects standard / MetaTrader 4 / MetaTrader 5: +**CSV**, free on all tiers, auto-detects standard / MetaTrader 4 / MetaTrader 5: ```python store = mbt.import_csv("EURUSD_1m.csv", symbol="EURUSD", symbol_id=1, interval="1m", asset_class="forex") ``` -**Exchange connectors** — Binance, Bybit, Hyperliquid, dYdX, Bitstamp (free); Databento, Massive (Pro): +**Exchange connectors**: Binance, Bybit, Hyperliquid, dYdX, Bitstamp (free); Databento, Massive (Pro): ```python store = mbt.ingest(provider="binance", symbol="BTCUSDT", symbol_id=1, @@ -133,17 +140,59 @@ manifoldbt ingest --provider binance --symbol BTCUSDT --symbol-id 1 --start ... ## Performance -EMA(12/26) + RSI(14) on 500K synthetic 1-min bars (manifoldbt/vectorbt: median of 5 runs; backtrader: median of 3): +Every number below comes from a benchmark that runs in public CI on a standard +GitHub runner, and links back to the run that produced it. It installs each +engine from PyPI the way a user would, generates its own data, checks that the +engines produced the **same result**, and only then reports how long each took: +a workload they disagree on gets no published timing at all. -| Engine | Time | vs ManifoldBT | -|--------|------|---------------| -| **ManifoldBT** (Rust) | **13 ms** | 1x | -| vectorbt (NumPy) | 4,662 ms | 353x slower | -| backtrader (Python) | 46,944 ms | ~3,556x slower | +**Latest run: [#11](https://github.com/manifoldbt/manifoldbt/actions/runs/32396472073)** +ran on Linux x86_64, 4 vCPU, Python 3.12, manifoldbt 0.17.3 / vectorbt 0.28.4 / +raptorbt 0.9.0, 3 interleaved repetitions. -ManifoldBT and vectorbt produce identical results (−30.23% vs −30.24% return, same trade count); backtrader's event-driven fills give a different PnL. +| Workload | Bars | ManifoldBT | vectorbt | raptorbt | +|---|---:|---:|---:|---:| +| SMA crossover | 10M | **317 ms** | 24.75 s (x78) | 878 ms (x2.8) | +| ...with drawdown, Sharpe, Sortino, volatility | 10M | **317 ms** | 97.46 s (**x308**) | 894 ms (x2.8) | +| ...with a 5 bps fee and 2 bps slippage | 10M | **316 ms** | 24.53 s (x78) | not supported | +| EMA + RSI filter, 5 bps fee | 1M | **52 ms** | 2.21 s (x41) | not supported | +| Five assets in one book | 1M | **140 ms** | 2.34 s (x17) | not supported | -Reproduce: `python benchmarks/bench_vs_competitors.py --rows 500000 --runs 5` +The second row is the one worth reading twice. Asking for a performance summary +costs ManifoldBT nothing measurable, because it computes one during the run +whether you read it or not, and costs vectorbt 73 seconds, because it defers the +equity curve until a risk metric needs it and then has to build one. + +The fifth row is the one where ManifoldBT does worst, and it is published for +that reason: broadcasting a column per asset is close to free for vectorbt, +while walking five books is not free for anything. + +### Parameter sweeps + +| Bars | Combinations | ManifoldBT | vectorbt | raptorbt | +|---:|---:|---:|---:|---:| +| 20,000 | 5,000 | **446 ms**, 40 MB | 5.84 s, 2.5 GB | 7.08 s | +| 200,000 | 10,000 | **9.96 s**, 79 MB | out of memory | 164.50 s | + +Past a certain grid the question stops being speed. vectorbt materialises the +simulation per combination, 1.57 MB of it at 20,000 bars, so the second row +would ask a machine for tens of gigabytes. ManifoldBT runs it in ten seconds +inside 79 MB. + +Reproduce any of it yourself: fork the repository and press **Run workflow** on +[the benchmark](https://github.com/manifoldbt/manifoldbt/actions/workflows/bench-vs-vectorbt.yml), +or run it locally from +[`benchmarks/vs_vectorbt/`](https://github.com/manifoldbt/manifoldbt/tree/master/benchmarks/vs_vectorbt). +The method, the parity gate and the known divergences are written up in +[its README](https://github.com/manifoldbt/manifoldbt/blob/master/benchmarks/vs_vectorbt/README.md). + +### Against an event-driven engine + +backtrader runs the same EMA(12/26) + RSI(14) strategy on 500K 1-minute bars in +**46,944 ms**, against **13 ms** for ManifoldBT: a factor of **3,556**. Measured +with `benchmarks/bench_vs_competitors.py`, median of 3 runs. It sits outside the +CI suite because its event-driven fills produce a different PnL, and the parity +gate publishes no timing for engines that did not do the same work. ### How it compares diff --git a/pyproject.toml b/pyproject.toml index 035edf8..25e2146 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "manifoldbt" -version = "0.17.3" +version = "0.18.0" description = "Rust-powered backtesting engine for quantitative research" requires-python = ">=3.9" license = { file = "LICENSE" } diff --git a/python/manifoldbt/__init__.py b/python/manifoldbt/__init__.py index 37a06d1..480ac07 100644 --- a/python/manifoldbt/__init__.py +++ b/python/manifoldbt/__init__.py @@ -5,6 +5,12 @@ from typing import Any, Dict, List, Optional, Tuple, Union import importlib as _importlib +# Avant TOUT chargement du module natif: la roue GPU charge NVRTC par son nom +# et ne le trouverait pas dans site-packages/nvidia/. Sans effet cote CPU. +from manifoldbt import _cuda_libs as _cuda_libs + +_cuda_libs.rendre_visible() + from manifoldbt._native import ( BacktestResult, BatchResultLite, @@ -189,6 +195,14 @@ def _require_pro_over_combos(n_combos: int, what: str) -> None: ) +#: Distances de bracket balayables par leur nom, en plus des ``param()`` +#: d'expression. Elles ne passent pas par ``param()`` parce qu'une distance de +#: bracket est un champ de configuration, pas un noeud d'expression : rien ne +#: l'evalue. Doit rester aligne sur ``ORDER_SWEEP_PARAMS`` (bt-core, +#: orchestrator.rs), qui fait la substitution par combinaison. +_ORDER_SWEEP_PARAMS = frozenset({"stop_loss", "take_profit", "trailing_stop"}) + + def _validate_swept_params(strategy: "Strategy", names, what: str) -> None: """Reject swept parameter names the strategy never declares. @@ -203,10 +217,10 @@ def _validate_swept_params(strategy: "Strategy", names, what: str) -> None: merges both into ``parameters`` (and is memoised, so this costs nothing). """ declared = set(strategy.to_json_dict().get("parameters") or {}) - unknown = [n for n in names if n not in declared] + unknown = [n for n in names if n not in declared and n not in _ORDER_SWEEP_PARAMS] if not unknown: return - known = ", ".join(sorted(declared)) if declared else "none" + known = ", ".join(sorted(declared | _ORDER_SWEEP_PARAMS)) raise StrategyError( f"{what}: parameter(s) {unknown} are not declared by strategy " f"'{strategy.name}' (declared: {known}). Sweeping them would run the " @@ -1173,17 +1187,42 @@ def run_walk_forward( Args: strategy: Strategy definition. wf_config: Walk-forward config dict with keys: - method (str): "Anchored" or "Rolling" - n_splits (int): Number of folds. - train_ratio (float): Fraction for training (0, 1). + geometry (str): "anchored" (default), "blocked", "pardo" or + "custom". + - "anchored"/"blocked" take ``n_splits`` + ``train_ratio``. + - "pardo"/"custom" take ``train``/``test`` window specs; the + fold count is DERIVED from the window lengths, never chosen. + n_splits (int): Number of folds (anchored/blocked only). + train_ratio (float): Training fraction in (0, 1) (anchored/blocked). + train (dict): pardo: ``{"length": Interval.days(365)}`` (fixed + sliding window W). custom: ``{"mode": "anchored", "min_length": + ...}`` or ``{"mode": "rolling", "length": ...}``. Every + duration also accepts a ``*_bars`` twin (signal bars). + test (dict): ``{"length": Interval.days(90), "step": + Interval.days(30)}``. ``step`` defaults to ``length`` (tests + tile end to end, the only shape whose OOS segments chain into + one tradable curve); ``step < length`` = overlapping windows, + flagged by ``folds_overlap``; ``step > length`` is refused. optimize_metric (str): e.g. "sharpe", "sortino". param_grid (dict): Parameter grid for optimization. max_parallelism (int): Max threads. + device (str): "auto" (default), "cpu" or "cuda". config: Backtest configuration. store: Data store. Returns: - Dict with ``folds`` and ``best_params_per_fold``. + Dict with ``folds``, ``best_params_per_fold``, ``n_folds``, + ``folds_overlap``, ``effective_folds`` (independent folds: overlapping + windows count for less) and ``walk_forward_efficiency`` (Pardo's WFE, + mean of per-fold ``oos.cagr / is.cagr``). + + Each fold's OOS run is WARMED UP: it simulates from the fold's train start + with trading suppressed until the test window, so indicators are hot at + the boundary instead of restarting empty. + + Note: the legacy ``method="Rolling"`` was renamed ``geometry="blocked"`` + (independent blocks separated by gaps, not Pardo's rolling); for Pardo's + walk-forward use ``geometry="pardo"``. """ # Pro gate (friendly message + clean exit). Real enforcement lives natively # in `py_run_walk_forward` (check_feature("walk_forward")), so this cannot be @@ -1511,7 +1550,11 @@ def register_exo( if provider: # Unified layout: {root}/{provider}/{timeframe}/{name}.arrow - target_dir = root / provider / timeframe + # Minuscules obligatoires: les deux ecrivains Rust (ingest.rs) et les + # deux lecteurs creent ce dossier en minuscules. Ecrire "BINANCE" ici + # produisait un second dossier, invisible aux lecteurs sur un systeme + # de fichiers sensible a la casse. + target_dir = root / provider.lower() / timeframe else: # Legacy layout: {root}/exo/{name}.arrow target_dir = root / "exo" diff --git a/python/manifoldbt/_cuda_libs.py b/python/manifoldbt/_cuda_libs.py new file mode 100644 index 0000000..5725592 --- /dev/null +++ b/python/manifoldbt/_cuda_libs.py @@ -0,0 +1,72 @@ +"""Rendre visibles les bibliotheques CUDA installees par pip. + +L'extra ``manifoldbt[gpu]`` installe ``nvidia-cuda-nvrtc-cu12``, qui depose +``libnvrtc.so.12`` / ``nvrtc64_120_0.dll`` dans ``site-packages/nvidia/``. +Ce dossier n'est ni dans le ``PATH`` (Windows) ni dans le chemin de recherche du +chargeur dynamique (Linux). Le coeur Rust charge NVRTC par son NOM, via +``libloading``, donc sans ce coup de pouce il ne trouve rien et le chemin GPU +echoue alors que la bibliotheque est bel et bien installee. + +C'est le meme probleme que PyTorch resout a son import, et par les memes moyens: +``os.add_dll_directory`` sous Windows, un pre-chargement ``RTLD_GLOBAL`` sous +Linux (une bibliotheque deja chargee sous son SONAME satisfait un ``dlopen`` +ulterieur qui la demande par ce nom). + +Sans effet quand l'extra n'est pas installe, ou sur une roue sans CUDA (macOS, +ARM, musl): les dossiers n'existent pas, tout est ignore. Aucune exception ne +remonte, un echec ici ne doit jamais empecher un import. +""" +import os +import sys +from pathlib import Path + +# Sous-dossiers de site-packages/nvidia/ qui portent des bibliotheques utiles au +# moteur. NVRTC compile les noyaux au runtime; le pilote lui-meme (libcuda) vient +# de l'installation systeme, jamais de pip. +_COMPOSANTS = ("cuda_nvrtc", "cuda_runtime") + +_fait = False + + +def _dossiers_candidats(): + """Les dossiers de bibliotheques des paquets nvidia-*, s'ils existent.""" + vus = set() + for base in sys.path: + if not base: + continue + racine = Path(base) / "nvidia" + if racine in vus or not racine.is_dir(): + continue + vus.add(racine) + for composant in _COMPOSANTS: + for feuille in ("bin", "lib"): + d = racine / composant / feuille + if d.is_dir(): + yield d + + +def rendre_visible(): + """Idempotent, silencieux, sans effet quand aucune lib pip n'est presente.""" + global _fait + if _fait: + return + _fait = True + + for d in _dossiers_candidats(): + try: + if sys.platform == "win32": + # add_dll_directory n'agit que sur les chargements ulterieurs, + # d'ou l'appel a l'import et non au premier usage du GPU. + os.add_dll_directory(str(d)) + else: + import ctypes + + for lib in sorted(d.glob("libnvrtc.so*")): + ctypes.CDLL(str(lib), mode=ctypes.RTLD_GLOBAL) + break + except Exception: + # Un dossier illisible, une DLL incompatible, une plateforme + # exotique: rien de tout cela ne justifie de casser l'import du + # paquet. Le chemin GPU rendra une erreur claire s'il ne trouve + # pas sa bibliotheque. + continue diff --git a/python/manifoldbt/_reprs.py b/python/manifoldbt/_reprs.py index 0d7cf7c..9f2fd31 100644 --- a/python/manifoldbt/_reprs.py +++ b/python/manifoldbt/_reprs.py @@ -81,10 +81,17 @@ class WalkForwardResult(dict): is_v = [_m(f, "is_metrics") for f in folds] oos_v = [_m(f, "oos_metrics") for f in folds] - lines = [ - f"> orange: full backtest is overfitted. - If orange >> blue: WFO optimization adds real value. + - Orange: OOS segments from each fold. When the test windows tile the + calendar end to end (anchored, pardo), segments are chained into ONE + curve: each one is rescaled to start at the previous segment's final + value, which is exactly return composition -- the account of someone + trading each fold's re-optimized winner in sequence. That chained curve + is the true out-of-sample performance of the WFO *policy*. + - When the test windows overlap (custom, step < length) or leave gaps + between them (blocked), two calendars cannot be traded at once and no + single account curve exists. Segments are then drawn separately, on + their own dates, and never chained: a single curve here would be a lie. + - Blue: full backtest with default params, restricted to the dates the + OOS windows actually cover. Comparing against the full period would + overlay months of compounding the OOS curve never had. """ with theme_context(): fig = new_figure(figsize) - # 1. Stitch OOS segments: chain so each starts where previous ended - stitched = [] - current_val = None - fold_boundaries = [] - for fold in folds: - oos_eq = fold.get("oos_equity", []) - if not oos_eq: - continue - oos = np.array(oos_eq, dtype=float) - if current_val is None: - stitched.extend(oos.tolist()) - current_val = oos[-1] - else: - scale = current_val / oos[0] if oos[0] != 0 else 1.0 - scaled = oos * scale - stitched.extend(scaled.tolist()) - current_val = scaled[-1] - fold_boundaries.append(len(stitched)) + # Segment geometry decides everything: chain only when the test + # windows tile the calendar without overlap or gap. The ranges come + # from one derivation in Rust, so exact equality is the right test. + ranges = [f.get("test_range") or {} for f in folds] + starts = [r.get("start") for r in ranges] + ends = [r.get("end") for r in ranges] + contiguous = ( + all(v is not None for v in starts + ends) + and all(starts[i + 1] == ends[i] for i in range(len(folds) - 1)) + and not wf_result.get("folds_overlap", False) + ) - if not stitched: + def _seg(fold): + eq = np.asarray(fold.get("oos_equity", []), dtype=float) + ts = np.asarray(fold.get("oos_timestamps", []), dtype="int64") + if len(ts) == len(eq) and len(ts) > 0: + return eq, ts.view("datetime64[ns]") + return eq, None + + segments = [_seg(f) for f in folds] + segments = [(eq, d) for eq, d in segments if len(eq) > 0] + if not segments: fig.update_layout(title_text="No OOS equity data available") return finalize(fig, show=show, save=save) + has_dates = all(d is not None for _, d in segments) - stitched = np.array(stitched) - x = np.arange(len(stitched)) + if contiguous: + # Chain: rescaling each segment to the previous final value IS + # return composition, valid because the windows are consecutive. + morceaux_eq, morceaux_dates = [], [] + current_val = None + for eq, d in segments: + if current_val is None: + scaled = eq + else: + scaled = eq * (current_val / eq[0]) if eq[0] != 0 else eq + morceaux_eq.append(scaled) + if has_dates: + morceaux_dates.append(d) + current_val = scaled[-1] - # 2. Full backtest equity (if provided) - if full_result is not None: - full_eq = np.array(full_result.equity_curve) - if len(full_eq) > 0: - indices = np.linspace(0, len(full_eq) - 1, len(stitched), dtype=int) - full_resampled = full_eq[indices].astype(float) - if full_resampled[0] != 0: - full_resampled = full_resampled * (stitched[0] / full_resampled[0]) + stitched = np.concatenate(morceaux_eq) + # Rester en numpy : `datetime64[ns].tolist()` rend des ENTIERS + # nanosecondes, pas des dates, et l'axe redeviendrait numerique. + x = (np.concatenate(morceaux_dates) if has_dates + else np.arange(len(stitched))) + fins = np.cumsum([len(e) for e in morceaux_eq]) - 1 + boundaries = [x[i] for i in fins] + + _overlay_full_backtest(fig, full_result, x, stitched, + has_dates=has_dates, color=is_color) + fig.add_trace(go.Scatter( + x=x, y=stitched, mode="lines", + name="Walk-forward (stitched OOS)", + line=dict(color=oos_color, width=1.0), opacity=0.85, + )) + for b in boundaries[:-1]: + fig.add_vline(x=b, line_color=DARK_GRAY, line_width=0.5, + line_dash="dash", opacity=0.3) + titre = title or "Walk-Forward: Stitched OOS vs Full Backtest" + else: + # Overlapping or gapped test windows: no single tradable account + # exists, draw each fold on its own dates instead of pretending. + raison = ("overlapping test windows" + if wf_result.get("folds_overlap", False) + else "gaps between test windows") + for i, (eq, d) in enumerate(segments): + x = d if d is not None else np.arange(len(eq)) fig.add_trace(go.Scatter( - x=x, y=full_resampled, mode="lines", - name="Full backtest (default params)", - line=dict(color=is_color, width=0.8), opacity=0.4, + x=x, y=eq, mode="lines", + name=f"Fold {i + 1} OOS", + line=dict(width=1.0), opacity=0.8, )) - - # 3. Plot stitched OOS on top - fig.add_trace(go.Scatter( - x=x, y=stitched, mode="lines", - name="Walk-forward (stitched OOS)", - line=dict(color=oos_color, width=1.0), opacity=0.85, - )) - - # Fold boundaries - for b in fold_boundaries[:-1]: - fig.add_vline(x=b, line_color=DARK_GRAY, line_width=0.5, - line_dash="dash", opacity=0.3) + fig.add_annotation( + x=0.5, y=1.06, xref="paper", yref="paper", showarrow=False, + text=f"segments not chained: {raison}", + font=dict(size=10, color=DARK_GRAY), + ) + titre = title or "Walk-Forward: OOS Segments (not tradable as one curve)" fig.update_layout( - title_text=title or "Walk-Forward: Stitched OOS vs Full Backtest", + title_text=titre, legend=dict(x=0.01, y=0.99), ) - fig.update_xaxes(title_text="Bars") + fig.update_xaxes(title_text="Date" if has_dates else "Bars") fig.update_yaxes(title_text="Equity") return finalize(fig, show=show, save=save) +def _overlay_full_backtest(fig, full_result, x, stitched, *, has_dates, color): + """Full-backtest baseline, restricted to the dates the OOS curve covers. + + The previous version resampled the FULL period onto the OOS length with + ``np.linspace``: it overlaid a year of compounding on a few months of + out-of-sample and the baseline crushed the OOS curve for purely + mechanical reasons. Date alignment is the only honest comparison, so + without dates on both sides nothing is drawn. + """ + if full_result is None: + return + if not has_dates: + import warnings + warnings.warn( + "walk_forward stitched: full_result ignored (the walk-forward " + "result carries no oos_timestamps; re-run it to get dated folds)", + stacklevel=3) + return + try: + full_dates, full_eq = equity_with_dates(full_result) + except Exception: + import warnings + warnings.warn( + "walk_forward stitched: full_result ignored (no positions table " + "to date its equity curve)", stacklevel=3) + return + if len(full_eq) == 0: + return + mask = (full_dates >= x[0]) & (full_dates <= x[-1]) + if not mask.any(): + return + fen_dates, fen_eq = full_dates[mask], full_eq[mask].astype(float) + # Meme point de depart que la courbe OOS : on compare des trajectoires, + # pas des niveaux absolus. + if fen_eq[0] != 0: + fen_eq = fen_eq * (stitched[0] / fen_eq[0]) + fig.add_trace(go.Scatter( + x=fen_dates, y=fen_eq, mode="lines", + name="Full backtest (default params, same window)", + line=dict(color=color, width=0.8), opacity=0.4, + )) + + # ── Parameter Stability ───────────────────────────────────────────────────── diff --git a/python/tests/test_sweep_validation.py b/python/tests/test_sweep_validation.py index 18dc56e..6d05343 100644 --- a/python/tests/test_sweep_validation.py +++ b/python/tests/test_sweep_validation.py @@ -56,7 +56,7 @@ def test_sweep_rejects_undeclared_param(): def test_walk_forward_rejects_undeclared_param(): wf = { - "method": "Rolling", "n_splits": 2, "train_ratio": 0.7, + "geometry": "blocked", "n_splits": 2, "train_ratio": 0.7, "optimize_metric": "sharpe", "param_grid": {"fast": [10, 20]}, } with pytest.raises((StrategyError, bt.LicenseError)) as exc: diff --git a/python/tests/test_walk_forward_exo.py b/python/tests/test_walk_forward_exo.py new file mode 100644 index 0000000..e798b08 --- /dev/null +++ b/python/tests/test_walk_forward_exo.py @@ -0,0 +1,87 @@ +"""Le walk-forward doit accepter les series EXOGENES, comme tous les autres +chemins du moteur. + +Il chargeait ses colonnes exo pour les deux runs qui tracent les courbes +d'equite, mais appelait le moteur avec des tables VIDES pour la selection du +reglage. Une branche sur trois etait oubliee, et c'etait celle qui decide : +toute strategie lisant `exo..` echouait sur "unknown input +column" alors que la meme strategie tourne dans `run`, `run_sweep_lite` et +`run_batch_lite`. + +Le second test verifie le point qui rend la correction sure : la selection lit +desormais des colonnes DECOUPEES une fois pour toutes, la ou les courbes les +rechargent par fenetre. Les deux chemins doivent rendre la meme metrique +d'in-sample pour le reglage retenu, sinon le decoupage est faux. +""" +import os + +import pytest + +import manifoldbt as bt + +pd = pytest.importorskip("pandas") +np = pytest.importorskip("numpy") + + +def _monte(tmp_path): + """Une serie horaire regulière, plus une moyenne posee en serie exogene.""" + n = 2400 + idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp") + pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0) + px = 100.0 + np.cumsum(pas) * 0.05 + df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996, + "close": px, "volume": np.full(n, 1000.0)}, index=idx) + racine = str(tmp_path / "data") + meta = str(tmp_path / "meta.sqlite") + os.makedirs(racine, exist_ok=True) + store = bt.import_dataframe(df.reset_index(), symbol="ZEXO", symbol_id=1, + interval="1h", asset_class="equity", + exchange="TEST", data_root=racine, metadata_db=meta) + moy = pd.Series(px).rolling(24).mean().to_numpy() + bt.register_exo("moyenne", pd.DataFrame({"timestamp": idx, "sma": moy}), + store=store, data_root=racine, timeframe="1h") + # la plage doit coller aux donnees : 2400 heures = 100 jours + tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11") + cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1, + initial_capital=1000.0, provider="TEST", + bar_interval=bt.Interval.hours(1), symbol_names={"ZEXO": 1}) + cfg.warmup_bars = 0 + cfg.exo_data = ["moyenne"] + return store, cfg + + +def _strategie(): + from manifoldbt.indicators import close, col + m = col("exo.moyenne.sma") + bande = m * (bt.lit(1.0) - bt.param("dev")) + return (bt.Strategy.create("s") + .signal("aux", bande) + .size(bt.when(close < bande, 1.0, bt.when(close > m, 0.0, bt.hold())))) + + +WF = {"method": "Anchored", "n_splits": 3, "train_ratio": 0.5, + "optimize_metric": "sharpe", + "param_grid": {"dev": [0.002, 0.005, 0.01]}} + + +def test_walk_forward_accepte_une_serie_exogene(tmp_path): + store, cfg = _monte(tmp_path) + r = bt.run_walk_forward(_strategie(), WF, cfg, store) + assert len(r["folds"]) == 3 + # chaque pli doit avoir EVALUE la grille, pas l'avoir sautee + for f in r["folds"]: + assert len(f["all_is_results"]) == 3, "la grille n'a pas ete evaluee" + + +def test_selection_et_courbes_voient_les_memes_colonnes(tmp_path): + """La selection tranche les colonnes une fois, les courbes les rechargent + par fenetre : le meme reglage doit donner le meme in-sample des deux cotes.""" + store, cfg = _monte(tmp_path) + r = bt.run_walk_forward(_strategie(), WF, cfg, store) + for f in r["folds"]: + meilleur = max(f["all_is_results"], + key=lambda x: x["metrics"].get("sharpe", float("-inf"))) + a = meilleur["metrics"]["sharpe"] + b = f["is_metrics"]["sharpe"] + assert a == b, "selection {} contre courbe {} au pli {}".format( + a, b, f["fold_index"]) diff --git a/python/tests/test_walk_forward_geometry.py b/python/tests/test_walk_forward_geometry.py new file mode 100644 index 0000000..6649d45 --- /dev/null +++ b/python/tests/test_walk_forward_geometry.py @@ -0,0 +1,128 @@ +"""Geometrie du walk-forward et chauffe hors echantillon. + +Trois contrats poses par la refonte : + +1. Le run OOS est CHAUFFE : il simule depuis le debut de l'apprentissage du + pli et ne trade qu'a partir du test. Le test le prouve avec un SMA plus + long que la fenetre de test : a froid l'indicateur resterait nul sur toute + la fenetre et l'equity serait PLATE ; chauffe, il est disponible des la + premiere barre tradable. + +2. Les geometries `pardo` et `custom` derivent le nombre de plis des + longueurs de fenetres, et `custom` sait exprimer des tests recouvrants -- + signales par `folds_overlap` et repondus par `effective_folds`. + +3. `method="Rolling"` est refuse avec un message qui nomme le remplacant : + ce mode faisait des blocs disjoints, pas le rolling de Pardo. +""" +import os + +import pytest + +import manifoldbt as bt + +pd = pytest.importorskip("pandas") +np = pytest.importorskip("numpy") + +JOUR_NS = 86_400 * 1_000_000_000 + + +def _monte(tmp_path): + """100 jours de barres horaires, prix cyclique pour garantir des trades.""" + n = 2400 + idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp") + pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0) + px = 100.0 + np.cumsum(pas) * 0.05 + df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996, + "close": px, "volume": np.full(n, 1000.0)}, index=idx) + racine = str(tmp_path / "data") + meta = str(tmp_path / "meta.sqlite") + os.makedirs(racine, exist_ok=True) + store = bt.import_dataframe(df.reset_index(), symbol="ZWFG", symbol_id=1, + interval="1h", asset_class="equity", + exchange="TEST", data_root=racine, metadata_db=meta) + tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11") + cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1, + initial_capital=1000.0, provider="TEST", + bar_interval=bt.Interval.hours(1), symbol_names={"ZWFG": 1}) + cfg.warmup_bars = 0 + return store, cfg + + +def _strategie_sma_long(): + """SMA plus long (400 barres) que toute fenetre de test des tests ci-dessous.""" + from manifoldbt.indicators import close, sma + m = sma(close, 400) * (bt.lit(1.0) + bt.param("dev") * 0.0) + return (bt.Strategy.create("s") + .signal("m", m) + .size(bt.when(close > m, 1.0, 0.0))) + + +def test_oos_est_chauffe_l_indicateur_est_disponible(tmp_path): + store, cfg = _monte(tmp_path) + wf = {"geometry": "anchored", "n_splits": 2, "train_ratio": 0.8, + "optimize_metric": "sharpe", "param_grid": {"dev": [0.0, 1.0]}} + r = bt.run_walk_forward(_strategie_sma_long(), wf, cfg, store) + assert r["n_folds"] == 2 + for f in r["folds"]: + eq = f["oos_equity"] + ts = f["oos_timestamps"] + # la courbe rendue couvre les seules barres du test, chauffe exclue + assert len(eq) == len(ts) > 0 + assert ts[0] >= f["test_range"]["start"] + assert ts[-1] < f["test_range"]["end"] + # fenetre de test = 10 jours = 240 barres < SMA(400) : a froid, + # l'indicateur serait nul sur TOUTE la fenetre et l'equity plate. + assert len(eq) <= 400, "le test doit etre plus court que le SMA" + assert max(eq) != min(eq), ( + "equity OOS plate : l'indicateur n'a pas ete chauffe") + + +def test_pardo_derive_le_nombre_de_plis(tmp_path): + store, cfg = _monte(tmp_path) + wf = {"geometry": "pardo", + "train": {"length": {"Days": 50}}, + "test": {"length": {"Days": 10}}, + "optimize_metric": "sharpe", "param_grid": {"dev": [0.0]}} + r = bt.run_walk_forward(_strategie_sma_long(), wf, cfg, store) + # 100 jours : premier test a j50, puis 5 fenetres de 10 jours + assert r["n_folds"] == 5 + assert r["folds_overlap"] is False + assert r["effective_folds"] == 5.0 + for f in r["folds"]: + tr, te = f["train_range"], f["test_range"] + assert te["start"] - tr["start"] == 50 * JOUR_NS + assert te["end"] - te["start"] == 10 * JOUR_NS + + +def test_custom_recouvrant_expose_les_plis_effectifs(tmp_path): + store, cfg = _monte(tmp_path) + wf = {"geometry": "custom", + "train": {"mode": "anchored", "min_length": {"Days": 60}}, + "test": {"length": {"Days": 10}, "step": {"Days": 5}}, + "optimize_metric": "sharpe", "param_grid": {"dev": [0.0]}} + r = bt.run_walk_forward(_strategie_sma_long(), wf, cfg, store) + # tests possibles de j60 a j90 par pas de 5 -> 7 plis, union 40 jours + assert r["n_folds"] == 7 + assert r["folds_overlap"] is True + assert r["effective_folds"] == pytest.approx(4.0) + + +def test_rolling_est_refuse_avec_le_remplacant_nomme(tmp_path): + store, cfg = _monte(tmp_path) + wf = {"method": "Rolling", "n_splits": 2, "train_ratio": 0.7, + "optimize_metric": "sharpe", "param_grid": {"dev": [0.0]}} + with pytest.raises(Exception) as exc: + bt.run_walk_forward(_strategie_sma_long(), wf, cfg, store) + msg = str(exc.value) + assert "blocked" in msg and "pardo" in msg + + +def test_wfe_est_rendu(tmp_path): + store, cfg = _monte(tmp_path) + wf = {"geometry": "anchored", "n_splits": 2, "train_ratio": 0.8, + "optimize_metric": "sharpe", "param_grid": {"dev": [0.0]}} + r = bt.run_walk_forward(_strategie_sma_long(), wf, cfg, store) + assert "walk_forward_efficiency" in r + for f in r["folds"]: + assert "wfe" in f