release: v0.8.6

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="assets/logo.png" width="110" alt="ManifoldBT logo">
+  <img src="https://raw.githubusercontent.com/Jimmy7892/manifoldbt/master/assets/logo.png" width="110" alt="ManifoldBT logo">
 </p>
 
 <p align="center">
@@ -14,7 +14,7 @@
 <p align="center">
   <a href="https://www.manifoldbt.com">Website</a> &middot;
   <a href="https://www.manifoldbt.com/docs/documentation.html">Documentation</a> &middot;
-  <a href="examples/">Examples</a>
+  <a href="https://github.com/Jimmy7892/manifoldbt/tree/master/examples">Examples</a>
 </p>
 
 ---
@@ -109,25 +109,25 @@ manifoldbt ingest --provider binance --symbol BTCUSDT --symbol-id 1 --start ...
 
 | # | Example | What it shows |
 |---|---------|---------------|
-| 00 | [Template](examples/00_template.py) | Minimal starting point |
+| 00 | [Template](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/00_template.py) | Minimal starting point |
-| 01 | [Trend Following](examples/01_trend_following.py) | EMA crossover, volume filter, stop-loss |
+| 01 | [Trend Following](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/01_trend_following.py) | EMA crossover, volume filter, stop-loss |
-| 02 | [Mean Reversion](examples/02_mean_reversion.py) | EMA crossover with parameter sweep |
+| 02 | [Mean Reversion](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/02_mean_reversion.py) | EMA crossover with parameter sweep |
-| 03 | [Multi-Asset Momentum](examples/03_multi_asset_momentum.py) | Cross-asset signals |
+| 03 | [Multi-Asset Momentum](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/03_multi_asset_momentum.py) | Cross-asset signals |
-| 04 | [Linear Regression](examples/04_linear_regression.py) | Regression-based signal |
+| 04 | [Linear Regression](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/04_linear_regression.py) | Regression-based signal |
-| 05 | [Statistical Arbitrage](examples/05_stat_arb.py) | Pairs trading, spread z-score |
+| 05 | [Statistical Arbitrage](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/05_stat_arb.py) | Pairs trading, spread z-score |
-| 06 | [Full Visualization](examples/06_full_visualization.py) | Tearsheet and charts |
+| 06 | [Full Visualization](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/06_full_visualization.py) | Tearsheet and charts |
-| 07 | [Walk-Forward](examples/07_walk_forward.py) | Out-of-sample validation |
+| 07 | [Walk-Forward](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/07_walk_forward.py) | Out-of-sample validation |
-| 08 | [2D Sweep](examples/08_sweep_2d_heatmap.py) | Parameter grid heatmap |
+| 08 | [2D Sweep](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/08_sweep_2d_heatmap.py) | Parameter grid heatmap |
-| 09 | [3D Surface](examples/09_surface_3d.py) | Parameter surface plot |
+| 09 | [3D Surface](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/09_surface_3d.py) | Parameter surface plot |
-| 10 | [Monte Carlo](examples/10_monte_carlo.py) | Permutation-based robustness |
+| 10 | [Monte Carlo](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/10_monte_carlo.py) | Permutation-based robustness |
-| 11 | [Portfolio](examples/11_portfolio.py) | Multi-strategy portfolio |
+| 11 | [Portfolio](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/11_portfolio.py) | Multi-strategy portfolio |
-| 12 | [Diagnostics](examples/12_diagnostics.py) | Lookahead & exposure safety checks |
+| 12 | [Diagnostics](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/12_diagnostics.py) | Lookahead & exposure safety checks |
-| 13 | [Stochastic Simulation](examples/13_stochastic_simulation.py) | SDE path simulation (GBM, Heston, …) |
+| 13 | [Stochastic Simulation](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/13_stochastic_simulation.py) | SDE path simulation (GBM, Heston, …) |
-| 14 | [Multi-Timeframe](examples/14_multi_timeframe.py) | Combining signals across timeframes |
+| 14 | [Multi-Timeframe](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/14_multi_timeframe.py) | Combining signals across timeframes |
-| 15 | [Cross-Exchange](examples/15_cross_exchange.py) | Signal on one venue, execute on another |
+| 15 | [Cross-Exchange](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/15_cross_exchange.py) | Signal on one venue, execute on another |
-| 16 | [Exogenous Data](examples/16_hashrate_exogene.py) | External series (e.g. hashrate) as a signal |
+| 16 | [Exogenous Data](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/16_hashrate_exogene.py) | External series (e.g. hashrate) as a signal |
-| 17 | [Per-Venue Fees](examples/17_per_venue_fees.py) | Per-venue funding & borrow costs |
+| 17 | [Per-Venue Fees](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/17_per_venue_fees.py) | Per-venue funding & borrow costs |
-| 18 | [CSV Import](examples/18_csv_import.py) | Load OHLCV from CSV (standard / MT4 / MT5) |
+| 18 | [CSV Import](https://github.com/Jimmy7892/manifoldbt/blob/master/examples/18_csv_import.py) | Load OHLCV from CSV (standard / MT4 / MT5) |
 
 ## Performance
 
@@ -164,4 +164,4 @@ Full API reference, indicator list, configuration guide, and best practices:
 
 Apache 2.0 with Commons Clause. The source is available, free to use,
 modify and self-host. Reselling the software or offering it as a paid
-hosted service is not permitted. See [LICENSE](LICENSE) for the full text.
+hosted service is not permitted. See [LICENSE](https://github.com/Jimmy7892/manifoldbt/blob/master/LICENSE) for the full text.

examples/03_multi_asset_momentum_backup.py

@@ -1,69 +0,0 @@
-"""Multi-Asset Momentum -- relative strength across 5 assets.
-
-Demonstrates:
-  - Multi-asset universe (5 symbols)
-  - Momentum via smoothed ROC on 12h bars
-  - Volatility-adjusted sizing
-
-Usage:
-    python examples/03_multi_asset_momentum.py
-"""
-import os
-import time
-import manifoldbt as mbt
-from manifoldbt.indicators import close, ema, roc, high, low
-from manifoldbt.helpers import time_range, Slippage, Interval
-
-# -- Indicators ---------------------------------------------------------------
-mom = ema(roc(close, 14), 6)                       # 7-day momentum, smoothed
-avg_range = (high - low).rolling_mean(14)
-norm_vol = avg_range / (close + mbt.lit(1e-12))    # normalized volatility
-safe_vol = mbt.when(norm_vol > 0.0005, norm_vol, 0.0005)
-
-# -- Strategy -----------------------------------------------------------------
-signal = mbt.when(mom > 0.0, mom / safe_vol, 0.0)
-
-strategy = (
-    mbt.Strategy.create("multi_momentum")
-    .signal("momentum", mom)
-    .signal("norm_vol", norm_vol)
-    .size(signal * 0.01)
-    .describe("Multi-asset momentum with volatility-adjusted sizing")
-)
-
-# -- Config -------------------------------------------------------------------
-start, end = time_range("2022-01-01", "2025-01-01")
-
-config = mbt.BacktestConfig(
-    universe=[1, 2, 3, 4, 5],
-    time_range_start=start,
-    time_range_end=end,
-    bar_interval=Interval.hours(12),
-    initial_capital=10_000,
-    execution=mbt.ExecutionConfig(
-        signal_delay=1,
-        max_position_pct=0.3,
-        allow_short=False,
-    ),
-    fees=mbt.FeeConfig.binance_perps(),
-    slippage=Slippage.fixed_bps(2),
-    warmup_bars=25,
-)
-
-# -- Run ----------------------------------------------------------------------
-if __name__ == "__main__":
-    root = os.path.join(os.path.dirname(__file__), "..")
-    data_root = os.path.abspath(os.path.join(root, "data"))
-    store = mbt.DataStore(
-        data_root=data_root,
-        metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")),
-        arrow_dir=os.path.join(data_root, "mega"),
-    )
-
-    t0 = time.perf_counter()
-    result = mbt.run(strategy, config, store)
-    elapsed = time.perf_counter() - t0
-
-    print(result.summary())
-    print(f"\nElapsed: {elapsed:.3f}s")
-    mbt.plot.summary(result, show=True)

examples/bench_mega_all_symbols.py

@@ -1,62 +0,0 @@
-"""Benchmark: all symbols, Arrow IPC store (bars_1m + bars_1h).
-
-Usage:
-    python examples/bench_mega_all_symbols.py
-"""
-import os
-import time
-import manifoldbt as mbt
-from manifoldbt.indicators import ema, close
-from manifoldbt.helpers import time_range, Slippage, Interval
-
-# -- Strategy -------------------------------------------------------------------
-fast = ema(close, 12)
-slow = ema(close, 200)
-trend = fast - slow
-
-strategy = (
-    mbt.Strategy.create("ema_crossover_all")
-    .signal("trend", trend)
-    .size(mbt.when(trend > 0.0, 0.5, 0.0))
-)
-
-# -- Config: all available Binance perp symbols, 3 years, 1h bars -----------------
-universe = {"binance": [
-    "BTC-USDT:perp", "ETH-USDT:perp", "LTC-USDT:perp", "BNB-USDT:perp",
-    "DOT-USDT:perp", "XRP-USDT:perp", "ADA-USDT:perp", "LINK-USDT:perp",
-    "DOGE-USDT:perp", "AVAX-USDT:perp",
-]}
-start, end = time_range("2022-01-01", "2025-01-01")
-
-config = mbt.BacktestConfig(
-    universe=universe,
-    time_range_start=start,
-    time_range_end=end,
-    bar_interval=Interval.minutes(60),
-    precise=True,
-    initial_capital=100_000,
-    execution=mbt.ExecutionConfig(
-        allow_short=False,
-        max_position_pct=0.05,
-        position_sizing_mode="FractionOfInitialCapital",
-    ),
-    fees=mbt.FeeConfig.binance_perps(),
-    slippage=Slippage.fixed_bps(2),
-    warmup_bars=30,
-)
-
-# -- Run -----------------------------------------------------------------------
-root = os.path.join(os.path.dirname(__file__), "..")
-data_root = os.path.abspath(os.path.join(root, "data"))
-metadata_db = os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite"))
-
-store = mbt.DataStore(data_root=data_root, metadata_db=metadata_db, arrow_dir=os.path.join(data_root, "mega"))
-
-t0 = time.perf_counter()
-result = mbt.run(strategy, config, store)
-elapsed = time.perf_counter() - t0
-
-print(result.profile_summary())
-print(f"\nWall clock: {elapsed:.3f}s")
-print(f"Trades: {result.trade_count}")
-print(f"Symbols: {len(universe['binance'])}")

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "manifoldbt"
-version = "0.8.5"
+version = "0.8.6"
 description = "Rust-powered backtesting engine for quantitative research"
 requires-python = ">=3.9"
 license = { file = "LICENSE" }

python/manifoldbt/_native.pyi

@@ -5,9 +5,17 @@ import pyarrow as pa
 
 
 class DataStore:
-    """Parquet data store with SQLite metadata."""
+    """Bar data store (Parquet by default, or Arrow IPC via ``arrow_dir``) with SQLite metadata."""
 
-    def __init__(self, data_root: str, metadata_db: str = "metadata/metadata.sqlite") -> None: ...
+    def __init__(
+        self,
+        data_root: str,
+        metadata_db: str = "metadata/metadata.sqlite",
+        dataset: str = "bars_1m",
+        mega: Optional[str] = None,
+        arrow_dir: Optional[str] = None,
+    ) -> None: ...
+    def dataset(self) -> str: ...
     def data_root(self) -> str: ...
     def metadata_db(self) -> str: ...
     def active_version(self, dataset: str) -> str: ...

python/manifoldbt/dataframe.py

@@ -98,8 +98,15 @@ def arrow_to_series(
     if backend == "polars":
         import polars as pl
 
-        if hasattr(array, "to_pylist"):
+        try:
-            return pl.Series(name=name, values=array.to_pylist())
+            import pyarrow as pa
+        except ImportError:
+            pa = None
+        # Zero-copy: hand the Arrow buffers straight to polars instead of boxing
+        # every value into a Python object via to_pylist() (copies the whole
+        # column). pl.from_arrow shares the underlying buffers.
+        if pa is not None and isinstance(array, (pa.Array, pa.ChunkedArray)):
+            return pl.from_arrow(array).rename(name)
         return pl.Series(name=name, values=list(array))
 
     return array

python/manifoldbt/diagnostics.py

@@ -13,6 +13,27 @@ _EMPTY_TS = np.array([], dtype="datetime64[ns]")
 _SAFETY_PRO_FEATURE = "Safety checks (lookahead, exposure)"
 
 
+def _prepare_for_diagnostics(config, strategy, store):
+    """Mirror ``run()``'s config/store preparation for the diagnostics path.
+
+    ``run()`` resolves the config and store before serializing
+    (``_cap_output_resolution`` -> ``_resolve_store`` -> ``_prepare_config``).
+    Diagnostics must do the same: in particular a dict ``universe`` has to be
+    resolved to a ``List[SymbolId]`` first, otherwise ``config.to_json()`` emits
+    a JSON map and the Rust loader rejects it ("invalid type: map, expected a
+    sequence"). Returns the prepared ``(config, store)``.
+    """
+    from manifoldbt import (
+        _cap_output_resolution,
+        _resolve_store,
+        _prepare_config,
+    )
+    config = _cap_output_resolution(config)
+    store = _resolve_store(config, store)
+    config = _prepare_config(config, strategy, store)
+    return config, store
+
+
 @dataclass
 class LookaheadReport:
     """Result of a single look-ahead bias test."""
@@ -289,6 +310,10 @@ def detect_lookahead(
         run_on_aligned as _run_on_aligned,
     )
 
+    # Resolve config/store exactly like run() (notably dict universe -> ids),
+    # otherwise config.to_json() emits a map the Rust loader rejects.
+    config, store = _prepare_for_diagnostics(config, strategy, store)
+
     period = config.time_range_end - config.time_range_start
 
     # Load data ONCE for the full range.
@@ -813,6 +838,10 @@ def check_exposure_stability(
         run_on_aligned as _run_on_aligned,
     )
 
+    # Resolve config/store exactly like run() (notably dict universe -> ids),
+    # otherwise config.to_json() emits a map the Rust loader rejects.
+    config, store = _prepare_for_diagnostics(config, strategy, store)
+
     period = config.time_range_end - config.time_range_start
 
     # Load data ONCE.

python/tests/test_diagnostics.py

@@ -0,0 +1,108 @@
+"""Regression tests for the diagnostics config-preparation path.
+
+Guards the fix for the bug where ``detect_lookahead`` / ``check_exposure_stability``
+crashed with a dict ``universe`` (e.g. ``{"binance": ["BTC-USDT:perp"]}``):
+they serialized the config without resolving the universe, so ``config.to_json()``
+emitted a JSON *map* while the Rust loader expects a *sequence*
+(``ValueError: invalid type: map, expected a sequence``).
+
+The fix routes diagnostics through the same preparation as ``run()`` via
+``_prepare_for_diagnostics``. These tests assert that helper resolves a dict
+universe into a list of integer SymbolIds (so serialization is a JSON array),
+without needing a Pro license or real market data.
+"""
+import json
+import sqlite3
+
+import manifoldbt as bt
+from manifoldbt.diagnostics import _prepare_for_diagnostics
+
+
+def _make_metadata_db(path):
+    """Create a minimal metadata sqlite with one resolvable symbol (id=1)."""
+    conn = sqlite3.connect(path)
+    conn.execute(
+        "CREATE TABLE symbols ("
+        "id INTEGER PRIMARY KEY, base_currency TEXT, quote_currency TEXT, "
+        "asset_class TEXT, exchange TEXT, ticker TEXT)"
+    )
+    conn.execute(
+        "INSERT INTO symbols VALUES (1, 'BTC', 'USDT', 'CryptoPerpetual', "
+        "'BINANCE', 'BTC-USDT:perp')"
+    )
+    conn.commit()
+    conn.close()
+    return str(path)
+
+
+class _StubStore:
+    """Minimal DataStore stand-in.
+
+    ``_resolve_normalized`` only needs ``metadata_db()`` (+ ``resolve_symbol``
+    as a fallback). ``dataset()`` raises so ``_resolve_store`` returns the store
+    unchanged instead of trying to swap datasets on disk.
+    """
+
+    def __init__(self, db_path):
+        self._db = db_path
+
+    def metadata_db(self):
+        return self._db
+
+    def dataset(self):
+        raise NotImplementedError
+
+    def resolve_symbol(self, name):  # fallback, not expected to be hit here
+        return 1
+
+
+def _simple_strategy():
+    return (
+        bt.Strategy.create("regression")
+        .signal("s", bt.lit(1.0))
+        .size(bt.col("s"))
+    )
+
+
+def test_prepare_for_diagnostics_resolves_dict_universe(tmp_path):
+    """A dict universe must become a list of ints before serialization."""
+    db = _make_metadata_db(tmp_path / "metadata.sqlite")
+    store = _StubStore(db)
+
+    config = bt.BacktestConfig(
+        universe={"binance": ["BTC-USDT:perp"]},
+        time_range_start=0,
+        time_range_end=4_000_000_000,
+        bar_interval={"Hours": 1},
+        initial_capital=1000.0,
+    )
+
+    prepared, _ = _prepare_for_diagnostics(config, _simple_strategy(), store)
+
+    # Core invariant: universe is a list of ints, never a dict.
+    assert isinstance(prepared.universe, list)
+    assert prepared.universe == [1]
+
+    # And the JSON the Rust loader sees is an array, not a map (the crash cause).
+    universe_json = json.loads(prepared.to_json())["universe"]
+    assert isinstance(universe_json, list)
+    assert universe_json == [1]
+
+
+def test_prepare_for_diagnostics_passes_through_list_universe(tmp_path):
+    """An already-resolved list universe is left intact."""
+    db = _make_metadata_db(tmp_path / "metadata.sqlite")
+    store = _StubStore(db)
+
+    config = bt.BacktestConfig(
+        universe=[1],
+        time_range_start=0,
+        time_range_end=4_000_000_000,
+        bar_interval={"Hours": 1},
+        initial_capital=1000.0,
+    )
+
+    prepared, _ = _prepare_for_diagnostics(config, _simple_strategy(), store)
+
+    assert prepared.universe == [1]
+    assert json.loads(prepared.to_json())["universe"] == [1]

python/tests/test_doc_api_contract.py

@@ -0,0 +1,60 @@
+"""Doc <-> code signature contract.
+
+These assertions encode the public signatures and helper outputs that the
+online documentation and the interactive notebook rely on. They are cheap,
+IO-free, and Pro-free, and exist to catch *doc drift*: if a documented kwarg,
+preset, or helper shape changes in the code, a doc snippet silently breaks.
+
+This guards, among others:
+  * ``plot.monte_carlo`` exposing ``n_simulations`` (NOT ``n_paths``) -- the
+    notebook bug where ``n_paths=`` raised TypeError.
+  * ``Slippage.volume_impact`` emitting ``impact_coeff``/``exponent`` -- the
+    notebook bug where ``{"coefficient": ...}`` failed Rust deserialization.
+  * ``DataStore`` accepting ``mega``/``arrow_dir`` -- the doc signature that
+    omitted them.
+"""
+import inspect
+
+import manifoldbt as bt
+
+
+def test_monte_carlo_uses_n_simulations_not_n_paths():
+    params = inspect.signature(bt.plot.monte_carlo).parameters
+    assert "n_simulations" in params
+    assert "n_paths" not in params  # the notebook snippet bug
+
+
+def test_slippage_helper_shapes_match_serde():
+    # Keys must match the Rust SlippageConfig serde variants exactly.
+    assert bt.Slippage.volume_impact(0.1) == {
+        "VolumeImpact": {"impact_coeff": 0.1, "exponent": 1.5}
+    }
+    assert bt.Slippage.fixed_bps(2.0) == {"FixedBps": {"bps": 2.0}}
+
+
+def test_interval_helper_shapes():
+    assert bt.Interval.seconds(1) == {"Seconds": 1}
+    assert bt.Interval.minutes(1) == {"Minutes": 1}
+    assert bt.Interval.hours(12) == {"Hours": 12}
+    assert bt.Interval.days(1) == {"Days": 1}
+
+
+def test_fee_presets_match_documented_values():
+    # Documented under #configuration > FeeConfig Presets.
+    perps = bt.FeeConfig.binance_perps()
+    assert (perps.maker_fee_bps, perps.taker_fee_bps) == (2.0, 5.0)
+    spot = bt.FeeConfig.binance_spot()
+    assert (spot.maker_fee_bps, spot.taker_fee_bps) == (10.0, 10.0)
+
+
+def test_datastore_accepts_mega_and_arrow_dir_kwargs(tmp_path):
+    # The real signature is (data_root, metadata_db, dataset, mega, arrow_dir).
+    # We only assert the kwargs are *accepted* (no TypeError for unknown kwarg);
+    # any runtime/IO error from opening an empty dir is fine for this contract.
+    for kw in ("mega", "arrow_dir"):
+        try:
+            bt.DataStore(str(tmp_path), dataset="bars_1m", **{kw: str(tmp_path)})
+        except TypeError as exc:  # unexpected keyword argument -> contract broken
+            raise AssertionError(f"DataStore rejected kwarg {kw!r}: {exc}")
+        except Exception:
+            pass  # non-TypeError (e.g. cannot open store) -> kwarg was accepted