release: v0.4.0

examples/bench_mega_all_symbols.py

@@ -0,0 +1,59 @@
+"""Benchmark: all symbols, Arrow IPC store (bars_1m + bars_1h).
+
+Usage:
+    python examples/bench_mega_all_symbols.py
+"""
+import os
+import time
+import manifoldbt as mbt
+from manifoldbt.indicators import ema, close
+from manifoldbt.helpers import time_range, Slippage, Interval
+
+# -- Strategy -------------------------------------------------------------------
+fast = ema(close, 12)
+slow = ema(close, 200)
+trend = fast - slow
+
+strategy = (
+    mbt.Strategy.create("ema_crossover_all")
+    .signal("trend", trend)
+    .size(mbt.when(trend > 0.0, 0.5, 0.0))
+)
+
+# -- Config: 21 crypto symbols, 3 years, 1h bars --------------------------------
+# SOL (3) starts 2024 only — excluded
+universe = [s for s in range(1, 23) if s != 3]
+start, end = time_range("2022-01-01", "2025-01-01")
+
+config = mbt.BacktestConfig(
+    universe=universe,
+    time_range_start=start,
+    time_range_end=end,
+    bar_interval=Interval.minutes(60),
+    precise=True,
+    initial_capital=100_000,
+    execution=mbt.ExecutionConfig(
+        allow_short=False,
+        max_position_pct=0.05,
+        position_sizing_mode="FractionOfInitialCapital",
+    ),
+    fees=mbt.FeeConfig.binance_perps(),
+    slippage=Slippage.fixed_bps(2),
+    warmup_bars=30,
+)
+
+# -- Run -----------------------------------------------------------------------
+root = os.path.join(os.path.dirname(__file__), "..")
+data_root = os.path.abspath(os.path.join(root, "data"))
+metadata_db = os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite"))
+
+store = mbt.DataStore(data_root=data_root, metadata_db=metadata_db, arrow_dir=os.path.join(data_root, "mega"))
+
+t0 = time.perf_counter()
+result = mbt.run(strategy, config, store)
+elapsed = time.perf_counter() - t0
+
+print(result.profile_summary())
+print(f"\nWall clock: {elapsed:.3f}s")
+print(f"Trades: {result.trade_count}")
+print(f"Symbols: {len(universe)}")

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "manifoldbt"
-version = "0.3.0"
+version = "0.4.0"
 description = "Rust-powered backtesting engine for quantitative research"
 requires-python = ">=3.9"
 license = "MIT"

python/manifoldbt/__init__.py

@@ -250,7 +250,7 @@ def _resolve_store(config: BacktestConfig, store: DataStore) -> DataStore:
       - **Normal** (default): dataset matches ``bar_interval`` exactly.
         If no exact match, picks the closest smaller dataset and sets
         ``resample_to`` so the engine resamples to bar_interval (no hybrid overhead).
-      - **Accuracy** (``accuracy=True`` on config): always loads ``bars_1m``.
+      - **Precise** (``precise=True`` on config): always loads ``bars_1m``.
         Signals on ``bar_interval``, simulation on 1-min bars.
         Required for precise SL/TP fills.
 
@@ -261,12 +261,17 @@ def _resolve_store(config: BacktestConfig, store: DataStore) -> DataStore:
     except Exception:
         return store
 
+    # ArrowIpcDataStore handles multi-resolution internally via bar_interval —
+    # skip Python-side dataset swapping. Detected by dataset() returning "arrow_ipc".
+    if current == "arrow_ipc":
+        return store
+
     # If user explicitly chose a non-default dataset, respect it
     if current != "bars_1m":
         return store
 
     # Accuracy mode: keep bars_1m (hybrid: signals on bar_interval, sim on 1m)
-    if getattr(config, "accuracy", False):
+    if getattr(config, "precise", False):
         return store
 
     # Normal mode: pick dataset <= bar_interval.
@@ -312,55 +317,118 @@ def _cap_output_resolution(config: BacktestConfig) -> BacktestConfig:
 
 def ingest(
     provider: str,
-    symbol: str,
+    symbol: Optional[str] = None,
-    symbol_id: int,
+    symbol_id: Optional[int] = None,
-    start: str,
+    start: str = "",
-    end: str,
+    end: str = "",
     *,
+    symbols: Optional[list] = None,
     interval: str = "1m",
     dataset: Optional[str] = None,
     data_root: str = "data",
     metadata_db: str = "metadata/metadata.sqlite",
     exchange: Optional[str] = None,
     asset_class: str = "crypto_spot",
+    progress: bool = True,
 ) -> DataStore:
-    """Ingest bars from a data provider into the local Parquet store.
+    """Ingest bars from a data provider into the Arrow IPC store.
 
     Providers: ``"binance"``, ``"hyperliquid"`` (free), ``"databento"``, ``"massive"`` (Pro).
 
     Returns a :class:`DataStore` ready for :func:`run`.
 
-    Example::
+    Example (single symbol)::
 
         store = bt.ingest(
-            provider="databento",
+            provider="binance",
-            symbol="ESH5",
+            symbol="BTCUSDT",
             symbol_id=1,
-            start="2025-01-01T00:00:00Z",
+            start="2020-01-01T00:00:00Z",
-            end="2025-01-31T00:00:00Z",
+            end="2025-01-01T00:00:00Z",
-            dataset="GLBX.MDP3",
+        )
-            exchange="CME",
+
-            asset_class="future",
+    Example (multiple symbols)::
+
+        store = bt.ingest(
+            provider="binance",
+            symbols=[("XMRUSDT", 26), ("VETUSDT", 27), ("ZECUSDT", 28)],
+            start="2020-06-01T00:00:00Z",
+            end="2026-03-01T00:00:00Z",
         )
-        result = bt.run(strategy, config, store)
     """
     _PRO_PROVIDERS = {"databento", "massive"}
     if provider in _PRO_PROVIDERS:
         _require_pro(f"Data connector: {provider}")
 
-    return _ingest_native(
+    # Build list of (symbol, symbol_id) pairs.
-        provider=provider,
+    if symbols is not None:
-        symbol=symbol,
+        pairs = [(s, sid) for s, sid in symbols]
-        symbol_id=symbol_id,
+    elif symbol is not None and symbol_id is not None:
-        start=start,
+        pairs = [(symbol, symbol_id)]
-        end=end,
+    else:
-        interval=interval,
+        raise ValueError("provide either symbol+symbol_id or symbols=[(ticker, id), ...]")
-        dataset=dataset,
+
-        data_root=data_root,
+    if len(pairs) == 1:
-        metadata_db=metadata_db,
+        return _ingest_single(
-        exchange=exchange,
+            provider=provider, symbol=pairs[0][0], symbol_id=pairs[0][1],
-        asset_class=asset_class,
+            start=start, end=end, interval=interval, dataset=dataset,
-    )
+            data_root=data_root, metadata_db=metadata_db,
+            exchange=exchange, asset_class=asset_class, progress=progress,
+        )
+
+    # Multi-symbol: show all symbols with pending ones in grey.
+    display = None
+    callbacks = {}
+    if progress:
+        from manifoldbt._progress import make_multi_progress
+        display, callbacks = make_multi_progress(pairs, provider)
+
+    store = None
+    try:
+        for sym, sid in pairs:
+            cb = callbacks.get(sym) if callbacks else None
+            store = _ingest_native(
+                provider=provider, symbol=sym, symbol_id=sid,
+                start=start, end=end, interval=interval, dataset=dataset,
+                data_root=data_root, metadata_db=metadata_db,
+                exchange=exchange, asset_class=asset_class,
+                progress_cb=cb,
+            )
+    finally:
+        if display is not None:
+            display.stop()
+
+    return store
+
+
+def _ingest_single(
+    *, provider, symbol, symbol_id, start, end, interval, dataset,
+    data_root, metadata_db, exchange, asset_class, progress,
+) -> DataStore:
+    cb = None
+    display = None
+    if progress:
+        from manifoldbt._progress import make_progress_display
+        display, cb = make_progress_display(symbol, provider)
+
+    try:
+        return _ingest_native(
+            provider=provider,
+            symbol=symbol,
+            symbol_id=symbol_id,
+            start=start,
+            end=end,
+            interval=interval,
+            dataset=dataset,
+            data_root=data_root,
+            metadata_db=metadata_db,
+            exchange=exchange,
+            asset_class=asset_class,
+            progress_cb=cb,
+        )
+    finally:
+        if display is not None:
+            display.stop()
 
 
 # ---------------------------------------------------------------------------

python/manifoldbt/_progress.py

@@ -0,0 +1,160 @@
+"""Rich progress display for data ingestion."""
+from __future__ import annotations
+
+import time
+
+
+def make_progress_display(symbol: str, provider: str):
+    """Create a rich progress display for a single symbol.
+
+    Returns (display, callback).
+    """
+    try:
+        from rich.progress import (
+            Progress,
+            SpinnerColumn,
+            BarColumn,
+            TextColumn,
+            TimeRemainingColumn,
+        )
+        from rich.console import Console
+
+        console = Console()
+        progress = Progress(
+            SpinnerColumn(),
+            TextColumn("{task.description}"),
+            BarColumn(bar_width=40),
+            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+            TextColumn("[dim]{task.fields[status]}"),
+            TimeRemainingColumn(),
+            console=console,
+            transient=False,
+        )
+        task_id = progress.add_task(
+            f"[bold blue]{symbol} ({provider})", total=10000, status="connecting..."
+        )
+        progress.start()
+
+        def callback(phase: str, fetched: int, pct: float, msg: str):
+            if phase == "done":
+                progress.update(task_id, completed=10000, status=f"done - {fetched:,} bars")
+                progress.stop()
+            elif phase == "store":
+                progress.update(task_id, completed=9900, status="writing...")
+            else:
+                progress.update(
+                    task_id,
+                    completed=max(1, int(pct * 9800)),
+                    status=f"{fetched:,} bars",
+                )
+
+        return progress, callback
+
+    except ImportError:
+        return _make_fallback(symbol, provider)
+
+
+def make_multi_progress(symbols: list[tuple[str, int]], provider: str):
+    """Create a rich progress display for multiple symbols.
+
+    All symbols are shown upfront: current in blue, pending in dim grey.
+    Returns (display, dict of {symbol: callback}).
+    """
+    try:
+        from rich.progress import (
+            Progress,
+            SpinnerColumn,
+            BarColumn,
+            TextColumn,
+            TimeRemainingColumn,
+        )
+        from rich.console import Console
+
+        console = Console()
+        progress = Progress(
+            SpinnerColumn(),
+            TextColumn("{task.description}"),
+            BarColumn(bar_width=40),
+            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+            TextColumn("[dim]{task.fields[status]}"),
+            TimeRemainingColumn(),
+            console=console,
+            transient=False,
+        )
+
+        task_ids = {}
+        for i, (sym, _sid) in enumerate(symbols):
+            if i == 0:
+                desc = f"[bold blue]{sym} ({provider})"
+                status = "connecting..."
+            else:
+                desc = f"[dim]{sym} ({provider})[/dim]"
+                status = "waiting"
+            task_ids[sym] = progress.add_task(desc, total=10000, status=status)
+
+        progress.start()
+
+        def make_callback(sym: str, idx: int):
+            tid = task_ids[sym]
+
+            def callback(phase: str, fetched: int, pct: float, msg: str):
+                if phase == "done":
+                    progress.update(
+                        tid,
+                        description=f"[green]{sym} ({provider})[/green]",
+                        completed=10000,
+                        status=f"{fetched:,} bars",
+                    )
+                    # Activate next symbol if any.
+                    next_idx = idx + 1
+                    if next_idx < len(symbols):
+                        next_sym = symbols[next_idx][0]
+                        next_tid = task_ids[next_sym]
+                        progress.update(
+                            next_tid,
+                            description=f"[bold blue]{next_sym} ({provider})",
+                            status="connecting...",
+                        )
+                elif phase == "store":
+                    progress.update(tid, completed=9900, status="writing...")
+                else:
+                    progress.update(
+                        tid,
+                        completed=max(1, int(pct * 9800)),
+                        status=f"{fetched:,} bars",
+                    )
+
+            return callback
+
+        callbacks = {sym: make_callback(sym, i) for i, (sym, _sid) in enumerate(symbols)}
+        return progress, callbacks
+
+    except ImportError:
+        return _make_multi_fallback(symbols, provider)
+
+
+def _make_fallback(symbol: str, provider: str):
+    """Plain print fallback when rich is not installed."""
+    t0 = time.perf_counter()
+    last_pct = [-1]
+
+    def callback(phase: str, fetched: int, pct: float, msg: str):
+        elapsed = time.perf_counter() - t0
+        current = int(pct * 100)
+        if phase == "done":
+            print(f"\r  {symbol} ({provider}): done - {fetched:,} bars [{elapsed:.1f}s]      ")
+        elif current > last_pct[0] + 4 or phase == "store":
+            last_pct[0] = current
+            label = "writing..." if phase == "store" else f"{fetched:,} bars"
+            print(f"\r  {symbol} ({provider}): {current}% {label} [{elapsed:.1f}s]", end="", flush=True)
+
+    return None, callback
+
+
+def _make_multi_fallback(symbols: list[tuple[str, int]], provider: str):
+    """Plain print fallback for multiple symbols."""
+    callbacks = {}
+    for sym, _sid in symbols:
+        _, cb = _make_fallback(sym, provider)
+        callbacks[sym] = cb
+    return None, callbacks

python/manifoldbt/config.py

@@ -187,8 +187,8 @@ class BacktestConfig:
     Allows indicators (EMA, SMA, etc.) to stabilise. During warmup,
     equity tracking runs but no trades are generated.
     Set to at least the longest indicator window (e.g. 25 for EMA(25))."""
-    accuracy: bool = False
+    precise: bool = False
-    """When True, simulation runs on 1-minute bars regardless of bar_interval.
+    """When True, always load finest resolution (1m) regardless of bar_interval.
     Signals are still evaluated at bar_interval resolution (hybrid mode).
     Use for precise SL/TP fills and intraday drawdown tracking. Slower."""
     extra_timeframes: Dict[str, Any] = field(default_factory=dict)
@@ -224,6 +224,8 @@ class BacktestConfig:
             d["symbol_names"] = self.symbol_names
         if self.warmup_bars > 0:
             d["warmup_bars"] = self.warmup_bars
+        if self.precise:
+            d["precise"] = True
         if self.extra_timeframes:
             d["extra_timeframes"] = self.extra_timeframes
         return d

python/manifoldbt/result.py

@@ -143,6 +143,24 @@ class Result:
                 if val is not None:
                     lines.append(f"    {label:<18s} {fmt(val):>12s}")
 
+            # Signal quality metrics (MAE/MFE)
+            sq = ts.get("signal_quality")
+            if isinstance(sq, dict):
+                lines.append("")
+                lines.append("  Signal Quality")
+                lines.append("  " + "-" * 38)
+                _sq_fmt = [
+                    ("Avg MAE", "avg_mae", _pct),
+                    ("Avg MFE", "avg_mfe", _pct),
+                    ("Edge Ratio", "edge_ratio", _f2),
+                    ("Entry Efficiency", "avg_entry_efficiency", _pct),
+                    ("Exit Efficiency", "avg_exit_efficiency", _pct),
+                ]
+                for label, key, fmt in _sq_fmt:
+                    val = sq.get(key)
+                    if val is not None:
+                        lines.append(f"    {label:<18s} {fmt(val):>12s}")
+
         return "\n".join(lines)
 
     def profile_summary(self) -> str: