gavindiaz/manifoldbt
1
0
Fork 0
mirror of https://github.com/manifoldbt/manifoldbt.git synced 2026-08-24 22:48:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

release: v0.10.0

Browse Source
This commit is contained in:
github-actions[bot]
2026-07-09 11:01:08 +00:00
parent a13b3f5104
commit 220fc1d434
8 changed files with 137 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/manifoldbt/__init__.py
+64 -5
View File
@@ -127,12 +127,59 @@ def _is_pro() -> bool:
def _require_pro(feature: str) -> None:
"""Print Pro warning and exit cleanly if not Pro."""
"""Raise LicenseError if the current license is not Pro.
This used to ``raise SystemExit(0)``, which reads as a clean exit in a
``.py`` script but, in Jupyter/IPython, aborts the current cell with a bare
``SystemExit: 0`` (plus a spurious "To exit, use ..." warning) and silently
skips the rest of the cell. ``LicenseError`` is a normal, catchable
exception: a single clean traceback in a notebook, a real error in scripts.
"""
if _is_pro():
return
print(f"\n\033[38;5;214m[!] {feature} -- Pro feature\033[0m")
print("\033[38;5;214m -> upgrade at www.manifoldbt.com\033[0m")
raise SystemExit(0)
raise LicenseError(
f"'{feature}' is a Pro feature. Upgrade to Pro at www.manifoldbt.com"
)
def _require_pro_for_gpu(device, feature: str) -> None:
"""Gate GPU acceleration (``device="cuda"``/``"gpu"``) behind Pro.
GPU paths are also enforced natively, but that surfaces a ``PermissionError``
with a full traceback (GPU sweep) or a bare ``ValueError`` (stochastic). Gating
in Python first gives every GPU entry point the same clean ``LicenseError`` as
the other Pro features. No-op for CPU or for Pro users.
"""
if isinstance(device, str) and device.lower() in ("cuda", "gpu"):
_require_pro(feature)
# Community fan-out budget: sweeps and batches may run up to this many backtests
# per call for free; beyond it requires Pro. Single run() is never affected.
# Keep in sync with the native bt_license::COMMUNITY_MAX_SWEEP_COMBOS.
_COMMUNITY_MAX_COMBOS = 500
def _grid_combos(param_grid) -> int:
"""Number of Cartesian combinations produced by a sweep param grid."""
n = 1
for values in param_grid.values():
n *= max(1, len(values))
return n
def _require_pro_over_combos(n_combos: int, what: str) -> None:
"""Raise LicenseError if a fan-out exceeds the Community combination limit.
No-op at or below the limit, or for Pro users. Mirrors the native
``require_combo_limit`` so Community and Pro see identical behaviour.
"""
if n_combos <= _COMMUNITY_MAX_COMBOS or _is_pro():
return
raise LicenseError(
f"{what} with {n_combos} runs exceeds the Community limit of "
f"{_COMMUNITY_MAX_COMBOS}. Upgrade to Pro at www.manifoldbt.com"
)
def _classify_error(exc: Exception) -> Exception:
@@ -516,7 +563,8 @@ def ingest(
) -> DataStore:
"""Ingest bars from a data provider into the Arrow IPC store.
Providers: ``"binance"``, ``"hyperliquid"`` (free), ``"databento"``, ``"massive"`` (Pro).
Providers (free): ``"binance"``, ``"bybit"``, ``"hyperliquid"``, ``"dydx"``,
``"bitstamp"``. Pro: ``"databento"``, ``"massive"``.
Returns a :class:`DataStore` ready for :func:`run`.
@@ -708,6 +756,7 @@ def run_sweep(
Returns:
A :class:`SweepResult` with ``.to_df()``, ``.best()``, ``.plot_metric()``.
"""
_require_pro_over_combos(_grid_combos(param_grid), "Parameter sweep")
try:
config = _cap_output_resolution(config)
store = _resolve_store(config, store)
@@ -749,6 +798,7 @@ def run_batch(
Returns:
One :class:`Result` per strategy, in input order.
"""
_require_pro_over_combos(len(strategies), "Batch backtesting")
try:
config = _prepare_config(config, None, store)
config = _cap_output_resolution(config)
@@ -787,6 +837,7 @@ def run_batch_lite(
Returns:
One :class:`BatchResultLite` per strategy (name, metrics, equity, trade_count).
"""
_require_pro_over_combos(len(strategies), "Batch backtesting")
try:
config = _prepare_config(config, None, store)
config = _cap_output_resolution(config)
@@ -834,6 +885,8 @@ def run_sweep_lite(
Returns:
One :class:`BatchResultLite` per combo (Cartesian product order).
"""
_require_pro_over_combos(_grid_combos(param_grid), "Parameter sweep")
_require_pro_for_gpu(device, "GPU sweep")
try:
config = _cap_output_resolution(config)
store = _resolve_store(config, store)
@@ -913,6 +966,10 @@ def run_sweep_2d(
Returns:
Dict with ``metric_grid`` (2D list), ``x_values``, ``y_values``, etc.
"""
_require_pro_over_combos(
len(sweep_config.get("x_values", [])) * len(sweep_config.get("y_values", [])),
"2D parameter sweep",
)
config = _prepare_config(config, strategy, store)
sweep_json = json.dumps(_convert_scalar_values_in_sweep(sweep_config))
return _run_sweep_2d_native(strategy.to_json(), sweep_json, config.to_json(), store)
@@ -939,6 +996,7 @@ def run_stability(
Returns:
Dict with ``stability_score``, ``metric_values``, ``mean_metric``, ``std_metric``.
"""
_require_pro_over_combos(len(stability_config.get("values", [])), "Parameter stability analysis")
config = _prepare_config(config, strategy, store)
stab_json = json.dumps(_convert_scalar_values_in_stability(stability_config))
return _run_stability_native(strategy.to_json(), stab_json, config.to_json(), store)
@@ -1025,6 +1083,7 @@ def run_stochastic(
... )
>>> result = mbt.run_stochastic(model, s0=100, n_paths=5000)
"""
_require_pro_for_gpu(device, "GPU stochastic simulation")
config: Dict[str, Any] = {
"s0": s0,
"n_paths": n_paths,
python/manifoldbt/diagnostics.py
+19 -212
View File
@@ -9,9 +9,6 @@ import numpy as np
_EMPTY_TS = np.array([], dtype="datetime64[ns]")
# Pro-gated feature label (see _require_pro). Single source to avoid drift.
_SAFETY_PRO_FEATURE = "Safety checks (lookahead, exposure)"
def _prepare_for_diagnostics(config, strategy, store):
"""Mirror ``run()``'s config/store preparation for the diagnostics path.
@@ -94,178 +91,6 @@ class DiagnosticsResult:
return "\n".join(parts)
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
def _ts_as_int64(arr: np.ndarray) -> np.ndarray:
"""View a datetime64 array as int64, or return as-is if already numeric."""
return arr.view(np.int64) if arr.dtype.kind == "M" else arr
def _filter_overlap(base_ts: np.ndarray, ext_ts: np.ndarray) -> np.ndarray:
"""Return indices of *ext* trades within the base period."""
if len(ext_ts) == 0 or len(base_ts) == 0:
return np.array([], dtype=np.int64)
cutoff = _ts_as_int64(base_ts)[-1]
return np.nonzero(_ts_as_int64(ext_ts) <= cutoff)[0]
def _compare_trades(
trades_base: Dict[str, np.ndarray],
trades_ext: Dict[str, np.ndarray],
overlap_indices: np.ndarray,
n_compare: int,
tolerance: float,
) -> tuple:
"""Compare trades pairwise. Returns (mismatched_count, details_list)."""
strict_fields = ["signal_timestamp", "execution_timestamp", "symbol_id", "side"]
float_fields = ["quantity", "fill_price", "fees"]
details: List[Dict[str, Any]] = []
mismatched = 0
for i in range(n_compare):
ext_i = overlap_indices[i]
mismatch = _find_mismatch(
trades_base, trades_ext, i, ext_i,
strict_fields, float_fields, tolerance,
)
if mismatch:
mismatched += 1
if len(details) < 20:
details.append(mismatch)
return mismatched, details
def _find_mismatch(
base: dict, ext: dict, i: int, ext_i: int,
strict_fields: list, float_fields: list, tolerance: float,
) -> dict | None:
"""Check one trade pair for mismatches. Returns detail dict or None."""
for f in strict_fields:
if f not in base or f not in ext:
continue
if base[f][i] != ext[f][ext_i]:
return {"index": i, "field": f,
"base": base[f][i], "extended": ext[f][ext_i]}
for f in float_fields:
if f not in base or f not in ext:
continue
bv, ev = float(base[f][i]), float(ext[f][ext_i])
if not np.isclose(bv, ev, atol=tolerance, rtol=tolerance):
return {"index": i, "field": f, "base": bv, "extended": ev}
return None
def _run_split_test(
strategy, config, store, split_ns: int, tolerance: float,
trades_full: dict, full_ts: np.ndarray, method: str,
) -> LookaheadReport:
"""Run strategy on [start, split] and compare against the full run."""
from manifoldbt import run
from manifoldbt.plot._convert import trades_arrays
short_config = copy.deepcopy(config)
short_config.time_range_end = split_ns
try:
result_short = run(strategy, short_config, store)
except (ValueError, RuntimeError):
# Some symbols may lack data for the truncated range — skip.
return LookaheadReport(
passed=True, total_trades_base=0,
total_trades_overlap=0, mismatched=0, method=method,
)
trades_short = trades_arrays(result_short)
short_ts = trades_short.get("execution_timestamp", _EMPTY_TS.copy())
n_short = len(short_ts)
if n_short == 0:
return LookaheadReport(
passed=True, total_trades_base=0,
total_trades_overlap=0, mismatched=0, method=method,
)
overlap = _filter_overlap(short_ts, full_ts)
n_overlap = len(overlap)
if n_short != n_overlap:
return LookaheadReport(
passed=False, total_trades_base=n_short,
total_trades_overlap=n_overlap,
mismatched=abs(n_short - n_overlap), method=method,
details=[{"index": 0, "field": "trade_count",
"base": n_short, "extended": n_overlap}],
)
mismatched, details = _compare_trades(
trades_short, trades_full, overlap, n_short, tolerance,
)
return LookaheadReport(
passed=(mismatched == 0), total_trades_base=n_short,
total_trades_overlap=n_overlap, mismatched=mismatched,
method=method, details=details,
)
def _run_aligned_split_test(
strategy, config, aligned, split_ns: int, tolerance: float,
trades_full: dict, full_ts: np.ndarray, method: str,
) -> LookaheadReport:
"""Run strategy on sliced aligned data [start, split] and compare."""
from manifoldbt.plot._convert import trades_arrays
from manifoldbt._native import run_on_aligned as _run_on_aligned
short_config = copy.deepcopy(config)
short_config.time_range_end = split_ns
try:
sliced = aligned.slice(config.time_range_start, split_ns)
result_short = _run_on_aligned(
strategy.to_json(), short_config.to_json(), sliced,
)
except (ValueError, RuntimeError):
return LookaheadReport(
passed=True, total_trades_base=0,
total_trades_overlap=0, mismatched=0, method=method,
)
trades_short = trades_arrays(result_short)
short_ts = trades_short.get("execution_timestamp", _EMPTY_TS.copy())
n_short = len(short_ts)
if n_short == 0:
return LookaheadReport(
passed=True, total_trades_base=0,
total_trades_overlap=0, mismatched=0, method=method,
)
overlap = _filter_overlap(short_ts, full_ts)
n_overlap = len(overlap)
if n_short != n_overlap:
return LookaheadReport(
passed=False, total_trades_base=n_short,
total_trades_overlap=n_overlap,
mismatched=abs(n_short - n_overlap), method=method,
details=[{"index": 0, "field": "trade_count",
"base": n_short, "extended": n_overlap}],
)
mismatched, details = _compare_trades(
trades_short, trades_full, overlap, n_short, tolerance,
)
return LookaheadReport(
passed=(mismatched == 0), total_trades_base=n_short,
total_trades_overlap=n_overlap, mismatched=mismatched,
method=method, details=details,
)
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
@@ -301,45 +126,33 @@ def detect_lookahead(
Returns:
DiagnosticsResult with ``.passed``, ``.assert_clean()``, ``print()``.
"""
# Pro feature. Friendly UX gate first (clean LicenseError in notebooks); the
# analysis itself is enforced natively (`safety_checks`) so it can't be
# bypassed by editing this file.
from manifoldbt import _require_pro
_require_pro(_SAFETY_PRO_FEATURE)
_require_pro("Look-ahead bias detection")
from manifoldbt.plot._convert import trades_arrays
from manifoldbt._native import (
load_and_align as _load_and_align,
run_on_aligned as _run_on_aligned,
)
from manifoldbt._native import py_detect_lookahead as _native_detect
# Resolve config/store exactly like run() (notably dict universe -> ids),
# otherwise config.to_json() emits a map the Rust loader rejects.
config, store = _prepare_for_diagnostics(config, strategy, store)
period = config.time_range_end - config.time_range_start
# Load data ONCE for the full range.
aligned = _load_and_align(config.to_json(), store)
# Full run on pre-loaded data (no disk I/O).
result_full = _run_on_aligned(strategy.to_json(), config.to_json(), aligned)
trades_full = trades_arrays(result_full)
full_ts = trades_full.get("execution_timestamp", _EMPTY_TS.copy())
reports: List[LookaheadReport] = []
if mode in ("all", "extension"):
split = config.time_range_start + int(period * 2 / 3)
reports.append(_run_aligned_split_test(
strategy, config, aligned, split, tolerance,
trades_full, full_ts, method="extension",
))
if mode in ("all", "truncation"):
split = config.time_range_start + int(period / 3)
reports.append(_run_aligned_split_test(
strategy, config, aligned, split, tolerance,
trades_full, full_ts, method="truncation",
))
# All the run + comparison logic lives in Rust now; this is a thin wrapper
# that rebuilds the report objects from the native JSON.
raw = _native_detect(strategy.to_json(), config.to_json(), store, mode, tolerance)
reports = [
LookaheadReport(
passed=r["passed"],
total_trades_base=r["total_trades_base"],
total_trades_overlap=r["total_trades_overlap"],
mismatched=r["mismatched"],
method=r["method"],
details=r["details"],
)
for r in raw
]
return DiagnosticsResult(reports=reports)
@@ -540,9 +353,6 @@ def risk_check(
print(report)
report.assert_clean()
"""
from manifoldbt import _require_pro
_require_pro(_SAFETY_PRO_FEATURE)
from manifoldbt.plot._convert import positions_arrays
pos = positions_arrays(result)
@@ -830,9 +640,6 @@ def check_exposure_stability(
print(report)
report.assert_clean()
"""
from manifoldbt import _require_pro
_require_pro(_SAFETY_PRO_FEATURE)
from manifoldbt._native import (
load_and_align as _load_and_align,
run_on_aligned as _run_on_aligned,
python/manifoldbt/exceptions.py
+18
View File
@@ -19,3 +19,21 @@ class ConfigError(BacktesterError):
class LicenseError(BacktesterError):
"""Raised when a Pro feature is used without a valid license."""
def _render_traceback_(self):
# Jupyter/IPython uses this hook (when present) to render an exception,
# replacing the default traceback: a Community user hitting a Pro gate
# sees a short, frame-free notice instead of an internal traceback (file
# paths, the raise site, etc.). Plain `.py` scripts still get the normal
# traceback.
#
# Bold + the theme's default foreground (no fixed colour): orange washes
# out on Jupyter's pink error background, the default fg stays readable
# on any theme. Split across two indented lines with blank lines around
# so the notice breathes instead of reading as a cramped, clipped strip.
head, _, tail = str(self).partition(". ")
lines = ["", f" \033[1m{head}\033[0m"]
if tail:
lines.append(f" {tail}")
lines.append("")
return lines
python/manifoldbt/plot/tearsheet.py
-3
View File
@@ -217,9 +217,6 @@ def tearsheet(
Returns the HTML string. Opens in browser when ``show=True``,
writes to disk when ``save`` is given.
"""
from manifoldbt import _require_pro
_require_pro("Tearsheets & export")
_ = benchmark # reserved for future benchmark overlay support
strategy_name = title or auto_title(result, "Backtest")
metrics = result.metrics if hasattr(result, "metrics") else {}
python/tests/test_diagnostics.py
+13
View File
@@ -14,8 +14,21 @@ without needing a Pro license or real market data.
import json
import sqlite3
import pytest
import manifoldbt as bt
from manifoldbt.diagnostics import _prepare_for_diagnostics
from manifoldbt.exceptions import LicenseError
def test_detect_lookahead_gated_on_community(monkeypatch):
"""Look-ahead detection is Pro: Community gets a clean LicenseError before any
work (the analysis itself is enforced natively; this is the friendly UX gate).
"""
monkeypatch.setattr(bt, "_is_pro", lambda: False)
with pytest.raises(LicenseError):
# Raises before touching strategy/config/store, so None args are fine.
bt.diagnostics.detect_lookahead(None, None, None)
def _make_metadata_db(path):
python/tests/test_golden_buy_and_hold.py
+11
View File
@@ -6,9 +6,20 @@ to the Rust-only golden test fixtures.
import json
import os
import pytest
import manifoldbt as bt
from manifoldbt import run_with_parquet
# The golden fixtures were generated at full (Pro) resolution; the Community
# resolution cap changes the equity-point count and the comparison is
# meaningless. CI unlocks via BT_UNLOCKED=1 (debug builds); locally this needs
# an activated Pro license.
pytestmark = pytest.mark.skipif(
bt.license_info()[0] != "Pro",
reason="requires Pro (fixtures generated at sub-daily resolution); activate a license or use a BT_UNLOCKED dev build",
)
def test_golden_buy_and_hold_matches_fixtures(golden_buy_hold_dir):
"""Mirror of Rust golden_buy_and_hold_equity_trade_metrics_and_manifest_match_fixture."""
python/tests/test_sweep.py
+11
View File
@@ -3,9 +3,20 @@ import json
import os
import time
import pytest
import manifoldbt as bt
from manifoldbt import run_sweep, run_with_parquet
# The golden fixtures are 1-minute bars; on a Community license the engine caps
# resolution to daily, so these runs produce zero trades and the assertions are
# meaningless. CI unlocks via BT_UNLOCKED=1 (debug builds); locally this needs
# an activated Pro license.
pytestmark = pytest.mark.skipif(
bt.license_info()[0] != "Pro",
reason="requires Pro (sub-daily resolution); activate a license or use a BT_UNLOCKED dev build",
)
def test_sweep_returns_one_result_per_combo(golden_buy_hold_dir):
"""Sweep with 2x2 grid returns 4 results."""