From 6fb2c3fea1cf4dc20427cf5bc191bdb27be02bc5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 28 Oct 2026 16:24:33 +0000 Subject: [PATCH] release: v0.30.0 --- .github/workflows/ci.yml | 9 +- README.md | 48 +++--- examples/10_monte_carlo.py | 19 ++- pyproject.toml | 2 +- python/manifoldbt/__init__.py | 219 +++++++++++++++++++++++++-- python/manifoldbt/_update.py | 8 +- python/manifoldbt/cli.py | 2 +- python/manifoldbt/cloud.py | 41 ++--- python/tests/test_cloud.py | 17 ++- python/tests/test_monte_carlo_api.py | 200 ++++++++++++++++++++++++ 10 files changed, 491 insertions(+), 74 deletions(-) create mode 100644 python/tests/test_monte_carlo_api.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d079b36..f958fa8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,7 @@ name: CI # (benchmarks/vs_vectorbt/test_report.py). They import no engine and # need no wheel; without this job nothing collects them, because the # root testpaths points at python/tests. PR #13 arrived with tests -# their author could not wire in, and this is the wiring. It also -# runs test_data.py, which pins that the generated bars are the -# same bits on every machine, the condition for joining the CPU and -# GPU runs on their data digest. +# their author could not wire in, and this is the wiring. # # The second job is the point. The suite shipped in this repository for months # without anything running it, so "there are tests" was a claim a reader had to @@ -56,8 +53,8 @@ jobs: python-version: "3.12" - name: Run the harness's rendering tests run: | - python -m pip install --quiet pytest numpy==2.4.3 pandas==2.3.3 - python -m pytest benchmarks/vs_vectorbt/test_report.py benchmarks/vs_vectorbt/test_data.py -q + python -m pip install --quiet pytest + python -m pytest benchmarks/vs_vectorbt/test_report.py -q verify: name: Verify package layout diff --git a/README.md b/README.md index 169d9ca..b82d601 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,9 @@ strategy once, in a fluent Python DSL, and hand it to the engine: the whole backtest then runs in Rust, signals, fills, fees, slippage, funding and equity included. Python stays out of the hot loop. -- **Fast.** 10 million bars with a full performance summary in **236 ms**, +- **Fast.** 10 million bars with a full performance summary in **233 ms**, where vectorbt takes 98.4 s. 10,000 parameter sets over 200,000 bars in - **3.5 s inside 323 MB**. Measured on a standard 4-vCPU GitHub runner, + **2.5 s inside 342 MB**. Measured on a standard 4-vCPU GitHub runner, [every run linked](#performance). - **Checked.** Entry price, exit price and exit reason are asserted against vectorbt in CI, on the wheel published to PyPI. A benchmark workload the @@ -141,19 +141,19 @@ and only then reports how long each took. ```text SMA crossover with drawdown, Sharpe, Sortino and volatility, 10M bars - Manifold-BT ▏ 0.24 s - raptorbt ▍ 0.90 s + Manifold-BT ▏ 0.23 s + raptorbt ▍ 0.89 s vectorbt ████████████████████████████████████████████████ 98.4 s Parameter sweep, 10,000 combinations over 200,000 bars - Manifold-BT █▋ 3.5 s - raptorbt ████████████████████████████████████████████████ 102.2 s + Manifold-BT █▎ 2.5 s + raptorbt ████████████████████████████████████████████████ 92.1 s vectorbt not run: it materialises every combination, this grid needs tens of GB ``` -**[Run 38319867927](https://github.com/manifoldbt/manifoldbt/actions/runs/38319867927)**: -Linux x86_64, 4 vCPU, Python 3.12, manifoldbt 0.27.1, vectorbt 0.28.4, -raptorbt 0.9.0. Seven interleaved repetitions (two for sweeps), medians +**[Run 40212418887](https://github.com/manifoldbt/manifoldbt/actions/runs/40212418887)**: +Linux x86_64, 4 vCPU, Python 3.12, manifoldbt 0.29.0, vectorbt 0.28.4, +raptorbt 0.9.0. Three interleaved repetitions, medians reported; the ratio is the median of the per-repetition ratios. All five rows below agree with vectorbt exactly (relative tolerance 1e-9). The figures here are copied from that run; the newest published run is always the one on @@ -162,11 +162,11 @@ are copied from that run; the newest published run is always the one on | Workload | Bars | Manifold-BT | vectorbt | raptorbt | |---|---:|---:|---:|---:| -| SMA 30/150 crossover | 10M | **237 ms** | 25.4 s (×106.5) | 895 ms (×3.8) | -| ...plus drawdown, Sharpe, Sortino, volatility | 10M | **236 ms** | 98.4 s (**×415**) | 897 ms (×3.8) | -| ...with a 5 bps fee and 2 bps slippage | 10M | **245 ms** | 25.3 s (×103.4) | not supported | -| EMA 12/26 + RSI(14) filter, 5 bps fee | 1M | **45 ms** | 2.22 s (×49.3) | not supported | -| Five assets in one book, 5 bps fee | 1M | **153 ms** | 2.48 s (×15.8) | not supported | +| SMA 30/150 crossover | 10M | **234 ms** | 24.7 s (×106.0) | 891 ms (×3.8) | +| ...plus drawdown, Sharpe, Sortino, volatility | 10M | **233 ms** | 98.4 s (**×422**) | 889 ms (×3.8) | +| ...with a 5 bps fee and 2 bps slippage | 10M | **313 ms** | 25.4 s (×81.6) | not supported | +| EMA 12/26 + RSI(14) filter, 5 bps fee | 1M | **42 ms** | 2.18 s (×52.8) | not supported | +| Five assets in one book, 5 bps fee | 1M | **154 ms** | 2.49 s (×16.2) | not supported | The second row is the one worth reading twice. A performance summary costs Manifold-BT nothing measurable, because it computes one during the run whether @@ -180,29 +180,30 @@ of an empty cell. | Bars | Combinations | Manifold-BT | vectorbt | raptorbt | |---:|---:|---:|---:|---:| -| 20,000 | 5,000 | **0.20 s**, 30 MB | 8.86 s, 2.5 GB | 4.20 s, 36 MB | -| 20,000 | 20,000 | **0.81 s**, 66 MB | not run | 17.0 s | -| 200,000 | 10,000 | **3.49 s**, 323 MB | not run | 102.2 s | +| 20,000 | 5,000 | **0.16 s**, 38 MB | 5.33 s, 2.5 GB | 3.59 s, 38 MB | +| 20,000 | 20,000 | **0.57 s**, 76 MB | not run | 13.4 s, 62 MB | +| 200,000 | 10,000 | **2.49 s**, 342 MB | not run | 92.1 s, 367 MB | +| 200,000 | 150,000 | **49.0 s**, 826 MB | not run | not run | Past a certain grid the question stops being speed. vectorbt materialises the simulation per combination, 1.57 MB of it at 20,000 bars, so the larger grids would ask the runner for tens of gigabytes. Each sweep is checked for parity before it is timed: every combination against raptorbt, and against vectorbt on the full grid where it fits, or on a 250-combination anchor grid where it -does not. The sweep job runs with a Pro licence, since Community sweeps stop +does not. The sweep job runs with a paid licence, since Community sweeps stop at 256 combinations. ### Where it does not win - **On a single backtest, the gap to raptorbt is about 4×, not orders of magnitude.** Both are compiled engines; the large ratios in the first table - are against vectorbt. The ×21 to ×29 on sweeps is a different story: raptorbt + are against vectorbt. The ×22 to ×37 on sweeps is a different story: raptorbt has no fan-out API, so its sweep is a Python loop paying a full backtest per combination. -- **Multi-asset is the smallest lead over vectorbt (×15.8).** Broadcasting a +- **Multi-asset is the smallest lead over vectorbt (×16.2).** Broadcasting a column per asset is close to free for vectorbt, while walking five books is not free for anything. -- **On a stop-loss/take-profit bracket, raptorbt comes closest** (×1.5 at 1M +- **On a stop-loss/take-profit bracket, raptorbt comes closest** (×1.3 at 1M bars, ×1.8 at 10M). Deciding which side triggers first inside a bar is a sequential walk in both engines. That workload is also not in the table above: the engines disagree on whether to re-enter on the bar a bracket @@ -315,7 +316,8 @@ concludes, so the difference is visible rather than asserted. **Data.** Built-in connectors, all returning a store ready for `mbt.run`: Binance, Bybit, Hyperliquid, dYdX, Bitstamp, Deribit, Yahoo Finance and -Dukascopy on every tier, Databento and Massive with Pro. Or bring your own CSV +Dukascopy on every tier, London Strategic Edge on every tier with your own free +key, Databento and Massive with Pro. Or bring your own CSV (standard, MetaTrader 4 and MetaTrader 5 layouts are detected): ```python @@ -492,7 +494,7 @@ paid tiers add scale and depth. See [pricing](https://www.manifoldbt.com/#pricin | | Community | Pro | Researcher | |---|:---:|:---:|:---:| | Single backtests (`mbt.run`) | Unlimited, full speed | Unlimited | Unlimited | -| Parameter sweeps and batches | 256 backtests per session, batches included | Unlimited | Unlimited | +| Parameter sweeps and batches | Up to 256 backtests per sweep | Unlimited | Unlimited | | Simulation resolution (`bar_interval`) | 1 minute and coarser | Down to 1 second | Down to 1 second | | Output resolution | Daily | Down to 1 second | Down to 1 second | | Monte Carlo | 1,000 simulations | Unlimited | Unlimited | diff --git a/examples/10_monte_carlo.py b/examples/10_monte_carlo.py index 94cc495..fd10f35 100644 --- a/examples/10_monte_carlo.py +++ b/examples/10_monte_carlo.py @@ -1,9 +1,11 @@ -"""Monte Carlo Simulation -- confidence intervals on equity paths (Pro). +"""Monte Carlo Simulation -- confidence intervals on equity paths. Demonstrates: - - py_run_monte_carlo() for bootstrapped equity paths - - Monte Carlo fan chart visualization + - mbt.monte_carlo() for bootstrapped equity paths, computed in the engine - Risk metrics from simulated distributions + - Monte Carlo fan chart visualization + +Community runs up to 1,000 paths; Pro has no ceiling. Data: shared store — real market data from `data/` (see examples/README.md) @@ -62,6 +64,15 @@ if __name__ == "__main__": print(result.summary()) print(f"Elapsed: {time.perf_counter() - t0:.3f}s\n") - # 2. Monte Carlo fan chart + # 2. Risk distribution, resampled in the engine: blocks of 20 bars keep the + # short-term autocorrelation that an IID bootstrap would destroy. + mc = mbt.monte_carlo(result, n_paths=1000, method="block", block_size=20, seed=42) + final = dict(mc["final_return"]["percentiles"]) + drawdown = dict(mc["max_drawdown"]["percentiles"]) + print(f"Final return 5th pct {final[0.05]:+.1%} median {final[0.5]:+.1%}") + print(f"Max drawdown median {drawdown[0.5]:.1%} 95th pct {drawdown[0.95]:.1%}") + print(f"P(ruin) {mc['prob_of_ruin']:.1%}\n") + + # 3. Monte Carlo fan chart if plots_available(): mbt.plot.monte_carlo(result, n_simulations=10000, seed=42) diff --git a/pyproject.toml b/pyproject.toml index f24d567..d671150 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [project] name = "manifoldbt" -version = "0.29.0" +version = "0.30.0" description = "Rust-powered backtesting engine for quantitative research" readme = "README.md" requires-python = ">=3.9" diff --git a/python/manifoldbt/__init__.py b/python/manifoldbt/__init__.py index 9e2e166..829421c 100644 --- a/python/manifoldbt/__init__.py +++ b/python/manifoldbt/__init__.py @@ -194,8 +194,7 @@ atexit.register(_print_pro_summary) # Fold this session's anonymous usage counters into their file on the way out. # Without it a script short enough never to reach the periodic flush (most # scripts) would count everything and persist nothing. Order does not matter: -# this neither prints nor reads anything the hooks around it touch, and it is a -# no-op under MANIFOLDBT_NO_TELEMETRY / DO_NOT_TRACK / CI. +# this neither prints nor reads anything the hooks around it touch. atexit.register(_flush_usage_native) @@ -266,9 +265,8 @@ def _require_grant_for_gpu(device, feature: str) -> None: ) -# Community fan-out budget: sweeps and batches may run up to this many backtests -# cumulatively per session for free; beyond it requires Pro. A single run() is -# never affected. Keep in step with the engine's own limit. +# Community sweep ceiling: beyond it requires Pro. A single run() is never +# affected. Keep in step with the engine's own limit. _COMMUNITY_MAX_COMBOS = 256 @@ -283,10 +281,8 @@ def _grid_combos(param_grid) -> int: def _require_pro_over_combos(n_combos: int, what: str) -> None: """Raise LicenseError if a fan-out exceeds the Community combination limit. - Fast-fail, so a call that could never fit the budget reports cleanly in a - notebook instead of part-way through the run. The budget is consumed - **cumulatively across the session**: a sequence of smaller calls draws on - the same allowance as one large one. + Fast-fail, so a call that could never fit reports cleanly in a notebook + instead of part-way through the run. """ if n_combos <= _COMMUNITY_MAX_COMBOS or _is_pro(): return @@ -995,7 +991,8 @@ def ingest( Providers (free): ``"binance"``, ``"bybit"``, ``"hyperliquid"``, ``"dydx"``, ``"bitstamp"``, ``"deribit"``, ``"yahoo"`` (alias ``"yfinance"``), - ``"dukascopy"``. Pro: ``"databento"``, ``"massive"``. + ``"dukascopy"``. Free with your own key: ``"lse"``. Pro: ``"databento"``, + ``"massive"``. Returns a :class:`DataStore` ready for :func:`run`. @@ -1034,6 +1031,35 @@ def ingest( to the listing date. Prices are dividend-adjusted like ``yfinance``'s ``auto_adjust=True``; pass ``dataset="raw"`` for unadjusted quotes. + Example (minute or second bars on stocks, FX, crypto, futures, indices, + commodities and US options, from London Strategic Edge, with your own free + key in ``LSE_API_KEY``):: + + store = mbt.ingest( + provider="lse", + symbol="AAPL", # or "EUR_USD" (also "EUR/USD"), "BTCUSDT" + symbol_id=1, + start="2015-01-01T00:00:00Z", + end="2026-01-01T00:00:00Z", + interval="1m", + asset_class="equity", + ) + + Symbols are spelt as the LSE catalogue lists them (``"BTC/USD"`` is also + understood as ``"BTCUSDT"``), and the market is found from the symbol the + way LSE's own client finds it: US stocks first, so ``"AAPL"`` is the + Nasdaq share and not its Canadian receipt. Pass ``dataset=`` to read + another market (``"us_stocks"``, + ``"uk_stocks"``, ``"ca_stocks"``, ``"forex"``, ``"crypto"``, + ``"futures"``, ``"indices"``, ``"commodities"``, ``"us_options"``, or + ``"cme"`` and ``"nasdaq"`` on a paid LSE plan). Intervals run from + ``"1s"`` to ``"1d"``; ingest ``"1d"`` and resample for longer bars. Prices + and volumes are split-adjusted, not dividend-adjusted. A daily bar is + stamped at its first minute with a trade, not at midnight. The key is + yours and so is the data: LSE allows it for your own research and trading + and forbids passing it on, and it limits each key to one download at a + time, so the ingest downloads its windows one after the other. + Example (FX, metals, indices and CFDs with BOTH sides of the book, free and without a key, from Dukascopy Bank):: @@ -2610,7 +2636,7 @@ def run_sweep_lite( sweep. **Researcher**: without it, ``device="cuda"`` raises ``LicenseError``, and ``"auto"`` runs a large grid on the CPU with a ``UserWarning`` saying so when a GPU is present; the CPU sweep - itself is full speed on every tier. ``"cuda"`` requires a build with ``--features cuda`` + itself is full speed on every tier. ``"cuda"`` requires a CUDA-enabled build and a CUDA device; for any unsupported strategy/config (or when no GPU is present at runtime) it falls back to the CPU sweep with a ``UserWarning`` naming the reason, so results are never affected. @@ -2993,7 +3019,7 @@ def run_stochastic( confidence_levels: Quantile levels for reporting. store_paths: Whether to store full price paths. device: ``"cpu"`` (default, Rayon parallel) or ``"cuda"``/``"gpu"`` - (CUDA GPU, requires build with ``--features cuda``). + (CUDA GPU, requires a CUDA-enabled build). precision: ``"f64"`` (default, double) or ``"f32"`` (float, ~10-20x faster on consumer GPUs, suitable for research/prototyping). @@ -3055,6 +3081,170 @@ def run_stochastic( raise _classify_error(exc) from exc +# --------------------------------------------------------------------------- +# Monte Carlo +# --------------------------------------------------------------------------- + +_MC_METHODS = { + "return": "return_bootstrap", + "return_bootstrap": "return_bootstrap", + "block": "block_bootstrap", + "block_bootstrap": "block_bootstrap", + "trade": "trade_bootstrap", + "trade_bootstrap": "trade_bootstrap", +} + +# Community ceiling, checked here so the refusal is a LicenseError that says +# what to do, not a strategy error. +_COMMUNITY_MAX_MC_PATHS = 1000 + + +def monte_carlo( + result, + *, + n_paths: int = 1000, + method: str = "block", + block_size: int = 20, + seed: Optional[int] = None, + initial_capital: Optional[float] = None, + confidence_levels: Optional[List[float]] = None, + cvar_levels: Optional[List[float]] = None, + dd_thresholds: Optional[List[float]] = None, + wipeout_threshold: float = -0.90, + store_paths: bool = False, + device: str = "cpu", +) -> Dict[str, Any]: + """Monte Carlo resampling of a finished backtest, in the engine. + + Post-processing only: the strategy is not re-run and no data is reloaded. + Every path is drawn from a seeded ChaCha8 generator, so one ``seed`` gives + the same paths on every machine and on either device. + + Args: + result: A :class:`Result` from :func:`run` (or its ``.raw``). + n_paths: Number of resampled paths. Community runs up to 1,000; + Pro and above have no ceiling. + method: ``"return"`` resamples bar returns with replacement, + ``"block"`` (default) draws contiguous blocks of ``block_size`` + bars to keep short-term autocorrelation, ``"trade"`` reshuffles the + round-trip trade P&L and rebuilds the capital curve. + block_size: Block length in bars, for ``method="block"``. + seed: RNG seed. ``None`` draws a fresh seed each call. + initial_capital: Capital the returns and ruin are measured against. + Defaults to the backtest's own ``initial_capital``. + confidence_levels: Quantiles to report, default + ``[0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95]``. + cvar_levels: Expected-shortfall levels, default ``[0.95, 0.99]``. + dd_thresholds: Drawdown levels to report ``P(max_drawdown > x)`` for, + e.g. ``[0.2, 0.3, 0.5]``. + wipeout_threshold: Final return below which a path counts as a + wipeout, default ``-0.90``. + store_paths: Also return every equity path, as a numpy array of shape + ``(n_paths, n_steps)``. Memory grows with ``n_paths`` x bars. + device: ``"cpu"`` (default) or ``"cuda"``. The GPU runs the return and + block bootstraps, bit-identical to the CPU; ``"trade"`` and + ``store_paths=True`` run on the CPU and say so in a warning. + + Returns: + Dict with ``final_return`` and ``max_drawdown`` (each with + ``percentiles``, ``cvar``, ``mean``, ``std``, ``min``, ``max``), + ``prob_of_ruin``, ``prob_of_wipeout``, ``dd_exceedance``, + ``n_paths``, ``method_name``, and ``paths`` (``None`` unless + ``store_paths=True``). + + Example: + >>> mc = mbt.monte_carlo(result, n_paths=1000, method="block", seed=7) + >>> mc["max_drawdown"]["percentiles"] + [[0.05, 0.081], [0.1, 0.093], ...] + >>> mc["prob_of_ruin"] + 0.012 + """ + raw = result.raw if isinstance(result, Result) else result + if not hasattr(raw, "equity_curve"): + raise TypeError( + "monte_carlo resamples a backtest result: pass what mbt.run returned " + f"(got {type(result).__name__})" + ) + + if isinstance(method, str) and method.lower() in _MC_METHODS: + method_type = _MC_METHODS[method.lower()] + else: + raise ValueError( + f"monte_carlo: unknown method={method!r}: use \"return\", \"block\" or \"trade\"" + ) + if isinstance(n_paths, bool) or not isinstance(n_paths, int) or n_paths < 1: + raise ValueError(f"monte_carlo: n_paths must be a positive integer, got {n_paths!r}") + if method_type == "block_bootstrap" and ( + isinstance(block_size, bool) or not isinstance(block_size, int) or block_size < 1 + ): + raise ValueError(f"monte_carlo: block_size must be a positive integer, got {block_size!r}") + + dev = device.lower() if isinstance(device, str) else device + if dev not in ("cpu", "cuda", "gpu"): + raise ValueError(f'monte_carlo: unknown device={device!r}: use "cpu" or "cuda"') + _require_grant_for_gpu(dev, "GPU Monte Carlo") + + if n_paths > _COMMUNITY_MAX_MC_PATHS and not _grant_couvre("monte_carlo_unlimited"): + raise LicenseError( + f"monte_carlo: {n_paths:,} paths requested; this licence runs up to " + f"{_COMMUNITY_MAX_MC_PATHS:,}. Pass n_paths={_COMMUNITY_MAX_MC_PATHS}, " + "or see what Pro includes: www.manifoldbt.com/pro" + ) + + if dev in ("cuda", "gpu"): + # The engine routes these two to the CPU without a word; say it here. + why = None + if method_type == "trade_bootstrap": + why = "the trade bootstrap has no GPU kernel" + elif store_paths: + why = "store_paths=True keeps every path on the host" + if why: + import warnings + + warnings.warn( + f'monte_carlo: device="cuda" was requested but this run used the CPU: ' + f"{why}. Results are identical either way, only slower.", + UserWarning, + stacklevel=2, + ) + + if initial_capital is None: + initial_capital = raw.manifest["config"]["initial_capital"] + + method_cfg: Dict[str, Any] = {"type": method_type} + if method_type == "block_bootstrap": + method_cfg["block_size"] = block_size + config: Dict[str, Any] = { + "n_paths": n_paths, + "method": method_cfg, + "rng_seed": seed, + "initial_capital": float(initial_capital), + "wipeout_threshold": float(wipeout_threshold), + "store_paths": bool(store_paths), + "device": "cuda" if dev in ("cuda", "gpu") else "cpu", + } + if confidence_levels is not None: + config["confidence_levels"] = list(confidence_levels) + if cvar_levels is not None: + config["cvar_levels"] = list(cvar_levels) + if dd_thresholds is not None: + config["dd_thresholds"] = list(dd_thresholds) + + try: + out = py_run_monte_carlo(raw, json.dumps(config)) + except (ValueError, RuntimeError) as exc: + raise _classify_error(exc) from exc + + paths = out.get("paths") + if paths is not None: + import numpy as np + + n_steps = out.get("paths_n_steps") or 0 + flat = np.asarray(paths.to_numpy(zero_copy_only=False)) + out["paths"] = flat.reshape(-1, n_steps) if n_steps else flat + return out + + # --------------------------------------------------------------------------- # Portfolio API # --------------------------------------------------------------------------- @@ -3484,8 +3674,9 @@ __all__ = [ "run_sweep_2d", "run_stability", "replay", - # NOT exported: py_run_monte_carlo, the raw native binding. Listing it in - # __all__ published an internal as API. + # Monte Carlo. py_run_monte_carlo, the raw native binding under it, stays + # out: listing it in __all__ published an internal as API. + "monte_carlo", # Stochastic simulation "run_stochastic", "StochasticModel", diff --git a/python/manifoldbt/_update.py b/python/manifoldbt/_update.py index cff624b..d707e71 100644 --- a/python/manifoldbt/_update.py +++ b/python/manifoldbt/_update.py @@ -11,12 +11,8 @@ three rules, in order of importance: 3. It is opt-out, via ``MANIFOLDBT_NO_UPDATE_CHECK=1``, and it stays quiet on CI runners where nobody reads the notice and nobody can act on it. -This is not telemetry and shares nothing with the engine's install ping: the -request is a plain GET of a public JSON document, carries no identifier, and its -body is thrown away except for one version string. ``DO_NOT_TRACK`` is therefore -*not* read here -- there is nothing to track -- while ``MANIFOLDBT_NO_TELEMETRY`` -is, because someone who set it meant "this library does not touch the network on -its own", which covers this too. +The request is a plain GET of a public JSON document, carries no identifier, and +its body is thrown away except for one version string. """ import atexit import json diff --git a/python/manifoldbt/cli.py b/python/manifoldbt/cli.py index 5e714fc..af71f28 100644 --- a/python/manifoldbt/cli.py +++ b/python/manifoldbt/cli.py @@ -23,7 +23,7 @@ def main() -> None: required=True, help=( "binance | bybit | hyperliquid | dydx | bitstamp | deribit | yahoo " - "| dukascopy | databento | massive" + "| dukascopy | lse | databento | massive" ), ) ing.add_argument("--symbol", required=True, help="e.g. BTCUSDT, ESH5") diff --git a/python/manifoldbt/cloud.py b/python/manifoldbt/cloud.py index 0ec0319..f695bdb 100644 --- a/python/manifoldbt/cloud.py +++ b/python/manifoldbt/cloud.py @@ -1,4 +1,4 @@ -"""Run a backtest or a sweep on managed capacity instead of this machine. +"""Run a backtest or a sweep on your organisation's workers instead of this machine. The call is the local one with `cloud.` in front: @@ -10,9 +10,12 @@ serialization a local run uses, so a job that runs on a worker is the run you would have got here, through the same code path. Nothing about the strategy changes because it moved. -Authentication is an API key from the Firm portal, read from `MANIFOLDBT_API_KEY` -or passed to `configure()`. The key is never logged, and never travels anywhere -but the Authorization header. +Jobs go to the job queue your deployment runs, next to its workers: its URL +is read from `MANIFOLDBT_CLOUD_URL` or passed to `configure()`, and there is no +default, because the queue, the machines and the data are yours. Authentication +is an API key that queue accepts, read from `MANIFOLDBT_API_KEY` or passed to +`configure()`. The key is never logged, and never travels anywhere but the +Authorization header. Submission returns as soon as the job is queued. `wait()` blocks until a worker has finished it; without it you can close the notebook and come back: @@ -32,8 +35,6 @@ from typing import Any, Dict, List, Optional from manifoldbt._serde import scalar_value_to_json from manifoldbt.exceptions import BacktesterError -DEFAULT_BASE_URL = "https://www.manifoldbt.com" - # Terminal states. Anything else means a worker still has it, or nobody has # taken it yet. _DONE = ("succeeded", "failed") @@ -84,15 +85,16 @@ class QuotaExceeded(CloudError): def configure(api_key: Optional[str] = None, base_url: Optional[str] = None) -> None: """Set the credentials for this session. - Both fall back to the environment (`MANIFOLDBT_API_KEY`, - `MANIFOLDBT_CLOUD_URL`), so a notebook that never calls this still works - when the environment is set. + `base_url` is your job queue, e.g. ``http://queue.internal:8790``. Both + fall back to the environment (`MANIFOLDBT_API_KEY`, `MANIFOLDBT_CLOUD_URL`), + so a notebook that never calls this still works when the environment is + set. An empty string clears what an earlier call set. """ global _api_key, _base_url if api_key is not None: _api_key = api_key.strip() if base_url is not None: - _base_url = base_url.rstrip("/") + _base_url = base_url.strip().rstrip("/") or None def _key() -> str: @@ -100,13 +102,18 @@ def _key() -> str: if not key: raise CloudError( "no API key. Pass it with mbt.cloud.configure(api_key=...) or set " - "MANIFOLDBT_API_KEY. Keys are created in the Firm portal, under Keys." + "MANIFOLDBT_API_KEY. The key is one your job queue was started with." ) return key def _url(path: str) -> str: - base = _base_url or os.environ.get("MANIFOLDBT_CLOUD_URL", DEFAULT_BASE_URL) + base = _base_url or os.environ.get("MANIFOLDBT_CLOUD_URL", "").strip() + if not base: + raise CloudError( + "no job queue. Pass its URL with mbt.cloud.configure(base_url=...) or set " + "MANIFOLDBT_CLOUD_URL to the queue your workers claim from." + ) return f"{base.rstrip('/')}{path}" @@ -148,15 +155,13 @@ def _call(path: str, method: str = "GET", body: Optional[dict] = None, timeout: def _raise_for(status: int, payload: dict): if status == 401: raise CloudError( - "the API key was refused. Check it, or create a new one in the portal.", 401 + "the API key was refused. Check it against the keys your job queue accepts.", 401 ) if status == 402: raise QuotaExceeded(payload.get("used", 0), payload.get("included", 0)) if status == 403: raise CloudError( - "this organisation has not finished portal setup: sign in to the portal once, " - "answer the setup questions, then retry.", - 403, + f"the job queue refused this request: {payload.get('error', 'no detail')}", 403 ) if status == 413: raise CloudError( @@ -196,7 +201,7 @@ def _sharpe_text(metrics: dict) -> str: class CloudJob: - """A job on managed capacity: its state, and its result once it has one.""" + """A job on your job queue: its state, and its result once it has one.""" def __init__(self, row: dict): self._row = row @@ -283,7 +288,7 @@ class CloudJob: except CloudError as err: # The job is running on a worker, not in this process: a server # error or a dropped connection while polling says nothing about - # it. Giving up here would abandon compute that is being billed. + # it. Giving up here would abandon a job that is still running. # A refusal (a revoked key, an unknown id) is different and ends # the wait immediately. if not err.transient: diff --git a/python/tests/test_cloud.py b/python/tests/test_cloud.py index e5e73c9..224b454 100644 --- a/python/tests/test_cloud.py +++ b/python/tests/test_cloud.py @@ -65,7 +65,7 @@ def stub(): yield state server.shutdown() server.server_close() - mbt.cloud.configure(api_key="", base_url=mbt.cloud.DEFAULT_BASE_URL) + mbt.cloud.configure(api_key="", base_url="") def submit(**kwargs): @@ -153,6 +153,21 @@ def test_a_missing_key_names_where_to_get_one(): mbt.cloud.job(1) +def test_a_missing_queue_url_names_how_to_set_it(monkeypatch): + monkeypatch.delenv("MANIFOLDBT_CLOUD_URL", raising=False) + mbt.cloud.configure(api_key="k", base_url="") + with pytest.raises(CloudError, match="MANIFOLDBT_CLOUD_URL"): + mbt.cloud.job(1) + + +def test_the_queue_url_comes_from_the_environment(stub, monkeypatch): + base = mbt.cloud._base_url + mbt.cloud.configure(base_url="") + monkeypatch.setenv("MANIFOLDBT_CLOUD_URL", base + "/") + submit(wait=False, timeout=5) + assert stub.requests[-1][0] == "/api/jobs" + + def test_an_unreachable_server_is_transient_not_a_refusal(): mbt.cloud.configure(api_key="k", base_url="http://127.0.0.1:1") with pytest.raises(CloudError) as excinfo: diff --git a/python/tests/test_monte_carlo_api.py b/python/tests/test_monte_carlo_api.py new file mode 100644 index 0000000..9553a3d --- /dev/null +++ b/python/tests/test_monte_carlo_api.py @@ -0,0 +1,200 @@ +"""``mbt.monte_carlo``: the engine's Monte Carlo, as a public function. + +The engine has resampled finished backtests natively for a long time (return, +block and trade bootstraps, the return and block ones on the GPU too), but +Python only reached it through ``py_run_monte_carlo``, a raw binding that takes +a JSON string and was taken out of ``__all__`` for that reason. These tests pin +the public wrapper: it returns exactly what the binding returns, it reads the +run's own capital, it refuses what it cannot do by name, and it says when a +``device="cuda"`` request ran on the CPU instead of letting the engine fall +back without a word. +""" +import json +import warnings + +import pytest + +import manifoldbt as bt +from manifoldbt.exceptions import LicenseError +from manifoldbt.expr import col, lit, when +from manifoldbt.helpers import Interval + +pd = pytest.importorskip("pandas") +np = pytest.importorskip("numpy") + +N = 400 + + +def _frame(seed): + rng = np.random.default_rng(seed) + ts = pd.date_range("2022-01-01", periods=N, freq="1h", tz="UTC") + close = np.maximum(100.0 + np.cumsum(rng.normal(0.0, 0.5, N)), 1.0) + return pd.DataFrame({ + "timestamp": ts, + "open": close, "high": close + 0.5, "low": close - 0.5, + "close": close, "volume": [10.0] * N, + }) + + +@pytest.fixture(scope="module") +def result(tmp_path_factory): + tmp = tmp_path_factory.mktemp("mc") + store = bt.import_dataframe( + _frame(3), symbol="AAA", symbol_id=1, interval="1h", + data_root=str(tmp / "data"), metadata_db=str(tmp / "meta.sqlite"), + ) + config = bt.BacktestConfig( + universe=["AAA"], + time_range_start=0, + time_range_end=4_102_444_800_000_000_000, + bar_interval=Interval.hours(1), + initial_capital=25_000.0, + warmup_bars=0, + fees=bt.FeeConfig.zero(), + ) + # In and out of the market, so the trade bootstrap has round trips to shuffle. + pos = when(col("close") > lit(100.0), lit(1.0), lit(0.0)) + strategy = bt.Strategy.create("mc").signal("position", pos).size(pos) + return bt.run(strategy, config, store) + + +def _raw(result, **config): + return bt.py_run_monte_carlo(result.raw, json.dumps(config)) + + +def test_is_public_and_the_binding_is_not(): + assert "monte_carlo" in bt.__all__ + assert "py_run_monte_carlo" not in bt.__all__ + + +def test_returns_what_the_binding_returns(result): + mc = bt.monte_carlo(result, n_paths=200, method="block", block_size=10, seed=7) + raw = _raw( + result, + n_paths=200, + method={"type": "block_bootstrap", "block_size": 10}, + rng_seed=7, + initial_capital=25_000.0, + store_paths=False, + device="cpu", + ) + assert mc == raw + assert mc["n_paths"] == 200 + for key in ("final_return", "max_drawdown"): + assert {"percentiles", "cvar", "mean", "std", "min", "max"} <= set(mc[key]) + assert 0.0 <= mc["prob_of_ruin"] <= 1.0 + + +def test_one_seed_gives_the_same_paths(result): + a = bt.monte_carlo(result, n_paths=100, seed=11) + b = bt.monte_carlo(result, n_paths=100, seed=11) + c = bt.monte_carlo(result, n_paths=100, seed=12) + assert a == b + assert a["final_return"] != c["final_return"] + + +@pytest.mark.parametrize("short, full", [("return", "return_bootstrap"), + ("block", "block_bootstrap"), + ("trade", "trade_bootstrap")]) +def test_short_and_full_method_names_agree(result, short, full): + assert bt.monte_carlo(result, n_paths=50, method=short, seed=1) == bt.monte_carlo( + result, n_paths=50, method=full, seed=1 + ) + + +def test_capital_defaults_to_the_runs_own(result): + # The return and block bootstraps resample returns, where capital is only a + # scale; the trade bootstrap rebuilds the curve from P&L in money, so it is + # the one that tells which capital was used. + mc = lambda **kw: bt.monte_carlo(result, n_paths=50, seed=3, method="trade", **kw) # noqa: E731 + assert mc() == mc(initial_capital=25_000.0) + assert mc() != mc(initial_capital=50_000.0) + + +def test_accepts_the_raw_result(result): + assert bt.monte_carlo(result.raw, n_paths=50, seed=5) == bt.monte_carlo( + result, n_paths=50, seed=5 + ) + + +def test_stored_paths_come_back_as_a_matrix(result): + mc = bt.monte_carlo(result, n_paths=30, method="return", seed=2, store_paths=True) + paths = mc["paths"] + assert isinstance(paths, np.ndarray) + assert paths.shape == (30, mc["paths_n_steps"]) + np.testing.assert_allclose(paths[:, 0], 25_000.0) + + +def test_risk_aggregates_are_passed_through(result): + mc = bt.monte_carlo(result, n_paths=100, seed=4, cvar_levels=[0.9], + dd_thresholds=[0.01, 0.5], confidence_levels=[0.5]) + assert [lvl for lvl, _ in mc["final_return"]["cvar"]] == [0.9] + assert [lvl for lvl, _ in mc["dd_exceedance"]] == [0.01, 0.5] + assert [lvl for lvl, _ in mc["max_drawdown"]["percentiles"]] == [0.5] + + +@pytest.mark.parametrize("kw, needle", [ + ({"method": "shuffle"}, "unknown method"), + ({"device": "tpu"}, "unknown device"), + ({"n_paths": 0}, "n_paths"), + ({"n_paths": 10.5}, "n_paths"), + ({"method": "block", "block_size": 0}, "block_size"), +]) +def test_refuses_what_it_cannot_do_by_name(result, kw, needle): + with pytest.raises(ValueError, match=needle): + bt.monte_carlo(result, **kw) + + +def test_rejects_something_that_is_not_a_backtest(): + with pytest.raises(TypeError, match="backtest result"): + bt.monte_carlo(object()) + + +def test_above_1000_paths_asks_the_licence_flag(result, monkeypatch): + asked = [] + + def grant(feature): + asked.append(feature) + return False + + monkeypatch.setattr(bt, "_grant_couvre", grant) + monkeypatch.setattr(bt, "py_run_monte_carlo", lambda *_: pytest.fail("reached the engine")) + with pytest.raises(LicenseError, match="runs up to 1,000"): + bt.monte_carlo(result, n_paths=1001) + assert asked == ["monte_carlo_unlimited"] + + +def test_1000_paths_never_consults_the_licence(result, monkeypatch): + monkeypatch.setattr(bt, "_grant_couvre", lambda _f: pytest.fail("licence consulted")) + assert bt.monte_carlo(result, n_paths=1000, seed=1)["n_paths"] == 1000 + + +def test_cuda_without_the_gpu_grant_is_a_licence_error(result, monkeypatch): + monkeypatch.setattr(bt, "_grant_couvre", lambda _f: False) + monkeypatch.setattr(bt, "py_run_monte_carlo", lambda *_: pytest.fail("reached the engine")) + with pytest.raises(LicenseError, match="Researcher"): + bt.monte_carlo(result, n_paths=10, device="cuda") + + +@pytest.mark.parametrize("kw, why", [ + ({"method": "trade"}, "trade bootstrap"), + ({"method": "block", "store_paths": True}, "store_paths"), +]) +def test_cuda_on_a_cpu_only_case_says_so(result, monkeypatch, kw, why): + sent = [] + monkeypatch.setattr(bt, "_grant_couvre", lambda _f: True) + monkeypatch.setattr( + bt, "py_run_monte_carlo", + lambda _raw, cfg: sent.append(json.loads(cfg)) or {"paths": None}, + ) + with pytest.warns(UserWarning, match=why): + bt.monte_carlo(result, n_paths=10, device="cuda", **kw) + assert sent[0]["device"] == "cuda" + + +def test_cuda_on_a_gpu_case_does_not_warn(result, monkeypatch): + monkeypatch.setattr(bt, "_grant_couvre", lambda _f: True) + monkeypatch.setattr(bt, "py_run_monte_carlo", lambda _raw, _cfg: {"paths": None}) + with warnings.catch_warnings(): + warnings.simplefilter("error") + bt.monte_carlo(result, n_paths=10, device="cuda", method="block")