From 6cae68628325413378ef4fa8a07c26b892675f62 Mon Sep 17 00:00:00 2001 From: Exocet92 <79667065+Jimmy7892@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:45:05 +0200 Subject: [PATCH] bench: add a cost workload and a multi-asset one, and go to three repetitions (#10) Two gaps a reader could name without running anything: costs appeared on one workload out of four, and nothing in the suite was a portfolio. Costs could not simply be switched on across the board, and the reason is measured. On FractionOfEquity sizing a 5 bps fee puts the engines 1.3e-4 of capital apart and 2 bps of slippage 2.1e-5, against a 1e-9 tolerance, while the round-trip counts stay identical: the trading agrees, the cost arithmetic does not, because one charges the fee on top of the notional and the other reserves it out of cash first. In fixed units both land exactly, to 1e-15. So `sma_cross_costs` carries a fee and slippage on the headline signal, sized in units, and the price of that is visible rather than hidden: x48.0 against x50.6 at 100k bars. `multi_asset` runs five independent series in one shared book. It is the workload manifoldbt does worst on, and it is here for that reason: going from one asset to five costs it 6.1x and vectorbt 1.4x, so the ratio falls from x36.7 to x8.8 at a million bars. Broadcasting a column per asset is close to free; walking five books is not. A portfolio is also what people actually run, and a suite that only measures where it wins is not evidence. Both are capped where a materialised five-column simulation would stop measuring the engine and start measuring the swap file, and `ema_rsi_fees` keeps the ceiling it got for going bankrupt. Repetitions go from two to three: the floor at which a median is a median rather than the mean of two. --- .github/workflows/bench-vs-vectorbt.yml | 16 ++++---- benchmarks/vs_vectorbt/README.md | 40 +++++++++++++++++- benchmarks/vs_vectorbt/data.py | 13 ++++++ benchmarks/vs_vectorbt/engine_mbt.py | 54 +++++++++++++++++++------ benchmarks/vs_vectorbt/engine_vbt.py | 52 ++++++++++++++++++++++-- benchmarks/vs_vectorbt/workloads.py | 53 ++++++++++++++++++++++++ 6 files changed, 203 insertions(+), 25 deletions(-) diff --git a/.github/workflows/bench-vs-vectorbt.yml b/.github/workflows/bench-vs-vectorbt.yml index bdc9d83..88c2928 100644 --- a/.github/workflows/bench-vs-vectorbt.yml +++ b/.github/workflows/bench-vs-vectorbt.yml @@ -19,14 +19,14 @@ on: reps: description: "Interleaved repetitions per point" required: false - # 2, not 7. The budget went into a longer series instead: at 10M bars - # one vectorbt call costs 99 s, so seven of them would put the job past - # its timeout on Windows. Two repetitions still bracket the number (min, - # median and max are all published, and the median of two is their mean) - # but the noise flag gets cruder, since an interquartile range wants at - # least four samples to mean anything. A point that looks surprising is - # worth re-running at a higher `reps` before it is quoted anywhere. - default: "2" + # 3, not 7. The budget went into a longer series and two more + # workloads instead: at 10M bars one vectorbt call costs 99 s, so seven + # of them would put the job past its timeout. Three is the floor at + # which a median is a median rather than the mean of two, though an + # interquartile range still wants four samples to mean anything, so the + # noise flag stays coarse. A point that looks surprising is worth + # re-running at a higher `reps` before it is quoted anywhere. + default: "3" release: types: [published] schedule: diff --git a/benchmarks/vs_vectorbt/README.md b/benchmarks/vs_vectorbt/README.md index 5a8ddda..f713ea8 100644 --- a/benchmarks/vs_vectorbt/README.md +++ b/benchmarks/vs_vectorbt/README.md @@ -86,10 +86,48 @@ store)`, the documented entry point, not through an internal fast path. | Workload | What it exercises | vectorbt | raptorbt | |---|---|---|---| | `sma_cross` | SMA 30/150 crossover, long-only, no cost | exact | exact | -| `ema_rsi_fees` | EMA 12/26 crossover with an RSI(14) filter and a 5 bps taker fee, capped at 1M bars | exact | unsupported | | `sma_cross_metrics` | the same simulation, plus max drawdown, Sharpe, Sortino and volatility | exact | exact | +| `sma_cross_costs` | the same signal with a 5 bps fee and 2 bps of slippage | exact | unsupported | +| `multi_asset` | five assets in one shared book, capped at 1M bars | exact | unsupported | +| `ema_rsi_fees` | EMA 12/26 with an RSI(14) filter and a 5 bps taker fee, capped at 1M bars | exact | unsupported | | `bracket_sl_tp` | the same entry with a 15 bps stop and a 30 bps target | documented | documented | +### Costs live in their own workload, and not by preference + +A cost model cannot simply be switched on across the board. Measured on the +headline workload, a fee or a slippage applied to `FractionOfEquity` sizing puts +the engines 1.3e-4 and 2.1e-5 of capital apart against a 1e-9 tolerance, while +the round-trip counts stay identical: the trading agrees, the cost arithmetic +does not, because one charges the fee on top of the notional and the other +reserves it out of cash first. In fixed units both land exactly, to 1e-15. + +So `sma_cross_costs` carries both costs on the same signal as the headline, and +the price of that is visible: x48.0 against x50.6 at 100k bars, x36.1 against +x36.7 at 1M. Costs change the result, not the ratio. + +### The multi-asset workload is the one manifoldbt does worst on + +It is here on purpose. A portfolio is what people actually run, and it is also +where the two designs differ in kind: manifoldbt walks a universe against one +shared book, while vectorbt broadcasts a column per asset and has to be told to +share cash at all. + +Broadcasting wins. + +| Workload | 100k bars | 1M bars | +|---|---:|---:| +| `sma_cross`, one asset | x50.6 | x36.7 | +| `multi_asset`, five assets | **x10.7** | **x8.8** | + +The absolute timings say why: going from one asset to five costs manifoldbt 6.1x +(26.4 ms to 162.3 ms at 1M bars) and vectorbt 1.4x (989 ms to 1415 ms). Five +columns in one vectorised pass is close to free for it; five books are not free +for anything that walks them. + +Publishing this lowers the average number on the page. It is the most useful +workload in the suite for anyone deciding whether to switch, which is the only +audience the benchmark has. + ### Why the fee workload stops at 1M bars A workload can stop being a comparison before it stops running. `ema_rsi_fees` diff --git a/benchmarks/vs_vectorbt/data.py b/benchmarks/vs_vectorbt/data.py index dcf1f1b..3bea180 100644 --- a/benchmarks/vs_vectorbt/data.py +++ b/benchmarks/vs_vectorbt/data.py @@ -61,6 +61,19 @@ def make_ohlcv( ) +def make_universe(rows: int, count: int, *, seed: int = DEFAULT_SEED) -> dict: + """`count` independent series, keyed by the symbol id each engine will use. + + Independent, not correlated: a portfolio of copies of one asset would let a + position-sizing bug cancel itself out across the book, which is exactly the + class of mistake a multi-asset workload exists to catch. The seeds are + spaced far apart and derived from the same base, so the whole universe is + reproducible from `seed` alone and the first symbol is bit-identical to the + single-asset series of the same length. + """ + return {i + 1: make_ohlcv(rows, seed=seed + 1000 * i) for i in range(count)} + + def digest(df: pd.DataFrame) -> str: """Short content fingerprint of the bars, recorded in the result envelope. diff --git a/benchmarks/vs_vectorbt/engine_mbt.py b/benchmarks/vs_vectorbt/engine_mbt.py index 02e0686..e8833f6 100644 --- a/benchmarks/vs_vectorbt/engine_mbt.py +++ b/benchmarks/vs_vectorbt/engine_mbt.py @@ -22,6 +22,7 @@ from __future__ import annotations import os from typing import Any, Callable, Dict +import data as data_mod import manifoldbt as bt from manifoldbt.expr import col, lit, when from manifoldbt.helpers import Interval, Slippage @@ -36,7 +37,8 @@ def probe() -> Dict[str, Any]: return {"engine": NAME, "version": bt.__version__} -def _config(df, *, sizing: str, fee_bps: float) -> "bt.BacktestConfig": +def _config(df, *, sizing: str, fee_bps: float, slippage_bps: float = 0.0, + universe=None) -> "bt.BacktestConfig": last_ns = int(df["timestamp"].iloc[-1].value) fees = ( bt.FeeConfig.zero() @@ -44,7 +46,7 @@ def _config(df, *, sizing: str, fee_bps: float) -> "bt.BacktestConfig": else bt.FeeConfig(maker_fee_bps=fee_bps, taker_fee_bps=fee_bps) ) return bt.BacktestConfig( - universe=[1], + universe=universe or [1], time_range_start=0, # A day past the last bar: the range is inclusive of everything generated. time_range_end=last_ns + 86_400_000_000_000, @@ -58,7 +60,8 @@ def _config(df, *, sizing: str, fee_bps: float) -> "bt.BacktestConfig": position_sizing_mode=sizing, ), fees=fees, - slippage=Slippage.none(), + slippage=(Slippage.none() if slippage_bps == 0.0 + else Slippage.fixed_bps(slippage_bps)), warmup_bars=0, ) @@ -84,6 +87,18 @@ def _strategy(key: str): .take_profit(pct=p["tp_pct"]) ) + if key in ("sma_cross_costs", "multi_asset"): + # The same crossover as the headline workload. What differs is the cost + # model and the number of symbols the config points at, neither of which + # is visible from the strategy: a universe is walked by the engine, not + # spelled out per asset, which is the whole point of the comparison. + return ( + bt.Strategy.create(key) + .signal("fast", sma(close_px, p["fast"])) + .signal("slow", sma(close_px, p["slow"])) + .size(when(col("fast") > col("slow"), lit(p["units"]), lit(0.0))) + ) + if key == "ema_rsi_fees": entry = ( (col("fast") > col("slow")) @@ -110,16 +125,28 @@ def prepare(key: str, df, workdir: str) -> Callable[[], Dict[str, Any]]: root = os.path.join(workdir, key) os.makedirs(root, exist_ok=True) - store = bt.import_dataframe( - df, - symbol="BENCH", - symbol_id=1, - interval="1m", - data_root=os.path.join(root, "data"), - metadata_db=os.path.join(root, "metadata.sqlite"), - ) + data_root = os.path.join(root, "data") + metadata_db = os.path.join(root, "metadata.sqlite") + assets = int(p.get("assets", 1)) + if assets > 1: + # The universe is derived from the same generator and the same base + # seed, so the first symbol is the single-asset series bit for bit and + # the whole set is reproducible from the digest already recorded. + frames = data_mod.make_universe(len(df), assets) + for symbol_id, frame in frames.items(): + store = bt.import_dataframe( + frame, symbol="A%d" % symbol_id, symbol_id=symbol_id, + interval="1m", data_root=data_root, metadata_db=metadata_db) + universe = list(frames) + else: + store = bt.import_dataframe( + df, symbol="BENCH", symbol_id=1, interval="1m", + data_root=data_root, metadata_db=metadata_db) + universe = [1] strategy = _strategy(key) - config = _config(df, sizing=sizing, fee_bps=fee_bps) + config = _config(df, sizing=sizing, fee_bps=fee_bps, + slippage_bps=float(p.get("slippage_bps", 0.0)), + universe=universe) wants_metrics = bool(p.get("metrics")) @@ -171,7 +198,8 @@ def diagnose(key: str, df, workdir: str) -> Dict[str, Any]: result = bt.run( _strategy(key), _config(df, sizing="Units" if "units" in p else "FractionOfEquity", - fee_bps=float(p.get("fee_bps", 0.0))), + fee_bps=float(p.get("fee_bps", 0.0)), + slippage_bps=float(p.get("slippage_bps", 0.0))), store, ) trades = result.trades_df() diff --git a/benchmarks/vs_vectorbt/engine_vbt.py b/benchmarks/vs_vectorbt/engine_vbt.py index 109e266..00a7121 100644 --- a/benchmarks/vs_vectorbt/engine_vbt.py +++ b/benchmarks/vs_vectorbt/engine_vbt.py @@ -30,6 +30,8 @@ from typing import Any, Callable, Dict import numpy as np import pandas as pd import vectorbt as vbt + +import data as data_mod from vectorbt.portfolio.enums import StopExitPrice from workloads import CAPITAL, FREQ, WORKLOADS @@ -72,7 +74,8 @@ def indicators(key: str, close: pd.Series) -> Dict[str, pd.Series]: """Indicator series for a workload. Used by the timed path and by the definition check, so the two can never drift apart.""" p = WORKLOADS[key].params - if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp"): + if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp", + "sma_cross_costs", "multi_asset"): return { "fast": close.rolling(p["fast"]).mean(), "slow": close.rolling(p["slow"]).mean(), @@ -88,7 +91,8 @@ def indicators(key: str, close: pd.Series) -> Dict[str, pd.Series]: def _level(key: str, ind: Dict[str, pd.Series]) -> pd.Series: p = WORKLOADS[key].params - if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp"): + if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp", + "sma_cross_costs", "multi_asset"): return (ind["fast"] > ind["slow"]).fillna(False) if key == "ema_rsi_fees": return ( @@ -113,10 +117,52 @@ def prepare(key: str, df, workdir: str | None = None) -> Callable[[], Dict[str, else: size, size_type = p["alloc"], "percent" fees = float(p.get("fee_bps", 0.0)) / 10_000.0 + slippage = float(p.get("slippage_bps", 0.0)) / 10_000.0 wants_metrics = bool(p.get("metrics")) + assets = int(p.get("assets", 1)) sl = p["sl_pct"] / 100.0 if "sl_pct" in p else None tp = p["tp_pct"] / 100.0 if "tp_pct" in p else None + if assets > 1: + # One book, not five. `from_signals` on a frame of columns builds five + # independent portfolios unless it is told otherwise, and five separate + # books is a different question from the one manifoldbt answers when it + # walks a universe. `group_by` with `cash_sharing` is the spelling that + # asks the same thing. With fixed-unit sizing the cash constraint never + # binds, which is what lets the two agree at all: on a fraction of + # equity they would also have to agree on which asset gets the cash + # first, and that is policy rather than arithmetic. + frames = data_mod.make_universe(len(df), assets) + closes = pd.DataFrame( + {"A%d" % sid: f["close"].to_numpy(dtype=np.float64) + for sid, f in frames.items()}, + index=index, + ) + + def run_multi() -> Dict[str, Any]: + fast = closes.rolling(p["fast"]).mean() + slow = closes.rolling(p["slow"]).mean() + level = (fast > slow).fillna(False) + portfolio = vbt.Portfolio.from_signals( + closes, entries=level, exits=~level, + init_cash=CAPITAL, size=size, size_type=size_type, + fees=fees, slippage=slippage, direction="longonly", + accumulate=False, freq=FREQ, + group_by=True, cash_sharing=True, + ) + total_return = float(portfolio.total_return()) + trades = portfolio.trades + return { + "total_return": total_return, + "final_equity": CAPITAL * (1.0 + total_return), + "round_trips": int(trades.closed.count()), + "fills": None, + "total_fees": float(trades.records["entry_fees"].sum() + + trades.records["exit_fees"].sum()), + } + + return run_multi + def run() -> Dict[str, Any]: level = _level(key, indicators(key, close)) portfolio = vbt.Portfolio.from_signals( @@ -130,7 +176,7 @@ def prepare(key: str, df, workdir: str | None = None) -> Callable[[], Dict[str, size=size, size_type=size_type, fees=fees, - slippage=0.0, + slippage=slippage, sl_stop=sl, tp_stop=tp, stop_exit_price=StopExitPrice.StopMarket, diff --git a/benchmarks/vs_vectorbt/workloads.py b/benchmarks/vs_vectorbt/workloads.py index c378c16..18b1bcb 100644 --- a/benchmarks/vs_vectorbt/workloads.py +++ b/benchmarks/vs_vectorbt/workloads.py @@ -134,6 +134,59 @@ WORKLOADS: Dict[str, Workload] = { "summary costs each engine.", params=dict(fast=30, slow=150, alloc=1.0, metrics=True), ), + Workload( + key="sma_cross_costs", + title="SMA 30/150 with a 5 bps fee and 2 bps of slippage", + why="The same signal as the headline workload, run against a cost " + "model. Costs were the easiest objection to make of a benchmark " + "that had them on one workload out of four, and the answer is " + "not to sprinkle them everywhere: measured, a fee or a slippage " + "on FractionOfEquity sizing makes the engines disagree by 1e-4 " + "of capital against a 1e-9 tolerance, because they resolve the " + "same policy differently. In fixed units the arithmetic is " + "comparable and both costs land exactly, so this is where they " + "belong.", + params=dict(fast=30, slow=150, units=5.0, fee_bps=5.0, slippage_bps=2.0), + notes={ + "raptorbt": Note( + "unsupported", + "Same blocker as the other fixed-quantity workload: raptorbt " + "has no units sizing, and a cost model on top of a fraction " + "of equity compares policy rather than arithmetic. See " + "`ema_rsi_fees` for the measurements behind that.", + ), + }, + ), + Workload( + key="multi_asset", + title="Five assets in one book, SMA 30/150, 5 bps fee", + why="A single-asset benchmark measures a loop; a portfolio measures " + "the thing people actually run. It is also where the two designs " + "differ in kind rather than in speed: manifoldbt walks a universe " + "against one shared book, while vectorbt broadcasts a column per " + "asset and has to be told to share cash at all. The five series " + "are independent rather than correlated, so a sizing bug cannot " + "cancel itself out across the book.", + params=dict(fast=30, slow=150, units=2.0, fee_bps=5.0, assets=5), + # Five columns of a materialised simulation, not one. vectorbt adds + # 1.55 GB on a 10M-bar single asset, so five of them would ask a + # 16 GB runner for around 8 GB on top of 2.4 GB of generated frames. + # The ceiling is set where the point is certain to be measuring the + # engine rather than the swap file. + max_bars=1_000_000, + notes={ + "raptorbt": Note( + "unsupported", + "No multi-instrument portfolio on the entry point this " + "harness drives: `run_single_backtest` is one instrument, and " + "`run_multi_backtest` broadcasts over strategies rather than " + "over assets. 0.9.0 does add `run_portfolio_backtest`, but " + "its allocation model is a different one and would need its " + "own parity work before any timing from it could be " + "published. The units blocker applies here too.", + ), + }, + ), Workload( key="bracket_sl_tp", title="SMA 10/50 entry with a 15 bps stop / 30 bps target bracket",