diff --git a/.github/workflows/bench-vs-vectorbt.yml b/.github/workflows/bench-vs-vectorbt.yml index bdc9d83..88c2928 100644 --- a/.github/workflows/bench-vs-vectorbt.yml +++ b/.github/workflows/bench-vs-vectorbt.yml @@ -19,14 +19,14 @@ on: reps: description: "Interleaved repetitions per point" required: false - # 2, not 7. The budget went into a longer series instead: at 10M bars - # one vectorbt call costs 99 s, so seven of them would put the job past - # its timeout on Windows. Two repetitions still bracket the number (min, - # median and max are all published, and the median of two is their mean) - # but the noise flag gets cruder, since an interquartile range wants at - # least four samples to mean anything. A point that looks surprising is - # worth re-running at a higher `reps` before it is quoted anywhere. - default: "2" + # 3, not 7. The budget went into a longer series and two more + # workloads instead: at 10M bars one vectorbt call costs 99 s, so seven + # of them would put the job past its timeout. Three is the floor at + # which a median is a median rather than the mean of two, though an + # interquartile range still wants four samples to mean anything, so the + # noise flag stays coarse. A point that looks surprising is worth + # re-running at a higher `reps` before it is quoted anywhere. + default: "3" release: types: [published] schedule: diff --git a/benchmarks/vs_vectorbt/README.md b/benchmarks/vs_vectorbt/README.md index 5a8ddda..f713ea8 100644 --- a/benchmarks/vs_vectorbt/README.md +++ b/benchmarks/vs_vectorbt/README.md @@ -86,10 +86,48 @@ store)`, the documented entry point, not through an internal fast path. | Workload | What it exercises | vectorbt | raptorbt | |---|---|---|---| | `sma_cross` | SMA 30/150 crossover, long-only, no cost | exact | exact | -| `ema_rsi_fees` | EMA 12/26 crossover with an RSI(14) filter and a 5 bps taker fee, capped at 1M bars | exact | unsupported | | `sma_cross_metrics` | the same simulation, plus max drawdown, Sharpe, Sortino and volatility | exact | exact | +| `sma_cross_costs` | the same signal with a 5 bps fee and 2 bps of slippage | exact | unsupported | +| `multi_asset` | five assets in one shared book, capped at 1M bars | exact | unsupported | +| `ema_rsi_fees` | EMA 12/26 with an RSI(14) filter and a 5 bps taker fee, capped at 1M bars | exact | unsupported | | `bracket_sl_tp` | the same entry with a 15 bps stop and a 30 bps target | documented | documented | +### Costs live in their own workload, and not by preference + +A cost model cannot simply be switched on across the board. Measured on the +headline workload, a fee or a slippage applied to `FractionOfEquity` sizing puts +the engines 1.3e-4 and 2.1e-5 of capital apart against a 1e-9 tolerance, while +the round-trip counts stay identical: the trading agrees, the cost arithmetic +does not, because one charges the fee on top of the notional and the other +reserves it out of cash first. In fixed units both land exactly, to 1e-15. + +So `sma_cross_costs` carries both costs on the same signal as the headline, and +the price of that is visible: x48.0 against x50.6 at 100k bars, x36.1 against +x36.7 at 1M. Costs change the result, not the ratio. + +### The multi-asset workload is the one manifoldbt does worst on + +It is here on purpose. A portfolio is what people actually run, and it is also +where the two designs differ in kind: manifoldbt walks a universe against one +shared book, while vectorbt broadcasts a column per asset and has to be told to +share cash at all. + +Broadcasting wins. + +| Workload | 100k bars | 1M bars | +|---|---:|---:| +| `sma_cross`, one asset | x50.6 | x36.7 | +| `multi_asset`, five assets | **x10.7** | **x8.8** | + +The absolute timings say why: going from one asset to five costs manifoldbt 6.1x +(26.4 ms to 162.3 ms at 1M bars) and vectorbt 1.4x (989 ms to 1415 ms). Five +columns in one vectorised pass is close to free for it; five books are not free +for anything that walks them. + +Publishing this lowers the average number on the page. It is the most useful +workload in the suite for anyone deciding whether to switch, which is the only +audience the benchmark has. + ### Why the fee workload stops at 1M bars A workload can stop being a comparison before it stops running. `ema_rsi_fees` diff --git a/benchmarks/vs_vectorbt/data.py b/benchmarks/vs_vectorbt/data.py index dcf1f1b..3bea180 100644 --- a/benchmarks/vs_vectorbt/data.py +++ b/benchmarks/vs_vectorbt/data.py @@ -61,6 +61,19 @@ def make_ohlcv( ) +def make_universe(rows: int, count: int, *, seed: int = DEFAULT_SEED) -> dict: + """`count` independent series, keyed by the symbol id each engine will use. + + Independent, not correlated: a portfolio of copies of one asset would let a + position-sizing bug cancel itself out across the book, which is exactly the + class of mistake a multi-asset workload exists to catch. The seeds are + spaced far apart and derived from the same base, so the whole universe is + reproducible from `seed` alone and the first symbol is bit-identical to the + single-asset series of the same length. + """ + return {i + 1: make_ohlcv(rows, seed=seed + 1000 * i) for i in range(count)} + + def digest(df: pd.DataFrame) -> str: """Short content fingerprint of the bars, recorded in the result envelope. diff --git a/benchmarks/vs_vectorbt/engine_mbt.py b/benchmarks/vs_vectorbt/engine_mbt.py index 02e0686..e8833f6 100644 --- a/benchmarks/vs_vectorbt/engine_mbt.py +++ b/benchmarks/vs_vectorbt/engine_mbt.py @@ -22,6 +22,7 @@ from __future__ import annotations import os from typing import Any, Callable, Dict +import data as data_mod import manifoldbt as bt from manifoldbt.expr import col, lit, when from manifoldbt.helpers import Interval, Slippage @@ -36,7 +37,8 @@ def probe() -> Dict[str, Any]: return {"engine": NAME, "version": bt.__version__} -def _config(df, *, sizing: str, fee_bps: float) -> "bt.BacktestConfig": +def _config(df, *, sizing: str, fee_bps: float, slippage_bps: float = 0.0, + universe=None) -> "bt.BacktestConfig": last_ns = int(df["timestamp"].iloc[-1].value) fees = ( bt.FeeConfig.zero() @@ -44,7 +46,7 @@ def _config(df, *, sizing: str, fee_bps: float) -> "bt.BacktestConfig": else bt.FeeConfig(maker_fee_bps=fee_bps, taker_fee_bps=fee_bps) ) return bt.BacktestConfig( - universe=[1], + universe=universe or [1], time_range_start=0, # A day past the last bar: the range is inclusive of everything generated. time_range_end=last_ns + 86_400_000_000_000, @@ -58,7 +60,8 @@ def _config(df, *, sizing: str, fee_bps: float) -> "bt.BacktestConfig": position_sizing_mode=sizing, ), fees=fees, - slippage=Slippage.none(), + slippage=(Slippage.none() if slippage_bps == 0.0 + else Slippage.fixed_bps(slippage_bps)), warmup_bars=0, ) @@ -84,6 +87,18 @@ def _strategy(key: str): .take_profit(pct=p["tp_pct"]) ) + if key in ("sma_cross_costs", "multi_asset"): + # The same crossover as the headline workload. What differs is the cost + # model and the number of symbols the config points at, neither of which + # is visible from the strategy: a universe is walked by the engine, not + # spelled out per asset, which is the whole point of the comparison. + return ( + bt.Strategy.create(key) + .signal("fast", sma(close_px, p["fast"])) + .signal("slow", sma(close_px, p["slow"])) + .size(when(col("fast") > col("slow"), lit(p["units"]), lit(0.0))) + ) + if key == "ema_rsi_fees": entry = ( (col("fast") > col("slow")) @@ -110,16 +125,28 @@ def prepare(key: str, df, workdir: str) -> Callable[[], Dict[str, Any]]: root = os.path.join(workdir, key) os.makedirs(root, exist_ok=True) - store = bt.import_dataframe( - df, - symbol="BENCH", - symbol_id=1, - interval="1m", - data_root=os.path.join(root, "data"), - metadata_db=os.path.join(root, "metadata.sqlite"), - ) + data_root = os.path.join(root, "data") + metadata_db = os.path.join(root, "metadata.sqlite") + assets = int(p.get("assets", 1)) + if assets > 1: + # The universe is derived from the same generator and the same base + # seed, so the first symbol is the single-asset series bit for bit and + # the whole set is reproducible from the digest already recorded. + frames = data_mod.make_universe(len(df), assets) + for symbol_id, frame in frames.items(): + store = bt.import_dataframe( + frame, symbol="A%d" % symbol_id, symbol_id=symbol_id, + interval="1m", data_root=data_root, metadata_db=metadata_db) + universe = list(frames) + else: + store = bt.import_dataframe( + df, symbol="BENCH", symbol_id=1, interval="1m", + data_root=data_root, metadata_db=metadata_db) + universe = [1] strategy = _strategy(key) - config = _config(df, sizing=sizing, fee_bps=fee_bps) + config = _config(df, sizing=sizing, fee_bps=fee_bps, + slippage_bps=float(p.get("slippage_bps", 0.0)), + universe=universe) wants_metrics = bool(p.get("metrics")) @@ -171,7 +198,8 @@ def diagnose(key: str, df, workdir: str) -> Dict[str, Any]: result = bt.run( _strategy(key), _config(df, sizing="Units" if "units" in p else "FractionOfEquity", - fee_bps=float(p.get("fee_bps", 0.0))), + fee_bps=float(p.get("fee_bps", 0.0)), + slippage_bps=float(p.get("slippage_bps", 0.0))), store, ) trades = result.trades_df() diff --git a/benchmarks/vs_vectorbt/engine_vbt.py b/benchmarks/vs_vectorbt/engine_vbt.py index 109e266..00a7121 100644 --- a/benchmarks/vs_vectorbt/engine_vbt.py +++ b/benchmarks/vs_vectorbt/engine_vbt.py @@ -30,6 +30,8 @@ from typing import Any, Callable, Dict import numpy as np import pandas as pd import vectorbt as vbt + +import data as data_mod from vectorbt.portfolio.enums import StopExitPrice from workloads import CAPITAL, FREQ, WORKLOADS @@ -72,7 +74,8 @@ def indicators(key: str, close: pd.Series) -> Dict[str, pd.Series]: """Indicator series for a workload. Used by the timed path and by the definition check, so the two can never drift apart.""" p = WORKLOADS[key].params - if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp"): + if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp", + "sma_cross_costs", "multi_asset"): return { "fast": close.rolling(p["fast"]).mean(), "slow": close.rolling(p["slow"]).mean(), @@ -88,7 +91,8 @@ def indicators(key: str, close: pd.Series) -> Dict[str, pd.Series]: def _level(key: str, ind: Dict[str, pd.Series]) -> pd.Series: p = WORKLOADS[key].params - if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp"): + if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp", + "sma_cross_costs", "multi_asset"): return (ind["fast"] > ind["slow"]).fillna(False) if key == "ema_rsi_fees": return ( @@ -113,10 +117,52 @@ def prepare(key: str, df, workdir: str | None = None) -> Callable[[], Dict[str, else: size, size_type = p["alloc"], "percent" fees = float(p.get("fee_bps", 0.0)) / 10_000.0 + slippage = float(p.get("slippage_bps", 0.0)) / 10_000.0 wants_metrics = bool(p.get("metrics")) + assets = int(p.get("assets", 1)) sl = p["sl_pct"] / 100.0 if "sl_pct" in p else None tp = p["tp_pct"] / 100.0 if "tp_pct" in p else None + if assets > 1: + # One book, not five. `from_signals` on a frame of columns builds five + # independent portfolios unless it is told otherwise, and five separate + # books is a different question from the one manifoldbt answers when it + # walks a universe. `group_by` with `cash_sharing` is the spelling that + # asks the same thing. With fixed-unit sizing the cash constraint never + # binds, which is what lets the two agree at all: on a fraction of + # equity they would also have to agree on which asset gets the cash + # first, and that is policy rather than arithmetic. + frames = data_mod.make_universe(len(df), assets) + closes = pd.DataFrame( + {"A%d" % sid: f["close"].to_numpy(dtype=np.float64) + for sid, f in frames.items()}, + index=index, + ) + + def run_multi() -> Dict[str, Any]: + fast = closes.rolling(p["fast"]).mean() + slow = closes.rolling(p["slow"]).mean() + level = (fast > slow).fillna(False) + portfolio = vbt.Portfolio.from_signals( + closes, entries=level, exits=~level, + init_cash=CAPITAL, size=size, size_type=size_type, + fees=fees, slippage=slippage, direction="longonly", + accumulate=False, freq=FREQ, + group_by=True, cash_sharing=True, + ) + total_return = float(portfolio.total_return()) + trades = portfolio.trades + return { + "total_return": total_return, + "final_equity": CAPITAL * (1.0 + total_return), + "round_trips": int(trades.closed.count()), + "fills": None, + "total_fees": float(trades.records["entry_fees"].sum() + + trades.records["exit_fees"].sum()), + } + + return run_multi + def run() -> Dict[str, Any]: level = _level(key, indicators(key, close)) portfolio = vbt.Portfolio.from_signals( @@ -130,7 +176,7 @@ def prepare(key: str, df, workdir: str | None = None) -> Callable[[], Dict[str, size=size, size_type=size_type, fees=fees, - slippage=0.0, + slippage=slippage, sl_stop=sl, tp_stop=tp, stop_exit_price=StopExitPrice.StopMarket, diff --git a/benchmarks/vs_vectorbt/workloads.py b/benchmarks/vs_vectorbt/workloads.py index c378c16..18b1bcb 100644 --- a/benchmarks/vs_vectorbt/workloads.py +++ b/benchmarks/vs_vectorbt/workloads.py @@ -134,6 +134,59 @@ WORKLOADS: Dict[str, Workload] = { "summary costs each engine.", params=dict(fast=30, slow=150, alloc=1.0, metrics=True), ), + Workload( + key="sma_cross_costs", + title="SMA 30/150 with a 5 bps fee and 2 bps of slippage", + why="The same signal as the headline workload, run against a cost " + "model. Costs were the easiest objection to make of a benchmark " + "that had them on one workload out of four, and the answer is " + "not to sprinkle them everywhere: measured, a fee or a slippage " + "on FractionOfEquity sizing makes the engines disagree by 1e-4 " + "of capital against a 1e-9 tolerance, because they resolve the " + "same policy differently. In fixed units the arithmetic is " + "comparable and both costs land exactly, so this is where they " + "belong.", + params=dict(fast=30, slow=150, units=5.0, fee_bps=5.0, slippage_bps=2.0), + notes={ + "raptorbt": Note( + "unsupported", + "Same blocker as the other fixed-quantity workload: raptorbt " + "has no units sizing, and a cost model on top of a fraction " + "of equity compares policy rather than arithmetic. See " + "`ema_rsi_fees` for the measurements behind that.", + ), + }, + ), + Workload( + key="multi_asset", + title="Five assets in one book, SMA 30/150, 5 bps fee", + why="A single-asset benchmark measures a loop; a portfolio measures " + "the thing people actually run. It is also where the two designs " + "differ in kind rather than in speed: manifoldbt walks a universe " + "against one shared book, while vectorbt broadcasts a column per " + "asset and has to be told to share cash at all. The five series " + "are independent rather than correlated, so a sizing bug cannot " + "cancel itself out across the book.", + params=dict(fast=30, slow=150, units=2.0, fee_bps=5.0, assets=5), + # Five columns of a materialised simulation, not one. vectorbt adds + # 1.55 GB on a 10M-bar single asset, so five of them would ask a + # 16 GB runner for around 8 GB on top of 2.4 GB of generated frames. + # The ceiling is set where the point is certain to be measuring the + # engine rather than the swap file. + max_bars=1_000_000, + notes={ + "raptorbt": Note( + "unsupported", + "No multi-instrument portfolio on the entry point this " + "harness drives: `run_single_backtest` is one instrument, and " + "`run_multi_backtest` broadcasts over strategies rather than " + "over assets. 0.9.0 does add `run_portfolio_backtest`, but " + "its allocation model is a different one and would need its " + "own parity work before any timing from it could be " + "published. The units blocker applies here too.", + ), + }, + ), Workload( key="bracket_sl_tp", title="SMA 10/50 entry with a 15 bps stop / 30 bps target bracket",