diff --git a/.github/workflows/bench-vs-vectorbt.yml b/.github/workflows/bench-vs-vectorbt.yml new file mode 100644 index 0000000..35a5986 --- /dev/null +++ b/.github/workflows/bench-vs-vectorbt.yml @@ -0,0 +1,112 @@ +# The benchmark runs here, in the open, on standard GitHub-hosted runners. +# +# Nothing in this workflow has access to the engine source: it installs the +# published wheel from PyPI, exactly like any user would. That is what makes the +# result independent rather than self-reported, and it is why anyone can fork +# this repository and press "Run workflow" to reproduce the numbers on their own +# runner. + +name: Benchmark vs vectorbt + +on: + workflow_dispatch: + inputs: + manifoldbt_version: + description: "manifoldbt version to install from PyPI (blank = latest)" + required: false + default: "" + reps: + description: "Interleaved repetitions per point" + required: false + default: "7" + release: + types: [published] + schedule: + # Weekly, to catch a slowdown introduced by a dependency rather than by us. + - cron: "17 5 * * 1" + +permissions: + contents: read + +concurrency: + group: bench-vs-vectorbt-${{ github.ref }} + cancel-in-progress: true + +jobs: + bench: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + bars: "10000 100000 1000000 5000000" + - os: windows-latest + bars: "10000 100000 1000000 5000000" + # macOS runners ship 7 GB of RAM against 16 GB elsewhere, and vectorbt + # materialises the simulation in memory (roughly 150 MB per million + # bars, measured). The top size is trimmed so a point is never lost to + # swapping, which would time the disk instead of the engine. + - os: macos-latest + bars: "10000 100000 1000000" + + env: + PYTHONUNBUFFERED: "1" + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install engines from PyPI + shell: bash + run: | + set -e + VERSION="${{ inputs.manifoldbt_version }}" + # A release run benchmarks the version that was just published. + if [ -z "$VERSION" ] && [ "${{ github.event_name }}" = "release" ]; then + VERSION="$(echo '${{ github.event.release.tag_name }}' | sed 's/^v//')" + fi + python -m pip install --upgrade pip + if [ -n "$VERSION" ]; then + pip install "manifoldbt==${VERSION}" + else + pip install manifoldbt + fi + pip install -r benchmarks/vs_vectorbt/requirements-lock.txt + + - name: Record the resolved environment + shell: bash + run: pip freeze | grep -iE '^(manifoldbt|vectorbt|numpy|numba|pandas|psutil)=' || true + + - name: Run the benchmark + shell: bash + working-directory: benchmarks/vs_vectorbt + run: | + python bench.py \ + --bars ${{ matrix.bars }} \ + --reps "${{ inputs.reps || '7' }}" \ + --cold-start-reps 3 \ + --memory-bars 2000000 \ + --out "results-${{ matrix.os }}.json" + + - name: Render the report + # Runs even when the benchmark exits non-zero: a parity failure is the + # most interesting thing that can happen here, and it must be readable + # in the job summary rather than buried in a red step. + if: always() + shell: bash + working-directory: benchmarks/vs_vectorbt + run: python report.py "results-${{ matrix.os }}.json" + + - name: Upload the raw result + if: always() + uses: actions/upload-artifact@v4 + with: + name: bench-${{ matrix.os }} + path: benchmarks/vs_vectorbt/results-*.json + if-no-files-found: warn diff --git a/benchmarks/vs_vectorbt/README.md b/benchmarks/vs_vectorbt/README.md new file mode 100644 index 0000000..e2045bb --- /dev/null +++ b/benchmarks/vs_vectorbt/README.md @@ -0,0 +1,177 @@ +# manifoldbt vs vectorbt + +An engine-to-engine benchmark you can re-run yourself. It installs both engines +from PyPI, generates its own data, checks that the two engines produced the +**same result**, and only then reports how long each took. + +```bash +pip install manifoldbt vectorbt +pip install -r requirements-lock.txt +python bench.py --bars 10000 100000 1000000 --reps 7 --cold-start-reps 3 --memory-bars 2000000 --out results.json +python report.py results.json +``` + +No dataset to download, no API key, no configuration. The same command runs in +GitHub Actions on a public runner, so every published number has a run URL +behind it. + +## The rule this harness is built around + +A speed comparison between two backtesters is worthless unless both engines did +the same work. So parity comes first: + +1. each workload runs once per engine; +2. total return, round-trip count and fees are compared; +3. **a workload the engines disagree on gets no published timing.** + +Three verdicts come out of that gate: + +| Verdict | Meaning | What gets published | +|---|---|---| +| `exact` | agreement down to float-reordering noise (relative tolerance 1e-9) | the timing, in the headline table | +| `documented` | the engines disagree, the workload declared it in advance, and the cause is written down | the timing, in an annex, with the cause and its measured size | +| `failed` | the engines disagree and nobody predicted it | nothing. The run exits non-zero | + +The `failed` path is not decoration. It is the reason the other numbers can be +trusted, and it makes the benchmark fail loudly if a future release of either +engine changes a fill rule. + +## Method + +**Interleaved repetitions.** The engines alternate inside each repetition +(A, B, A, B, ...) rather than running in two blocks. On a shared cloud runner +that slows down halfway through, two blocks would hand the penalty to whichever +engine ran second; alternating splits it evenly. + +**The ratio is the headline, milliseconds are context.** Each ratio comes from +two measurements taken seconds apart on the same machine. Absolute timings from +a shared runner are worth much less than the ratio between them. + +**Dispersion is published, and noise is flagged.** Every point carries min, +median, max and interquartile range. If the IQR exceeds 15% of the median the +point is marked `noisy` and is not headline material, however good it looks. + +**Warmup is discarded, and that favours vectorbt on purpose.** The first call of +each engine is thrown away, which is where vectorbt pays its numba compilation. +Charging a one-off JIT cost to every repetition would inflate the result. + +**Data loading is excluded on both sides.** manifoldbt is handed a prepared +store, vectorbt is handed prepared Series. What is timed is indicators plus +simulation plus reading the headline metrics, nothing else. + +**Only public APIs.** manifoldbt is driven through `bt.run(strategy, config, +store)`, the documented entry point, not through an internal fast path. + +## What is compared + +| Workload | What it exercises | Parity | +|---|---|---| +| `sma_cross` | SMA 10/50 crossover, long-only, no cost | exact | +| `ema_rsi_fees` | EMA 12/26 crossover with an RSI(14) filter and a 5 bps taker fee | exact | +| `sma_cross_metrics` | the same simulation, plus max drawdown, Sharpe, Sortino and volatility | exact | +| `bracket_sl_tp` | the same entry with a 15 bps stop and a 30 bps target | documented divergence | + +Each of those runs across a range of series lengths. Two further axes, cold +start and memory, are measured in their own processes because they cannot be +measured honestly inside the main one. + +**Scope, stated twice on purpose.** `sma_cross` and `sma_cross_metrics` run the +identical simulation; only the second one also produces a performance summary. +manifoldbt computes that summary inside `run()` whether or not you read it, +while vectorbt defers the equity curve until a risk metric asks for it and then +pays to materialise it. Reporting both scopes is the only honest way to present +the result: a reader who only wants a total return should look at the first +number, and a reader who wants a Sharpe should look at the second. Parity on the +summary workload is gated on total return, round-trip count and max drawdown, +which match exactly; the ratios agree to about 3e-4, because manifoldbt buckets +its daily returns slightly differently, and that residual is reported rather +than smoothed over. + +The vectorbt side of the summary is written out in pandas rather than through +`pf.sharpe_ratio()` for two reasons, both in vectorbt's favour or neutral. +manifoldbt computes its ratios on daily returns annualised by sqrt(365) and its +drawdown at full bar resolution, so the native accessors would return different +numbers and the comparison would be timing two different computations; and the +hand-written version is measurably faster than a single native accessor on the +same data, so vectorbt is credited with the quicker of its two paths. + +**Cold start.** The steady-state table discards a warmup call, which is where +vectorbt compiles its numba kernels. A user pays that cost in every new +notebook, script or CI job, so it is measured rather than waved away: a fresh +process, an engine it has never imported, one backtest. The Python, numpy and +pandas baseline is measured the same way and reported alongside, so the engine's +own share can be read off instead of argued about. + +**Memory added by the run.** Resident memory sampled while the backtest +executes, after a warmup. vectorbt materialises the simulation as arrays and its +footprint grows with the series; manifoldbt streams bars out of its store. What +is compared is what running a backtest costs *on top of* already holding the +data: building each engine's data representation is a different question and is +excluded on both sides, in manifoldbt's case a deliberately unflattering choice +since its one-off ingest is the memory-hungry part. + +### Why the fee workload sizes in units + +With `FractionOfEquity` sizing and a non-zero fee the engines size differently: +manifoldbt charges the fee on top of a full-equity notional, vectorbt reserves +it out of cash first. Both are legitimate product decisions, and comparing them +would compare policy rather than speed or correctness. Sizing in fixed units +isolates the fee arithmetic, which is the thing both engines must agree on. + +### The documented divergence, in full + +When a bracket fires intrabar and the entry condition still holds at that bar's +close, manifoldbt books two orders on that bar: the stop or target exit, then a +fresh entry at the close. vectorbt processes one order per bar and re-enters on +the next bar instead. Neither is wrong. On controlled bars the bracket fills +themselves match exactly, which the cross-engine parity suite shipped with the +library pins test by test; the divergence is purely about *when* a re-entry is +allowed. + +The harness counts the affected round-trips rather than hand-waving at them, so +the report states what share of the trades the difference touches. + +## Alignment choices, and why each one exists + +Two engines only produce identical numbers if they are told to do the same +thing. These are the conventions the harness sets, all of them visible in +[engine_mbt.py](engine_mbt.py) and [engine_vbt.py](engine_vbt.py): + +- `signal_delay=0` with `execution_price="AtClose"`, matching what + `Portfolio.from_signals` does by default: a signal fills at the close of the + bar that produced it. +- `warmup_bars=0`, so the indicator's own NaN warmup is what suppresses early + signals, identically on both sides. +- Signals are fed to vectorbt as a *level* (`entries` = the condition holds, + `exits` = it no longer holds) rather than as transitions, which reproduces + manifoldbt's target-position semantics. +- Indicators are mirrored on manifoldbt's exact definitions. The EMA seeds on + the first observation with `alpha = 2/(span+1)`, which `ewm(span=n, + adjust=False)` reproduces exactly. The RSI is Wilder's, seeded with the + *simple* average of the first `period` deltas and emitted from bar `period`: + a plain `ewm(alpha=1/period)` over the whole delta series is a different + indicator, and using it would have made the engines disagree for a reason that + has nothing to do with either engine. +- The generated bars have no opening gap (`open == previous close`). A bar that + gaps through a stop is the one case where two engines can legitimately book + different fill prices while both being correct, so that class of false + failures is removed from the data rather than argued about in the report. + +## Reading the numbers honestly + +- On a shared runner with 4 vCPUs, an engine that parallelises is understated. + These are floors, not peaks. +- The published wheels are CPU-only, and GitHub-hosted runners have no GPU, so + nothing here says anything about GPU performance. +- vectorbt is the open-source package (`pip install vectorbt`), not vectorbtpro. + +## Files + +- `data.py` - deterministic OHLCV generator and its content digest +- `probe_child.py` - the cold-start and memory probes, each in a fresh process +- `workloads.py` - the parameters both engines read, and the declared parity status +- `engine_mbt.py` / `engine_vbt.py` - one adapter per engine +- `parity.py` - the gate +- `bench.py` - the runner +- `report.py` - JSON to Markdown, and to the GitHub job summary +- [`.github/workflows/bench-vs-vectorbt.yml`](../../.github/workflows/bench-vs-vectorbt.yml) - the workflow that runs all of the above on a GitHub-hosted runner diff --git a/benchmarks/vs_vectorbt/bench.py b/benchmarks/vs_vectorbt/bench.py new file mode 100644 index 0000000..344137d --- /dev/null +++ b/benchmarks/vs_vectorbt/bench.py @@ -0,0 +1,499 @@ +"""The runner: interleaved A/B timing with a parity gate in front of it. + +Design decisions that are the whole point of this harness +--------------------------------------------------------- +*Parity first.* Every workload runs once per engine before any measurement, and +the results are compared. A workload the engines disagree on gets no published +timing (see ``parity.py``). + +*Interleaved repetitions.* The engines alternate within each repetition rather +than running in two blocks. A cloud runner that slows down halfway through +penalises both engines equally instead of whichever one happened to be second. + +*Ratios are the headline, milliseconds are context.* The per-repetition ratio is +computed from two measurements taken seconds apart on the same machine, so it +survives the noise that absolute timings on shared hardware do not. + +*Dispersion is published.* Every point carries min, median, max and IQR. A point +whose IQR exceeds 15% of its median is flagged noisy, and a flagged point is not +headline material no matter how good it looks. + +Usage +----- + python bench.py --bars 10000 100000 1000000 --reps 7 --out results.json +""" +from __future__ import annotations + +import argparse +import json +import os +import platform +import statistics +import subprocess +import sys +import tempfile +import time +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import data as data_mod # noqa: E402 +import engine_mbt # noqa: E402 +import engine_vbt # noqa: E402 +import parity as parity_mod # noqa: E402 +from workloads import DEFAULT_KEYS, SCOPE_PAIR, WORKLOADS # noqa: E402 + +SCHEMA_VERSION = 1 +NOISE_THRESHOLD = 0.15 + + +# --------------------------------------------------------------------------- # +# Environment: what a reader needs to know before trusting a number +# --------------------------------------------------------------------------- # +def _cpu_model() -> str: + try: + if sys.platform.startswith("linux"): + with open("/proc/cpuinfo") as fh: + for line in fh: + if line.startswith("model name"): + return line.split(":", 1)[1].strip() + elif sys.platform == "darwin": + return subprocess.check_output( + ["sysctl", "-n", "machdep.cpu.brand_string"], text=True + ).strip() + elif sys.platform.startswith("win"): + return os.environ.get("PROCESSOR_IDENTIFIER", platform.processor()) + except Exception: + pass + return platform.processor() or "unknown" + + +def _ram_gb(): + try: + import psutil + + return round(psutil.virtual_memory().total / 1e9, 1) + except Exception: + pass + try: + if sys.platform.startswith("linux"): + with open("/proc/meminfo") as fh: + for line in fh: + if line.startswith("MemTotal"): + return round(int(line.split()[1]) * 1024 / 1e9, 1) + except Exception: + pass + return None + + +def _versions() -> Dict[str, str]: + import importlib.metadata as md + + out = {} + for name in ("manifoldbt", "vectorbt", "numpy", "numba", "pandas"): + try: + out[name] = md.version(name) + except Exception: + out[name] = "absent" + return out + + +def environment() -> Dict[str, Any]: + run_id = os.environ.get("GITHUB_RUN_ID") + server = os.environ.get("GITHUB_SERVER_URL", "https://github.com") + repo = os.environ.get("GITHUB_REPOSITORY") + run_url = None + if run_id and repo: + run_url = server + "/" + repo + "/actions/runs/" + run_id + return { + "os": platform.system(), + "os_release": platform.release(), + "arch": platform.machine(), + "cpu": _cpu_model(), + "logical_cores": os.cpu_count(), + "ram_gb": _ram_gb(), + "python": platform.python_version(), + "versions": _versions(), + "runner": os.environ.get("RUNNER_NAME") or "local", + "ci": bool(run_id), + "run_url": run_url, + "commit": os.environ.get("GITHUB_SHA"), + } + + +# --------------------------------------------------------------------------- # +# Measurement +# --------------------------------------------------------------------------- # +def _time_once(fn: Callable[[], Dict[str, Any]]): + start = time.perf_counter() + metrics = fn() + return time.perf_counter() - start, metrics + + +def _cpu_seconds() -> float: + """Process CPU time, user plus system. Returns 0.0 if psutil is missing.""" + try: + import psutil + + times = psutil.Process().cpu_times() + return times.user + times.system + except Exception: + return 0.0 + + +class _Parallelism: + """CPU time over wall time, accumulated across every repetition. + + This is the number that decides whether a result measured on a 20-core + workstation transposes to a 4-vCPU cloud runner. Close to 1.0 means the + engine used one thread and the core count barely matters; well above 1.0 + means the measurement is a function of the machine it ran on, and a + reader on smaller hardware should expect less. Accumulating over all + repetitions rather than timing each call keeps the coarse resolution of + the OS accounting clocks from dominating short runs. + + Below `MIN_WALL_S` of accumulated work the answer is withheld rather than + guessed. Process CPU accounting advances in scheduler ticks of roughly 15 ms, + so a sub-millisecond run lands either on zero ticks or on one whole tick, and + the ratio comes out as 0.00 or as 11.0 for work that is plainly single + threaded. A benchmark that prints "this engine used 11 cores" once has + spent its credibility on a rounding artefact. + """ + + MIN_WALL_S = 0.5 + + def __init__(self) -> None: + self.cpu = 0.0 + self.wall = 0.0 + + def record(self, fn: Callable[[], Dict[str, Any]]): + cpu_before = _cpu_seconds() + elapsed, metrics = _time_once(fn) + self.cpu += _cpu_seconds() - cpu_before + self.wall += elapsed + return elapsed, metrics + + @property + def ratio(self): + if self.wall < self.MIN_WALL_S: + return None + return self.cpu / self.wall + + +def _summarise(samples: List[float]) -> Dict[str, Any]: + ordered = sorted(samples) + median = statistics.median(ordered) + if len(ordered) >= 4: + mid = len(ordered) // 2 + lower = statistics.median(ordered[:mid]) + upper = statistics.median(ordered[-mid:]) + iqr = upper - lower + else: + iqr = ordered[-1] - ordered[0] + return { + "samples_s": samples, + "min_s": ordered[0], + "median_s": median, + "max_s": ordered[-1], + "iqr_s": iqr, + "iqr_over_median": iqr / median if median else 0.0, + } + + +def measure_pair(keys: List[str], bars: int, reps: int, workdir: str) -> List[Dict[str, Any]]: + """Measure two workloads inside ONE interleaved loop. + + The report puts these two side by side to show what a performance summary + costs each engine. That subtraction is only legitimate if all four timings + come from the same stretch of machine time: measured in separate blocks, the + drift in absolute timings is larger than the difference being reported, and + the table ends up claiming the version doing more work is the faster one. + """ + frame = data_mod.make_ohlcv(bars) + runners = {} + entries = {} + for key in keys: + run_mbt = engine_mbt.prepare(key, frame, workdir) + run_vbt = engine_vbt.prepare(key, frame, workdir) + _, warm_mbt = _time_once(run_mbt) + _, warm_vbt = _time_once(run_vbt) + verdict = parity_mod.compare(warm_mbt, warm_vbt, key) + runners[key] = (run_mbt, run_vbt) + entries[key] = { + "workload": key, + "title": WORKLOADS[key].title, + "bars": bars, + "data_digest": data_mod.digest(frame), + "parity": verdict, + "paired_with": [k for k in keys if k != key], + } + + samples: Dict[str, Dict[str, List[float]]] = { + key: {"manifoldbt": [], "vectorbt": [], "ratio": []} for key in keys + } + threading_use = {key: {"manifoldbt": _Parallelism(), "vectorbt": _Parallelism()} + for key in keys} + for _ in range(reps): + for key in keys: + run_mbt, run_vbt = runners[key] + t_mbt, _ = threading_use[key]["manifoldbt"].record(run_mbt) + t_vbt, _ = threading_use[key]["vectorbt"].record(run_vbt) + samples[key]["manifoldbt"].append(t_mbt) + samples[key]["vectorbt"].append(t_vbt) + samples[key]["ratio"].append(t_vbt / t_mbt if t_mbt > 0 else float("nan")) + + out = [] + for key in keys: + entry = entries[key] + if entry["parity"]["status"] == "failed": + entry["timings"] = None + entry["note"] = "timing withheld: unexplained disagreement between engines" + out.append(entry) + continue + mbt_stats = _summarise(samples[key]["manifoldbt"]) + vbt_stats = _summarise(samples[key]["vectorbt"]) + ratios = samples[key]["ratio"] + entry["timings"] = {"manifoldbt": mbt_stats, "vectorbt": vbt_stats} + entry["speedup"] = { + "median_of_ratios": statistics.median(ratios), + "min": min(ratios), + "max": max(ratios), + } + entry["noisy"] = ( + mbt_stats["iqr_over_median"] > NOISE_THRESHOLD + or vbt_stats["iqr_over_median"] > NOISE_THRESHOLD + ) + entry["cpu_over_wall"] = { + engine: threading_use[key][engine].ratio for engine in ("manifoldbt", "vectorbt") + } + out.append(entry) + return out + + +def measure(key: str, bars: int, reps: int, workdir: str) -> Dict[str, Any]: + frame = data_mod.make_ohlcv(bars) + run_mbt = engine_mbt.prepare(key, frame, workdir) + run_vbt = engine_vbt.prepare(key, frame, workdir) + + # Warmup, discarded: numba compiles on vectorbt's first call, and the engine + # warms its own caches. Both are one-off costs, reported separately rather + # than smeared across every repetition. + _, warm_mbt = _time_once(run_mbt) + _, warm_vbt = _time_once(run_vbt) + + verdict = parity_mod.compare(warm_mbt, warm_vbt, key) + entry: Dict[str, Any] = { + "workload": key, + "title": WORKLOADS[key].title, + "bars": bars, + "data_digest": data_mod.digest(frame), + "parity": verdict, + } + if verdict["status"] == "documented": + entry["divergence_scale"] = engine_mbt.diagnose(key, frame, workdir) + + if verdict["status"] == "failed": + entry["timings"] = None + entry["note"] = "timing withheld: unexplained disagreement between engines" + return entry + + mbt_samples: List[float] = [] + vbt_samples: List[float] = [] + ratios: List[float] = [] + threading_use = {"manifoldbt": _Parallelism(), "vectorbt": _Parallelism()} + for _ in range(reps): + t_mbt, _ = threading_use["manifoldbt"].record(run_mbt) + t_vbt, _ = threading_use["vectorbt"].record(run_vbt) + mbt_samples.append(t_mbt) + vbt_samples.append(t_vbt) + ratios.append(t_vbt / t_mbt if t_mbt > 0 else float("nan")) + + mbt_stats = _summarise(mbt_samples) + vbt_stats = _summarise(vbt_samples) + entry["timings"] = {"manifoldbt": mbt_stats, "vectorbt": vbt_stats} + entry["speedup"] = { + "median_of_ratios": statistics.median(ratios), + "min": min(ratios), + "max": max(ratios), + } + entry["noisy"] = ( + mbt_stats["iqr_over_median"] > NOISE_THRESHOLD + or vbt_stats["iqr_over_median"] > NOISE_THRESHOLD + ) + entry["cpu_over_wall"] = { + engine: threading_use[engine].ratio for engine in ("manifoldbt", "vectorbt") + } + return entry + + +# --------------------------------------------------------------------------- # +# Probes that need a fresh process (see probe_child.py) +# --------------------------------------------------------------------------- # +CHILD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "probe_child.py") + + +def _probe(mode: str, engine: str, workload: str, bars: int) -> Dict[str, Any]: + out = subprocess.run( + [sys.executable, CHILD, mode, engine, workload, str(bars)], + capture_output=True, text=True, check=True, + ).stdout + for line in out.splitlines(): + if line.startswith("PROBE_RESULT "): + return json.loads(line[len("PROBE_RESULT "):]) + raise RuntimeError("probe produced no result: " + out[-500:]) + + +def cold_start(workload: str, bars: int, reps: int) -> Dict[str, Any]: + """Time to a first backtest in a process that has never seen either engine. + + This is the cost the steady-state benchmark throws away as warmup, and the + one a user actually waits through every time they open a notebook. The + engines alternate here too, and the interpreter/numpy/pandas baseline is + measured alongside so the engine's own share is visible. + """ + samples: Dict[str, List[float]] = {"manifoldbt": [], "vectorbt": [], "baseline": []} + for _ in range(reps): + samples["manifoldbt"].append(_probe("coldstart", "mbt", workload, bars)["seconds"]) + samples["vectorbt"].append(_probe("coldstart", "vbt", workload, bars)["seconds"]) + samples["baseline"].append(_probe("baseline", "none", workload, bars)["seconds"]) + medians = {k: statistics.median(v) for k, v in samples.items()} + base = medians["baseline"] + return { + "workload": workload, + "bars": bars, + "samples_s": samples, + "median_s": medians, + "engine_share_s": { + "manifoldbt": medians["manifoldbt"] - base, + "vectorbt": medians["vectorbt"] - base, + }, + "ratio": medians["vectorbt"] / medians["manifoldbt"] if medians["manifoldbt"] else None, + } + + +def memory(workload: str, bars: int) -> Dict[str, Any]: + """Resident memory each engine adds while running one backtest. + + vectorbt materialises the simulation as arrays, so its footprint grows with + the series; manifoldbt streams bars out of its store. The number reported is + what the *run* adds, measured after a warmup, not the process total: the + one-off cost of building each engine's data representation is a different + question and is excluded on both sides. + """ + mbt = _probe("memory", "mbt", workload, bars) + vbt = _probe("memory", "vbt", workload, bars) + return { + "workload": workload, + "bars": bars, + "manifoldbt": mbt, + "vectorbt": vbt, + } + + +# --------------------------------------------------------------------------- # +# Entry point +# --------------------------------------------------------------------------- # +def main() -> int: + parser = argparse.ArgumentParser(description="manifoldbt vs vectorbt") + parser.add_argument("--bars", type=int, nargs="+", default=[10_000, 100_000, 1_000_000]) + parser.add_argument("--workloads", nargs="+", default=DEFAULT_KEYS, choices=DEFAULT_KEYS) + parser.add_argument("--reps", type=int, default=7) + parser.add_argument("--out", default="results.json") + parser.add_argument("--workdir", default=None, help="where the engine store is built") + parser.add_argument("--cold-start-reps", type=int, default=3, + help="0 disables the cold-start probe") + parser.add_argument("--memory-bars", type=int, default=0, + help="bars for the memory probe; 0 disables it") + parser.add_argument("--pin-cores", type=int, default=0, + help="restrict the process to N logical cores, so a big " + "workstation can reproduce what a small cloud runner " + "sees; 0 leaves the machine alone") + args = parser.parse_args() + + pinned = None + if args.pin_cores: + try: + import psutil + + everything = list(range(os.cpu_count() or 1)) + psutil.Process().cpu_affinity(everything[: args.pin_cores]) + pinned = args.pin_cores + except Exception as exc: # macOS has no affinity API + print("could not pin to {} cores: {}".format(args.pin_cores, exc)) + + workdir = args.workdir or tempfile.mkdtemp(prefix="mbt_vs_vbt_") + env = environment() + env["pinned_cores"] = pinned + versions = env["versions"] + print("manifoldbt " + versions["manifoldbt"] + " vs vectorbt " + versions["vectorbt"]) + print("{cpu} | {cores} logical cores | {ram} GB | {os} {arch}".format( + cpu=env["cpu"], cores=env["logical_cores"], ram=env["ram_gb"], + os=env["os"], arch=env["arch"])) + if pinned: + print("pinned to {} logical cores for this run".format(pinned)) + print(str(args.reps) + " interleaved repetitions per point\n") + + def announce(entry: Dict[str, Any]) -> bool: + status = entry["parity"]["status"] + print(" {k:18s} {b:>9,} bars ... ".format(k=entry["workload"], b=entry["bars"]), + end="", flush=True) + if entry.get("timings"): + print("{s:10s} manifoldbt x{v:.1f}{m}".format( + s=status, v=entry["speedup"]["median_of_ratios"], + m=" NOISY" if entry.get("noisy") else "")) + return False + print("{s:10s} timing withheld".format(s=status)) + return True + + # The scope pair is measured together; everything else one workload at a time. + paired = [k for k in SCOPE_PAIR if k in args.workloads] + singles = [k for k in args.workloads if k not in paired] + + results = [] + failures = 0 + for bars in args.bars: + if len(paired) > 1: + for entry in measure_pair(paired, bars, args.reps, workdir): + results.append(entry) + failures += announce(entry) + for key in singles + (paired if len(paired) == 1 else []): + for bars in args.bars: + entry = measure(key, bars, args.reps, workdir) + results.append(entry) + failures += announce(entry) + + cold = None + if args.cold_start_reps: + print("") + print(" cold start ... ", end="", flush=True) + cold = cold_start(args.workloads[0], 20_000, args.cold_start_reps) + print("manifoldbt {:.2f} s vs vectorbt {:.2f} s (x{:.1f})".format( + cold["median_s"]["manifoldbt"], cold["median_s"]["vectorbt"], cold["ratio"])) + + mem = None + if args.memory_bars: + print(" memory ... ", end="", flush=True) + mem = memory(args.workloads[0], args.memory_bars) + print("manifoldbt +{:.0f} MB vs vectorbt +{:.0f} MB at {:,} bars".format( + mem["manifoldbt"]["added_mb"], mem["vectorbt"]["added_mb"], args.memory_bars)) + + payload = { + "schema_version": SCHEMA_VERSION, + "generated_at": datetime.now(timezone.utc).isoformat(), + "environment": env, + "reps": args.reps, + "results": results, + "cold_start": cold, + "memory": mem, + } + with open(args.out, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2) + print("\nwrote " + args.out) + + # A parity failure is a finding, and CI should go red on it. + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/vs_vectorbt/data.py b/benchmarks/vs_vectorbt/data.py new file mode 100644 index 0000000..dcf1f1b --- /dev/null +++ b/benchmarks/vs_vectorbt/data.py @@ -0,0 +1,74 @@ +"""Deterministic synthetic OHLCV bars, identical for every engine. + +One generator, one seed, one fingerprint. Both engines receive the *same* +DataFrame object; nothing about the data can differ between them. + +Two properties are deliberate, not incidental: + +* ``open == previous close`` (no overnight gap). A bar that gaps through a stop + is the one place where two engines can legitimately disagree on the fill price + while both being correct. Removing gaps removes that whole class of false + parity failures, so a real semantic drift is the only thing left that can trip + the gate. +* the intrabar range is wide enough that percentage stops and targets actually + trigger, otherwise the bracket workload would measure an empty branch. +""" +from __future__ import annotations + +import hashlib + +import numpy as np +import pandas as pd + +DEFAULT_SEED = 20260816 + + +def make_ohlcv( + rows: int, + *, + seed: int = DEFAULT_SEED, + freq: str = "1min", + start: str = "2020-01-01", + vol: float = 3e-4, + drift: float = 2e-7, +) -> pd.DataFrame: + """A gap-free random walk of ``rows`` bars, reproducible from ``seed``.""" + rng = np.random.default_rng(seed) + + log_ret = rng.normal(drift, vol, size=rows) + close = 100.0 * np.exp(np.cumsum(log_ret)) + + open_ = np.empty(rows, dtype=np.float64) + open_[0] = 100.0 + open_[1:] = close[:-1] + + # Intrabar excursion beyond the open/close body, as a fraction of price. + wick = rng.uniform(0.2, 1.8, size=rows) * vol * close + body_hi = np.maximum(open_, close) + body_lo = np.minimum(open_, close) + high = body_hi + wick + low = np.maximum(body_lo - wick, 1e-8) + + return pd.DataFrame( + { + "timestamp": pd.date_range(start, periods=rows, freq=freq, tz="UTC"), + "open": open_, + "high": high, + "low": low, + "close": close, + "volume": rng.uniform(100.0, 10_000.0, size=rows), + } + ) + + +def digest(df: pd.DataFrame) -> str: + """Short content fingerprint of the bars, recorded in the result envelope. + + Anyone re-running the harness can compare this before comparing timings: a + different digest means a different dataset, which makes the numbers + incomparable no matter how clean the machine was. + """ + h = hashlib.sha256() + for column in ("open", "high", "low", "close", "volume"): + h.update(np.ascontiguousarray(df[column].to_numpy(dtype=np.float64)).tobytes()) + return h.hexdigest()[:16] diff --git a/benchmarks/vs_vectorbt/engine_mbt.py b/benchmarks/vs_vectorbt/engine_mbt.py new file mode 100644 index 0000000..1190d32 --- /dev/null +++ b/benchmarks/vs_vectorbt/engine_mbt.py @@ -0,0 +1,191 @@ +"""manifoldbt adapter. + +Public API only. The timed region is exactly what a user writes: +``bt.run(strategy, config, store)`` plus reading the headline metrics off the +result. Building the store from the DataFrame happens once, before timing, and +is excluded on both sides (vectorbt is likewise handed arrays it does not have +to load). + +Execution conventions, chosen to line up with vectorbt rather than to flatter +either engine (same conventions as the parity suite shipped with the library): + +* ``signal_delay=0`` and ``execution_price="AtClose"`` -> a market signal fills + at the close of the signal bar, which is what ``from_signals`` does by default. +* ``warmup_bars=0`` -> the indicator's own NaN warmup is what suppresses early + signals, identically on both sides. +* ``FractionOfEquity`` sizing is taken at the signal-bar close, which for a + market entry equals the fill price, so vectorbt ``size_type="percent"`` is the + matching mode. +""" +from __future__ import annotations + +import os +from typing import Any, Callable, Dict + +import manifoldbt as bt +from manifoldbt.expr import col, lit, when +from manifoldbt.helpers import Interval, Slippage +from manifoldbt.indicators import close as close_px, ema, rsi, sma + +from workloads import CAPITAL, WORKLOADS + +NAME = "manifoldbt" + + +def probe() -> Dict[str, Any]: + return {"engine": NAME, "version": bt.__version__} + + +def _config(df, *, sizing: str, fee_bps: float) -> "bt.BacktestConfig": + last_ns = int(df["timestamp"].iloc[-1].value) + fees = ( + bt.FeeConfig.zero() + if fee_bps == 0.0 + else bt.FeeConfig(maker_fee_bps=fee_bps, taker_fee_bps=fee_bps) + ) + return bt.BacktestConfig( + universe=[1], + time_range_start=0, + # A day past the last bar: the range is inclusive of everything generated. + time_range_end=last_ns + 86_400_000_000_000, + bar_interval=Interval.minutes(1), + initial_capital=CAPITAL, + execution=bt.ExecutionConfig( + signal_delay=0, + execution_price="AtClose", + max_position_pct=1.0, + allow_short=False, + position_sizing_mode=sizing, + ), + fees=fees, + slippage=Slippage.none(), + warmup_bars=0, + ) + + +def _strategy(key: str): + p = WORKLOADS[key].params + + if key in ("sma_cross", "sma_cross_metrics"): + return ( + bt.Strategy.create(key) + .signal("fast", sma(close_px, p["fast"])) + .signal("slow", sma(close_px, p["slow"])) + .size(when(col("fast") > col("slow"), lit(p["alloc"]), lit(0.0))) + ) + + if key == "bracket_sl_tp": + return ( + bt.Strategy.create("bracket_sl_tp") + .signal("fast", sma(close_px, p["fast"])) + .signal("slow", sma(close_px, p["slow"])) + .size(when(col("fast") > col("slow"), lit(p["alloc"]), lit(0.0))) + .stop_loss(pct=p["sl_pct"]) + .take_profit(pct=p["tp_pct"]) + ) + + if key == "ema_rsi_fees": + entry = ( + (col("fast") > col("slow")) + & (col("rsi") > lit(p["rsi_lo"])) + & (col("rsi") < lit(p["rsi_hi"])) + ) + return ( + bt.Strategy.create("ema_rsi_fees") + .signal("fast", ema(close_px, p["fast"])) + .signal("slow", ema(close_px, p["slow"])) + .signal("rsi", rsi(close_px, p["rsi_period"])) + .signal("entry", entry) + .size(when(col("entry"), lit(p["units"]), lit(0.0))) + ) + + raise KeyError(f"unknown workload {key!r}") + + +def prepare(key: str, df, workdir: str) -> Callable[[], Dict[str, Any]]: + """Untimed setup; returns the closure the harness times.""" + p = WORKLOADS[key].params + fee_bps = float(p.get("fee_bps", 0.0)) + sizing = "Units" if "units" in p else "FractionOfEquity" + + root = os.path.join(workdir, key) + os.makedirs(root, exist_ok=True) + store = bt.import_dataframe( + df, + symbol="BENCH", + symbol_id=1, + interval="1m", + data_root=os.path.join(root, "data"), + metadata_db=os.path.join(root, "metadata.sqlite"), + ) + strategy = _strategy(key) + config = _config(df, sizing=sizing, fee_bps=fee_bps) + + wants_metrics = bool(p.get("metrics")) + + def run() -> Dict[str, Any]: + result = bt.run(strategy, config, store) + m = result.metrics + ts = m.get("trade_stats") or {} + out = { + "total_return": float(m["total_return"]), + "final_equity": CAPITAL * (1.0 + float(m["total_return"])), + "round_trips": int(ts.get("round_trips", 0)), + "fills": int(ts.get("total_trades", 0)), + "total_fees": float(ts.get("total_fees", 0.0)), + } + if wants_metrics: + # Already computed by run(): reading them costs nothing measurable, + # which is the whole point of the comparison. + out.update({ + "max_drawdown": float(m["max_drawdown"]), + "sharpe": float(m["sharpe"]), + "sortino": float(m["sortino"]), + "volatility": float(m["volatility"]), + }) + return out + + return run + + +def diagnose(key: str, df, workdir: str) -> Dict[str, Any]: + """Untimed measurement of *how much* a documented divergence actually bites. + + For the bracket workload this counts the round-trips whose entry lands on the + same bar as the previous exit, which is precisely the population where + vectorbt takes the next bar instead. Reporting the count turns "the engines + differ" into a number a reader can weigh. + """ + if WORKLOADS[key].parity != "documented": + return {} + + p = WORKLOADS[key].params + root = os.path.join(workdir, key + "_diag") + os.makedirs(root, exist_ok=True) + store = bt.import_dataframe( + df, symbol="BENCH", symbol_id=1, interval="1m", + data_root=os.path.join(root, "data"), + metadata_db=os.path.join(root, "metadata.sqlite"), + ) + result = bt.run( + _strategy(key), + _config(df, sizing="Units" if "units" in p else "FractionOfEquity", + fee_bps=float(p.get("fee_bps", 0.0))), + store, + ) + trades = result.trades_df() + ts = trades["execution_timestamp"].to_numpy() + # Fills alternate entry, exit, entry, exit ... An entry that shares a bar + # with the exit before it is one the other engine would defer by one bar. + entries, exits = ts[0::2], ts[1::2] + n = min(len(exits), len(entries) - 1) + same_bar = int((exits[:n] == entries[1 : n + 1]).sum()) if n > 0 else 0 + stats = result.metrics.get("trade_stats") or {} + round_trips = int(stats.get("round_trips", 0)) + return { + "reentries_on_exit_bar": same_bar, + "round_trips": round_trips, + "share_of_round_trips": (same_bar / round_trips) if round_trips else 0.0, + "sl_exits": int(stats.get("sl_exits", 0)), + "tp_exits": int(stats.get("tp_exits", 0)), + } diff --git a/benchmarks/vs_vectorbt/engine_vbt.py b/benchmarks/vs_vectorbt/engine_vbt.py new file mode 100644 index 0000000..109e266 --- /dev/null +++ b/benchmarks/vs_vectorbt/engine_vbt.py @@ -0,0 +1,187 @@ +"""vectorbt adapter. + +The timed region is what a vectorbt user writes: compute the indicators, derive +the signals, call ``Portfolio.from_signals``, read the headline metrics. The +DataFrame-to-Series conversion happens once, before timing, so neither engine +pays for data marshalling inside the measurement. + +Indicator definitions are mirrored on manifoldbt's, not approximated +------------------------------------------------------------------- +* SMA: rolling mean, NaN for the first ``n-1`` bars. Identical by construction. +* EMA: ``alpha = 2/(span+1)``, seeded on the first observation, emitted from the + first bar. That is exactly ``ewm(span=n, adjust=False)``. +* RSI: Wilder, seeded with the *simple* average of the first ``period`` deltas + and emitted from bar ``period`` onwards. A plain ``ewm(alpha=1/period)`` over + the whole delta series is a different indicator (it seeds on the first delta + and emits from bar 1), which is why the seed is written explicitly here. The + recursion itself still runs through pandas' C ``ewm``, so this is not a Python + loop handicapping vectorbt. + +Signals are fed as a *level*, not as transitions: ``entries`` is "the condition +holds", ``exits`` is "it no longer holds". With ``accumulate=False`` vectorbt +enters when flat and the level is true, which reproduces manifoldbt's +target-position semantics, including a re-entry after a bracket exit while the +condition is still true. +""" +from __future__ import annotations + +from typing import Any, Callable, Dict + +import numpy as np +import pandas as pd +import vectorbt as vbt +from vectorbt.portfolio.enums import StopExitPrice + +from workloads import CAPITAL, FREQ, WORKLOADS + +NAME = "vectorbt" + + +def probe() -> Dict[str, Any]: + return {"engine": NAME, "version": vbt.__version__} + + +def _wilder_rsi(close: pd.Series, period: int) -> pd.Series: + """RSI with manifoldbt's exact seeding (see module docstring).""" + values = close.to_numpy(dtype=np.float64) + delta = np.empty_like(values) + delta[0] = np.nan + delta[1:] = values[1:] - values[:-1] + + gain = np.where(delta > 0.0, delta, 0.0) + loss = np.where(delta < 0.0, -delta, 0.0) + + # Seed at index `period` with the simple mean of the first `period` deltas, + # then let ewm(alpha=1/period) carry Wilder's recursion from there. + gain[:period] = np.nan + loss[:period] = np.nan + gain[period] = np.nanmean(np.where(delta[1 : period + 1] > 0.0, delta[1 : period + 1], 0.0)) + loss[period] = np.nanmean(np.where(delta[1 : period + 1] < 0.0, -delta[1 : period + 1], 0.0)) + + alpha = 1.0 / period + avg_gain = pd.Series(gain, index=close.index).ewm(alpha=alpha, adjust=False).mean() + avg_loss = pd.Series(loss, index=close.index).ewm(alpha=alpha, adjust=False).mean() + + rs = avg_gain / avg_loss + out = 100.0 - 100.0 / (1.0 + rs) + # avg_loss == 0 -> RSI is 100 by definition (matches the engine). + return out.where(avg_loss != 0.0, 100.0) + + +def indicators(key: str, close: pd.Series) -> Dict[str, pd.Series]: + """Indicator series for a workload. Used by the timed path and by the + definition check, so the two can never drift apart.""" + p = WORKLOADS[key].params + if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp"): + return { + "fast": close.rolling(p["fast"]).mean(), + "slow": close.rolling(p["slow"]).mean(), + } + if key == "ema_rsi_fees": + return { + "fast": close.ewm(span=p["fast"], adjust=False).mean(), + "slow": close.ewm(span=p["slow"], adjust=False).mean(), + "rsi": _wilder_rsi(close, p["rsi_period"]), + } + raise KeyError(f"unknown workload {key!r}") + + +def _level(key: str, ind: Dict[str, pd.Series]) -> pd.Series: + p = WORKLOADS[key].params + if key in ("sma_cross", "sma_cross_metrics", "bracket_sl_tp"): + return (ind["fast"] > ind["slow"]).fillna(False) + if key == "ema_rsi_fees": + return ( + (ind["fast"] > ind["slow"]) + & (ind["rsi"] > p["rsi_lo"]) + & (ind["rsi"] < p["rsi_hi"]) + ).fillna(False) + raise KeyError(f"unknown workload {key!r}") + + +def prepare(key: str, df, workdir: str | None = None) -> Callable[[], Dict[str, Any]]: + """Untimed setup; returns the closure the harness times.""" + p = WORKLOADS[key].params + index = pd.DatetimeIndex(df["timestamp"]) + close = pd.Series(df["close"].to_numpy(dtype=np.float64), index=index) + open_ = pd.Series(df["open"].to_numpy(dtype=np.float64), index=index) + high = pd.Series(df["high"].to_numpy(dtype=np.float64), index=index) + low = pd.Series(df["low"].to_numpy(dtype=np.float64), index=index) + + if "units" in p: + size, size_type = p["units"], "amount" + else: + size, size_type = p["alloc"], "percent" + fees = float(p.get("fee_bps", 0.0)) / 10_000.0 + wants_metrics = bool(p.get("metrics")) + sl = p["sl_pct"] / 100.0 if "sl_pct" in p else None + tp = p["tp_pct"] / 100.0 if "tp_pct" in p else None + + def run() -> Dict[str, Any]: + level = _level(key, indicators(key, close)) + portfolio = vbt.Portfolio.from_signals( + close, + entries=level, + exits=~level, + open=open_, + high=high, + low=low, + init_cash=CAPITAL, + size=size, + size_type=size_type, + fees=fees, + slippage=0.0, + sl_stop=sl, + tp_stop=tp, + stop_exit_price=StopExitPrice.StopMarket, + direction="longonly", + accumulate=False, + freq=FREQ, + ) + total_return = float(portfolio.total_return()) + trades = portfolio.trades + out = { + "total_return": total_return, + "final_equity": CAPITAL * (1.0 + total_return), + "round_trips": int(trades.closed.count()), + "fills": None, # vectorbt books round-trips, not individual fills + "total_fees": float(trades.records["entry_fees"].sum() + + trades.records["exit_fees"].sum()), + } + if wants_metrics: + out.update(_summary(portfolio)) + return out + + return run + + +def _summary(portfolio) -> Dict[str, Any]: + """The same performance summary manifoldbt returns from every run. + + Written out by hand rather than through ``pf.sharpe_ratio()`` and friends for + two reasons. First, basis: manifoldbt computes its ratios on *daily* returns + annualised by sqrt(365) and its drawdown at full bar resolution, while + vectorbt's accessors annualise at the data's own frequency, so the native + calls would return different numbers and the comparison would be timing two + different computations. Second, speed: this version is measurably faster than + a single native accessor on the same data, so vectorbt is credited with the + quicker of the two paths available to it. + + The cost that dominates either way is materialising the equity curve, which + ``from_signals`` defers until a risk metric asks for it. + """ + equity = portfolio.value() + drawdown = float((equity / equity.cummax() - 1.0).min()) + + daily = equity.resample("1D").last().dropna() + returns = daily.pct_change().dropna() + annualiser = np.sqrt(365.0) + deviation = returns.std(ddof=1) + downside = returns[returns < 0].std(ddof=1) + mean = returns.mean() + return { + "max_drawdown": drawdown, + "sharpe": float(mean / deviation * annualiser), + "sortino": float(mean / downside * annualiser), + "volatility": float(deviation * annualiser), + } diff --git a/benchmarks/vs_vectorbt/parity.py b/benchmarks/vs_vectorbt/parity.py new file mode 100644 index 0000000..f8cb4f7 --- /dev/null +++ b/benchmarks/vs_vectorbt/parity.py @@ -0,0 +1,88 @@ +"""The gate: no speed number is published for a workload the engines disagree on. + +A benchmark between two backtesters is only a benchmark if both engines did the +same work. This module compares what each engine produced and classifies the +result, and ``bench.py`` refuses to report a timing for anything it classifies +as a failure. + +Three verdicts: + +``exact`` + Agreement down to float-reordering noise. The timing is publishable. +``documented`` + The engines disagree, the workload declared it in advance, and the reason is + written down. The timing goes to the annex with the reason attached. +``failed`` + The engines disagree and nobody predicted it. That is a finding about the + engines, not about their speed: the timing is withheld. +""" +from __future__ import annotations + +from typing import Any, Dict + +from workloads import CAPITAL, WORKLOADS + +# Float reordering across two implementations of the same arithmetic lands +# around 1e-13 of the account on a million bars. Anything above this is a +# different decision somewhere, not a different summation order. +REL_TOL = 1e-9 + + +def _rel(a: float, b: float) -> float: + """Plain relative difference, for quantities that are not money.""" + return abs(a - b) / max(1e-12, abs(b)) + + +def _vs_capital(a: float, b: float) -> float: + """Difference as a fraction of the money at risk, not of the result itself. + + Anchoring on the result breaks exactly when the result is interesting: a + strategy that ends near zero equity, or near zero return, turns a difference + of a hundredth of a cent into a 1% relative error and fails a comparison the + engines actually passed. The account size is the stable yardstick.""" + return abs(a - b) / CAPITAL + + +def compare(mbt: Dict[str, Any], vbt: Dict[str, Any], key: str) -> Dict[str, Any]: + expected = WORKLOADS[key].parity + + diffs = { + "final_equity_vs_capital": _vs_capital(mbt["final_equity"], vbt["final_equity"]), + "round_trips_delta": mbt["round_trips"] - vbt["round_trips"], + "total_fees_vs_capital": _vs_capital(mbt["total_fees"], vbt["total_fees"]), + } + agrees = ( + diffs["final_equity_vs_capital"] <= REL_TOL + and diffs["round_trips_delta"] == 0 + and diffs["total_fees_vs_capital"] <= REL_TOL + ) + + # Workloads that also produce a performance summary are gated on the + # drawdown, which both engines compute at full bar resolution and which must + # match. The ratios are reported but not gated: manifoldbt buckets its daily + # returns slightly differently, a difference worth stating rather than + # hiding, and worth nothing at all as an argument about speed. + if "max_drawdown" in mbt and "max_drawdown" in vbt: + diffs["max_drawdown_rel"] = _rel(mbt["max_drawdown"], vbt["max_drawdown"]) + agrees = agrees and diffs["max_drawdown_rel"] <= REL_TOL + diffs["advisory_ratio_rel"] = { + name: _rel(mbt[name], vbt[name]) + for name in ("sharpe", "sortino", "volatility") + if name in mbt and name in vbt + } + + if agrees: + status = "exact" + elif expected == "documented": + status = "documented" + else: + status = "failed" + + return { + "status": status, + "expected": expected, + "publishable": status == "exact", + "diffs": diffs, + "metrics": {"manifoldbt": mbt, "vectorbt": vbt}, + "note": WORKLOADS[key].divergence if status == "documented" else "", + } diff --git a/benchmarks/vs_vectorbt/probe_child.py b/benchmarks/vs_vectorbt/probe_child.py new file mode 100644 index 0000000..f6c940e --- /dev/null +++ b/benchmarks/vs_vectorbt/probe_child.py @@ -0,0 +1,124 @@ +"""One measurement, one fresh process, one JSON line on stdout. + +Two things cannot be measured honestly inside the main harness process: + +*Cold start* - the wait between typing "run" and seeing a result. vectorbt +compiles its numba kernels on the first call, which is a real cost a user pays +in every new notebook or script, and which the steady-state benchmark +deliberately discards. Measuring it requires a process that has never imported +either engine. + +*Memory* - peak resident memory attributable to the run. Once one engine has +run in a process, the allocator has already grown and the other engine's +measurement is meaningless. + +The ``baseline`` mode measures the same process doing everything except calling +an engine (interpreter start, numpy and pandas import, data generation) so the +engine's own share can be read off rather than argued about. + + python probe_child.py coldstart mbt sma_cross 20000 + python probe_child.py memory vbt sma_cross 5000000 + python probe_child.py baseline none sma_cross 20000 +""" +from __future__ import annotations + +import gc +import json +import os +import sys +import tempfile +import threading +import time + +START = time.perf_counter() + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + + +def _rss_mb() -> float: + import psutil + + return psutil.Process().memory_info().rss / 1e6 + + +def _build(engine: str, workload: str, bars: int): + import data as data_mod + + frame = data_mod.make_ohlcv(bars) + if engine == "mbt": + import engine_mbt + + return engine_mbt.prepare(workload, frame, tempfile.mkdtemp(prefix="mbt_probe_")) + if engine == "vbt": + import engine_vbt + + return engine_vbt.prepare(workload, frame, None) + return lambda: {} + + +def cold_start(engine: str, workload: str, bars: int) -> dict: + """Wall time from process start to a finished backtest, engine import included.""" + run = _build(engine, workload, bars) + run() + return {"seconds": time.perf_counter() - START} + + +def memory(engine: str, workload: str, bars: int) -> dict: + """Resident memory the run itself adds, sampled while it runs. + + A warmup call first, so what is measured is the steady-state cost of running + a backtest rather than the one-off growth of a cold allocator. + """ + run = _build(engine, workload, bars) + run() + gc.collect() + time.sleep(0.3) + + peak = [_rss_mb()] + stop = threading.Event() + + def sample(): + while not stop.is_set(): + peak[0] = max(peak[0], _rss_mb()) + time.sleep(0.002) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + before = _rss_mb() + started = time.perf_counter() + run() + elapsed = time.perf_counter() - started + stop.set() + sampler.join() + + delta = peak[0] - before + return { + "before_mb": before, + "peak_mb": peak[0], + "added_mb": delta, + "added_mb_per_million_bars": delta / (bars / 1e6), + "seconds": elapsed, + } + + +def baseline(engine: str, workload: str, bars: int) -> dict: + """Everything except the engine: interpreter, numpy, pandas, data generation.""" + _build("none", workload, bars) + return {"seconds": time.perf_counter() - START} + + +MODES = {"coldstart": cold_start, "memory": memory, "baseline": baseline} + + +def main() -> int: + mode, engine, workload, bars = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) + payload = MODES[mode](engine, workload, bars) + payload.update({"mode": mode, "engine": engine, "workload": workload, "bars": bars}) + # A marker prefix: the engines print a banner on import, and the parent must + # not have to guess which line is the result. + sys.stdout.write("\nPROBE_RESULT " + json.dumps(payload) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/vs_vectorbt/report.py b/benchmarks/vs_vectorbt/report.py new file mode 100644 index 0000000..a6f2db8 --- /dev/null +++ b/benchmarks/vs_vectorbt/report.py @@ -0,0 +1,218 @@ +"""Turn a results JSON into the Markdown a human reads. + +Prints to stdout and, when running under GitHub Actions, appends the same text +to the job summary so the numbers are visible without downloading an artifact. + + python report.py results.json + +This is a run output, not an article: tables, and only the glue needed to read +them. Every "why" belongs in README.md, which is written once instead of being +reprinted underneath every single run. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Dict, List + +METHOD_LINK = "benchmarks/vs_vectorbt/README.md" + + +def _ms(seconds: float) -> str: + if seconds < 1.0: + return "{:.1f} ms".format(seconds * 1e3) + return "{:.2f} s".format(seconds) + + +def _header(payload: Dict[str, Any]) -> List[str]: + env = payload["environment"] + versions = env["versions"] + cores = str(env["logical_cores"]) + if env.get("pinned_cores"): + cores += " (pinned to {})".format(env["pinned_cores"]) + lines = [ + "# manifoldbt {} vs vectorbt {}".format(versions["manifoldbt"], versions["vectorbt"]), + "", + "`{os} {arch}` | {cpu} | {cores} cores | {ram} GB | python {py} | " + "numpy {np} / numba {nb} / pandas {pd} | {reps} interleaved reps | {when}".format( + os=env["os"], arch=env["arch"], cpu=env["cpu"], cores=cores, ram=env["ram_gb"], + py=env["python"], np=versions["numpy"], nb=versions["numba"], + pd=versions["pandas"], reps=payload["reps"], when=payload["generated_at"][:19], + ), + ] + if env.get("run_url"): + lines += ["", env["run_url"]] + lines.append("") + return lines + + +def _speed_table(rows: List[Dict[str, Any]], title: str) -> List[str]: + if not rows: + return [] + lines = [ + "## " + title, + "", + "| Workload | Bars | manifoldbt | vectorbt | Ratio |", + "|---|---:|---:|---:|---:|", + ] + for row in rows: + timings = row["timings"] + lines.append( + "| {w} | {b:,} | {m} | {v} | **x{s:.1f}**{f} |".format( + w=row["workload"], b=row["bars"], + m=_ms(timings["manifoldbt"]["median_s"]), + v=_ms(timings["vectorbt"]["median_s"]), + s=row["speedup"]["median_of_ratios"], + f=" ~" if row.get("noisy") else "", + ) + ) + lines.append("") + return lines + + +def _summary_cost(exact: List[Dict[str, Any]]) -> List[str]: + plain = {r["bars"]: r for r in exact if r["workload"] == "sma_cross"} + summarised = {r["bars"]: r for r in exact if r["workload"] == "sma_cross_metrics"} + # Only subtract timings that were measured in the same interleaved loop. + shared = sorted( + b for b in set(plain) & set(summarised) + if "sma_cross_metrics" in (plain[b].get("paired_with") or []) + ) + if not shared: + return [] + + lines = [ + "## Cost of the performance summary", + "", + "Same simulation, with and without max drawdown / Sharpe / Sortino / volatility.", + "", + "| Bars | Engine | Without | With | Delta |", + "|---:|---|---:|---:|---:|", + ] + for bars in shared: + for engine in ("manifoldbt", "vectorbt"): + without = plain[bars]["timings"][engine]["median_s"] + with_ = summarised[bars]["timings"][engine]["median_s"] + lines.append( + "| {b:,} | {e} | {a} | {c} | {d} |".format( + b=bars, e=engine, a=_ms(without), c=_ms(with_), + d=("+" + _ms(with_ - without)) if with_ > without else "none measurable", + ) + ) + lines.append("") + advisory = summarised[shared[-1]]["parity"]["diffs"].get("advisory_ratio_rel") + if advisory: + lines += [ + "Gated on total return, round-trips and max drawdown (exact). Sharpe, Sortino " + "and volatility agree to {:.1e}, from daily bucketing.".format(max(advisory.values())), + "", + ] + return lines + + +def _side_measures(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> List[str]: + cold = payload.get("cold_start") + mem = payload.get("memory") + threading_rows = [ + r for r in results + if r.get("cpu_over_wall") + and all(r["cpu_over_wall"].get(e) is not None for e in ("manifoldbt", "vectorbt")) + ] + if not (cold or mem or threading_rows): + return [] + + lines = ["## Cold start, memory, threads", "", "| | manifoldbt | vectorbt |", "|---|---:|---:|"] + if cold: + medians, share = cold["median_s"], cold["engine_share_s"] + lines.append("| Fresh process to first backtest | {:.2f} s | {:.2f} s |".format( + medians["manifoldbt"], medians["vectorbt"])) + lines.append("| ... minus the {:.2f} s python baseline | {:.2f} s | {:.2f} s |".format( + medians["baseline"], share["manifoldbt"], share["vectorbt"])) + if mem: + lines.append("| RAM added by the run, per 1M bars | {:.0f} MB | {:.0f} MB |".format( + mem["manifoldbt"]["added_mb_per_million_bars"], + mem["vectorbt"]["added_mb_per_million_bars"])) + if threading_rows: + biggest = max(threading_rows, key=lambda r: r["bars"]) + ratio = biggest["cpu_over_wall"] + lines.append("| CPU over wall time at {:,} bars | {:.2f} | {:.2f} |".format( + biggest["bars"], ratio["manifoldbt"], ratio["vectorbt"])) + lines.append("") + return lines + + +def render(payload: Dict[str, Any]) -> str: + results = payload["results"] + exact = [r for r in results if r["parity"]["status"] == "exact" and r.get("timings")] + documented = [r for r in results if r["parity"]["status"] == "documented"] + failed = [r for r in results if r["parity"]["status"] == "failed"] + + lines = _header(payload) + lines += _speed_table(exact, "Same results, both engines") + lines += _summary_cost(exact) + lines += _side_measures(payload, results) + + timed = [r for r in documented if r.get("timings")] + if timed: + lines += _speed_table(timed, "Results differ, kept out of the headline") + scales = [r for r in documented if r.get("divergence_scale")] + if scales: + row = scales[0] + scale = row["divergence_scale"] + lines += [ + "`{w}`: {n} of {t} round-trips re-enter on the exit bar ({p:.0%}) at {b:,} " + "bars, from {sl} stop and {tp} target exits. Cause in {link}.".format( + w=row["workload"], n=scale["reentries_on_exit_bar"], + t=scale["round_trips"], p=scale["share_of_round_trips"], + b=row["bars"], sl=scale["sl_exits"], tp=scale["tp_exits"], + link=METHOD_LINK, + ), + "", + ] + + if failed: + lines += ["## Timing withheld", ""] + for row in failed: + diffs = row["parity"]["diffs"] + lines.append( + "- `{w}` at {b:,} bars: final equity differs by {d:.2e} of capital, " + "round-trips by {t}.".format( + w=row["workload"], b=row["bars"], + d=diffs["final_equity_vs_capital"], t=diffs["round_trips_delta"], + ) + ) + lines.append("") + + lines += [ + "---", + "", + "`~` = IQR above 15% of the median, indicative only. Ratios are medians of " + "per-repetition ratios with the engines interleaved; data loading excluded, warmup " + "discarded. Method and caveats: {}".format(METHOD_LINK), + "", + ] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description="render a benchmark result") + parser.add_argument("results", nargs="?", default="results.json") + args = parser.parse_args() + + with open(args.results, encoding="utf-8") as fh: + payload = json.load(fh) + + text = render(payload) + sys.stdout.write(text + "\n") + + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as fh: + fh.write(text + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/vs_vectorbt/requirements-lock.txt b/benchmarks/vs_vectorbt/requirements-lock.txt new file mode 100644 index 0000000..450568c --- /dev/null +++ b/benchmarks/vs_vectorbt/requirements-lock.txt @@ -0,0 +1,12 @@ +# Pinned on purpose: a benchmark whose dependency set moves under it is not a +# benchmark. These are the exact versions the harness was validated against, and +# they are recorded again inside every results JSON so a reader can tell whether +# two runs are comparable. +# +# manifoldbt itself is NOT pinned here: the workflow installs the version under +# test (`pip install manifoldbt==X`), so the same lock file serves every release. +vectorbt==0.28.4 +numpy==2.4.3 +numba==0.64.0 +pandas==2.3.3 +psutil==7.2.2 diff --git a/benchmarks/vs_vectorbt/workloads.py b/benchmarks/vs_vectorbt/workloads.py new file mode 100644 index 0000000..72fce73 --- /dev/null +++ b/benchmarks/vs_vectorbt/workloads.py @@ -0,0 +1,111 @@ +"""Workload definitions: the numbers both engines read, in one place. + +Nothing here is engine-specific. Each adapter (``engine_mbt``, ``engine_vbt``) +reads the same constants, so a workload cannot drift between the two sides by +someone editing one file and forgetting the other. + +Sizing policy, and why it changes when fees are on +-------------------------------------------------- +With ``FractionOfEquity`` sizing and a non-zero fee, the two engines size a +position differently: manifoldbt charges the fee on top of a full-equity +notional, vectorbt reserves it out of cash first. Both are defensible product +decisions, and comparing them would compare *policy*, not speed or correctness. +The fee workload therefore sizes in fixed units, which isolates the fee +arithmetic itself. This mirrors the choice already made in the cross-engine +parity suite shipped with the library. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict + +CAPITAL = 100_000.0 + +# Bar interval of the generated data. Both engines are told the same thing: +# manifoldbt through `Interval.minutes(1)`, vectorbt through `freq="1min"` +# (annualisation only; it does not touch the simulation). +FREQ = "1min" + + +@dataclass(frozen=True) +class Workload: + key: str + title: str + why: str + params: Dict[str, Any] = field(default_factory=dict) + # "exact": the engines must agree to float-reordering noise, or the timing is + # not published. "documented": they are known to disagree for a reason + # written down in `divergence`; the timing goes to the annex, never to the + # headline table. + parity: str = "exact" + divergence: str = "" + + +WORKLOADS: Dict[str, Workload] = { + w.key: w + for w in ( + Workload( + key="sma_cross", + title="SMA 10/50 crossover, long-only, no cost", + why="The canonical baseline. Unambiguous indicator, no fee policy, " + "no stop semantics: if the engines disagree here, nothing else " + "in the suite is worth reading.", + params=dict(fast=10, slow=50, alloc=1.0), + ), + Workload( + key="ema_rsi_fees", + title="EMA 12/26 crossover + RSI(14) filter, 5 bps taker fee", + why="A realistic signal stack with a real cost model, sized in fixed " + "units so the fee arithmetic is comparable across engines. The " + "unit count is small relative to capital on purpose: at 1-minute " + "resolution this strategy turns over often enough that a larger " + "size would spend the whole account on fees, and comparing two " + "engines on a wiped-out account compares rounding noise.", + params=dict(fast=12, slow=26, rsi_period=14, rsi_lo=30.0, rsi_hi=70.0, + units=5.0, fee_bps=5.0), + ), + Workload( + key="sma_cross_metrics", + title="SMA 10/50 crossover, with a performance summary", + why="The same simulation as `sma_cross`, but both engines are asked " + "for what a user actually reads: max drawdown, Sharpe, Sortino " + "and volatility alongside the return. manifoldbt computes them " + "inside run() whether you ask or not; vectorbt defers the equity " + "curve until a risk metric needs it, and every one of them pays " + "for materialising it. This is a scope difference, not a trick: " + "`sma_cross` above is the same work without the summary, and the " + "two are reported side by side so the reader can see what the " + "summary costs each engine.", + params=dict(fast=10, slow=50, alloc=1.0, metrics=True), + ), + Workload( + key="bracket_sl_tp", + title="SMA 10/50 entry with a 15 bps stop / 30 bps target bracket", + why="Brackets are where two engines most easily disagree: the stop " + "level, the fill on the triggering bar, and whether a re-entry " + "is allowed on the bar after an exit.", + params=dict(fast=10, slow=50, alloc=1.0, sl_pct=0.15, tp_pct=0.30), + parity="documented", + divergence=( + "Re-entry on the exit bar. When a bracket fires intrabar and the " + "entry condition still holds at that bar's close, manifoldbt books " + "two orders on that bar (the stop or target exit, then a fresh " + "entry at the close); vectorbt processes one order per bar and " + "re-enters on the next bar instead. Neither is wrong, and on " + "controlled bars the bracket fills themselves match exactly (see " + "the cross-engine parity suite shipped with the library). The " + "harness counts the affected round-trips so the size of the " + "divergence is measured, not asserted." + ), + ), + ) +} + +DEFAULT_KEYS = list(WORKLOADS) + +# Two workloads that the report compares directly against each other, so they +# must be measured inside ONE interleaved loop rather than in two blocks minutes +# apart. Comparing medians across blocks is exactly the mistake the interleaving +# exists to prevent: absolute timings drift between blocks, and the drift once +# produced a table claiming the version doing MORE work was the faster one. +SCOPE_PAIR = ("sma_cross", "sma_cross_metrics")