From a4040375e23e94c7b523c7c2c61de693c0de45b2 Mon Sep 17 00:00:00 2001 From: Exocet92 <79667065+Jimmy7892@users.noreply.github.com> Date: Thu, 20 Aug 2026 17:00:41 +0200 Subject: [PATCH] bench: fix the red runs, trim the matrix, add a grid job (#7) Five consecutive red runs, two unrelated causes. Four of them never reached an engine: the workflow installs the tag it is handed, but 0.18.0rc1 and rc2 were previews that never reached PyPI. Pre-releases are now skipped, and the report step checks for its input file instead of dying on a missing one and reporting the wrong cause twice. The fifth came from adding 10M bars, which broke a workload whose validity depended on the ladder stopping at 5M. ema_rsi_fees sizes in fixed units and pays 5 bps a side, so over 10M one-minute bars the fees compound into the whole account: -15% of capital at 1M, -74% at 5M, exactly -100% at 10M, where fees reach 99,611 of the 100,000 it started with. Both engines then sit at zero and disagree by 9,085 round-trips about how many worthless trades to book on a dead account. Workloads can now declare a ceiling, and the runner skips past it out loud. Fewer points per axis: three series lengths instead of five, a decade apart each step. 10k measured the clock rather than the work, and 5M sat between two points that already bracketed it. sma_cross crosses on 30/150 rather than 10/50, worth about 15% on the ratio because it books a third of the trades. Grids get their own job, licensed through ci_activate.py, which refuses to run unlicensed rather than time a wait. Three points, not a matrix: across the plane the four-core ratio moves only between x32 and x38. --- .github/workflows/bench-vs-vectorbt.yml | 100 ++++++++++++++++++++++-- benchmarks/vs_vectorbt/README.md | 62 ++++++++++++++- benchmarks/vs_vectorbt/bench.py | 38 ++++++++- benchmarks/vs_vectorbt/workloads.py | 20 ++++- 4 files changed, 204 insertions(+), 16 deletions(-) diff --git a/.github/workflows/bench-vs-vectorbt.yml b/.github/workflows/bench-vs-vectorbt.yml index e45c00b..7617c49 100644 --- a/.github/workflows/bench-vs-vectorbt.yml +++ b/.github/workflows/bench-vs-vectorbt.yml @@ -44,15 +44,26 @@ jobs: bench: name: ${{ matrix.os }} runs-on: ${{ matrix.os }} - timeout-minutes: 90 + # A published release is not always a released version. The preview tags + # 0.18.0rc1 and rc2 went to the public repository without ever reaching + # PyPI, so `pip install manifoldbt==0.18.0rc2` failed and this workflow went + # red four times in two days for a reason that had nothing to do with any + # engine. Pre-releases are skipped; a manual dispatch can still benchmark + # one by naming the version, if it ever exists on PyPI. + if: github.event_name != 'release' || github.event.release.prerelease == false + timeout-minutes: 60 strategy: fail-fast: false matrix: include: + # Three lengths, not five. 10k was sub-millisecond on the engine + # side, which measures the clock rather than the work, and 5M sat + # between two points that already bracket it. What is left is a + # decade apart each step, which is what makes the trend readable. - os: ubuntu-latest - bars: "10000 100000 1000000 5000000 10000000" + bars: "100000 1000000 10000000" - os: windows-latest - bars: "10000 100000 1000000 5000000 10000000" + bars: "100000 1000000 10000000" # macOS runners ship 7 GB of RAM against 16 GB elsewhere, and vectorbt # materialises the simulation in memory (roughly 150 MB per million # bars, measured). The top size is trimmed so a point is never lost to @@ -60,7 +71,7 @@ jobs: # arithmetic is why 10M bars is added on the other two and not here: # measured, that point adds 1.55 GB on vectorbt's side alone. - os: macos-latest - bars: "10000 100000 1000000" + bars: "100000 1000000" env: PYTHONUNBUFFERED: "1" @@ -115,11 +126,19 @@ jobs: - name: Render the report # Runs even when the benchmark exits non-zero: a parity failure is the # most interesting thing that can happen here, and it must be readable - # in the job summary rather than buried in a red step. + # in the job summary rather than buried in a red step. But only if there + # is something to render: when the install step failed, this used to die + # on a missing file and put a FileNotFoundError on top of the real + # error, which is how a run reports the wrong cause twice. if: always() shell: bash working-directory: benchmarks/vs_vectorbt - run: python report.py "results-${{ matrix.os }}.json" + run: | + if [ -f "results-${{ matrix.os }}.json" ]; then + python report.py "results-${{ matrix.os }}.json" + else + echo "no result file: the benchmark did not get far enough to write one" + fi - name: Upload the raw result if: always() @@ -128,3 +147,72 @@ jobs: name: bench-${{ matrix.os }} path: benchmarks/vs_vectorbt/results-*.json if-no-files-found: warn + + # ------------------------------------------------------------------------ # + # Parameter grids, which need a licence and therefore a job of their own + # ------------------------------------------------------------------------ # + sweeps: + name: sweeps (ubuntu) + runs-on: ubuntu-latest + if: github.event_name != 'release' || github.event.release.prerelease == false + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install engines from PyPI + shell: bash + run: | + set -e + VERSION="${{ inputs.manifoldbt_version }}" + if [ -z "$VERSION" ] && [ "${{ github.event_name }}" = "release" ]; then + VERSION="$(echo '${{ github.event.release.tag_name }}' | sed 's/^v//')" + fi + python -m pip install --upgrade pip + if [ -n "$VERSION" ]; then pip install "manifoldbt==${VERSION}"; else pip install manifoldbt; fi + pip install -r benchmarks/vs_vectorbt/requirements-lock.txt + + # A grid benchmark without a licence does not fail, it produces a wrong + # number: every unlicensed fan-out call waits a fixed interval before any + # work starts, so the stopwatch would time the wait. This step exits + # non-zero rather than let that happen, and the harness refuses again on + # its own if the tier is not what it expects. + - name: Activate the benchmark licence + env: + MANIFOLDBT_CI_LICENSE: ${{ secrets.MANIFOLDBT_CI_LICENSE }} + shell: bash + working-directory: benchmarks/vs_vectorbt + run: python ci_activate.py + + - name: Run the grids + shell: bash + working-directory: benchmarks/vs_vectorbt + # Three points, chosen from a measured map of the bars-by-combinations + # plane rather than picked: across it the ratio moves between x32 and + # x38, so a denser matrix would spend runner time re-measuring the same + # number. What the three do carry is the shape of the thing: two grid + # sizes at one series length, and one grid vectorbt cannot hold at all. + run: | + python bench.py --workloads sma_cross --bars 100000 --reps 1 --cold-start-reps 0 --sweep 20000:2500 20000:5000 200000:2500:oos --sweep-reps "${{ inputs.reps || '2' }}" --out "results-sweeps.json" + + - name: Render the report + if: always() + shell: bash + working-directory: benchmarks/vs_vectorbt + run: | + if [ -f results-sweeps.json ]; then + python report.py results-sweeps.json + else + echo "no result file: the grids did not get far enough to write one" + fi + + - name: Upload the raw result + if: always() + uses: actions/upload-artifact@v4 + with: + name: bench-sweeps + path: benchmarks/vs_vectorbt/results-sweeps.json + if-no-files-found: warn diff --git a/benchmarks/vs_vectorbt/README.md b/benchmarks/vs_vectorbt/README.md index 6700c34..c40c7e7 100644 --- a/benchmarks/vs_vectorbt/README.md +++ b/benchmarks/vs_vectorbt/README.md @@ -85,11 +85,47 @@ store)`, the documented entry point, not through an internal fast path. | Workload | What it exercises | vectorbt | raptorbt | |---|---|---|---| -| `sma_cross` | SMA 10/50 crossover, long-only, no cost | exact | exact | -| `ema_rsi_fees` | EMA 12/26 crossover with an RSI(14) filter and a 5 bps taker fee | exact | unsupported | +| `sma_cross` | SMA 30/150 crossover, long-only, no cost | exact | exact | +| `ema_rsi_fees` | EMA 12/26 crossover with an RSI(14) filter and a 5 bps taker fee, capped at 1M bars | exact | unsupported | | `sma_cross_metrics` | the same simulation, plus max drawdown, Sharpe, Sortino and volatility | exact | exact | | `bracket_sl_tp` | the same entry with a 15 bps stop and a 30 bps target | documented | documented | +### Why the fee workload stops at 1M bars + +A workload can stop being a comparison before it stops running. `ema_rsi_fees` +sizes in fixed units and pays 5 bps a side, and at 1-minute resolution it turns +over often enough that the fees compound into the account: measured, it ends at +-15% of capital on 1M bars, -74% on 5M, and exactly -100% on 10M, where fees +reach 99,611 of the 100,000 it started with. + +Past that point the engines still agree on the equity, because both are sitting +at zero, and disagree by thousands of round-trips about how many more worthless +trades to book on a dead account. That is a fact about a bankrupt strategy, not +about either engine, so the workload carries a ceiling and the runner skips it +above that with the reason printed. The other workloads have no ceiling. + +### What the windows are, and why they moved + +`sma_cross` crosses on 30/150 rather than 10/50. The two were measured against +each other on the same 5M bars, and the slower pair is worth about 15% on the +ratio (x267 against x232 with a performance summary) because it books a third of +the trades and manifoldbt's cost, unlike vectorbt's, moves with the trade count. + +That is a real effect and a small one, and it is worth knowing which way the +knobs turn before anyone quotes a number: + +| Turn up | Effect on the ratio | Why | +|---|---|---| +| series length | **widens** | vectorbt materialises the simulation; its cost is linear in bars | +| asking for the summary | **widens sharply** | it has to build the equity curve it deferred | +| number of trades | narrows | near-free for vectorbt's per-bar loop, real for manifoldbt | +| number of indicators | narrows | same reason | + +Measured at 5M bars on four levels of turnover, the ratio runs from x40 at +480,000 round-trips to x71 at 2,500. The floor matters more than the peak: even +in the busiest configuration tested, with half a million round-trips, the gap +holds at x40, and x151 with a performance summary. + Each of those runs across a range of series lengths. Two further axes, cold start and memory, are measured in their own processes because they cannot be measured honestly inside the main one. @@ -254,8 +290,30 @@ On the raptorbt side specifically: - `bench.py` - the runner - `sweep_child.py` - one parameter-grid point, in its own process - `report.py` - JSON to Markdown, and to the GitHub job summary +- `ci_activate.py` - activates the licence the grid job needs, and refuses to + continue without one - `ci/bench-vs-vectorbt.yml` - the workflow, deployed to the public repository +## Parameter grids + +Grids run in their own CI job, because they need a licence: an unlicensed +fan-out call waits out a fixed interval before doing any work, so a stopwatch +would be timing the wait rather than the engine. The harness refuses to produce +a number in that state rather than producing a wrong one. + +Three points, chosen from a measured map of the plane rather than picked. Across +bars from 5,000 to 200,000 and grids from 500 to 10,000 combinations, the ratio +on four cores moves only between x32 and x38, so a denser matrix would spend +runner time re-measuring the same number. What the three points carry is the +shape: two grid sizes at one series length, and one grid vectorbt cannot hold at +all (it materialises the simulation per combination, measured at 3.93 MB per +combination on 50,000 bars). + +Grid ratios are much more sensitive to core count than single backtests are, +because manifoldbt is the only one of the three that fans out across cores: +measured on the same point, x38 on four cores and x146 on twenty. Numbers from +the CI job are four-core numbers, and they are the conservative ones. + The directory is still named `vs_vectorbt` and the workflow file still `bench-vs-vectorbt.yml`: renaming either would break the path the public repository runs and start a fresh, empty run history. diff --git a/benchmarks/vs_vectorbt/bench.py b/benchmarks/vs_vectorbt/bench.py index 2b02594..bc102eb 100644 --- a/benchmarks/vs_vectorbt/bench.py +++ b/benchmarks/vs_vectorbt/bench.py @@ -55,6 +55,19 @@ from workloads import ( # noqa: E402 unsupported_by, ) + +def _in_range(key: str, bars: int) -> bool: + """Is this workload still a comparison at this series length? + + A ceiling is not a performance limit, it is a validity one. The fee workload + bankrupts its account past a certain length, and two engines agreeing that a + dead account is worth zero is not a measurement. Skipping is loud in the run + output rather than silent, because a table that is short by one row reads + like a choice. + """ + ceiling = WORKLOADS[key].max_bars + return ceiling is None or bars <= ceiling + # 2: timings, parity and speedups became per-engine maps when the harness grew # past two engines. `report.py` reads version 1 files as well, so the results # archived under results/ stay readable. @@ -655,15 +668,31 @@ def main() -> int: paired = [k for k in SCOPE_PAIR if k in args.workloads] singles = [k for k in args.workloads if k not in paired] + def skip(key: str, bars: int) -> None: + print(" {k:18s} {b:>12,} bars ... skipped beyond this workload's " + "ceiling of {c:,} bars".format( + k=key, b=bars, c=WORKLOADS[key].max_bars)) + results = [] failures = 0 for bars in args.bars: - if len(paired) > 1: - for entry in measure_pair(paired, bars, args.reps, workdir, active): + runnable = [k for k in paired if _in_range(k, bars)] + for key in paired: + if key not in runnable: + skip(key, bars) + if len(runnable) > 1: + for entry in measure_pair(runnable, bars, args.reps, workdir, active): results.append(entry) failures += announce(entry) - for key in singles + (paired if len(paired) == 1 else []): + elif runnable: + entry = measure(runnable[0], bars, args.reps, workdir, active) + results.append(entry) + failures += announce(entry) + for key in singles: for bars in args.bars: + if not _in_range(key, bars): + skip(key, bars) + continue entry = measure(key, bars, args.reps, workdir, active) results.append(entry) failures += announce(entry) @@ -672,7 +701,8 @@ def main() -> int: # a cold-start table cannot come back missing a column because the workload # happened to be one somebody sits out. probe_workload = next( - (k for k in args.workloads if all(supported(k, n) for n in active)), + (k for k in args.workloads + if all(supported(k, n) for n in active) and _in_range(k, 20_000)), args.workloads[0], ) diff --git a/benchmarks/vs_vectorbt/workloads.py b/benchmarks/vs_vectorbt/workloads.py index 9a9597d..c378c16 100644 --- a/benchmarks/vs_vectorbt/workloads.py +++ b/benchmarks/vs_vectorbt/workloads.py @@ -59,6 +59,10 @@ class Workload: title: str why: str params: Dict[str, Any] = field(default_factory=dict) + # Longest series this workload is valid on, or None for no ceiling. A + # workload can stop being a comparison before it stops running: see the fee + # workload, whose account this exists to keep alive. + max_bars: int | None = None # Engine name -> Note. An engine with no entry here is expected to agree # with the reference down to float-reordering noise, and a disagreement is # a failure that withholds the timing. @@ -70,11 +74,11 @@ WORKLOADS: Dict[str, Workload] = { for w in ( Workload( key="sma_cross", - title="SMA 10/50 crossover, long-only, no cost", + title="SMA 30/150 crossover, long-only, no cost", why="The canonical baseline. Unambiguous indicator, no fee policy, " "no stop semantics: if the engines disagree here, nothing else " "in the suite is worth reading.", - params=dict(fast=10, slow=50, alloc=1.0), + params=dict(fast=30, slow=150, alloc=1.0), ), Workload( key="ema_rsi_fees", @@ -87,6 +91,14 @@ WORKLOADS: Dict[str, Workload] = { "engines on a wiped-out account compares rounding noise.", params=dict(fast=12, slow=26, rsi_period=14, rsi_lo=30.0, rsi_hi=70.0, units=5.0, fee_bps=5.0), + # The account has to survive, or the engines are being compared on + # rounding noise around zero. Measured: -15% of capital at 1M bars, + # -74% at 5M, and exactly -100% at 10M, where fees reach 99,611 of + # the 100,000 started with. Past that the two disagree by thousands + # of round-trips while both sit at zero equity, which is a fact + # about a bankrupt strategy and not about either engine. The ceiling + # is set where the comparison still means something. + max_bars=1_000_000, notes={ "raptorbt": Note( "unsupported", @@ -110,7 +122,7 @@ WORKLOADS: Dict[str, Workload] = { ), Workload( key="sma_cross_metrics", - title="SMA 10/50 crossover, with a performance summary", + title="SMA 30/150 crossover, with a performance summary", why="The same simulation as `sma_cross`, but both engines are asked " "for what a user actually reads: max drawdown, Sharpe, Sortino " "and volatility alongside the return. manifoldbt computes them " @@ -120,7 +132,7 @@ WORKLOADS: Dict[str, Workload] = { "`sma_cross` above is the same work without the summary, and the " "two are reported side by side so the reader can see what the " "summary costs each engine.", - params=dict(fast=10, slow=50, alloc=1.0, metrics=True), + params=dict(fast=30, slow=150, alloc=1.0, metrics=True), ), Workload( key="bracket_sl_tp",