manifoldbt/.github/workflows/bench-vs-vectorbt.yml

# The benchmark runs here, in the open, on standard GitHub-hosted runners.
#
# Nothing in this workflow has access to the engine source: it installs the
# published wheel from PyPI, exactly like any user would. That is what makes the
# result independent rather than self-reported, and it is why anyone can fork
# this repository and press "Run workflow" to reproduce the numbers on their own
# runner. Same for the engines it is compared against: they come from PyPI at
# pinned versions, and the run prints the versions it resolved.

name: Benchmark vs vectorbt and raptorbt

on:
  workflow_dispatch:
    inputs:
      manifoldbt_version:
        description: "manifoldbt version to install from PyPI (blank = latest)"
        required: false
        default: ""
      reps:
        description: "Interleaved repetitions per point"
        required: false
        # 2, not 7. The budget went into a longer series instead: at 10M bars
        # one vectorbt call costs 99 s, so seven of them would put the job past
        # its timeout on Windows. Two repetitions still bracket the number (min,
        # median and max are all published, and the median of two is their mean)
        # but the noise flag gets cruder, since an interquartile range wants at
        # least four samples to mean anything. A point that looks surprising is
        # worth re-running at a higher `reps` before it is quoted anywhere.
        default: "2"
  release:
    types: [published]
  schedule:
    # Weekly, to catch a slowdown introduced by a dependency rather than by us.
    - cron: "17 5 * * 1"

permissions:
  contents: read

concurrency:
  group: bench-vs-vectorbt-${{ github.ref }}
  cancel-in-progress: true

jobs:
  bench:
    name: ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    # A published release is not always a released version. The preview tags
    # 0.18.0rc1 and rc2 went to the public repository without ever reaching
    # PyPI, so `pip install manifoldbt==0.18.0rc2` failed and this workflow went
    # red four times in two days for a reason that had nothing to do with any
    # engine. Pre-releases are skipped; a manual dispatch can still benchmark
    # one by naming the version, if it ever exists on PyPI.
    if: github.event_name != 'release' || github.event.release.prerelease == false
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        include:
          # Three lengths, not five. 10k was sub-millisecond on the engine
          # side, which measures the clock rather than the work, and 5M sat
          # between two points that already bracket it. What is left is a
          # decade apart each step, which is what makes the trend readable.
          - os: ubuntu-latest
            bars: "100000 1000000 10000000"
          - os: windows-latest
            bars: "100000 1000000 10000000"
          # macOS runners ship 7 GB of RAM against 16 GB elsewhere, and vectorbt
          # materialises the simulation in memory (roughly 150 MB per million
          # bars, measured). The top size is trimmed so a point is never lost to
          # swapping, which would time the disk instead of the engine. The same
          # arithmetic is why 10M bars is added on the other two and not here:
          # measured, that point adds 1.55 GB on vectorbt's side alone.
          - os: macos-latest
            bars: "100000 1000000"

    env:
      PYTHONUNBUFFERED: "1"

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          # 3.12, not 3.13, and it is raptorbt that pins it: it is built against
          # pyo3 0.20.3, whose maximum supported CPython is 3.12. No release up
          # to 0.9.0 publishes a cp313 wheel and a source build refuses outright
          # ("the configured Python interpreter version (3.13) is newer than
          # PyO3's maximum supported version (3.12)"). Comparing engines means
          # running them in one environment, and the environment has to be one
          # they all support. Runs before 2026-08-20 used 3.13 with two engines,
          # so their absolute timings are not directly comparable with these.
          python-version: "3.12"

      - name: Install engines from PyPI
        shell: bash
        run: |
          set -e
          VERSION="${{ inputs.manifoldbt_version }}"
          # A release run benchmarks the version that was just published.
          if [ -z "$VERSION" ] && [ "${{ github.event_name }}" = "release" ]; then
            VERSION="$(echo '${{ github.event.release.tag_name }}' | sed 's/^v//')"
          fi
          python -m pip install --upgrade pip
          if [ -n "$VERSION" ]; then
            pip install "manifoldbt==${VERSION}"
          else
            pip install manifoldbt
          fi
          pip install -r benchmarks/vs_vectorbt/requirements-lock.txt

      - name: Record the resolved environment
        shell: bash
        run: pip freeze | grep -iE '^(manifoldbt|vectorbt|raptorbt|numpy|numba|pandas|psutil)=' || true

      - name: Run the benchmark
        shell: bash
        working-directory: benchmarks/vs_vectorbt
        run: |
          python bench.py \
            --bars ${{ matrix.bars }} \
            --reps "${{ inputs.reps || '7' }}" \
            --cold-start-reps 3 \
            --memory-bars 2000000 \
            --out "results-${{ matrix.os }}.json"

      - name: Render the report
        # Runs even when the benchmark exits non-zero: a parity failure is the
        # most interesting thing that can happen here, and it must be readable
        # in the job summary rather than buried in a red step. But only if there
        # is something to render: when the install step failed, this used to die
        # on a missing file and put a FileNotFoundError on top of the real
        # error, which is how a run reports the wrong cause twice.
        if: always()
        shell: bash
        working-directory: benchmarks/vs_vectorbt
        run: |
          if [ -f "results-${{ matrix.os }}.json" ]; then
            python report.py "results-${{ matrix.os }}.json"
          else
            echo "no result file: the benchmark did not get far enough to write one"
          fi

      - name: Upload the raw result
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: bench-${{ matrix.os }}
          path: benchmarks/vs_vectorbt/results-*.json
          if-no-files-found: warn

  # ------------------------------------------------------------------------ #
  # Parameter grids, which need a licence and therefore a job of their own
  # ------------------------------------------------------------------------ #
  sweeps:
    name: sweeps (ubuntu)
    runs-on: ubuntu-latest
    if: github.event_name != 'release' || github.event.release.prerelease == false
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install engines from PyPI
        shell: bash
        run: |
          set -e
          VERSION="${{ inputs.manifoldbt_version }}"
          if [ -z "$VERSION" ] && [ "${{ github.event_name }}" = "release" ]; then
            VERSION="$(echo '${{ github.event.release.tag_name }}' | sed 's/^v//')"
          fi
          python -m pip install --upgrade pip
          if [ -n "$VERSION" ]; then pip install "manifoldbt==${VERSION}"; else pip install manifoldbt; fi
          pip install -r benchmarks/vs_vectorbt/requirements-lock.txt

      # A grid benchmark without a licence does not fail, it produces a wrong
      # number: every unlicensed fan-out call waits a fixed interval before any
      # work starts, so the stopwatch would time the wait. This step exits
      # non-zero rather than let that happen, and the harness refuses again on
      # its own if the tier is not what it expects.
      - name: Activate the benchmark licence
        env:
          MANIFOLDBT_CI_LICENSE: ${{ secrets.MANIFOLDBT_CI_LICENSE }}
        shell: bash
        working-directory: benchmarks/vs_vectorbt
        run: python ci_activate.py

      - name: Run the grids
        shell: bash
        working-directory: benchmarks/vs_vectorbt
        # Three points, chosen from a measured map of the bars-by-combinations
        # plane rather than picked: across it the ratio moves between x32 and
        # x38, so a denser matrix would spend runner time re-measuring the same
        # number. What the three do carry is the shape of the thing: two grid
        # sizes at one series length, and one grid vectorbt cannot hold at all.
        run: |
          python bench.py             --workloads sma_cross             --bars 100000             --reps 1             --cold-start-reps 0             --sweep 20000:2500 20000:5000 200000:2500:oos             --sweep-reps "${{ inputs.reps || '2' }}"             --out "results-sweeps.json"

      - name: Render the report
        if: always()
        shell: bash
        working-directory: benchmarks/vs_vectorbt
        run: |
          if [ -f results-sweeps.json ]; then
            python report.py results-sweeps.json
          else
            echo "no result file: the grids did not get far enough to write one"
          fi

      - name: Upload the raw result
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: bench-sweeps
          path: benchmarks/vs_vectorbt/results-sweeps.json
          if-no-files-found: warn