mirror of
https://github.com/manifoldbt/manifoldbt.git
synced 2026-11-02 07:56:11 +00:00
bench: generate the same bars on every machine
The site joins the GPU run with the CPU run on the digest of their data, and that digest depended on the processor. The walk was exp(cumsum(log returns)), and numpy's exp rounds differently on an AVX-512 CPU, without AVX-512 and on Windows: one seed, three datasets. The CPU run of 2026-10-25 landed on an EPYC 9V45 (Zen 5) and the GPU run of 2026-10-26 on a Modal host without AVX-512, so their points no longer matched and the ratios dropped out. The walk is now 100 * cumprod(1 + r): additions and multiplications only, rounded exactly whatever the CPU. Checked on numpy 2.4.3: the normal and uniform draws and the new close are the same bits on Linux and Windows, where exp was not. test_data.py pins it (the walk equals a plain Python product of the draws, and the digest of the default dataset is fixed), and the bench-report CI job now runs it. The GPU report also names the runner's processor beside the ratio, since GitHub hands out different generations and the ratio moves with them: the same L4 times read x17.7 against an EPYC 9V45 and x36.3 against an EPYC 7763. The dataset changes, so digests recorded before this commit will not match runs made after it.
This commit is contained in:
@@ -19,7 +19,10 @@ name: CI
|
||||
# (benchmarks/vs_vectorbt/test_report.py). They import no engine and
|
||||
# need no wheel; without this job nothing collects them, because the
|
||||
# root testpaths points at python/tests. PR #13 arrived with tests
|
||||
# their author could not wire in, and this is the wiring.
|
||||
# their author could not wire in, and this is the wiring. It also
|
||||
# runs test_data.py, which pins that the generated bars are the
|
||||
# same bits on every machine, the condition for joining the CPU and
|
||||
# GPU runs on their data digest.
|
||||
#
|
||||
# The second job is the point. The suite shipped in this repository for months
|
||||
# without anything running it, so "there are tests" was a claim a reader had to
|
||||
@@ -53,8 +56,8 @@ jobs:
|
||||
python-version: "3.12"
|
||||
- name: Run the harness's rendering tests
|
||||
run: |
|
||||
python -m pip install --quiet pytest
|
||||
python -m pytest benchmarks/vs_vectorbt/test_report.py -q
|
||||
python -m pip install --quiet pytest numpy==2.4.3 pandas==2.3.3
|
||||
python -m pytest benchmarks/vs_vectorbt/test_report.py benchmarks/vs_vectorbt/test_data.py -q
|
||||
|
||||
verify:
|
||||
name: Verify package layout
|
||||
|
||||
@@ -40,6 +40,7 @@ def cpu_reference(published: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"points": points,
|
||||
"version": (env.get("versions") or {}).get("manifoldbt"),
|
||||
"cores": env.get("logical_cores"),
|
||||
"cpu_model": env.get("cpu"),
|
||||
"run_url": (published or {}).get("run_url", ""),
|
||||
}
|
||||
|
||||
@@ -69,9 +70,11 @@ def render(payload: Dict[str, Any], cpu: Optional[Dict[str, Any]] = None) -> str
|
||||
if cpu["points"]:
|
||||
lines.append(
|
||||
"CPU column: the CPU sweeps job's published run{}, manifoldbt {} on a "
|
||||
"{}-vCPU GitHub runner.{}".format(
|
||||
"{}-vCPU GitHub runner{}. GitHub does not always hand out the same "
|
||||
"processor, so the ratio moves with it.{}".format(
|
||||
" ({})".format(cpu["run_url"]) if cpu["run_url"] else "",
|
||||
cpu["version"] or "?", cpu["cores"] or "?",
|
||||
" ({})".format(cpu["cpu_model"]) if cpu.get("cpu_model") else "",
|
||||
"" if same_version else
|
||||
" Not the version measured here, so no ratio is given."))
|
||||
lines.append("")
|
||||
|
||||
@@ -12,6 +12,14 @@ Two properties are deliberate, not incidental:
|
||||
the gate.
|
||||
* the intrabar range is wide enough that percentage stops and targets actually
|
||||
trigger, otherwise the bracket workload would measure an empty branch.
|
||||
|
||||
The bars are also the same on every machine, which is what lets two runs on two
|
||||
machines be joined on the digest. The walk is built from additions and
|
||||
multiplications only, each rounded exactly by IEEE 754 whatever the CPU. It
|
||||
used to be ``exp(cumsum(log returns))``, and numpy's ``exp`` is not that: its
|
||||
AVX-512 kernel, glibc's and the Windows runtime's each round some values
|
||||
differently, so the same seed gave three datasets (and two GitHub runners of
|
||||
different generations, two digests).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -35,8 +43,9 @@ def make_ohlcv(
|
||||
"""A gap-free random walk of ``rows`` bars, reproducible from ``seed``."""
|
||||
rng = np.random.default_rng(seed)
|
||||
|
||||
log_ret = rng.normal(drift, vol, size=rows)
|
||||
close = 100.0 * np.exp(np.cumsum(log_ret))
|
||||
ret = rng.normal(drift, vol, size=rows)
|
||||
# A sequential product: no transcendental function, no reassociation.
|
||||
close = 100.0 * np.cumprod(1.0 + ret)
|
||||
|
||||
open_ = np.empty(rows, dtype=np.float64)
|
||||
open_[0] = 100.0
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""The generated bars are the same on every machine.
|
||||
|
||||
The CPU benchmark runs on a GitHub runner and the GPU benchmark on a rented
|
||||
card; the site joins their times on the digest of the data. That join only
|
||||
holds if the same seed gives the same bits on every CPU. It did not while the
|
||||
walk went through ``numpy.exp``: an AVX-512 runner, a runner without AVX-512
|
||||
and Windows gave three digests for one seed. These tests pin the property, not
|
||||
the machine: the walk is a sequential product, and the digest of the default
|
||||
dataset is fixed for the numpy release in requirements-lock.txt.
|
||||
"""
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import data
|
||||
|
||||
# numpy==2.4.3 (requirements-lock.txt); identical on Linux and Windows.
|
||||
PINNED = {20_000: "0d360424244ea1ee", 200_000: "41728ac4d4fb6c04"}
|
||||
|
||||
|
||||
def test_the_walk_is_a_sequential_product_of_the_draws():
|
||||
rows = 5_000
|
||||
frame = data.make_ohlcv(rows)
|
||||
ret = np.random.default_rng(data.DEFAULT_SEED).normal(2e-7, 3e-4, size=rows)
|
||||
level = 1.0
|
||||
expected = []
|
||||
for r in ret:
|
||||
level *= 1.0 + r
|
||||
expected.append(100.0 * level)
|
||||
assert frame["close"].to_numpy().tobytes() == np.asarray(expected).tobytes()
|
||||
|
||||
|
||||
def test_open_is_the_previous_close():
|
||||
frame = data.make_ohlcv(1_000)
|
||||
assert frame["open"].iloc[0] == 100.0
|
||||
assert (frame["open"].to_numpy()[1:] == frame["close"].to_numpy()[:-1]).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(np.__version__ != "2.4.3", reason="digest pinned for numpy 2.4.3")
|
||||
@pytest.mark.parametrize("rows, digest", sorted(PINNED.items()))
|
||||
def test_the_digest_does_not_depend_on_the_machine(rows, digest):
|
||||
assert data.digest(data.make_ohlcv(rows)) == digest
|
||||
Reference in New Issue
Block a user