bench: print the measured size of every documented divergence

render() wrote the size of the first documented point and stopped, so a
run with three bracket_sl_tp timings published one measured line for
three rows. A divergence can grow with the number of bars, so each point
now gets its own line, as a list item, and the cause follows once.

test_report.py renders hand-built payloads and imports no engine.
This commit is contained in:
Jimmy7892
2026-10-23 23:03:30 +02:00
parent 8126c84120
commit 65910ba388
2 changed files with 129 additions and 12 deletions
+16 -12
View File
@@ -376,26 +376,29 @@ def _sweep_section(payload: Dict[str, Any]) -> List[str]:
def _divergence_note(row: Dict[str, Any], payload: Dict[str, Any]) -> List[str]:
"""The measured size of a documented divergence, engine by engine."""
"""The measured size of one documented divergence, engine by engine.
A list item, so that the notes of several points stay one per line once
rendered; the cause is common to all of them and render() states it once.
"""
scales = row.get("divergence_scale") or {}
reference = _reference(payload)
view = scales.get(reference)
if not view:
return []
lines = [
"`{w}` at {b:,} bars: {n} of {t} {ref} round-trips re-enter on the exit "
note = (
"- `{w}` at {b:,} bars: {n} of {t} {ref} round-trips re-enter on the exit "
"bar ({p:.0%}), from {sl} stop and {tp} target exits.".format(
w=row["workload"], b=row["bars"], n=view["reentries_on_exit_bar"],
t=view["round_trips"], p=view["share_of_round_trips"],
sl=view["sl_exits"], tp=view["tp_exits"], ref=reference,
),
]
)
)
for engine, scale in scales.items():
if engine == reference or "round_trips" not in scale:
continue
lines[-1] += " {e} books {n}.".format(e=engine, n=scale["round_trips"])
lines += ["", "Cause in {}.".format(METHOD_LINK), ""]
return lines
note += " {e} books {n}.".format(e=engine, n=scale["round_trips"])
return [note]
def render(payload: Dict[str, Any]) -> str:
@@ -415,10 +418,11 @@ def render(payload: Dict[str, Any]) -> str:
timed = [r for r in documented if r.get("timings")]
if timed:
lines += _speed_table(timed, "Results differ, kept out of the headline", payload)
for row in documented:
if row.get("divergence_scale"):
lines += _divergence_note(row, payload)
break
# Every point carries its own size: a divergence can grow with the
# number of bars, so the first point does not speak for the others.
notes = [note for row in documented for note in _divergence_note(row, payload)]
if notes:
lines += notes + ["", "Cause in {}.".format(METHOD_LINK), ""]
if failed:
lines += ["## Timing withheld", ""]
+113
View File
@@ -0,0 +1,113 @@
"""Rendering tests for report.py, on payloads built by hand.
No engine is imported and nothing is timed: these only check what the
published Markdown says about a given results file.
python -m pytest benchmarks/vs_vectorbt/test_report.py -q
"""
from __future__ import annotations
import os
import sys
from typing import Any, Dict, List
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import report # noqa: E402
ENGINES = ["manifoldbt", "vectorbt", "raptorbt"]
def _timing(seconds: float) -> Dict[str, Any]:
return {"median_s": seconds, "min_s": seconds, "max_s": seconds, "iqr_s": 0.0}
def _bracket_point(bars: int, reentries: int, trips: int, raptorbt_trips: int,
timed: bool = True) -> Dict[str, Any]:
"""A documented ``bracket_sl_tp`` point, with or without a measured size."""
point: Dict[str, Any] = {
"workload": "bracket_sl_tp",
"bars": bars,
"status": "documented",
"engines": list(ENGINES),
"parity": {},
"divergence_scale": {
"manifoldbt": {
"reentries_on_exit_bar": reentries,
"round_trips": trips,
"share_of_round_trips": reentries / trips,
"sl_exits": trips // 2,
"tp_exits": trips - trips // 2,
},
"raptorbt": {"round_trips": raptorbt_trips},
},
}
if timed:
point["timings"] = {e: _timing(0.01 * (i + 1)) for i, e in enumerate(ENGINES)}
point["speedup"] = {
"vectorbt": {"median_of_ratios": 2.0},
"raptorbt": {"median_of_ratios": 3.0},
}
return point
def _results_file(points: List[Dict[str, Any]]) -> Dict[str, Any]:
versions = {e: "1.0" for e in ENGINES}
versions.update(numpy="2.0", numba="0.60", pandas="2.2")
return {
"schema_version": 2,
"reference": "manifoldbt",
"engines": list(ENGINES),
"reps": 3,
"generated_at": "2026-01-01T00:00:00Z",
"environment": {
"os": "Linux", "arch": "x86_64", "cpu": "test cpu", "logical_cores": 4,
"ram_gb": 16, "python": "3.12", "versions": versions,
},
"results": points,
}
def _size_lines(text: str) -> List[str]:
return [line for line in text.splitlines() if "re-enter on the exit bar" in line]
def test_each_documented_point_states_its_own_size():
points = [
_bracket_point(10_000, 90, 230, 140),
_bracket_point(100_000, 925, 2309, 1384),
_bracket_point(1_000_000, 9100, 23000, 21000),
]
lines = _size_lines(report.render(_results_file(points)))
assert len(lines) == 3
for bars, line in zip(("10,000", "100,000", "1,000,000"), lines):
assert "at {} bars".format(bars) in line
def test_the_sizes_are_list_items_under_one_cause():
points = [_bracket_point(10_000, 90, 230, 140), _bracket_point(100_000, 925, 2309, 1384)]
text = report.render(_results_file(points))
assert all(line.startswith("- `bracket_sl_tp`") for line in _size_lines(text))
assert text.count("Cause in ") == 1
def test_a_point_without_a_size_adds_no_line():
sized = _bracket_point(10_000, 90, 230, 140)
unsized = _bracket_point(100_000, 925, 2309, 1384)
del unsized["divergence_scale"]
text = report.render(_results_file([sized, unsized]))
assert len(_size_lines(text)) == 1
assert text.count("| bracket_sl_tp |") == 2
def test_no_size_at_all_means_no_cause_line():
point = _bracket_point(10_000, 90, 230, 140)
del point["divergence_scale"]
assert "Cause in " not in report.render(_results_file([point]))