mirror of
https://github.com/manifoldbt/manifoldbt.git
synced 2026-11-02 07:56:11 +00:00
bench: print the measured size of every documented divergence
render() wrote the size of the first documented point and stopped, so a run with three bracket_sl_tp timings published one measured line for three rows. A divergence can grow with the number of bars, so each point now gets its own line, as a list item, and the cause follows once. test_report.py renders hand-built payloads and imports no engine.
This commit is contained in:
@@ -376,26 +376,29 @@ def _sweep_section(payload: Dict[str, Any]) -> List[str]:
|
||||
|
||||
|
||||
def _divergence_note(row: Dict[str, Any], payload: Dict[str, Any]) -> List[str]:
|
||||
"""The measured size of a documented divergence, engine by engine."""
|
||||
"""The measured size of one documented divergence, engine by engine.
|
||||
|
||||
A list item, so that the notes of several points stay one per line once
|
||||
rendered; the cause is common to all of them and render() states it once.
|
||||
"""
|
||||
scales = row.get("divergence_scale") or {}
|
||||
reference = _reference(payload)
|
||||
view = scales.get(reference)
|
||||
if not view:
|
||||
return []
|
||||
lines = [
|
||||
"`{w}` at {b:,} bars: {n} of {t} {ref} round-trips re-enter on the exit "
|
||||
note = (
|
||||
"- `{w}` at {b:,} bars: {n} of {t} {ref} round-trips re-enter on the exit "
|
||||
"bar ({p:.0%}), from {sl} stop and {tp} target exits.".format(
|
||||
w=row["workload"], b=row["bars"], n=view["reentries_on_exit_bar"],
|
||||
t=view["round_trips"], p=view["share_of_round_trips"],
|
||||
sl=view["sl_exits"], tp=view["tp_exits"], ref=reference,
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
for engine, scale in scales.items():
|
||||
if engine == reference or "round_trips" not in scale:
|
||||
continue
|
||||
lines[-1] += " {e} books {n}.".format(e=engine, n=scale["round_trips"])
|
||||
lines += ["", "Cause in {}.".format(METHOD_LINK), ""]
|
||||
return lines
|
||||
note += " {e} books {n}.".format(e=engine, n=scale["round_trips"])
|
||||
return [note]
|
||||
|
||||
|
||||
def render(payload: Dict[str, Any]) -> str:
|
||||
@@ -415,10 +418,11 @@ def render(payload: Dict[str, Any]) -> str:
|
||||
timed = [r for r in documented if r.get("timings")]
|
||||
if timed:
|
||||
lines += _speed_table(timed, "Results differ, kept out of the headline", payload)
|
||||
for row in documented:
|
||||
if row.get("divergence_scale"):
|
||||
lines += _divergence_note(row, payload)
|
||||
break
|
||||
# Every point carries its own size: a divergence can grow with the
|
||||
# number of bars, so the first point does not speak for the others.
|
||||
notes = [note for row in documented for note in _divergence_note(row, payload)]
|
||||
if notes:
|
||||
lines += notes + ["", "Cause in {}.".format(METHOD_LINK), ""]
|
||||
|
||||
if failed:
|
||||
lines += ["## Timing withheld", ""]
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Rendering tests for report.py, on payloads built by hand.
|
||||
|
||||
No engine is imported and nothing is timed: these only check what the
|
||||
published Markdown says about a given results file.
|
||||
|
||||
python -m pytest benchmarks/vs_vectorbt/test_report.py -q
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, List
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import report # noqa: E402
|
||||
|
||||
ENGINES = ["manifoldbt", "vectorbt", "raptorbt"]
|
||||
|
||||
|
||||
def _timing(seconds: float) -> Dict[str, Any]:
|
||||
return {"median_s": seconds, "min_s": seconds, "max_s": seconds, "iqr_s": 0.0}
|
||||
|
||||
|
||||
def _bracket_point(bars: int, reentries: int, trips: int, raptorbt_trips: int,
|
||||
timed: bool = True) -> Dict[str, Any]:
|
||||
"""A documented ``bracket_sl_tp`` point, with or without a measured size."""
|
||||
point: Dict[str, Any] = {
|
||||
"workload": "bracket_sl_tp",
|
||||
"bars": bars,
|
||||
"status": "documented",
|
||||
"engines": list(ENGINES),
|
||||
"parity": {},
|
||||
"divergence_scale": {
|
||||
"manifoldbt": {
|
||||
"reentries_on_exit_bar": reentries,
|
||||
"round_trips": trips,
|
||||
"share_of_round_trips": reentries / trips,
|
||||
"sl_exits": trips // 2,
|
||||
"tp_exits": trips - trips // 2,
|
||||
},
|
||||
"raptorbt": {"round_trips": raptorbt_trips},
|
||||
},
|
||||
}
|
||||
if timed:
|
||||
point["timings"] = {e: _timing(0.01 * (i + 1)) for i, e in enumerate(ENGINES)}
|
||||
point["speedup"] = {
|
||||
"vectorbt": {"median_of_ratios": 2.0},
|
||||
"raptorbt": {"median_of_ratios": 3.0},
|
||||
}
|
||||
return point
|
||||
|
||||
|
||||
def _results_file(points: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
versions = {e: "1.0" for e in ENGINES}
|
||||
versions.update(numpy="2.0", numba="0.60", pandas="2.2")
|
||||
return {
|
||||
"schema_version": 2,
|
||||
"reference": "manifoldbt",
|
||||
"engines": list(ENGINES),
|
||||
"reps": 3,
|
||||
"generated_at": "2026-01-01T00:00:00Z",
|
||||
"environment": {
|
||||
"os": "Linux", "arch": "x86_64", "cpu": "test cpu", "logical_cores": 4,
|
||||
"ram_gb": 16, "python": "3.12", "versions": versions,
|
||||
},
|
||||
"results": points,
|
||||
}
|
||||
|
||||
|
||||
def _size_lines(text: str) -> List[str]:
|
||||
return [line for line in text.splitlines() if "re-enter on the exit bar" in line]
|
||||
|
||||
|
||||
def test_each_documented_point_states_its_own_size():
|
||||
points = [
|
||||
_bracket_point(10_000, 90, 230, 140),
|
||||
_bracket_point(100_000, 925, 2309, 1384),
|
||||
_bracket_point(1_000_000, 9100, 23000, 21000),
|
||||
]
|
||||
|
||||
lines = _size_lines(report.render(_results_file(points)))
|
||||
|
||||
assert len(lines) == 3
|
||||
for bars, line in zip(("10,000", "100,000", "1,000,000"), lines):
|
||||
assert "at {} bars".format(bars) in line
|
||||
|
||||
|
||||
def test_the_sizes_are_list_items_under_one_cause():
|
||||
points = [_bracket_point(10_000, 90, 230, 140), _bracket_point(100_000, 925, 2309, 1384)]
|
||||
|
||||
text = report.render(_results_file(points))
|
||||
|
||||
assert all(line.startswith("- `bracket_sl_tp`") for line in _size_lines(text))
|
||||
assert text.count("Cause in ") == 1
|
||||
|
||||
|
||||
def test_a_point_without_a_size_adds_no_line():
|
||||
sized = _bracket_point(10_000, 90, 230, 140)
|
||||
unsized = _bracket_point(100_000, 925, 2309, 1384)
|
||||
del unsized["divergence_scale"]
|
||||
|
||||
text = report.render(_results_file([sized, unsized]))
|
||||
|
||||
assert len(_size_lines(text)) == 1
|
||||
assert text.count("| bracket_sl_tp |") == 2
|
||||
|
||||
|
||||
def test_no_size_at_all_means_no_cause_line():
|
||||
point = _bracket_point(10_000, 90, 230, 140)
|
||||
del point["divergence_scale"]
|
||||
|
||||
assert "Cause in " not in report.render(_results_file([point]))
|
||||
Reference in New Issue
Block a user