bench: report an engine that was held back, or has no drawdown, in words

The report assumed every engine ran on every row. With a ceiling that is
no longer true, so:

- "Not run, and why" prints one reason once for all the workloads it
  covers, instead of once per workload.
- The CPU-over-wall row reads each engine at the longest series it ran,
  where it used to need every engine at one size and would have vanished.
- A memory figure taken at an engine's ceiling carries its bar count.
- An engine with no maximum drawdown is said to have none, rather than
  "it was not compared", and its summary workload is described as gated on
  total return and round-trips.
- A documented divergence where an engine takes the same trades and sizes
  one differently reports how many re-entries and how far apart.
- Probes that died for a challenger are listed under the table.

test_harness.py adds guards for the gate (an engine without a drawdown,
the documented bracket divergence), the ceilings, the sweep scope rules
(including that a recipe saying only `:oos` cannot start Nautilus on a
big grid), the out-of-scope reasons and these report changes. Like
test_report.py it needs no engine installed.

benchmarks/vs_vectorbt/report.py

@@ -201,15 +201,23 @@ def _summary_cost(exact: List[Dict[str, Any]], payload: Dict[str, Any]) -> List[
             )
     lines.append("")
 
-    top = summarised[shared[-1]]
     advisory = {}
     basis_differs = {}
-    for engine, verdict in top["parity"].items():
-        diffs = verdict.get("diffs", {})
-        if diffs.get("advisory_ratio_rel"):
-            advisory[engine] = max(diffs["advisory_ratio_rel"].values())
-        elif diffs.get("ratio_basis"):
-            basis_differs[engine] = diffs.get("max_drawdown_rel")
+    no_drawdown = set()
+    # Each engine is read at the longest series it ran on: one held back by its
+    # ceiling has no verdict at the longest size, and reading only that row would
+    # leave it out of the sentence.
+    for bars in reversed(shared):
+        for engine, verdict in summarised[bars]["parity"].items():
+            if engine in advisory or engine in basis_differs:
+                continue
+            diffs = verdict.get("diffs", {})
+            if diffs.get("max_drawdown_absent"):
+                no_drawdown.add(engine)
+            if diffs.get("advisory_ratio_rel"):
+                advisory[engine] = max(diffs["advisory_ratio_rel"].values())
+            elif diffs.get("ratio_basis"):
+                basis_differs[engine] = diffs.get("max_drawdown_rel")
     if advisory:
         lines += [
             "Gated on total return, round-trips and max drawdown (exact). Sharpe, "
@@ -217,6 +225,17 @@ def _summary_cost(exact: List[Dict[str, Any]], payload: Dict[str, Any]) -> List[
             "",
         ]
     for engine, drawdown_rel in basis_differs.items():
+        if engine in no_drawdown:
+            # Not "it was not compared": it does not exist. The engine offers
+            # volatility, Sharpe and Sortino and no drawdown statistic, and the
+            # harness does not build one for it.
+            lines += [
+                "{e} annualises its ratios on its own basis and reports no maximum "
+                "drawdown, so this workload is gated on total return and "
+                "round-trips only (exact).".format(e=engine),
+                "",
+            ]
+            continue
         if drawdown_rel is None:
             agreement = "was not compared"
         elif drawdown_rel == 0.0:
@@ -235,12 +254,19 @@ def _side_measures(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> Li
     cold = payload.get("cold_start")
     mem = payload.get("memory")
     engines = _engines(payload)
-    threading_rows = [
-        r for r in results
-        if r.get("cpu_over_wall")
-        and all(r["cpu_over_wall"].get(e) is not None for e in engines)
-    ]
-    if not (cold or mem or threading_rows):
+
+    # CPU over wall time, per engine, at the longest series that engine ran and
+    # produced a reading for. An engine held back by its ceiling has no reading at
+    # the longest size, and requiring every engine at one size would drop the row
+    # for all of them.
+    threading: Dict[str, Any] = {}
+    for engine in engines:
+        rows = [r for r in results
+                if (r.get("cpu_over_wall") or {}).get(engine) is not None]
+        if rows:
+            best = max(rows, key=lambda r: r["bars"])
+            threading[engine] = (best["cpu_over_wall"][engine], best["bars"])
+    if not (cold or mem or threading):
         return []
 
     lines = [
@@ -257,15 +283,38 @@ def _side_measures(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> Li
                      + " | ".join("{:.2f} s".format(share[e]) if e in share else "-"
                                   for e in engines) + " |")
     if mem:
-        lines.append("| RAM added by the run, per 1M bars | " + " | ".join(
-            "{:.0f} MB".format(mem[e]["added_mb_per_million_bars"]) if e in mem else "-"
-            for e in engines) + " |")
-    if threading_rows:
-        biggest = max(threading_rows, key=lambda r: r["bars"])
-        ratio = biggest["cpu_over_wall"]
-        lines.append("| CPU over wall time at {:,} bars | ".format(biggest["bars"])
-                     + " | ".join("{:.2f}".format(ratio[e]) for e in engines) + " |")
+        def memory_cell(engine: str) -> str:
+            if engine not in mem or not isinstance(mem[engine], dict):
+                return "-"
+            probe = mem[engine]
+            text = "{:.0f} MB".format(probe["added_mb_per_million_bars"])
+            # An engine held back by its ceiling is measured at the ceiling, and
+            # the figure is scaled to a million bars like the others. Say so
+            # beside the number rather than leave a reader to assume the same
+            # series length.
+            if probe.get("bars") and probe["bars"] != mem.get("bars"):
+                text += " (at {:,} bars)".format(probe["bars"])
+            return text
+
+        lines.append("| RAM added by the run, per 1M bars | "
+                     + " | ".join(memory_cell(e) for e in engines) + " |")
+    if threading:
+        sizes = {bars for _, bars in threading.values()}
+        if len(sizes) == 1:
+            lines.append("| CPU over wall time at {:,} bars | ".format(sizes.pop())
+                         + " | ".join("{:.2f}".format(threading[e][0]) if e in threading else "-"
+                                      for e in engines) + " |")
+        else:
+            lines.append("| CPU over wall time, at the longest series each ran | " + " | ".join(
+                "{:.2f} (at {:,})".format(*threading[e]) if e in threading else "-"
+                for e in engines) + " |")
     lines.append("")
+    if mem and mem.get("failed"):
+        for engine, why in mem["failed"].items():
+            lines += ["- Memory probe for {}: {}".format(engine, why), ""]
+    if cold and cold.get("failed"):
+        for engine, why in cold["failed"].items():
+            lines += ["- Cold-start probe for {}: {}".format(engine, why), ""]
     return lines
 
 
@@ -275,7 +324,9 @@ def _not_run(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> List[str
     Kept as prose rather than a table because the reason is the content. An
     engine that cannot express a workload has told you something about itself,
     and compressing that into an empty cell would throw away the only part worth
-    reading.
+    reading. An engine held back by its ceiling gets the same treatment, and the
+    same reason is printed once with the list of workloads it applies to, not
+    once per workload.
     """
     seen: Dict[str, Dict[str, str]] = {}
     for row in results:
@@ -287,8 +338,12 @@ def _not_run(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> List[str
 
     lines = ["## Not run, and why", ""]
     for engine, entries in seen.items():
+        by_reason: Dict[str, List[str]] = {}
         for workload, why in entries.items():
-            lines += ["- **{e} on `{w}`.** {why}".format(e=engine, w=workload, why=why), ""]
+            by_reason.setdefault(why, []).append(workload)
+        for why, workloads in by_reason.items():
+            lines += ["- **{e} on {w}.** {why}".format(
+                e=engine, w=", ".join("`{}`".format(w) for w in workloads), why=why), ""]
     return lines
 
 
@@ -398,6 +453,15 @@ def _divergence_note(row: Dict[str, Any], payload: Dict[str, Any]) -> List[str]:
         if engine == reference or "round_trips" not in scale:
             continue
         note += " {e} books {n}.".format(e=engine, n=scale["round_trips"])
+        # An engine that takes the same trades and sizes one of them differently
+        # says how far: a count and a median, not an adjective.
+        if scale.get("reentries_on_exit_bar") is not None:
+            note += (" Of those, {n} re-enter on the exit bar, with a quantity that "
+                     "sits a median of {m:.1e} (at most {x:.1e}), in relative terms, "
+                     "away from the position closed on that bar.".format(
+                         n=scale["reentries_on_exit_bar"],
+                         m=scale.get("reentry_quantity_gap_median", 0.0),
+                         x=scale.get("reentry_quantity_gap_max", 0.0)))
     return [note]
 
 

benchmarks/vs_vectorbt/test_harness.py

@@ -0,0 +1,362 @@
+"""Guards for the parts of the harness that decide what gets published.
+
+These run without any engine installed: they exercise the gate, the ceilings and
+the sweep plumbing, which are plain Python. Run with
+``pytest benchmarks/vs_vectorbt/test_harness.py``.
+"""
+from __future__ import annotations
+
+import argparse
+
+import pytest
+
+import bench
+import parity
+import report
+import sweep_child
+from engines import ENGINES, REFERENCE, ceiling
+
+
+# --------------------------------------------------------------------------- #
+# The gate, for an engine that has no drawdown of its own
+# --------------------------------------------------------------------------- #
+def _metrics(**over):
+    base = {
+        "total_return": 0.01, "final_equity": 101_000.0, "round_trips": 10,
+        "fills": 20, "total_fees": 0.0,
+        "max_drawdown": -0.02, "sharpe": 1.0, "sortino": 2.0, "volatility": 0.1,
+    }
+    base.update(over)
+    return base
+
+
+def test_an_engine_without_a_drawdown_is_gated_on_money_and_round_trips():
+    verdict = parity.compare(
+        _metrics(), _metrics(max_drawdown=None, sharpe=9.0), "sma_cross_metrics",
+        "nautilus_trader")
+
+    assert verdict["status"] == "exact"
+    assert verdict["diffs"]["max_drawdown_absent"] is True
+    assert "max_drawdown_rel" not in verdict["diffs"]
+    # Its ratios sit on their own basis, so they are not subtracted from ours.
+    assert verdict["diffs"]["ratio_basis"] == "native"
+    assert "advisory_ratio_rel" not in verdict["diffs"]
+
+
+def test_an_engine_without_a_drawdown_still_fails_on_money():
+    verdict = parity.compare(
+        _metrics(), _metrics(max_drawdown=None, final_equity=101_050.0),
+        "sma_cross_metrics", "nautilus_trader")
+
+    assert verdict["status"] == "failed"
+
+
+def test_a_drawdown_that_is_present_is_still_gated():
+    verdict = parity.compare(
+        _metrics(), _metrics(max_drawdown=-0.03), "sma_cross_metrics", "raptorbt")
+
+    assert verdict["status"] == "failed"
+    assert verdict["diffs"]["max_drawdown_rel"] > parity.REL_TOL
+
+
+def test_the_documented_bracket_divergence_is_expected_for_nautilus():
+    verdict = parity.compare(
+        _metrics(), _metrics(final_equity=101_000.7, max_drawdown=None),
+        "bracket_sl_tp", "nautilus_trader")
+
+    assert verdict["expected"] == "documented"
+    assert verdict["status"] == "documented"
+
+
+# --------------------------------------------------------------------------- #
+# Ceilings
+# --------------------------------------------------------------------------- #
+@pytest.fixture(autouse=True)
+def _no_overrides():
+    bench.CEILING_OVERRIDES.clear()
+    bench.SWEEP_CEILING_OVERRIDES.clear()
+    yield
+    bench.CEILING_OVERRIDES.clear()
+    bench.SWEEP_CEILING_OVERRIDES.clear()
+
+
+def test_only_the_event_driven_engine_has_a_ceiling_by_default():
+    limited = {n for n in ENGINES if ceiling(n) is not None}
+
+    assert limited == {"nautilus_trader"}
+    assert REFERENCE not in limited
+
+
+def test_an_engine_runs_up_to_its_ceiling_and_not_beyond():
+    limit = ceiling("nautilus_trader")
+
+    assert bench._within_ceiling("nautilus_trader", limit)
+    assert not bench._within_ceiling("nautilus_trader", limit + 1)
+    assert bench._within_ceiling("vectorbt", 10_000_000)
+
+
+def test_an_override_moves_a_ceiling_and_zero_removes_it():
+    bench.CEILING_OVERRIDES["nautilus_trader"] = 1_000_000
+    assert bench._within_ceiling("nautilus_trader", 1_000_000)
+    assert not bench._within_ceiling("nautilus_trader", 1_000_001)
+
+    bench.CEILING_OVERRIDES["nautilus_trader"] = 0
+    assert bench._within_ceiling("nautilus_trader", 100_000_000)
+
+
+def test_a_held_back_engine_gets_a_reason_where_the_site_already_looks_for_one():
+    """`unsupported` is the field the website prints under "Not run, and why"."""
+    active = ["manifoldbt", "vectorbt", "nautilus_trader"]
+    long_run = bench._entry("sma_cross", 1_000_000, frame=_Frame(1_000_000), verdicts={},
+                            engines_run=["manifoldbt", "vectorbt"], active=active)
+    short_run = bench._entry("sma_cross", 100_000, frame=_Frame(100_000), verdicts={},
+                             engines_run=active, active=active)
+
+    assert "nautilus_trader" in long_run["unsupported"]
+    assert long_run["unsupported"]["nautilus_trader"].startswith("Not run above 100,000 bars")
+    assert long_run["beyond_ceiling"] == {"nautilus_trader": 100_000}
+    assert "unsupported" not in short_run
+    assert "beyond_ceiling" not in short_run
+
+
+def test_a_ceiling_is_not_reported_for_a_workload_the_engine_cannot_run_anyway():
+    """raptorbt's blocker is the reason on those rows, not a ceiling."""
+    entry = bench._entry("sma_cross_costs", 1_000_000, frame=_Frame(1_000_000), verdicts={},
+                         engines_run=["manifoldbt", "vectorbt"],
+                         active=["manifoldbt", "vectorbt", "raptorbt"])
+
+    assert set(entry["unsupported"]) == {"raptorbt"}
+    assert "beyond_ceiling" not in entry
+
+
+class _Frame:
+    """Stands in for a DataFrame where only the length and a digest are read."""
+
+    def __init__(self, rows):
+        self.rows = rows
+
+    def __len__(self):
+        return self.rows
+
+    def __getitem__(self, column):
+        import numpy as np
+
+        return _Column(np.zeros(self.rows))
+
+
+class _Column:
+    def __init__(self, values):
+        self.values = values
+
+    def to_numpy(self, dtype=None):
+        return self.values
+
+
+# --------------------------------------------------------------------------- #
+# Sweep points
+# --------------------------------------------------------------------------- #
+def test_a_plain_point_runs_everyone():
+    assert bench.parse_point("20000:5000") == (20_000, 5_000, ())
+
+
+def test_a_bare_oos_still_means_vectorbt():
+    assert bench.parse_point("20000:20000:oos") == (20_000, 20_000, ("vectorbt",))
+
+
+def test_oos_can_name_several_engines():
+    assert bench.parse_point("20000:5000:oos=vectorbt,nautilus_trader") == (
+        20_000, 5_000, ("vectorbt", "nautilus_trader"))
+
+
+def test_the_reference_and_strangers_cannot_be_declared_out_of_scope():
+    with pytest.raises(argparse.ArgumentTypeError):
+        bench.parse_point("20000:5000:oos=manifoldbt")
+    with pytest.raises(argparse.ArgumentTypeError):
+        bench.parse_point("20000:5000:oos=zipline")
+    with pytest.raises(argparse.ArgumentTypeError):
+        bench.parse_point("20000:5000:maybe")
+
+
+ALL = ["manifoldbt", "vectorbt", "raptorbt", "nautilus_trader"]
+
+
+def test_a_big_grid_holds_back_the_looped_engine_without_being_told_to():
+    """The recipe on the website only says `:oos`; it must not run Nautilus for hours."""
+    assert bench._sweep_out_of_scope(5_000, (), ALL) == ["nautilus_trader"]
+    assert bench._sweep_out_of_scope(5_000, ("vectorbt",), ALL) == ["vectorbt", "nautilus_trader"]
+
+
+def test_a_small_grid_is_run_by_everyone():
+    assert bench._sweep_out_of_scope(36, (), ALL) == []
+    assert bench._sweep_out_of_scope(64, (), ALL) == []
+    assert bench._sweep_out_of_scope(65, (), ALL) == ["nautilus_trader"]
+
+
+def test_the_sweep_ceiling_can_be_moved_or_lifted():
+    bench.SWEEP_CEILING_OVERRIDES["nautilus_trader"] = 500
+    assert bench._sweep_out_of_scope(500, (), ALL) == []
+    bench.SWEEP_CEILING_OVERRIDES["nautilus_trader"] = 0
+    assert bench._sweep_out_of_scope(50_000, (), ALL) == []
+
+
+def test_an_engine_that_is_not_running_is_not_listed_as_out_of_scope():
+    assert bench._sweep_out_of_scope(5_000, ("vectorbt",), ["manifoldbt", "raptorbt"]) == []
+
+
+def test_the_nautilus_anchor_spans_the_grid_it_stands_in_for():
+    values = list(range(100))
+
+    picked = sweep_child.spread(values, 4)
+
+    assert picked[0] == 0 and picked[-1] == 99
+    assert len(picked) == 4
+    assert picked == sorted(set(picked))
+
+
+def test_a_small_grid_is_its_own_anchor():
+    assert sweep_child.spread([5, 7], 4) == [5, 7]
+
+
+def test_the_reason_for_a_looped_engine_carries_what_was_measured():
+    text = sweep_child.oos_reason("nautilus_trader", 20_000, 5_000,
+                                  anchor_cells=16, anchor_s=16.0)
+
+    assert "1.00 s per backtest at 20,000 bars" in text
+    assert "5,000 combinations" in text
+    assert "about 83 minutes" in text
+
+
+def test_a_grid_held_back_by_the_sweep_ceiling_says_so_even_when_it_is_cheap():
+    text = sweep_child.oos_reason("nautilus_trader", 5_000, 100, anchor_cells=16,
+                                  anchor_s=3.2, loop_limit=64)
+
+    assert "about 20 seconds per call" in text
+    assert "held to 64 combinations" in text
+    assert "past the time budget" not in text
+
+
+def test_the_vectorbt_reason_is_the_memory_one():
+    text = sweep_child.oos_reason("vectorbt", 20_000, 20_000, anchor_cells=250, anchor_s=1.0)
+
+    assert text == sweep_child.OOS_VBT_REASON
+
+
+@pytest.mark.parametrize("seconds,expected", [
+    (42.0, "42 seconds"), (600.0, "10 minutes"), (9_000.0, "2.5 hours"),
+])
+def test_durations_read_in_the_unit_a_person_would_use(seconds, expected):
+    assert sweep_child._duration(seconds) == expected
+
+
+# --------------------------------------------------------------------------- #
+# The report, for an engine that was held back
+# --------------------------------------------------------------------------- #
+def _row(workload, bars, engines, **extra):
+    row = {
+        "workload": workload, "title": workload, "bars": bars, "status": "exact",
+        "engines": engines,
+        "timings": {e: {"median_s": 0.01, "iqr_over_median": 0.01} for e in engines},
+        "parity": {e: {"status": "exact", "publishable": True, "diffs": {}}
+                   for e in engines if e != "manifoldbt"},
+        "speedup": {e: {"median_of_ratios": 2.0, "min": 1.0, "max": 3.0}
+                    for e in engines if e != "manifoldbt"},
+    }
+    row.update(extra)
+    return row
+
+
+def _payload(rows, **extra):
+    engines = ["manifoldbt", "nautilus_trader"]
+    payload = {
+        "schema_version": 2, "reference": "manifoldbt", "engines": engines, "reps": 3,
+        "generated_at": "2026-10-22T00:00:00+00:00",
+        "environment": {
+            "os": "Linux", "arch": "x86_64", "cpu": "a cpu", "logical_cores": 4,
+            "ram_gb": 16.0, "python": "3.12.0", "pinned_cores": None,
+            "versions": {"manifoldbt": "0.28.0", "nautilus_trader": "1.231.0",
+                         "numpy": "2.4.3", "numba": "0.64.0", "pandas": "2.3.3"},
+        },
+        "results": rows,
+    }
+    payload.update(extra)
+    return payload
+
+
+WHY = "Not run above 100,000 bars. Event-driven."
+
+
+def test_one_reason_is_printed_once_for_all_the_workloads_it_covers():
+    rows = [
+        _row("sma_cross", 1_000_000, ["manifoldbt"], unsupported={"nautilus_trader": WHY}),
+        _row("multi_asset", 1_000_000, ["manifoldbt"], unsupported={"nautilus_trader": WHY}),
+    ]
+
+    text = report.render(_payload(rows))
+
+    assert text.count(WHY) == 1
+    assert "`sma_cross`, `multi_asset`" in text
+
+
+def test_an_engine_without_a_drawdown_is_said_not_to_have_one():
+    summarised = _row("sma_cross_metrics", 100_000, ["manifoldbt", "nautilus_trader"],
+                      paired_with=["sma_cross"])
+    summarised["parity"]["nautilus_trader"]["diffs"] = {
+        "max_drawdown_absent": True, "ratio_basis": "native"}
+    plain = _row("sma_cross", 100_000, ["manifoldbt", "nautilus_trader"],
+                 paired_with=["sma_cross_metrics"])
+    # The longest row is one the engine did not run on: its verdict is read from
+    # the longest row it did.
+    longer = _row("sma_cross_metrics", 1_000_000, ["manifoldbt"],
+                  paired_with=["sma_cross"], unsupported={"nautilus_trader": WHY})
+    longer_plain = _row("sma_cross", 1_000_000, ["manifoldbt"],
+                        paired_with=["sma_cross_metrics"], unsupported={"nautilus_trader": WHY})
+
+    text = report.render(_payload([plain, summarised, longer_plain, longer]))
+
+    assert "reports no maximum drawdown" in text
+    assert "was not compared" not in text
+
+
+def test_a_memory_figure_taken_at_the_ceiling_says_so():
+    memory = {
+        "workload": "sma_cross", "bars": 2_000_000, "engines": ["manifoldbt", "nautilus_trader"],
+        "manifoldbt": {"added_mb_per_million_bars": 0.006, "bars": 2_000_000},
+        "nautilus_trader": {"added_mb_per_million_bars": 4.8, "bars": 100_000},
+    }
+
+    text = report.render(_payload([_row("sma_cross", 100_000, ["manifoldbt"])], memory=memory))
+
+    assert "5 MB (at 100,000 bars)" in text
+    assert "0 MB |" in text
+
+
+def test_the_cpu_row_survives_an_engine_that_did_not_run_the_longest_series():
+    rows = [
+        _row("sma_cross", 100_000, ["manifoldbt", "nautilus_trader"],
+             cpu_over_wall={"manifoldbt": 1.0, "nautilus_trader": 0.99}),
+        _row("sma_cross", 10_000_000, ["manifoldbt"], cpu_over_wall={"manifoldbt": 3.5}),
+    ]
+
+    text = report.render(_payload(rows, memory=None, cold_start=None))
+
+    assert "CPU over wall time, at the longest series each ran" in text
+    assert "3.50 (at 10,000,000)" in text
+    assert "0.99 (at 100,000)" in text
+
+
+def test_a_reentry_that_differs_only_in_quantity_reports_how_far():
+    row = _row("bracket_sl_tp", 100_000, ["manifoldbt", "nautilus_trader"], status="documented")
+    row["parity"]["nautilus_trader"] = {"status": "documented", "publishable": False,
+                                        "diffs": {"final_equity_vs_capital": 7e-6}}
+    row["divergence_scale"] = {
+        "manifoldbt": {"reentries_on_exit_bar": 900, "round_trips": 2300,
+                       "share_of_round_trips": 0.39, "sl_exits": 400, "tp_exits": 300},
+        "nautilus_trader": {"round_trips": 2300, "reentries_on_exit_bar": 900,
+                            "reentry_quantity_gap_median": 3e-4,
+                            "reentry_quantity_gap_max": 1e-3},
+    }
+
+    text = report.render(_payload([row]))
+
+    assert "nautilus_trader books 2300." in text
+    assert "a median of 3.0e-04" in text