mirror of
https://github.com/manifoldbt/manifoldbt.git
synced 2026-08-24 14:38:04 +00:00
bench: fix the red runs, trim the matrix, add a grid job (#7)
Five consecutive red runs, two unrelated causes. Four of them never reached an engine: the workflow installs the tag it is handed, but 0.18.0rc1 and rc2 were previews that never reached PyPI. Pre-releases are now skipped, and the report step checks for its input file instead of dying on a missing one and reporting the wrong cause twice. The fifth came from adding 10M bars, which broke a workload whose validity depended on the ladder stopping at 5M. ema_rsi_fees sizes in fixed units and pays 5 bps a side, so over 10M one-minute bars the fees compound into the whole account: -15% of capital at 1M, -74% at 5M, exactly -100% at 10M, where fees reach 99,611 of the 100,000 it started with. Both engines then sit at zero and disagree by 9,085 round-trips about how many worthless trades to book on a dead account. Workloads can now declare a ceiling, and the runner skips past it out loud. Fewer points per axis: three series lengths instead of five, a decade apart each step. 10k measured the clock rather than the work, and 5M sat between two points that already bracketed it. sma_cross crosses on 30/150 rather than 10/50, worth about 15% on the ratio because it books a third of the trades. Grids get their own job, licensed through ci_activate.py, which refuses to run unlicensed rather than time a wait. Three points, not a matrix: across the plane the four-core ratio moves only between x32 and x38.
This commit is contained in:
@@ -55,6 +55,19 @@ from workloads import ( # noqa: E402
|
||||
unsupported_by,
|
||||
)
|
||||
|
||||
|
||||
def _in_range(key: str, bars: int) -> bool:
|
||||
"""Is this workload still a comparison at this series length?
|
||||
|
||||
A ceiling is not a performance limit, it is a validity one. The fee workload
|
||||
bankrupts its account past a certain length, and two engines agreeing that a
|
||||
dead account is worth zero is not a measurement. Skipping is loud in the run
|
||||
output rather than silent, because a table that is short by one row reads
|
||||
like a choice.
|
||||
"""
|
||||
ceiling = WORKLOADS[key].max_bars
|
||||
return ceiling is None or bars <= ceiling
|
||||
|
||||
# 2: timings, parity and speedups became per-engine maps when the harness grew
|
||||
# past two engines. `report.py` reads version 1 files as well, so the results
|
||||
# archived under results/ stay readable.
|
||||
@@ -655,15 +668,31 @@ def main() -> int:
|
||||
paired = [k for k in SCOPE_PAIR if k in args.workloads]
|
||||
singles = [k for k in args.workloads if k not in paired]
|
||||
|
||||
def skip(key: str, bars: int) -> None:
|
||||
print(" {k:18s} {b:>12,} bars ... skipped beyond this workload's "
|
||||
"ceiling of {c:,} bars".format(
|
||||
k=key, b=bars, c=WORKLOADS[key].max_bars))
|
||||
|
||||
results = []
|
||||
failures = 0
|
||||
for bars in args.bars:
|
||||
if len(paired) > 1:
|
||||
for entry in measure_pair(paired, bars, args.reps, workdir, active):
|
||||
runnable = [k for k in paired if _in_range(k, bars)]
|
||||
for key in paired:
|
||||
if key not in runnable:
|
||||
skip(key, bars)
|
||||
if len(runnable) > 1:
|
||||
for entry in measure_pair(runnable, bars, args.reps, workdir, active):
|
||||
results.append(entry)
|
||||
failures += announce(entry)
|
||||
for key in singles + (paired if len(paired) == 1 else []):
|
||||
elif runnable:
|
||||
entry = measure(runnable[0], bars, args.reps, workdir, active)
|
||||
results.append(entry)
|
||||
failures += announce(entry)
|
||||
for key in singles:
|
||||
for bars in args.bars:
|
||||
if not _in_range(key, bars):
|
||||
skip(key, bars)
|
||||
continue
|
||||
entry = measure(key, bars, args.reps, workdir, active)
|
||||
results.append(entry)
|
||||
failures += announce(entry)
|
||||
@@ -672,7 +701,8 @@ def main() -> int:
|
||||
# a cold-start table cannot come back missing a column because the workload
|
||||
# happened to be one somebody sits out.
|
||||
probe_workload = next(
|
||||
(k for k in args.workloads if all(supported(k, n) for n in active)),
|
||||
(k for k in args.workloads
|
||||
if all(supported(k, n) for n in active) and _in_range(k, 20_000)),
|
||||
args.workloads[0],
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user