mirror of
https://github.com/manifoldbt/manifoldbt.git
synced 2026-11-02 07:56:11 +00:00
Jimmy7892
23421cd5ca
The report assumed every engine ran on every row. With a ceiling that is no longer true, so: - "Not run, and why" prints one reason once for all the workloads it covers, instead of once per workload. - The CPU-over-wall row reads each engine at the longest series it ran, where it used to need every engine at one size and would have vanished. - A memory figure taken at an engine's ceiling carries its bar count. - An engine with no maximum drawdown is said to have none, rather than "it was not compared", and its summary workload is described as gated on total return and round-trips. - A documented divergence where an engine takes the same trades and sizes one differently reports how many re-entries and how far apart. - Probes that died for a challenger are listed under the table. test_harness.py adds guards for the gate (an engine without a drawdown, the documented bracket divergence), the ceilings, the sweep scope rules (including that a recipe saying only `:oos` cannot start Nautilus on a big grid), the out-of-scope reasons and these report changes. Like test_report.py it needs no engine installed.
540 lines
22 KiB
Python
540 lines
22 KiB
Python
"""Turn a results JSON into the Markdown a human reads.
|
|
|
|
Prints to stdout and, when running under GitHub Actions, appends the same text
|
|
to the job summary so the numbers are visible without downloading an artifact.
|
|
|
|
python report.py results.json
|
|
|
|
This is a run output, not an article: tables, and only the glue needed to read
|
|
them. Every "why" belongs in README.md, which is written once instead of being
|
|
reprinted underneath every single run.
|
|
|
|
Two result schemas are accepted. Version 2 keys everything by engine name;
|
|
version 1, written while the harness compared exactly two engines, is normalised
|
|
into that shape on load so the results archived under ``results/`` keep
|
|
rendering after the third engine arrived.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from typing import Any, Dict, List
|
|
|
|
METHOD_LINK = "benchmarks/vs_vectorbt/README.md"
|
|
|
|
# What version 1 files were, before the shape became a map.
|
|
V1_REFERENCE = "manifoldbt"
|
|
V1_CHALLENGER = "vectorbt"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Reading
|
|
# --------------------------------------------------------------------------- #
|
|
def normalise(payload: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Bring a version 1 payload up to the per-engine shape used below."""
|
|
if payload.get("schema_version", 1) >= 2:
|
|
return payload
|
|
|
|
payload["reference"] = V1_REFERENCE
|
|
payload["engines"] = [V1_REFERENCE, V1_CHALLENGER]
|
|
for row in payload.get("results", []):
|
|
verdict = row.get("parity") or {}
|
|
row["status"] = verdict.get("status", "exact")
|
|
row["parity"] = {V1_CHALLENGER: verdict}
|
|
row["engines"] = [V1_REFERENCE, V1_CHALLENGER]
|
|
if row.get("speedup"):
|
|
row["speedup"] = {V1_CHALLENGER: row["speedup"]}
|
|
if row.get("divergence_scale"):
|
|
row["divergence_scale"] = {V1_REFERENCE: row["divergence_scale"]}
|
|
|
|
cold = payload.get("cold_start")
|
|
if cold:
|
|
cold["engines"] = [V1_REFERENCE, V1_CHALLENGER]
|
|
if not isinstance(cold.get("ratio"), dict):
|
|
cold["ratio"] = {V1_CHALLENGER: cold.get("ratio")}
|
|
mem = payload.get("memory")
|
|
if mem:
|
|
mem["engines"] = [V1_REFERENCE, V1_CHALLENGER]
|
|
for point in payload.get("sweeps") or []:
|
|
timings = point.get("timings")
|
|
if timings and "seconds" not in timings:
|
|
point["timings"] = {
|
|
"seconds": {
|
|
V1_REFERENCE: timings.get("manifoldbt_s"),
|
|
V1_CHALLENGER: timings.get("vectorbt_s"),
|
|
},
|
|
"ratio": ({V1_CHALLENGER: timings["ratio"]}
|
|
if timings.get("ratio") is not None else {}),
|
|
}
|
|
memory = point.get("memory")
|
|
if memory and "manifoldbt_added_mb" in memory:
|
|
point["memory"] = {
|
|
V1_REFERENCE: memory.get("manifoldbt_added_mb"),
|
|
V1_CHALLENGER: memory.get("vectorbt_added_mb"),
|
|
}
|
|
if point.get(V1_CHALLENGER) and "status" in point[V1_CHALLENGER]:
|
|
point["out_of_scope"] = {V1_CHALLENGER: point[V1_CHALLENGER]}
|
|
if point.get("parity") and "status" in point["parity"]:
|
|
point["parity"] = {V1_CHALLENGER: point["parity"]}
|
|
return payload
|
|
|
|
|
|
def _engines(payload: Dict[str, Any]) -> List[str]:
|
|
return payload.get("engines") or [V1_REFERENCE, V1_CHALLENGER]
|
|
|
|
|
|
def _reference(payload: Dict[str, Any]) -> str:
|
|
return payload.get("reference", V1_REFERENCE)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Rendering
|
|
# --------------------------------------------------------------------------- #
|
|
def _ms(seconds: float) -> str:
|
|
if seconds < 1.0:
|
|
return "{:.1f} ms".format(seconds * 1e3)
|
|
return "{:.2f} s".format(seconds)
|
|
|
|
|
|
def _header(payload: Dict[str, Any]) -> List[str]:
|
|
env = payload["environment"]
|
|
versions = env["versions"]
|
|
cores = str(env["logical_cores"])
|
|
if env.get("pinned_cores"):
|
|
cores += " (pinned to {})".format(env["pinned_cores"])
|
|
lines = [
|
|
"# " + " vs ".join(
|
|
"{} {}".format(name, versions.get(name, "?")) for name in _engines(payload)),
|
|
"",
|
|
"`{os} {arch}` | {cpu} | {cores} cores | {ram} GB | python {py} | "
|
|
"numpy {np} / numba {nb} / pandas {pd} | {reps} interleaved reps | {when}".format(
|
|
os=env["os"], arch=env["arch"], cpu=env["cpu"], cores=cores, ram=env["ram_gb"],
|
|
py=env["python"], np=versions["numpy"], nb=versions["numba"],
|
|
pd=versions["pandas"], reps=payload["reps"], when=payload["generated_at"][:19],
|
|
),
|
|
]
|
|
if env.get("run_url"):
|
|
lines += ["", env["run_url"]]
|
|
lines.append("")
|
|
return lines
|
|
|
|
|
|
def _speed_table(rows: List[Dict[str, Any]], title: str, payload: Dict[str, Any]) -> List[str]:
|
|
"""One row per workload and size, one column per engine, then the ratios.
|
|
|
|
Columns are taken from the run rather than hardcoded, and a cell is only
|
|
empty when that engine was withheld or sat the workload out. Those cases get
|
|
a marker and a sentence of their own further down, because a blank in a
|
|
speed table reads as a defeat.
|
|
"""
|
|
if not rows:
|
|
return []
|
|
reference = _reference(payload)
|
|
engines = _engines(payload)
|
|
challengers = [e for e in engines if e != reference]
|
|
|
|
head = "| Workload | Bars | " + " | ".join(engines) + " | "
|
|
head += " | ".join("vs " + c for c in challengers) + " |"
|
|
lines = [
|
|
"## " + title,
|
|
"",
|
|
head,
|
|
"|---|---:|" + "---:|" * (len(engines) + len(challengers)),
|
|
]
|
|
for row in rows:
|
|
timings = row["timings"]
|
|
cells = [
|
|
_ms(timings[e]["median_s"]) if e in timings else "-" for e in engines
|
|
]
|
|
speedup = row.get("speedup") or {}
|
|
# The noise flag rides on the last ratio, where a reader's eye already
|
|
# is when deciding whether to believe the number.
|
|
mark = " ~" if row.get("noisy") else ""
|
|
ratios = [
|
|
"**x{:.1f}**".format(speedup[c]["median_of_ratios"]) if c in speedup else "-"
|
|
for c in challengers
|
|
]
|
|
# On the last ratio that actually has a number: hung on a "-" it would
|
|
# look like a comment on the engine that did not run.
|
|
present = [i for i, cell in enumerate(ratios) if cell != "-"]
|
|
if mark and present:
|
|
ratios[present[-1]] += mark
|
|
lines.append("| {w} | {b:,} | {cells} |".format(
|
|
w=row["workload"], b=row["bars"], cells=" | ".join(cells + ratios)))
|
|
lines.append("")
|
|
return lines
|
|
|
|
|
|
def _summary_cost(exact: List[Dict[str, Any]], payload: Dict[str, Any]) -> List[str]:
|
|
plain = {r["bars"]: r for r in exact if r["workload"] == "sma_cross"}
|
|
summarised = {r["bars"]: r for r in exact if r["workload"] == "sma_cross_metrics"}
|
|
# Only subtract timings that were measured in the same interleaved loop.
|
|
shared = sorted(
|
|
b for b in set(plain) & set(summarised)
|
|
if "sma_cross_metrics" in (plain[b].get("paired_with") or [])
|
|
)
|
|
if not shared:
|
|
return []
|
|
|
|
lines = [
|
|
"## Cost of the performance summary",
|
|
"",
|
|
"Same simulation, with and without max drawdown / Sharpe / Sortino / volatility.",
|
|
"",
|
|
"| Bars | Engine | Without | With | Delta |",
|
|
"|---:|---|---:|---:|---:|",
|
|
]
|
|
for bars in shared:
|
|
for engine in _engines(payload):
|
|
without = (plain[bars]["timings"] or {}).get(engine)
|
|
with_ = (summarised[bars]["timings"] or {}).get(engine)
|
|
if not (without and with_):
|
|
continue
|
|
a, c = without["median_s"], with_["median_s"]
|
|
lines.append(
|
|
"| {b:,} | {e} | {a} | {c} | {d} |".format(
|
|
b=bars, e=engine, a=_ms(a), c=_ms(c),
|
|
d=("+" + _ms(c - a)) if c > a else "none measurable",
|
|
)
|
|
)
|
|
lines.append("")
|
|
|
|
advisory = {}
|
|
basis_differs = {}
|
|
no_drawdown = set()
|
|
# Each engine is read at the longest series it ran on: one held back by its
|
|
# ceiling has no verdict at the longest size, and reading only that row would
|
|
# leave it out of the sentence.
|
|
for bars in reversed(shared):
|
|
for engine, verdict in summarised[bars]["parity"].items():
|
|
if engine in advisory or engine in basis_differs:
|
|
continue
|
|
diffs = verdict.get("diffs", {})
|
|
if diffs.get("max_drawdown_absent"):
|
|
no_drawdown.add(engine)
|
|
if diffs.get("advisory_ratio_rel"):
|
|
advisory[engine] = max(diffs["advisory_ratio_rel"].values())
|
|
elif diffs.get("ratio_basis"):
|
|
basis_differs[engine] = diffs.get("max_drawdown_rel")
|
|
if advisory:
|
|
lines += [
|
|
"Gated on total return, round-trips and max drawdown (exact). Sharpe, "
|
|
"Sortino and volatility agree to {:.1e}.".format(max(advisory.values())),
|
|
"",
|
|
]
|
|
for engine, drawdown_rel in basis_differs.items():
|
|
if engine in no_drawdown:
|
|
# Not "it was not compared": it does not exist. The engine offers
|
|
# volatility, Sharpe and Sortino and no drawdown statistic, and the
|
|
# harness does not build one for it.
|
|
lines += [
|
|
"{e} annualises its ratios on its own basis and reports no maximum "
|
|
"drawdown, so this workload is gated on total return and "
|
|
"round-trips only (exact).".format(e=engine),
|
|
"",
|
|
]
|
|
continue
|
|
if drawdown_rel is None:
|
|
agreement = "was not compared"
|
|
elif drawdown_rel == 0.0:
|
|
agreement = "is identical"
|
|
else:
|
|
agreement = "agrees to {:.1e}".format(drawdown_rel)
|
|
lines += [
|
|
"{e} annualises its ratios on its own basis, so only the drawdown is "
|
|
"compared there: it {a}.".format(e=engine, a=agreement),
|
|
"",
|
|
]
|
|
return lines
|
|
|
|
|
|
def _side_measures(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> List[str]:
|
|
cold = payload.get("cold_start")
|
|
mem = payload.get("memory")
|
|
engines = _engines(payload)
|
|
|
|
# CPU over wall time, per engine, at the longest series that engine ran and
|
|
# produced a reading for. An engine held back by its ceiling has no reading at
|
|
# the longest size, and requiring every engine at one size would drop the row
|
|
# for all of them.
|
|
threading: Dict[str, Any] = {}
|
|
for engine in engines:
|
|
rows = [r for r in results
|
|
if (r.get("cpu_over_wall") or {}).get(engine) is not None]
|
|
if rows:
|
|
best = max(rows, key=lambda r: r["bars"])
|
|
threading[engine] = (best["cpu_over_wall"][engine], best["bars"])
|
|
if not (cold or mem or threading):
|
|
return []
|
|
|
|
lines = [
|
|
"## Cold start, memory, threads",
|
|
"",
|
|
"| | " + " | ".join(engines) + " |",
|
|
"|---|" + "---:|" * len(engines),
|
|
]
|
|
if cold:
|
|
medians, share = cold["median_s"], cold["engine_share_s"]
|
|
lines.append("| Fresh process to first backtest | " + " | ".join(
|
|
"{:.2f} s".format(medians[e]) if e in medians else "-" for e in engines) + " |")
|
|
lines.append("| ... minus the {:.2f} s python baseline | ".format(medians["baseline"])
|
|
+ " | ".join("{:.2f} s".format(share[e]) if e in share else "-"
|
|
for e in engines) + " |")
|
|
if mem:
|
|
def memory_cell(engine: str) -> str:
|
|
if engine not in mem or not isinstance(mem[engine], dict):
|
|
return "-"
|
|
probe = mem[engine]
|
|
text = "{:.0f} MB".format(probe["added_mb_per_million_bars"])
|
|
# An engine held back by its ceiling is measured at the ceiling, and
|
|
# the figure is scaled to a million bars like the others. Say so
|
|
# beside the number rather than leave a reader to assume the same
|
|
# series length.
|
|
if probe.get("bars") and probe["bars"] != mem.get("bars"):
|
|
text += " (at {:,} bars)".format(probe["bars"])
|
|
return text
|
|
|
|
lines.append("| RAM added by the run, per 1M bars | "
|
|
+ " | ".join(memory_cell(e) for e in engines) + " |")
|
|
if threading:
|
|
sizes = {bars for _, bars in threading.values()}
|
|
if len(sizes) == 1:
|
|
lines.append("| CPU over wall time at {:,} bars | ".format(sizes.pop())
|
|
+ " | ".join("{:.2f}".format(threading[e][0]) if e in threading else "-"
|
|
for e in engines) + " |")
|
|
else:
|
|
lines.append("| CPU over wall time, at the longest series each ran | " + " | ".join(
|
|
"{:.2f} (at {:,})".format(*threading[e]) if e in threading else "-"
|
|
for e in engines) + " |")
|
|
lines.append("")
|
|
if mem and mem.get("failed"):
|
|
for engine, why in mem["failed"].items():
|
|
lines += ["- Memory probe for {}: {}".format(engine, why), ""]
|
|
if cold and cold.get("failed"):
|
|
for engine, why in cold["failed"].items():
|
|
lines += ["- Cold-start probe for {}: {}".format(engine, why), ""]
|
|
return lines
|
|
|
|
|
|
def _not_run(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> List[str]:
|
|
"""Workloads an engine sits out, and why.
|
|
|
|
Kept as prose rather than a table because the reason is the content. An
|
|
engine that cannot express a workload has told you something about itself,
|
|
and compressing that into an empty cell would throw away the only part worth
|
|
reading. An engine held back by its ceiling gets the same treatment, and the
|
|
same reason is printed once with the list of workloads it applies to, not
|
|
once per workload.
|
|
"""
|
|
seen: Dict[str, Dict[str, str]] = {}
|
|
for row in results:
|
|
for engine, why in (row.get("unsupported") or {}).items():
|
|
if engine in _engines(payload):
|
|
seen.setdefault(engine, {})[row["workload"]] = why
|
|
if not seen:
|
|
return []
|
|
|
|
lines = ["## Not run, and why", ""]
|
|
for engine, entries in seen.items():
|
|
by_reason: Dict[str, List[str]] = {}
|
|
for workload, why in entries.items():
|
|
by_reason.setdefault(why, []).append(workload)
|
|
for why, workloads in by_reason.items():
|
|
lines += ["- **{e} on {w}.** {why}".format(
|
|
e=engine, w=", ".join("`{}`".format(w) for w in workloads), why=why), ""]
|
|
return lines
|
|
|
|
|
|
def _gb(mb: float) -> str:
|
|
return "{:.1f} GB".format(mb / 1024) if mb >= 1024 else "{:.0f} MB".format(mb)
|
|
|
|
|
|
def _sweep_section(payload: Dict[str, Any]) -> List[str]:
|
|
"""The parameter-grid table, plus the memory that decides what is runnable.
|
|
|
|
Memory is reported next to the timings rather than in its own annex because
|
|
on a grid it is not a footnote: it is the first thing to run out. A machine
|
|
that cannot hold the grid does not produce a slow number, it produces no
|
|
number, and a reader sizing a job needs both columns side by side.
|
|
"""
|
|
sweeps = payload.get("sweeps")
|
|
if not sweeps:
|
|
return []
|
|
|
|
reference = _reference(payload)
|
|
engines = _engines(payload)
|
|
challengers = [e for e in engines if e != reference]
|
|
timed = [s for s in sweeps if s.get("timings")]
|
|
untimed = [s for s in sweeps if not s.get("timings")]
|
|
oos = [s for s in timed if s.get("out_of_scope")]
|
|
lines = ["## Parameter sweeps", ""]
|
|
|
|
if timed:
|
|
header = "| Bars | Combinations | " + " | ".join(engines) + " | "
|
|
header += " | ".join("vs " + c for c in challengers) + " | "
|
|
header += " | ".join("RAM " + e for e in engines) + " |"
|
|
lines += [
|
|
header,
|
|
# bars, combinations, one column per engine, one ratio per
|
|
# challenger, then one RAM column per engine.
|
|
"|---:|---:|" + "---:|" * (2 * len(engines) + len(challengers)),
|
|
]
|
|
for s in timed:
|
|
mem = s.get("memory") or {}
|
|
seconds = s["timings"]["seconds"]
|
|
ratios = s["timings"].get("ratio") or {}
|
|
cells = [_ms(seconds[e]) if seconds.get(e) is not None else "not run"
|
|
for e in engines]
|
|
cells += ["**x{:.1f}**".format(ratios[c]) if ratios.get(c) is not None else "-"
|
|
for c in challengers]
|
|
cells += [_gb(mem[e]) if mem.get(e) is not None else "-" for e in engines]
|
|
lines.append("| {b:,} | {c:,} | {cells} |".format(
|
|
b=s["bars"], c=s["combos"], cells=" | ".join(cells)))
|
|
lines += [
|
|
"",
|
|
"Each grid is checked cell by cell before any of it is timed: the "
|
|
"engines are joined on the parameter pair they actually ran, not on "
|
|
"position, and the worst disagreement in the grid is what the gate "
|
|
"sees. A single wrong cell among thousands is exactly the failure a "
|
|
"sweep can have and a single backtest cannot.",
|
|
"",
|
|
]
|
|
|
|
if oos:
|
|
lines += ["### Where a challenger was not run", ""]
|
|
for s in oos:
|
|
for engine, detail in s["out_of_scope"].items():
|
|
verdict = (s.get("parity") or {}).get(engine) or {}
|
|
lines.append(
|
|
"- **{e}, {c:,} combinations at {b:,} bars.** {why} The "
|
|
"cross-engine check for this point therefore covers the same "
|
|
"code path at {a:,} combinations, not this grid: agreement "
|
|
"was `{st}`.".format(
|
|
e=engine, c=s["combos"], b=s["bars"], why=detail["reason"],
|
|
a=verdict.get("checked_at_combos", 0),
|
|
st=verdict.get("status", "?"),
|
|
)
|
|
)
|
|
lines.append("")
|
|
|
|
if untimed:
|
|
lines += ["No timing for these points:", ""]
|
|
for s in untimed:
|
|
lines.append("- {b:,} bars x {c:,} combinations: {why}".format(
|
|
b=s["bars"], c=s["combos"],
|
|
why=s.get("reason") or s.get("note") or "no result",
|
|
))
|
|
lines.append("")
|
|
return lines
|
|
|
|
|
|
def _divergence_note(row: Dict[str, Any], payload: Dict[str, Any]) -> List[str]:
|
|
"""The measured size of one documented divergence, engine by engine.
|
|
|
|
A list item, so that the notes of several points stay one per line once
|
|
rendered; the cause is common to all of them and render() states it once.
|
|
"""
|
|
scales = row.get("divergence_scale") or {}
|
|
reference = _reference(payload)
|
|
view = scales.get(reference)
|
|
if not view:
|
|
return []
|
|
note = (
|
|
"- `{w}` at {b:,} bars: {n} of {t} {ref} round-trips re-enter on the exit "
|
|
"bar ({p:.0%}), from {sl} stop and {tp} target exits.".format(
|
|
w=row["workload"], b=row["bars"], n=view["reentries_on_exit_bar"],
|
|
t=view["round_trips"], p=view["share_of_round_trips"],
|
|
sl=view["sl_exits"], tp=view["tp_exits"], ref=reference,
|
|
)
|
|
)
|
|
for engine, scale in scales.items():
|
|
if engine == reference or "round_trips" not in scale:
|
|
continue
|
|
note += " {e} books {n}.".format(e=engine, n=scale["round_trips"])
|
|
# An engine that takes the same trades and sizes one of them differently
|
|
# says how far: a count and a median, not an adjective.
|
|
if scale.get("reentries_on_exit_bar") is not None:
|
|
note += (" Of those, {n} re-enter on the exit bar, with a quantity that "
|
|
"sits a median of {m:.1e} (at most {x:.1e}), in relative terms, "
|
|
"away from the position closed on that bar.".format(
|
|
n=scale["reentries_on_exit_bar"],
|
|
m=scale.get("reentry_quantity_gap_median", 0.0),
|
|
x=scale.get("reentry_quantity_gap_max", 0.0)))
|
|
return [note]
|
|
|
|
|
|
def render(payload: Dict[str, Any]) -> str:
|
|
payload = normalise(payload)
|
|
results = payload["results"]
|
|
exact = [r for r in results if r.get("status") == "exact" and r.get("timings")]
|
|
documented = [r for r in results if r.get("status") == "documented"]
|
|
failed = [r for r in results if r.get("status") == "failed"]
|
|
|
|
lines = _header(payload)
|
|
lines += _speed_table(exact, "Same results, every engine", payload)
|
|
lines += _summary_cost(exact, payload)
|
|
lines += _side_measures(payload, results)
|
|
lines += _not_run(payload, results)
|
|
lines += _sweep_section(payload)
|
|
|
|
timed = [r for r in documented if r.get("timings")]
|
|
if timed:
|
|
lines += _speed_table(timed, "Results differ, kept out of the headline", payload)
|
|
# Every point carries its own size: a divergence can grow with the
|
|
# number of bars, so the first point does not speak for the others.
|
|
notes = [note for row in timed for note in _divergence_note(row, payload)]
|
|
if notes:
|
|
lines += notes + ["", "Cause in {}.".format(METHOD_LINK), ""]
|
|
|
|
if failed:
|
|
lines += ["## Timing withheld", ""]
|
|
for row in failed:
|
|
for engine, verdict in row["parity"].items():
|
|
if verdict["status"] != "failed":
|
|
continue
|
|
diffs = verdict["diffs"]
|
|
lines.append(
|
|
"- `{w}` at {b:,} bars, {e}: final equity differs by {d:.2e} of "
|
|
"capital, round-trips by {t}.".format(
|
|
w=row["workload"], b=row["bars"], e=engine,
|
|
d=diffs["final_equity_vs_capital"],
|
|
t=diffs["round_trips_delta"],
|
|
)
|
|
)
|
|
lines.append("")
|
|
|
|
lines += [
|
|
"---",
|
|
"",
|
|
"`~` = IQR above 15% of the median, indicative only. Ratios are medians of "
|
|
"per-repetition ratios against {ref}, with the engines interleaved; data loading "
|
|
"excluded, warmup discarded. Method and caveats: {link}".format(
|
|
ref=_reference(payload), link=METHOD_LINK),
|
|
"",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="render a benchmark result")
|
|
parser.add_argument("results", nargs="?", default="results.json")
|
|
args = parser.parse_args()
|
|
|
|
with open(args.results, encoding="utf-8") as fh:
|
|
payload = json.load(fh)
|
|
|
|
text = render(payload)
|
|
sys.stdout.write(text + "\n")
|
|
|
|
summary = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
if summary:
|
|
with open(summary, "a", encoding="utf-8") as fh:
|
|
fh.write(text + "\n")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|