diff --git a/.github/workflows/bench-vs-vectorbt.yml b/.github/workflows/bench-vs-vectorbt.yml index 64dd9cc..953adc2 100644 --- a/.github/workflows/bench-vs-vectorbt.yml +++ b/.github/workflows/bench-vs-vectorbt.yml @@ -236,3 +236,63 @@ jobs: name: bench-sweeps path: benchmarks/vs_vectorbt/results-sweeps.json if-no-files-found: warn + + # ------------------------------------------------------------------------ # + # Publish, so the numbers live somewhere that is not an artifact + # ------------------------------------------------------------------------ # + publish: + name: publish results + runs-on: ubuntu-latest + # Only a run where both halves came back green gets published. A partial + # result is worse than a stale one: the website renders whatever this file + # says, and a missing sweep table reads as a choice rather than a crash. + needs: [bench, sweeps] + # A dispatch that pins an old version is a question somebody asked, not the + # current state of the engine, so it measures and reports without becoming + # the published number. + if: github.event_name != 'workflow_dispatch' || inputs.manifoldbt_version == '' + permissions: + contents: write + steps: + # The default branch explicitly: a release run is checked out at a tag, + # and there is nothing to push a commit onto there. + - uses: actions/checkout@v4 + with: + ref: master + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - uses: actions/download-artifact@v4 + with: + pattern: bench-* + path: artifacts + merge-multiple: true + + - name: Merge the run into one published result + shell: bash + run: | + python benchmarks/vs_vectorbt/publish.py artifacts \ + --out benchmarks/vs_vectorbt/results/latest.json + + # Committed, not uploaded: an artifact needs a token to download and + # expires after ninety days, so anything outside GitHub that wants these + # numbers needs them at a plain URL. This is that URL. + - name: Commit it, if it moved + shell: bash + run: | + set -e + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add benchmarks/vs_vectorbt/results/latest.json + if git diff --cached --quiet; then + echo "identical to the published result: nothing to commit" + exit 0 + fi + # [skip ci] because a results file is not a code change, and running + # the test suite over it would only add a red herring to the log. + git commit -m "bench: publish results from run ${GITHUB_RUN_ID} [skip ci]" + # Another run may have landed while this one was measuring. + git pull --rebase origin master + git push origin HEAD:master diff --git a/benchmarks/vs_vectorbt/publish.py b/benchmarks/vs_vectorbt/publish.py new file mode 100644 index 0000000..60b6c7f --- /dev/null +++ b/benchmarks/vs_vectorbt/publish.py @@ -0,0 +1,84 @@ +"""Merge a run's result artifacts into the one file the website reads. + + python publish.py --out results/latest.json + +An artifact is not a publication. It needs a token to download, it expires +after ninety days, and nothing outside GitHub can link to it, so a number that +only exists as an artifact is a number nobody can check. This writes the same +measurements to a path in the repository instead: fetchable by anyone, over +plain HTTPS, for as long as the repository exists. + +The two jobs produce two payloads with two environments, because they run on two +runners. They are stored side by side rather than merged into one table: a +timing measured on one machine and a timing measured on another are not rows of +the same table, and pretending otherwise is how a benchmark starts lying +quietly. Provenance goes on top, from the workflow environment, so the file +names the run it came from without anyone having to trust the filename. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + +# artifact file name -> key in the published file. +SOURCES = { + "results-ubuntu-latest.json": "backtests", + "results-sweeps.json": "sweeps", +} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifacts", help="directory the artifacts were downloaded into") + parser.add_argument("--out", required=True, help="file to write") + args = parser.parse_args() + + server = os.environ.get("GITHUB_SERVER_URL", "https://github.com") + repo = os.environ.get("GITHUB_REPOSITORY", "manifoldbt/manifoldbt") + run_id = os.environ.get("GITHUB_RUN_ID", "") + + published = { + "repo": repo, + "run_id": run_id, + "run_url": "{}/{}/actions/runs/{}".format(server, repo, run_id) if run_id else "", + "commit": os.environ.get("GITHUB_SHA", ""), + "run_started_at": os.environ.get("GITHUB_RUN_STARTED_AT", ""), + # This job only runs when both measuring jobs came back green, so the + # conclusion is not read from anywhere: it is the precondition. + "conclusion": "success", + "synced_at": datetime.now(timezone.utc).isoformat(), + } + + root = Path(args.artifacts) + for name, key in SOURCES.items(): + # download-artifact with merge-multiple flattens the two artifacts into + # one directory, but a plain download nests each under its own name. + # Take whichever layout the caller produced. + found = next(iter(sorted(root.rglob(name))), None) + if found is None: + print("! {}: no {} under {}".format(key, name, root), file=sys.stderr) + continue + payload = json.loads(found.read_text(encoding="utf-8")) + if payload.get("schema_version", 1) < 2: + raise SystemExit( + "{} is schema {}, expected 2 or later".format( + found, payload.get("schema_version", 1)) + ) + published[key] = payload + + if "backtests" not in published and "sweeps" not in published: + raise SystemExit("neither artifact was found: nothing to publish") + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(published, indent=2) + "\n", encoding="utf-8") + print("wrote {} ({:.0f} KB)".format(out, out.stat().st_size / 1024)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())