mirror of
https://github.com/manifoldbt/manifoldbt.git
synced 2026-08-24 14:38:04 +00:00
Exocet92andGitHub
52cbe1ba54
The harness compared two engines everywhere; it now compares N against a reference. manifoldbt is the reference: every parity check and every ratio is a challenger against it, never two challengers against each other. raptorbt 0.9.0 joins on three of the four workloads. Its sma_cross comes back bit-identical to the reference's final equity, and its rsi matches to the last bit; its ema seeds on a different warmup and it has no fixed-quantity sizing, so the fee workload records it as unsupported with the reason rather than leaving a blank cell. On the bracket it diverges in its own documented way: it never re-arms while the entry level holds, so it books exactly the reference's round-trips minus the ones that re-enter on the exit bar. Python moves to 3.12, which raptorbt pins rather than we do: it is built against pyo3 0.20.3, whose maximum supported CPython is 3.12. Timings from runs before this change are therefore not directly comparable. The bar matrix gains 10M and the repetition default drops from 7 to 2. Measured, those two almost cancel: the job stays around 16 minutes. macOS keeps its old ceiling, since 10M bars adds 1.55 GB on vectorbt's side alone and that runner has 7 GB.
194 lines
9.4 KiB
Python
194 lines
9.4 KiB
Python
"""Workload definitions: the numbers every engine reads, in one place.
|
|
|
|
Nothing here is engine-specific. Each adapter (``engine_mbt``, ``engine_vbt``,
|
|
``engine_rbt``) reads the same constants, so a workload cannot drift between two
|
|
sides by someone editing one file and forgetting another.
|
|
|
|
What a workload *does* carry per engine is a note: the places where an engine is
|
|
known in advance to disagree, or is unable to run the workload at all. Those are
|
|
written down here, next to the parameters they apply to, rather than in the
|
|
adapters, so a reader can see the whole map of "who runs what, and where they
|
|
part ways" without opening three files.
|
|
|
|
Sizing policy, and why it changes when fees are on
|
|
--------------------------------------------------
|
|
With ``FractionOfEquity`` sizing and a non-zero fee, engines size a position
|
|
differently: manifoldbt charges the fee on top of a full-equity notional,
|
|
vectorbt reserves it out of cash first. Both are defensible product decisions,
|
|
and comparing them would compare *policy*, not speed or correctness. The fee
|
|
workload therefore sizes in fixed units, which isolates the fee arithmetic
|
|
itself. This mirrors the choice already made in the cross-engine parity suite
|
|
shipped with the library. It is also the reason raptorbt sits that workload out:
|
|
it has no fixed-quantity sizing to offer (see its note below).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Dict
|
|
|
|
CAPITAL = 100_000.0
|
|
|
|
# Bar interval of the generated data. Both engines are told the same thing:
|
|
# manifoldbt through `Interval.minutes(1)`, vectorbt through `freq="1min"`
|
|
# (annualisation only; it does not touch the simulation).
|
|
FREQ = "1min"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Note:
|
|
"""What is known in advance about one engine on one workload.
|
|
|
|
``documented``
|
|
The engine is known to disagree with the reference, and ``why`` says on
|
|
what. It still gets timed, but in the annex, never in the headline
|
|
table, and always with the reason printed next to the number.
|
|
``unsupported``
|
|
The engine cannot express this workload at all. It is not run, and the
|
|
report says so rather than leaving a blank a reader would read as a
|
|
loss. An unsupported entry is a finding about the engine's API, so
|
|
``why`` has to be specific enough to be checked.
|
|
"""
|
|
|
|
status: str
|
|
why: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Workload:
|
|
key: str
|
|
title: str
|
|
why: str
|
|
params: Dict[str, Any] = field(default_factory=dict)
|
|
# Engine name -> Note. An engine with no entry here is expected to agree
|
|
# with the reference down to float-reordering noise, and a disagreement is
|
|
# a failure that withholds the timing.
|
|
notes: Dict[str, Note] = field(default_factory=dict)
|
|
|
|
|
|
WORKLOADS: Dict[str, Workload] = {
|
|
w.key: w
|
|
for w in (
|
|
Workload(
|
|
key="sma_cross",
|
|
title="SMA 10/50 crossover, long-only, no cost",
|
|
why="The canonical baseline. Unambiguous indicator, no fee policy, "
|
|
"no stop semantics: if the engines disagree here, nothing else "
|
|
"in the suite is worth reading.",
|
|
params=dict(fast=10, slow=50, alloc=1.0),
|
|
),
|
|
Workload(
|
|
key="ema_rsi_fees",
|
|
title="EMA 12/26 crossover + RSI(14) filter, 5 bps taker fee",
|
|
why="A realistic signal stack with a real cost model, sized in fixed "
|
|
"units so the fee arithmetic is comparable across engines. The "
|
|
"unit count is small relative to capital on purpose: at 1-minute "
|
|
"resolution this strategy turns over often enough that a larger "
|
|
"size would spend the whole account on fees, and comparing two "
|
|
"engines on a wiped-out account compares rounding noise.",
|
|
params=dict(fast=12, slow=26, rsi_period=14, rsi_lo=30.0, rsi_hi=70.0,
|
|
units=5.0, fee_bps=5.0),
|
|
notes={
|
|
"raptorbt": Note(
|
|
"unsupported",
|
|
"Two blockers, either one sufficient. Sizing: raptorbt has no "
|
|
"fixed-quantity mode. `position_sizes` is a fraction of equity "
|
|
"(measured: 0.5 buys exactly half the equity of the bar before "
|
|
"the entry), `lot_size` rounds a computed size down to a "
|
|
"multiple, and `alloted_capital` fixes the notional, not the "
|
|
"quantity. Reproducing `units=5` would mean feeding a fraction "
|
|
"derived from an equity curve that does not exist until the run "
|
|
"is over. Indicator: `raptorbt.ema` seeds on a simple mean of "
|
|
"the first `period` bars and emits from bar `period-1`, while "
|
|
"manifoldbt seeds on the first observation and emits from bar 0 "
|
|
"(measured: 11 leading NaN for span 12, and ema[11] equal to "
|
|
"sma(12)[11] to the last bit). The two are the same recursion "
|
|
"with a different warmup, so the signal differs early and the "
|
|
"round-trip count with it. Its `sma` and `rsi` do match, which "
|
|
"is why the other three workloads run.",
|
|
),
|
|
},
|
|
),
|
|
Workload(
|
|
key="sma_cross_metrics",
|
|
title="SMA 10/50 crossover, with a performance summary",
|
|
why="The same simulation as `sma_cross`, but both engines are asked "
|
|
"for what a user actually reads: max drawdown, Sharpe, Sortino "
|
|
"and volatility alongside the return. manifoldbt computes them "
|
|
"inside run() whether you ask or not; vectorbt defers the equity "
|
|
"curve until a risk metric needs it, and every one of them pays "
|
|
"for materialising it. This is a scope difference, not a trick: "
|
|
"`sma_cross` above is the same work without the summary, and the "
|
|
"two are reported side by side so the reader can see what the "
|
|
"summary costs each engine.",
|
|
params=dict(fast=10, slow=50, alloc=1.0, metrics=True),
|
|
),
|
|
Workload(
|
|
key="bracket_sl_tp",
|
|
title="SMA 10/50 entry with a 15 bps stop / 30 bps target bracket",
|
|
why="Brackets are where two engines most easily disagree: the stop "
|
|
"level, the fill on the triggering bar, and whether a re-entry "
|
|
"is allowed on the bar after an exit.",
|
|
params=dict(fast=10, slow=50, alloc=1.0, sl_pct=0.15, tp_pct=0.30),
|
|
notes={
|
|
"vectorbt": Note(
|
|
"documented",
|
|
"Re-entry on the exit bar. When a bracket fires intrabar and "
|
|
"the entry condition still holds at that bar's close, "
|
|
"manifoldbt books two orders on that bar (the stop or target "
|
|
"exit, then a fresh entry at the close); vectorbt processes one "
|
|
"order per bar and re-enters on the next bar instead. Neither "
|
|
"is wrong, and on controlled bars the bracket fills themselves "
|
|
"match exactly (see the cross-engine parity suite shipped with "
|
|
"the library). The harness counts the affected round-trips so "
|
|
"the size of the divergence is measured, not asserted.",
|
|
),
|
|
"raptorbt": Note(
|
|
"documented",
|
|
"Same fork in the road, taken further: raptorbt does not "
|
|
"re-arm at all. Once a bracket closes a position, the entry "
|
|
"level being still true is not enough to open another one; it "
|
|
"waits for the level to go false and true again. So on the same "
|
|
"bars the three engines book a different number of round-trips "
|
|
"from the same signal, manifoldbt re-entering on the exit bar, "
|
|
"vectorbt on the next bar, raptorbt not until the next crossing. "
|
|
"The count of affected round-trips is measured rather than "
|
|
"asserted, and it is the same population for both challengers.",
|
|
),
|
|
},
|
|
),
|
|
)
|
|
}
|
|
|
|
def expectation(key: str, engine: str) -> str:
|
|
"""What this engine is expected to do on this workload, before it runs."""
|
|
note = WORKLOADS[key].notes.get(engine)
|
|
return note.status if note else "exact"
|
|
|
|
|
|
def why(key: str, engine: str) -> str:
|
|
note = WORKLOADS[key].notes.get(engine)
|
|
return note.why if note else ""
|
|
|
|
|
|
def supported(key: str, engine: str) -> bool:
|
|
return expectation(key, engine) != "unsupported"
|
|
|
|
|
|
def unsupported_by(key: str) -> Dict[str, str]:
|
|
"""Engines that sit this workload out, and the reason each one gives."""
|
|
return {
|
|
name: note.why
|
|
for name, note in WORKLOADS[key].notes.items()
|
|
if note.status == "unsupported"
|
|
}
|
|
|
|
|
|
DEFAULT_KEYS = list(WORKLOADS)
|
|
|
|
# Two workloads that the report compares directly against each other, so they
|
|
# must be measured inside ONE interleaved loop rather than in two blocks minutes
|
|
# apart. Comparing medians across blocks is exactly the mistake the interleaving
|
|
# exists to prevent: absolute timings drift between blocks, and the drift once
|
|
# produced a table claiming the version doing MORE work was the faster one.
|
|
SCOPE_PAIR = ("sma_cross", "sma_cross_metrics")
|