mirror of
https://github.com/manifoldbt/manifoldbt.git
synced 2026-11-02 07:56:11 +00:00
release: v0.27.1
This commit is contained in:
committed by
Jimmy7892
parent
6de6e3ae2e
commit
8ebd171f71
@@ -259,9 +259,11 @@ precomputed with future bars and imported as data). Bar T sees the same T+1
|
||||
in every window, so the shorter run trades exactly like the full one and the
|
||||
re-runs report PASS, next to a Sharpe in the hundreds. No choice of split
|
||||
changes that. The detector therefore also walks the strategy's expressions
|
||||
and fails any explicit `lead()`, naming the signal; for an implicit one, the
|
||||
check that works is to perturb the bars after some K and require the equity
|
||||
up to K to be bit-identical, which `examples/25_lookahead_trap.py` does.
|
||||
and fails any explicit `lead()`, naming the signal. For an implicit one, the
|
||||
check belongs to the pipeline that built the column: perturb the raw data
|
||||
after some bar K, rebuild the column from it, and require the equity up to K
|
||||
to be bit-identical. Perturbing the bars alone leaves a precomputed column
|
||||
untouched and sees nothing, as `examples/25_lookahead_trap.py` shows.
|
||||
|
||||
A parameter derived from the data *before* the backtest is a different
|
||||
question: a threshold computed over the whole history in a notebook and then
|
||||
|
||||
@@ -3618,9 +3618,16 @@ store = mbt.DataStore(data_root="data", metadata_db="...", dataset="bars_1m")
|
||||
# Look-ahead bias detection: a static walk for every non-causal operator,
|
||||
# then two re-runs over shorter windows. The two halves catch different
|
||||
# things. The re-runs cannot see a fixed read-ahead (close.lead(1) trades
|
||||
# the same in every window); the static walk names it. The static walk
|
||||
# only knows the operators, so it cannot see a leak that comes from the
|
||||
# data itself; the re-runs can.
|
||||
# the same in every window); the static walk names it. The re-runs catch a
|
||||
# strategy whose past trades change with how much data the run is given.
|
||||
# NEITHER half sees a leak carried by the data itself: a column computed
|
||||
# with future bars and imported, or a constant fitted on the whole history,
|
||||
# reads the same in every window and passes both. That one is caught
|
||||
# upstream, by re-deriving every data-dependent input on the window under
|
||||
# test (examples/25_lookahead_trap.py).
|
||||
#
|
||||
# A re-run with nothing to compare (a strategy that never trades, a cut
|
||||
# run that fails) reports INCONCLUSIVE, not PASS.
|
||||
#
|
||||
# The static walk reads a registry in which every DSL operator is
|
||||
# classified causal or not, so a new operator cannot be added without
|
||||
|
||||
@@ -19,8 +19,9 @@ from _bootstrap import open_store, plots_available
|
||||
|
||||
# -- Strategy with tunable parameters ----------------------------------------
|
||||
# The sweep engine substitutes each grid value into the param() references at
|
||||
# runtime. The period MUST be param("..."): a hardcoded int compiles every
|
||||
# combo to the same strategy, so the sweep becomes a silent no-op.
|
||||
# runtime. The period MUST be param("..."): with a hardcoded int no expression
|
||||
# reads the grid key, every combo would be the same strategy, and the engine
|
||||
# refuses the run with a StrategyError naming the undeclared parameter.
|
||||
fast = ema(close, mbt.param("fast", default=12))
|
||||
slow = ema(close, mbt.param("slow", default=26))
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Demonstrates:
|
||||
- param() in indicator periods (engine re-compiles per combo)
|
||||
- run_sweep() for Cartesian grid search
|
||||
- run_sweep_lite() for Cartesian grid search
|
||||
- Heatmap visualization with mbt.plot.heatmap_2d()
|
||||
|
||||
Data: shared store — real market data from `data/` (see examples/README.md)
|
||||
@@ -18,8 +18,11 @@ from manifoldbt.helpers import time_range, Slippage, Interval
|
||||
from _bootstrap import open_store, plots_available
|
||||
|
||||
# -- Strategy (single definition, param() in periods) ------------------------
|
||||
fast = ema(close, mbt.param("fast"))
|
||||
slow = ema(close, mbt.param("slow"))
|
||||
# The sweep sets both periods for every combo. The defaults are what a plain
|
||||
# mbt.run(strategy, ...) or a diagnostic uses: without one, a period has no
|
||||
# value outside the sweep.
|
||||
fast = ema(close, mbt.param("fast", default=20))
|
||||
slow = ema(close, mbt.param("slow", default=100))
|
||||
|
||||
signal = mbt.when(fast > slow, 0.25, mbt.when(fast < slow, -0.25, 0.0))
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Look-ahead — the leak no re-run method can see, and the one that does.
|
||||
"""Look-ahead — the leak no re-run method can see, and how to backtest without it.
|
||||
|
||||
The classic beginner's leak, and the one that survives review: computing a
|
||||
statistic over the *whole* dataset in the notebook, then using it as a strategy
|
||||
@@ -16,7 +16,9 @@ What this example shows, in order:
|
||||
1. the leak triples the return and doubles the Sharpe;
|
||||
2. `detect_lookahead` sees nothing;
|
||||
3. perturbing every future bar sees nothing either;
|
||||
4. the one method that catches it: re-derive the parameter on truncated data.
|
||||
4. what to do instead: re-derive the parameter on the window under test.
|
||||
That is not a detector (two different constants give two different
|
||||
curves by construction); it is the backtest you can report.
|
||||
|
||||
**Why the first two are blind.** Both re-run the *same strategy* on shorter or
|
||||
altered data. The mean is not recomputed by the engine: it is a number baked
|
||||
@@ -33,7 +35,7 @@ Self-contained: generates its own synthetic data in a temp store.
|
||||
Demonstrates:
|
||||
- a parameter computed over the whole dataset, used from bar 0
|
||||
- two audit methods that do not see it, and why
|
||||
- the one that does: re-deriving the parameter per window
|
||||
- the method that avoids it: re-deriving the parameter per window
|
||||
|
||||
Data: synthetic (seed 11) — generated by this file, reproducible.
|
||||
The fixture DETERMINES the outcome: see examples/README.md.
|
||||
@@ -157,7 +159,7 @@ if __name__ == "__main__":
|
||||
except LicenseError:
|
||||
# A Pro diagnostic. The file's argument does not need it to run: this
|
||||
# is the audit the leak is INVISIBLE to (run under Pro, it reports
|
||||
# SEES NOTHING here), and the audit that does catch it, [3], is free.
|
||||
# SEES NOTHING here), and the method that avoids it, [3], is free.
|
||||
print("\n [1] detect_lookahead .......... skipped (Pro diagnostic)")
|
||||
|
||||
# -- 3. Perturbing the future ---------------------------------------------
|
||||
@@ -186,15 +188,16 @@ if __name__ == "__main__":
|
||||
store_of(truncated, "t1")))
|
||||
with_past = equity(mbt.run(leaky(honest_mean), config_of(truncated),
|
||||
store_of(truncated, "t2")))
|
||||
m = min(len(with_future), len(with_past))
|
||||
gap = float(np.abs(with_future[:m] - with_past[:m]).max())
|
||||
verdict = CAUGHT if gap > 0 else BLIND
|
||||
print(f" [3] re-deriving the parameter . {verdict}")
|
||||
# Not a detector: the two runs differ by one constant, so their curves
|
||||
# differ whenever the constants do, leak or not. What [3] gives is the
|
||||
# backtest that only uses what bar 500 knew: the one you can report.
|
||||
print(" [3] re-deriving the parameter . the honest backtest")
|
||||
print(f" mean from the whole set : {global_mean:.4f} -> "
|
||||
f"final equity {with_future[-1]:,.0f}")
|
||||
print(f" mean from the past only : {honest_mean:.4f} -> "
|
||||
f"final equity {with_past[-1]:,.0f}")
|
||||
print(f" same window, same bars, equity differs by {gap:.0f}")
|
||||
print("\n Verdict: the leak is real, and only [3] found it. A")
|
||||
print(" 'SEES NOTHING' is a statement about the method, not about")
|
||||
print(" the strategy.")
|
||||
print(" (a different constant gives a different curve by construction:")
|
||||
print(" the second line is the number to report, not a test verdict)")
|
||||
print("\n Verdict: the leak is real, and no audit of the finished")
|
||||
print(" strategy can see it. A 'SEES NOTHING' is a statement about the")
|
||||
print(" method, not about the strategy: the defence is upstream, [3].")
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@
|
||||
|
||||
[project]
|
||||
name = "manifoldbt"
|
||||
version = "0.27.0"
|
||||
version = "0.27.1"
|
||||
description = "Rust-powered backtesting engine for quantitative research"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
||||
@@ -545,6 +545,8 @@ def _reject_intrabar_price_without_delay(cfg: BacktestConfig) -> None:
|
||||
Measured on a driftless random walk, "long when close > open" returns
|
||||
+1913% under `AtOpen`, +361% under `AtVwap`, `MidPrice` and
|
||||
`custom("low")`, against +5% under the defaults. The engine said nothing.
|
||||
`fill_model.intra_bar_price` (`TypicalPrice`, `OhlcAverage`) is the same
|
||||
average reached by another key, and is refused the same way.
|
||||
|
||||
**`custom(<a signal the strategy defines>)` is exempt, and deliberately.**
|
||||
There the level is one the DSL computed, and the fill only books if the bar
|
||||
@@ -556,6 +558,23 @@ def _reject_intrabar_price_without_delay(cfg: BacktestConfig) -> None:
|
||||
execution = getattr(cfg, "execution", None)
|
||||
if execution is None or getattr(execution, "signal_delay", 1) != 0:
|
||||
return
|
||||
# The same question under a second key: `fill_model.intra_bar_price`
|
||||
# averages the execution bar ((H+L+C)/3, (O+H+L+C)/4) whatever
|
||||
# `execution_price` says, and at delay zero that bar is the signal's own.
|
||||
# Read first, because the default price (the close) returns early below,
|
||||
# and that is the very combination that slipped through: Sharpe 37 and 65
|
||||
# on a driftless walk, against 1.3 at the close.
|
||||
fill = getattr(execution, "fill_model", None)
|
||||
intra = fill.get("intra_bar_price", "SinglePoint") if isinstance(fill, dict) else "SinglePoint"
|
||||
if intra not in (None, "SinglePoint"):
|
||||
raise ConfigError(
|
||||
f'fill_model["intra_bar_price"]={intra!r} needs signal_delay >= 1: '
|
||||
"with signal_delay=0 the fill is an average over the same bar whose "
|
||||
"close produced the signal, prices the decision came after. Set "
|
||||
"ExecutionConfig(signal_delay=1) to decide on one bar and execute on "
|
||||
'the next, or drop intra_bar_price ("SinglePoint", the default) to '
|
||||
"fill at the very close the signal was computed on."
|
||||
)
|
||||
price = getattr(execution, "execution_price", "AtClose")
|
||||
# `ExecutionPrice.custom(name)` serialises as {"Custom": name}.
|
||||
label = next(iter(price)) if isinstance(price, dict) else price
|
||||
@@ -1947,6 +1966,28 @@ def _by_day_arg(by_day) -> Optional[bool]:
|
||||
raise ValueError(f'by_day takes "auto", True or False, got {by_day!r}')
|
||||
|
||||
|
||||
def _reject_unset_params(strategy) -> None:
|
||||
"""A plain run needs a value for every param() its expressions read.
|
||||
|
||||
Without a default, a param() is set by a sweep and by nothing else. A run
|
||||
read it as an empty series: an EMA of no period, no signal, no trade, and
|
||||
no error (example 08's strategy run on its own made 0 trades, and
|
||||
detect_lookahead answered PASS on it). Sweeps never come through here.
|
||||
"""
|
||||
unset = getattr(strategy, "_unset_params", lambda: [])()
|
||||
if not unset:
|
||||
return
|
||||
names = ", ".join(repr(n) for n in unset)
|
||||
first = unset[0]
|
||||
raise StrategyError(
|
||||
f"strategy {strategy.name!r}: param({names}) has no value in a plain run. "
|
||||
"A param() without a default is set by a sweep; run() would read it as "
|
||||
f"empty and the strategy would make no trade. Give it one, "
|
||||
f"param({first!r}, default=...), or declare it with "
|
||||
f"Strategy.param({first!r}, default=...), or sweep it (run_sweep_lite)."
|
||||
)
|
||||
|
||||
|
||||
def run(
|
||||
strategy: Strategy,
|
||||
config: BacktestConfig,
|
||||
@@ -1988,6 +2029,7 @@ def run(
|
||||
graphics card, with the same result; a build or a machine without CUDA
|
||||
prepares them on the CPU and says so in a warning.
|
||||
"""
|
||||
_reject_unset_params(strategy)
|
||||
if _quotes(strategy):
|
||||
return _run_quotes(strategy, config, store, by_day=by_day, device=device)
|
||||
if not (isinstance(by_day, str) and by_day == "auto") or device != "cpu":
|
||||
|
||||
@@ -513,6 +513,8 @@ class ExecutionConfig:
|
||||
"""Fill model configuration. None = Rust defaults (atomic fill, single point).
|
||||
Example: {"max_participation_rate": 0.1, "intra_bar_price": "TypicalPrice"}
|
||||
intra_bar_price options: "SinglePoint", "TypicalPrice", "OhlcAverage"
|
||||
(the two averages need signal_delay >= 1: at delay 0 they would average
|
||||
the bar whose close produced the signal, and run() refuses them)
|
||||
passive_fill options: "touch" (default: a maker fill books when the bar
|
||||
reaches its level) or "traverse" (the bar must trade through the level;
|
||||
conservative, and the run stays on the general loop). The result's
|
||||
|
||||
@@ -41,9 +41,18 @@ class LookaheadReport:
|
||||
mismatched: int
|
||||
method: str = ""
|
||||
details: List[Dict[str, Any]] = field(default_factory=list)
|
||||
inconclusive: Optional[str] = None
|
||||
"""Why a re-run sub-test compared nothing (a strategy that never trades, a
|
||||
truncated run that failed). Such a report is not a pass: ``passed`` is
|
||||
False, and it prints INCONCLUSIVE rather than PASS or FAIL."""
|
||||
|
||||
def assert_clean(self) -> None:
|
||||
"""Raise AssertionError if look-ahead bias was detected."""
|
||||
"""Raise AssertionError if look-ahead bias was detected, or if the
|
||||
sub-test could not look for it."""
|
||||
if self.inconclusive:
|
||||
raise AssertionError(
|
||||
f"Look-ahead check inconclusive ({self.method}): {self.inconclusive}"
|
||||
)
|
||||
if not self.passed:
|
||||
if self.method == "static":
|
||||
msg = (
|
||||
@@ -63,6 +72,8 @@ class LookaheadReport:
|
||||
raise AssertionError(msg)
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.inconclusive:
|
||||
return f" [{self.method}] INCONCLUSIVE ({self.inconclusive})"
|
||||
status = "PASS" if self.passed else "FAIL"
|
||||
if self.method == "static":
|
||||
# "non-causal", pas "reads ahead" : le compte couvre maintenant
|
||||
@@ -102,7 +113,12 @@ class DiagnosticsResult:
|
||||
r.assert_clean()
|
||||
|
||||
def __str__(self) -> str:
|
||||
status = "PASS" if self.passed else "FAIL"
|
||||
if self.passed:
|
||||
status = "PASS"
|
||||
elif all(r.passed or r.inconclusive for r in self.reports):
|
||||
status = "INCONCLUSIVE"
|
||||
else:
|
||||
status = "FAIL"
|
||||
parts = [f"Lookahead diagnostics: {status}"]
|
||||
for r in self.reports:
|
||||
parts.append(str(r))
|
||||
@@ -135,15 +151,25 @@ def detect_lookahead(
|
||||
classified in a registry the engine checks exhaustively, so a new
|
||||
operator cannot be added without answering the question.
|
||||
* **extension** -- re-run on the first 2/3 of the data and compare the
|
||||
trades with the full run. Catches look-ahead that depends on how much
|
||||
data the run was given (a statistic over the whole series, a
|
||||
normalisation by the last value).
|
||||
trades with the full run. Catches a strategy whose past trades change
|
||||
with how much data the run was given: in this engine that takes a
|
||||
non-causal operator over the whole column, which the static test
|
||||
already names. A statistic computed outside the engine (a z-score or
|
||||
a normalisation fitted on the whole history, handed in as a constant
|
||||
or a column) reads the same in every window and is NOT caught: see
|
||||
the warning and the note below.
|
||||
* **truncation** -- the same comparison, split at 1/3.
|
||||
|
||||
The re-runs load the data once and slice it (no redundant I/O). The
|
||||
verdict is the conjunction: ``.passed`` is False as soon as one sub-test
|
||||
fails.
|
||||
|
||||
A re-run sub-test that has nothing to compare reports INCONCLUSIVE, not
|
||||
PASS, and ``.passed`` is False: a strategy that never trades (a
|
||||
``param()`` left without a value is enough), a truncated run that fails,
|
||||
or no trade before the split. A cut run with no trade while the full run
|
||||
traded before the split is a FAIL: the same bars gave two answers.
|
||||
|
||||
.. warning::
|
||||
**What the re-runs cannot see.** A leak that reads a *fixed* number of
|
||||
bars ahead (``close.lead(1)``, or a column precomputed with future bars
|
||||
@@ -158,9 +184,12 @@ def detect_lookahead(
|
||||
clean bill of health.
|
||||
|
||||
The ``static`` sub-test is the one that names an explicit ``lead()``.
|
||||
For an implicit one (a column built outside the engine) the check that
|
||||
works is to perturb the bars after some bar K and require the equity
|
||||
up to K to be bit-identical: ``examples/25_lookahead_trap.py`` does it.
|
||||
For an implicit one (a column built outside the engine) the check
|
||||
belongs to the pipeline that built it: perturb the raw data after some
|
||||
bar K, REBUILD the column from it, and require the equity up to K to be
|
||||
bit-identical. Perturbing the bars alone leaves a precomputed column
|
||||
untouched, so it sees nothing: ``examples/25_lookahead_trap.py`` shows
|
||||
exactly that.
|
||||
|
||||
.. note::
|
||||
**Scope.** The re-runs re-run the *same strategy* over a different
|
||||
@@ -227,6 +256,7 @@ def detect_lookahead(
|
||||
mismatched=r["mismatched"],
|
||||
method=r["method"],
|
||||
details=r["details"],
|
||||
inconclusive=r.get("inconclusive"),
|
||||
)
|
||||
for r in raw
|
||||
]
|
||||
@@ -746,9 +776,14 @@ def check_exposure_stability(
|
||||
|
||||
Sub-tests:
|
||||
* **extension** — compare full period vs first 2/3.
|
||||
Catches global normalization (e.g. zscore over entire series).
|
||||
* **truncation** — compare full period vs first 1/3.
|
||||
Catches rolling window that peeks ahead.
|
||||
|
||||
Both catch sizing whose past values change with how much data the run is
|
||||
given, which in this engine takes a non-causal operator over the whole
|
||||
column. Neither catches a fixed read-ahead (a window shifted by ``lead``
|
||||
reads the same bars in every window), nor a statistic fitted outside the
|
||||
engine on the whole history and handed in as a constant or a column: see
|
||||
:func:`detect_lookahead` for what each method can and cannot see.
|
||||
|
||||
Args:
|
||||
strategy: Strategy definition.
|
||||
|
||||
@@ -637,6 +637,34 @@ class Strategy:
|
||||
"""Order config dict (stop-loss, take-profit, trailing), or None."""
|
||||
return self._orders
|
||||
|
||||
def _expression_params(self) -> "_Declarations":
|
||||
"""The param() references this strategy's expressions read."""
|
||||
declared = _Declarations()
|
||||
for expr in self.signals.values():
|
||||
_collect_params(expr, declared)
|
||||
_collect_params(self.position_sizing, declared)
|
||||
for q in self._quotes:
|
||||
for key in ("price", "size", "enabled"):
|
||||
if key in q:
|
||||
_collect_params(q[key], declared)
|
||||
return declared
|
||||
|
||||
def _unset_params(self) -> List[str]:
|
||||
"""Parameters an expression reads that have no value outside a sweep.
|
||||
|
||||
A param() without a default serialises with a null default. A sweep
|
||||
sets every name it sweeps, but a plain run reads the null, which the
|
||||
engine evaluated as an empty series: no signal, no trade, no error. A
|
||||
name only declared with ``.param()`` and read by no expression is not
|
||||
reported: nothing reads it.
|
||||
"""
|
||||
specs = self.to_json_dict()["parameters"]
|
||||
return sorted(
|
||||
name
|
||||
for name in self._expression_params().metas
|
||||
if specs.get(name, {}).get("default") == "Null"
|
||||
)
|
||||
|
||||
def to_json_dict(self) -> dict:
|
||||
"""Serialize to a dict matching Rust ``StrategyDef`` serde format."""
|
||||
# `signals` is a plain public dict, so a caller can drop anything into it
|
||||
@@ -649,14 +677,7 @@ class Strategy:
|
||||
# Auto-collect params from this strategy's own expressions: the
|
||||
# default of a param() is the one written in the expression that uses
|
||||
# it, never one another strategy of the process declared.
|
||||
declared = _Declarations()
|
||||
for expr in self.signals.values():
|
||||
_collect_params(expr, declared)
|
||||
_collect_params(self.position_sizing, declared)
|
||||
for q in self._quotes:
|
||||
for key in ("price", "size", "enabled"):
|
||||
if key in q:
|
||||
_collect_params(q[key], declared)
|
||||
declared = self._expression_params()
|
||||
|
||||
# Merge: explicit .param() calls override auto-collected. One without
|
||||
# a default keeps the default the expressions give (like a param()
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Les diagnostics doivent accepter les series EXOGENES, comme `run`.
|
||||
|
||||
`detect_lookahead` relancait la strategie avec des tables exo vides, et
|
||||
`AlignedData` (sur lequel `check_exposure_stability` relance ses runs) n'en
|
||||
portait aucune : une strategie lisant `exo.<nom>.<colonne>` levait
|
||||
"unknown input column" dans les deux, alors que `mbt.run` la faisait tourner.
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import manifoldbt as bt
|
||||
|
||||
# Safety diagnostics are a Pro feature.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
bt.license_info()[0] != "Pro",
|
||||
reason="requires Pro: skipped on a Community wheel",
|
||||
)
|
||||
|
||||
pd = pytest.importorskip("pandas")
|
||||
np = pytest.importorskip("numpy")
|
||||
|
||||
|
||||
def _monte(tmp_path):
|
||||
"""Serie horaire reguliere, plus une moyenne posee en serie exogene."""
|
||||
n = 2400
|
||||
idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp")
|
||||
pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0)
|
||||
px = 100.0 + np.cumsum(pas) * 0.05
|
||||
df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996,
|
||||
"close": px, "volume": np.full(n, 1000.0)}, index=idx)
|
||||
racine = str(tmp_path / "data")
|
||||
meta = str(tmp_path / "meta.sqlite")
|
||||
os.makedirs(racine, exist_ok=True)
|
||||
store = bt.import_dataframe(df.reset_index(), symbol="ZDEX", symbol_id=1,
|
||||
interval="1h", asset_class="equity",
|
||||
exchange="TEST", data_root=racine, metadata_db=meta)
|
||||
# Moyenne CAUSALE (fenetre glissante sur le passe) : aucune fuite.
|
||||
moy = pd.Series(px).rolling(24).mean().to_numpy()
|
||||
bt.register_exo("moyenne", pd.DataFrame({"timestamp": idx, "sma": moy}),
|
||||
store=store, data_root=racine, timeframe="1h")
|
||||
tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11")
|
||||
cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1,
|
||||
initial_capital=1000.0, provider="TEST",
|
||||
bar_interval=bt.Interval.hours(1), symbol_names={"ZDEX": 1})
|
||||
cfg.warmup_bars = 0
|
||||
cfg.exo_data = ["moyenne"]
|
||||
return store, cfg
|
||||
|
||||
|
||||
def _strategie():
|
||||
from manifoldbt.indicators import close, col
|
||||
m = col("exo.moyenne.sma")
|
||||
return (bt.Strategy.create("s")
|
||||
.signal("m", m)
|
||||
.size(bt.when(close > m, 1.0, 0.0)))
|
||||
|
||||
|
||||
def test_la_strategie_a_exo_trade_dans_run(tmp_path):
|
||||
store, cfg = _monte(tmp_path)
|
||||
r = bt.run(_strategie(), cfg, store)
|
||||
assert r.trade_count > 0
|
||||
|
||||
|
||||
def test_detect_lookahead_accepte_une_serie_exogene(tmp_path):
|
||||
store, cfg = _monte(tmp_path)
|
||||
rapport = bt.diagnostics.detect_lookahead(_strategie(), cfg, store, mode="all")
|
||||
# Une moyenne glissante sur le passe ne fuit pas : les re-runs tronques
|
||||
# doivent retrouver les trades du run complet.
|
||||
assert rapport.passed, rapport
|
||||
|
||||
|
||||
def test_check_exposure_stability_accepte_une_serie_exogene(tmp_path):
|
||||
store, cfg = _monte(tmp_path)
|
||||
rapport = bt.diagnostics.check_exposure_stability(_strategie(), cfg, store)
|
||||
assert rapport.passed, rapport
|
||||
@@ -149,6 +149,37 @@ def test_le_message_nomme_le_prix_et_la_sortie(store):
|
||||
assert "AtClose" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moyenne", ["TypicalPrice", "OhlcAverage"])
|
||||
def test_une_moyenne_intrabarre_sans_delai_est_refusee(store, moyenne):
|
||||
"""`fill_model.intra_bar_price` averages the execution bar whatever the
|
||||
price says: at delay 0, with the default close, it filled at (H+L+C)/3 or
|
||||
(O+H+L+C)/4 of the signal's own bar (Sharpe 37 and 65 on a driftless
|
||||
walk, `detect_lookahead` PASS). Refused like `AtVwap`."""
|
||||
with pytest.raises(ConfigError, match="needs signal_delay >= 1") as exc:
|
||||
bt.run(_reads_its_own_bar(),
|
||||
_config(signal_delay=0, fill_model={"intra_bar_price": moyenne}),
|
||||
store)
|
||||
assert moyenne in str(exc.value)
|
||||
assert "signal_delay=1" in str(exc.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moyenne", ["TypicalPrice", "OhlcAverage"])
|
||||
def test_une_moyenne_intrabarre_avec_delai_reste_permise(store, moyenne):
|
||||
"""With a delay the averaged bar follows the signal: nothing to refuse."""
|
||||
result = bt.run(_reads_its_own_bar(),
|
||||
_config(signal_delay=1, fill_model={"intra_bar_price": moyenne}),
|
||||
store)
|
||||
assert abs(result.metrics["total_return"]) < 0.5
|
||||
|
||||
|
||||
def test_single_point_sans_delai_reste_permis(store):
|
||||
"""`SinglePoint` is the default: the price named by `execution_price`."""
|
||||
result = bt.run(_reads_its_own_bar(),
|
||||
_config(signal_delay=0, fill_model={"intra_bar_price": "SinglePoint"}),
|
||||
store)
|
||||
assert abs(result.metrics["total_return"]) < 0.5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("price", ["AtClose", "NextBarClose"])
|
||||
def test_la_cloture_reste_permise_sans_delai(store, price):
|
||||
"""These two fill at the close that produced the signal: nothing to refuse.
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Un sous-test de re-run qui n'a rien compare ne rend pas PASS.
|
||||
|
||||
detect_lookahead rendait PASS sur une strategie qui ne trade jamais (un
|
||||
param() sans valeur suffit), ou quand le run tronque echouait : un verdict
|
||||
vide lu comme un feu vert. Il rend maintenant INCONCLUSIVE, `.passed` vaut
|
||||
False et `assert_clean()` leve, en disant pourquoi.
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import manifoldbt as bt
|
||||
|
||||
# Safety diagnostics are a Pro feature.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
bt.license_info()[0] != "Pro",
|
||||
reason="requires Pro: skipped on a Community wheel",
|
||||
)
|
||||
|
||||
pd = pytest.importorskip("pandas")
|
||||
np = pytest.importorskip("numpy")
|
||||
|
||||
|
||||
def _monte(tmp_path):
|
||||
n = 2400
|
||||
idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp")
|
||||
pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0)
|
||||
px = 100.0 + np.cumsum(pas) * 0.05
|
||||
df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996,
|
||||
"close": px, "volume": np.full(n, 1000.0)}, index=idx)
|
||||
racine = str(tmp_path / "data")
|
||||
os.makedirs(racine, exist_ok=True)
|
||||
store = bt.import_dataframe(df.reset_index(), symbol="ZLSV", symbol_id=1,
|
||||
interval="1h", asset_class="equity",
|
||||
exchange="TEST", data_root=racine,
|
||||
metadata_db=str(tmp_path / "meta.sqlite"))
|
||||
tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11")
|
||||
cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1,
|
||||
initial_capital=1000.0, provider="TEST",
|
||||
bar_interval=bt.Interval.hours(1), symbol_names={"ZLSV": 1})
|
||||
cfg.warmup_bars = 0
|
||||
return store, cfg
|
||||
|
||||
|
||||
def _jamais():
|
||||
from manifoldbt.indicators import close
|
||||
return (bt.Strategy.create("jamais")
|
||||
.signal("c", close)
|
||||
.size(bt.lit(0.0)))
|
||||
|
||||
|
||||
def _croise():
|
||||
from manifoldbt.indicators import close, sma
|
||||
m = sma(close, 24)
|
||||
return bt.Strategy.create("croise").signal("m", m).size(bt.when(close > m, 1.0, 0.0))
|
||||
|
||||
|
||||
def test_une_strategie_qui_ne_trade_pas_est_inconclusive(tmp_path):
|
||||
store, cfg = _monte(tmp_path)
|
||||
rapport = bt.diagnostics.detect_lookahead(_jamais(), cfg, store, mode="all")
|
||||
assert not rapport.passed
|
||||
re_runs = [r for r in rapport.reports if r.method != "static"]
|
||||
assert {r.method for r in re_runs} == {"extension", "truncation"}
|
||||
for r in re_runs:
|
||||
assert r.inconclusive and "made no trade" in r.inconclusive
|
||||
assert "INCONCLUSIVE" in str(r)
|
||||
statique = [r for r in rapport.reports if r.method == "static"]
|
||||
assert statique and statique[0].passed
|
||||
assert str(rapport).startswith("Lookahead diagnostics: INCONCLUSIVE")
|
||||
with pytest.raises(AssertionError, match="inconclusive"):
|
||||
rapport.assert_clean()
|
||||
|
||||
|
||||
def test_une_strategie_causale_qui_trade_reste_pass(tmp_path):
|
||||
store, cfg = _monte(tmp_path)
|
||||
rapport = bt.diagnostics.detect_lookahead(_croise(), cfg, store, mode="all")
|
||||
assert rapport.passed, rapport
|
||||
assert all(r.inconclusive is None for r in rapport.reports)
|
||||
assert str(rapport).startswith("Lookahead diagnostics: PASS")
|
||||
@@ -0,0 +1,81 @@
|
||||
"""run() refuse un param() lu sans valeur, au lieu de ne rien trader.
|
||||
|
||||
Un param() sans defaut n'a de valeur que dans un balayage. Un run seul le
|
||||
lisait comme une serie vide : EMA sans periode, aucun signal, 0 trade, aucune
|
||||
erreur (la strategie de l'exemple 08 lancee seule). Les balayages, eux,
|
||||
fixent chaque nom balaye et ne passent pas par cette garde.
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import manifoldbt as bt
|
||||
from manifoldbt.exceptions import StrategyError
|
||||
from manifoldbt.indicators import close, ema
|
||||
|
||||
pd = pytest.importorskip("pandas")
|
||||
np = pytest.importorskip("numpy")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def monte(tmp_path_factory):
|
||||
tmp = tmp_path_factory.mktemp("param")
|
||||
n = 2400
|
||||
idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp")
|
||||
pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0)
|
||||
px = 100.0 + np.cumsum(pas) * 0.05
|
||||
df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996,
|
||||
"close": px, "volume": np.full(n, 1000.0)}, index=idx)
|
||||
racine = str(tmp / "data")
|
||||
os.makedirs(racine, exist_ok=True)
|
||||
store = bt.import_dataframe(df.reset_index(), symbol="ZPSV", symbol_id=1,
|
||||
interval="1h", asset_class="equity",
|
||||
exchange="TEST", data_root=racine,
|
||||
metadata_db=str(tmp / "meta.sqlite"))
|
||||
tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11")
|
||||
cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1,
|
||||
initial_capital=1000.0, provider="TEST",
|
||||
bar_interval=bt.Interval.hours(1), symbol_names={"ZPSV": 1})
|
||||
return store, cfg
|
||||
|
||||
|
||||
def _croisement(fast, slow):
|
||||
f, s = ema(close, fast), ema(close, slow)
|
||||
return (bt.Strategy.create("croisement").signal("f", f).signal("s", s)
|
||||
.size(bt.when(f > s, 0.25, bt.when(f < s, -0.25, 0.0))))
|
||||
|
||||
|
||||
def test_un_param_lu_sans_valeur_est_refuse_et_nomme(monte):
|
||||
store, cfg = monte
|
||||
with pytest.raises(StrategyError, match="has no value in a plain run") as exc:
|
||||
bt.run(_croisement(bt.param("fast"), bt.param("slow")), cfg, store)
|
||||
assert "'fast'" in str(exc.value) and "'slow'" in str(exc.value)
|
||||
|
||||
|
||||
def test_un_param_avec_defaut_tourne(monte):
|
||||
store, cfg = monte
|
||||
r = bt.run(_croisement(bt.param("fast", default=12), bt.param("slow", default=26)),
|
||||
cfg, store)
|
||||
assert r.trade_count > 0
|
||||
|
||||
|
||||
def test_une_declaration_strategy_param_donne_la_valeur(monte):
|
||||
store, cfg = monte
|
||||
s = (_croisement(bt.param("fast"), bt.param("slow"))
|
||||
.param("fast", default=12).param("slow", default=26))
|
||||
assert bt.run(s, cfg, store).trade_count > 0
|
||||
|
||||
|
||||
def test_un_param_declare_mais_lu_par_rien_ne_bloque_pas(monte):
|
||||
store, cfg = monte
|
||||
s = _croisement(12, 26).param("seuil")
|
||||
assert bt.run(s, cfg, store).trade_count > 0
|
||||
|
||||
|
||||
def test_un_balayage_fixe_les_valeurs_et_passe(monte):
|
||||
store, cfg = monte
|
||||
if bt.license_info()[0] != "Pro":
|
||||
pytest.skip("requires Pro beyond the Community combination limit")
|
||||
b = bt.run_sweep_lite(_croisement(bt.param("fast"), bt.param("slow")),
|
||||
{"fast": [8, 12], "slow": [26]}, cfg, store)
|
||||
assert len(b) == 2
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Le walk-forward doit accepter les sources exogenes PAR SYMBOLE (`exo_sources`).
|
||||
|
||||
Une strategie lisant `exo.<colonne>` d'une source par symbole tournait dans
|
||||
`run`, mais le walk-forward passait des sources vides a ses trois runs
|
||||
(selection, run d'apprentissage, run hors echantillon) et levait
|
||||
"unknown input column".
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import manifoldbt as bt
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
bt.license_info()[0] != "Pro",
|
||||
reason="requires Pro: skipped on a Community wheel",
|
||||
)
|
||||
|
||||
pd = pytest.importorskip("pandas")
|
||||
np = pytest.importorskip("numpy")
|
||||
|
||||
|
||||
def _monte(tmp_path):
|
||||
"""Le titre trade, une seconde source porte une moyenne causale du titre."""
|
||||
n = 2400
|
||||
idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp")
|
||||
pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0)
|
||||
px = 100.0 + np.cumsum(pas) * 0.05
|
||||
aux = pd.Series(px).rolling(24, min_periods=1).mean().to_numpy()
|
||||
|
||||
def barres(p):
|
||||
return pd.DataFrame({"open": p, "high": p * 1.004, "low": p * 0.996,
|
||||
"close": p, "volume": np.full(n, 1000.0)},
|
||||
index=idx).reset_index()
|
||||
|
||||
racine = str(tmp_path / "data")
|
||||
meta = str(tmp_path / "meta.sqlite")
|
||||
os.makedirs(racine, exist_ok=True)
|
||||
store = bt.import_dataframe(barres(px), symbol="ZESM", symbol_id=1, interval="1h",
|
||||
asset_class="equity", exchange="TEST",
|
||||
data_root=racine, metadata_db=meta)
|
||||
bt.import_dataframe(barres(aux), symbol="ZESA", symbol_id=2, interval="1h",
|
||||
asset_class="equity", exchange="AUX",
|
||||
data_root=racine, metadata_db=meta)
|
||||
tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11")
|
||||
cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1,
|
||||
initial_capital=1000.0, provider="TEST",
|
||||
bar_interval=bt.Interval.hours(1), symbol_names={"ZESM": 1})
|
||||
cfg.warmup_bars = 0
|
||||
cfg.exo_sources = {1: ("AUX", "ZESA")}
|
||||
return store, cfg
|
||||
|
||||
|
||||
def _strategie():
|
||||
from manifoldbt.indicators import close, col
|
||||
m = col("exo.close")
|
||||
bande = m * (bt.lit(1.0) - bt.param("dev", default=0.002))
|
||||
return (bt.Strategy.create("s")
|
||||
.signal("b", bande)
|
||||
.size(bt.when(close < bande, 1.0, bt.when(close > m, 0.0, bt.hold()))))
|
||||
|
||||
|
||||
WF = {"method": "Anchored", "n_splits": 3, "train_ratio": 0.5,
|
||||
"optimize_metric": "sharpe", "param_grid": {"dev": [0.002, 0.005, 0.01]}}
|
||||
|
||||
|
||||
def test_la_strategie_trade_dans_run(tmp_path):
|
||||
store, cfg = _monte(tmp_path)
|
||||
assert bt.run(_strategie(), cfg, store).trade_count > 0
|
||||
|
||||
|
||||
def test_walk_forward_accepte_une_source_par_symbole(tmp_path):
|
||||
store, cfg = _monte(tmp_path)
|
||||
r = bt.run_walk_forward(_strategie(), WF, cfg, store)
|
||||
assert r["n_folds"] == 3
|
||||
# La source est lue dans les trois runs : la selection (metriques IS) et
|
||||
# les deux courbes. Sans elle, aucun pli ne trade.
|
||||
assert any(f["oos_metrics"]["total_return"] != 0.0 for f in r["folds"])
|
||||
assert any(len(set(f["is_equity"])) > 1 for f in r["folds"])
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Walk-forward sur un signal plus grossier que le store.
|
||||
|
||||
Un signal 12h sur des barres 1h (mode hybride) rendait, en sortie
|
||||
automatique, une courbe OOS sous-echantillonnee au pas du signal. Sa longueur
|
||||
ne valait plus celle des barres natives, et le walk-forward retombait sur les
|
||||
metriques du run ENTIER, chauffe plate comprise : Sharpe hors echantillon
|
||||
dilue, WFE faux, sans un avertissement. Avec `output_resolution` a 1h, le meme
|
||||
walk-forward donnait le bon chiffre.
|
||||
|
||||
Le contrat : les metriques hors echantillon ne dependent pas de la resolution
|
||||
de sortie, et la courbe rendue suit celle-ci, avec autant d'horodatages que de
|
||||
points.
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import manifoldbt as bt
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
bt.license_info()[0] != "Pro",
|
||||
reason="requires Pro: skipped on a Community wheel",
|
||||
)
|
||||
|
||||
pd = pytest.importorskip("pandas")
|
||||
np = pytest.importorskip("numpy")
|
||||
|
||||
HEURE_NS = 3_600 * 1_000_000_000
|
||||
|
||||
|
||||
def _magasin_horaire(tmp_path):
|
||||
"""100 jours de barres horaires, prix cyclique pour garantir des trades."""
|
||||
n = 2400
|
||||
idx = pd.date_range("2021-01-01", periods=n, freq="1h", tz="UTC", name="timestamp")
|
||||
pas = np.sin(np.arange(n) / 37.0) * 2.0 + np.cos(np.arange(n) / 11.0)
|
||||
px = 100.0 + np.cumsum(pas) * 0.05
|
||||
df = pd.DataFrame({"open": px, "high": px * 1.004, "low": px * 0.996,
|
||||
"close": px, "volume": np.full(n, 1000.0)}, index=idx)
|
||||
racine = str(tmp_path / "data")
|
||||
meta = str(tmp_path / "meta.sqlite")
|
||||
os.makedirs(racine, exist_ok=True)
|
||||
store = bt.import_dataframe(df.reset_index(), symbol="ZWFR", symbol_id=1,
|
||||
interval="1h", asset_class="equity",
|
||||
exchange="TEST", data_root=racine, metadata_db=meta)
|
||||
return store
|
||||
|
||||
|
||||
def _config(signal):
|
||||
tr0, tr1 = bt.time_range("2021-01-01", "2021-04-11")
|
||||
cfg = bt.BacktestConfig(universe=[1], time_range_start=tr0, time_range_end=tr1,
|
||||
initial_capital=1000.0, provider="TEST",
|
||||
bar_interval=signal, symbol_names={"ZWFR": 1})
|
||||
cfg.warmup_bars = 0
|
||||
return cfg
|
||||
|
||||
|
||||
def _strategie():
|
||||
from manifoldbt.indicators import close, sma
|
||||
m = sma(close, bt.param("n"))
|
||||
return (bt.Strategy.create("s")
|
||||
.signal("m", m)
|
||||
.size(bt.when(close > m, 1.0, 0.0)))
|
||||
|
||||
|
||||
WF = {"geometry": "anchored", "n_splits": 2, "train_ratio": 0.6,
|
||||
"optimize_metric": "total_return", "param_grid": {"n": [4, 8]}}
|
||||
|
||||
|
||||
def test_les_metriques_oos_ne_dependent_pas_de_la_sortie(tmp_path):
|
||||
store = _magasin_horaire(tmp_path)
|
||||
auto = bt.run_walk_forward(_strategie(), WF, _config(bt.Interval.hours(12)), store)
|
||||
cfg_native = _config(bt.Interval.hours(12))
|
||||
cfg_native.output_resolution = bt.Interval.hours(1)
|
||||
native = bt.run_walk_forward(_strategie(), WF, cfg_native, store)
|
||||
|
||||
assert auto["n_folds"] == native["n_folds"] == 2
|
||||
for fa, fn in zip(auto["folds"], native["folds"]):
|
||||
assert fa["best_params"] == fn["best_params"]
|
||||
for k in ("sharpe", "cagr", "total_return", "volatility", "max_drawdown"):
|
||||
assert fa["oos_metrics"][k] == pytest.approx(fn["oos_metrics"][k], rel=1e-12), k
|
||||
assert fa["oos_metrics"]["sharpe"] != 0.0, "le pli doit trader"
|
||||
if fn["wfe"] is None:
|
||||
assert fa["wfe"] is None
|
||||
else:
|
||||
assert fa["wfe"] == pytest.approx(fn["wfe"], rel=1e-12)
|
||||
|
||||
|
||||
def test_la_courbe_oos_suit_la_resolution_de_sortie(tmp_path):
|
||||
store = _magasin_horaire(tmp_path)
|
||||
auto = bt.run_walk_forward(_strategie(), WF, _config(bt.Interval.hours(12)), store)
|
||||
cfg_native = _config(bt.Interval.hours(12))
|
||||
cfg_native.output_resolution = bt.Interval.hours(1)
|
||||
native = bt.run_walk_forward(_strategie(), WF, cfg_native, store)
|
||||
for fa, fn in zip(auto["folds"], native["folds"]):
|
||||
eq, ts = fa["oos_equity"], fa["oos_timestamps"]
|
||||
assert len(eq) == len(ts) > 0
|
||||
assert ts[0] >= fa["test_range"]["start"]
|
||||
assert ts[-1] < fa["test_range"]["end"]
|
||||
# Un point par seau de sortie, au plus 12h d'ecart...
|
||||
pas = np.diff(np.asarray(ts))
|
||||
assert pas.max() == 12 * HEURE_NS and pas.min() > 0
|
||||
# ... et chaque point est celui de la courbe native a cet instant.
|
||||
natif = dict(zip(fn["oos_timestamps"], fn["oos_equity"]))
|
||||
assert len(natif) == len(fn["oos_timestamps"]) > len(ts)
|
||||
for t, e in zip(ts, eq):
|
||||
assert natif[t] == e
|
||||
Reference in New Issue
Block a user