gavindiazandClaude Code
1bcfe6e374
research/ now runs end-to-end (fetch -> features/labels -> walk-forward -> train -> OOS evaluate -> save), and backtest_strategy() replays the same signal(ctx) strategy over a historical frame with the same manage_position hook the live loop calls, simulating SL/TP, trailing, partial closes and cost per bar. --backtest wires it into the CLI. Co-Authored-By: Claude Code <noreply@anthropic.com>
446 lines
17 KiB
Python
446 lines
17 KiB
Python
"""Walk-forward backtesting: delegate to mtdata-cli ``strategy_validate``.
|
|
|
|
``strategy_validate`` runs anchored fixed-candidate OOS validation with
|
|
horizon-safe outcomes (barrier next-open execution, purge/embargo to prevent
|
|
leakage). This module wraps it as a subprocess and also provides a tiny local
|
|
walk-forward stub so the pipeline can run without a live broker connection.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import subprocess
|
|
from typing import Any, Optional
|
|
|
|
import pandas as pd
|
|
|
|
from .config import ResearchConfig
|
|
|
|
|
|
class BacktestError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _run_cli(config: ResearchConfig, *args: str, timeout: Optional[float] = None) -> dict[str, Any]:
|
|
cmd = [config.mtdata_cli, *args, "--json"]
|
|
try:
|
|
proc = subprocess.run(
|
|
cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout or config.cli_timeout_seconds,
|
|
)
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise BacktestError(f"mtdata-cli timed out: {' '.join(cmd)}") from exc
|
|
except FileNotFoundError as exc:
|
|
raise BacktestError(f"mtdata-cli not found: {config.mtdata_cli!r}") from exc
|
|
if proc.returncode != 0:
|
|
raise BacktestError(f"mtdata-cli failed ({proc.returncode}): {proc.stderr.strip()}")
|
|
try:
|
|
payload = json.loads(proc.stdout)
|
|
except json.JSONDecodeError as exc:
|
|
raise BacktestError(f"mtdata-cli returned non-JSON: {proc.stdout[:200]!r}") from exc
|
|
return payload
|
|
|
|
|
|
def run_strategy_validate(
|
|
config: ResearchConfig,
|
|
symbol: str,
|
|
candidates: list[dict[str, Any]],
|
|
*,
|
|
timeframe: Optional[str] = None,
|
|
lookback: Optional[int] = None,
|
|
) -> dict[str, Any]:
|
|
"""Invoke mtdata-cli strategy_validate with cost-model flags and n_splits."""
|
|
args = ["strategy_validate", symbol]
|
|
args += ["--timeframe", timeframe or config.timeframe]
|
|
if lookback is not None:
|
|
args += ["--lookback", str(lookback)]
|
|
elif config.wf_min_train_bars > 0:
|
|
args += ["--lookback", str(config.wf_min_train_bars * max(config.wf_n_splits, 1))]
|
|
args += ["--n-splits", str(config.wf_n_splits)]
|
|
args += ["--candidates", json.dumps(candidates)]
|
|
if config.commission_bps_per_side > 0 or config.spread_bps > 0:
|
|
args += ["--cost-model", "fixed"]
|
|
args += ["--spread-bps", str(config.spread_bps)]
|
|
if config.commission_bps_per_side > 0:
|
|
args += ["--commission-bps-per-side", str(config.commission_bps_per_side)]
|
|
if config.slippage_bps > 0:
|
|
args += ["--slippage-bps", str(config.slippage_bps)]
|
|
return _run_cli(config, *args)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# local walk-forward stub (no broker needed)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def local_walk_forward(frame: pd.DataFrame, config: ResearchConfig) -> dict[str, Any]:
|
|
"""Chronological fold summary over a local frame.
|
|
|
|
Placeholder: splits the frame into ``wf_n_splits`` chronological folds and
|
|
reports per-fold bar counts plus the label distribution. Real strategy/model
|
|
scoring plugs in here or, preferably, through ``run_strategy_validate``.
|
|
"""
|
|
if len(frame) < config.wf_min_train_bars:
|
|
raise BacktestError(
|
|
f"need >= {config.wf_min_train_bars} bars for walk-forward, got {len(frame)}"
|
|
)
|
|
from .feature_engineering import build_labels
|
|
|
|
labels = build_labels(frame, config)
|
|
n = len(frame)
|
|
fold_size = n // config.wf_n_splits
|
|
folds: list[dict[str, Any]] = []
|
|
for i in range(config.wf_n_splits):
|
|
lo = i * fold_size
|
|
hi = n if i == config.wf_n_splits - 1 else (i + 1) * fold_size
|
|
part = labels.iloc[lo:hi].dropna()
|
|
counts = part.value_counts().to_dict()
|
|
folds.append(
|
|
{
|
|
"fold": i,
|
|
"start_idx": lo,
|
|
"end_idx": hi,
|
|
"bars": hi - lo,
|
|
"labels": {int(k): int(v) for k, v in counts.items()},
|
|
"hit_rate": float((part == 1).mean()) if len(part) else float("nan"),
|
|
}
|
|
)
|
|
return {"method": "local_walk_forward_stub", "n_splits": config.wf_n_splits, "folds": folds}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# offline strategy backtest (reuses the *same* signal(ctx) as live trading)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _candle_row(row: pd.Series) -> dict[str, Any]:
|
|
"""Map one frame row to the candle dict the shell's SignalContext expects."""
|
|
return {
|
|
"time": str(row.name) if hasattr(row.name, "strftime") else str(row.name),
|
|
"open": float(row.get("open", row["close"])),
|
|
"high": float(row.get("high", row["close"])),
|
|
"low": float(row.get("low", row["close"])),
|
|
"close": float(row["close"]),
|
|
"tick_volume": int(row.get("tick_volume", 0) or 0),
|
|
}
|
|
|
|
|
|
def _cost_pct(config: ResearchConfig) -> tuple[float, float]:
|
|
"""Round-trip cost as a fraction: (entry_side, exit_side)."""
|
|
entry = (config.slippage_bps + config.commission_bps_per_side) * 0.0001
|
|
exit_ = (config.slippage_bps + config.commission_bps_per_side) * 0.0001
|
|
entry += config.spread_bps * 0.0001 # spread opens against the trade
|
|
return entry, exit_
|
|
|
|
|
|
def _make_context(
|
|
frame: pd.DataFrame,
|
|
config: ResearchConfig,
|
|
index: int,
|
|
position: Optional[dict],
|
|
symbol_spec: Optional[dict],
|
|
) -> Any:
|
|
"""Rebuild a :class:`trading.types.SignalContext` for bar ``index``, with the
|
|
current open position and broker spec — matching the live loop's shape."""
|
|
from trading.types import SignalContext
|
|
|
|
window = frame.iloc[: index + 1].reset_index(drop=True)
|
|
candles = [_candle_row(window.iloc[k]) for k in range(len(window))]
|
|
px = float(frame["close"].iloc[index])
|
|
return SignalContext(
|
|
symbol=getattr(config, "symbol", "EURUSD"),
|
|
timeframe=config.timeframe,
|
|
candles=candles,
|
|
quote={"bid": px, "ask": px},
|
|
account={},
|
|
positions=[position] if position is not None else [],
|
|
symbol_spec=symbol_spec or {},
|
|
)
|
|
|
|
|
|
def _sim_position(direction: int, entry: float, sl: float, tp: float, volume: float, reason: str) -> dict:
|
|
"""Simulated position in the shape exits.py and strategies read."""
|
|
return {
|
|
"type": "BUY" if direction == 1 else "SELL",
|
|
"open_price": entry,
|
|
"price_current": entry,
|
|
"stop_loss": sl,
|
|
"take_profit": tp,
|
|
"volume": volume,
|
|
"signal_reason": reason,
|
|
}
|
|
|
|
|
|
def _pnl_pct(position: dict, exit_price: float, entry_cost: float, exit_cost: float) -> float:
|
|
direction = 1 if position["type"] == "BUY" else -1
|
|
entry = float(position["open_price"])
|
|
gross = direction * (exit_price / entry - 1.0) * 100.0
|
|
return gross - entry_cost * 100.0 - exit_cost * 100.0
|
|
|
|
|
|
def _r_multiple(position: dict, exit_price: float) -> float:
|
|
direction = 1 if position["type"] == "BUY" else -1
|
|
entry = float(position["open_price"])
|
|
risk = abs(entry - float(position["stop_loss"]))
|
|
if risk <= 0:
|
|
return 0.0
|
|
return direction * (exit_price / entry - 1.0) / (risk / entry)
|
|
|
|
|
|
def apply_position_action(
|
|
position: dict,
|
|
action: Optional[Any],
|
|
price: float,
|
|
entry_cost: float,
|
|
exit_cost: float,
|
|
) -> tuple[Optional[dict], Optional[dict]]:
|
|
"""Apply a strategy :class:`trading.types.PositionAction` to a simulated position.
|
|
|
|
Returns ``(new_position, realized_trade)`` — ``realized_trade`` is ``None``
|
|
for "hold"/"modify" (no PnL locked yet); for "close" it clears the position
|
|
and locks full PnL; for "partial_close" it reduces ``volume`` by
|
|
``close_volume`` and locks PnL for that slice only. This mirrors exactly
|
|
what the live loop's Executor does with the same action.
|
|
"""
|
|
if action is None or action.kind == "hold":
|
|
return position, None
|
|
if action.kind == "modify":
|
|
if action.stop_loss is not None:
|
|
position["stop_loss"] = float(action.stop_loss)
|
|
if action.take_profit is not None:
|
|
position["take_profit"] = float(action.take_profit)
|
|
return position, None
|
|
if action.kind == "close":
|
|
net = _pnl_pct(position, price, entry_cost, exit_cost)
|
|
return None, {
|
|
"exit": round(price, 5),
|
|
"reason": "close",
|
|
"pnl_pct": net,
|
|
"r_multiple": _r_multiple(position, price),
|
|
}
|
|
if action.kind == "partial_close":
|
|
volume = float(position.get("volume", 0.0) or 0.0)
|
|
close_vol = min(float(action.close_volume or 0.0), volume)
|
|
if close_vol <= 0:
|
|
return position, None
|
|
fraction = close_vol / volume if volume else 1.0
|
|
net = _pnl_pct(position, price, entry_cost, exit_cost) * fraction
|
|
position["volume"] = volume - close_vol
|
|
trade = {
|
|
"exit": round(price, 5),
|
|
"reason": "partial_close",
|
|
"pnl_pct": net,
|
|
"r_multiple": _r_multiple(position, price) * fraction,
|
|
"close_volume": round(close_vol, 6),
|
|
}
|
|
return (None if position["volume"] <= 1e-12 else position), trade
|
|
return position, None
|
|
|
|
|
|
def _record_trade(
|
|
trades: list[dict],
|
|
equity_pct: list[float],
|
|
position: dict,
|
|
entry_bar: int,
|
|
exit_idx: int,
|
|
exit_price: float,
|
|
reason: str,
|
|
net_pct: float,
|
|
r_mul: float,
|
|
*,
|
|
close_volume: Optional[float] = None,
|
|
) -> None:
|
|
"""Append one closed-trade row and advance the cumulative equity curve."""
|
|
entry = float(position["open_price"])
|
|
row = {
|
|
"side": "BUY" if position["type"] == "BUY" else "SELL",
|
|
"entry_idx": entry_bar,
|
|
"entry": round(entry, 5),
|
|
"exit_idx": exit_idx,
|
|
"exit": round(exit_price, 5),
|
|
"reason": reason,
|
|
"bars_held": exit_idx - entry_bar,
|
|
"pnl_pct": round(net_pct, 4),
|
|
"r_multiple": round(r_mul, 4),
|
|
"signal_reason": position.get("signal_reason", ""),
|
|
}
|
|
if close_volume is not None:
|
|
row["close_volume"] = round(close_volume, 6)
|
|
trades.append(row)
|
|
equity_pct.append(equity_pct[-1] + net_pct)
|
|
|
|
|
|
def backtest_strategy(
|
|
strategy: Any,
|
|
frame: pd.DataFrame,
|
|
config: ResearchConfig,
|
|
*,
|
|
same_bar_policy: str = "sl_first",
|
|
symbol_spec: Optional[dict] = None,
|
|
) -> dict[str, Any]:
|
|
"""Backtest any ``signal(ctx)`` strategy on a historical frame.
|
|
|
|
This is the key abstraction for "time spent on strategy research": the
|
|
*same* strategy object used by the live shell (a ``trading.strategies``
|
|
instance) is called bar-by-bar over historical data with a reconstructed
|
|
:class:`trading.types.SignalContext`. No dataset-label coupling — trades
|
|
are simulated from the signal's own SL/TP levels with realistic cost.
|
|
|
|
Position management is modeled too: if the strategy overrides
|
|
``manage_position`` (the hook the live loop calls), every bar that does not
|
|
hit the server-side SL/TP first runs it and applies the returned
|
|
``PositionAction`` with the same semantics as the live Executor — so a
|
|
trailing stop, breakeven, partial close or early exit behaves identically
|
|
in research and live. ``symbol_spec`` (a ``symbols_describe`` payload) is
|
|
passed into every context, as the live loop does.
|
|
|
|
Execution model (close-to-next-open to avoid lookahead):
|
|
1. evaluate the strategy at bar *i* (close)
|
|
2. open the trade at bar *i+1*, if a signal fired
|
|
3. per bar: server-side SL/TP first (prices moved first), then
|
|
``manage_position`` at the bar close; unresolved trades close at the
|
|
last bar's close.
|
|
|
|
Returns summary stats plus per-trade rows.
|
|
"""
|
|
required = {"close"}
|
|
missing = required - set(frame.columns)
|
|
if missing:
|
|
raise BacktestError(f"frame missing columns {sorted(missing)} for backtest")
|
|
if len(frame) < 3:
|
|
raise BacktestError(f"need >= 3 bars to backtest, got {len(frame)}")
|
|
|
|
trades: list[dict[str, Any]] = []
|
|
equity_pct: list[float] = [0.0] # cumulative return % after each closed trade
|
|
entry_cost, exit_cost = _cost_pct(config)
|
|
|
|
n_bars = len(frame)
|
|
position: Optional[dict] = None
|
|
entry_bar = -1
|
|
last_close = 0.0
|
|
|
|
for j in range(n_bars):
|
|
bar = frame.iloc[j]
|
|
last_close = float(bar["close"])
|
|
high = float(bar["high"]) if "high" in frame.columns else last_close
|
|
low = float(bar["low"]) if "low" in frame.columns else last_close
|
|
|
|
if position is not None and j > entry_bar:
|
|
# (a) server-side SL/TP decide first — prices moved first in real time
|
|
direction = 1 if position["type"] == "BUY" else -1
|
|
sl = float(position["stop_loss"])
|
|
tp = float(position["take_profit"])
|
|
hit_sl = low <= sl if direction == 1 else high >= sl
|
|
hit_tp = high >= tp if direction == 1 else low <= tp
|
|
if hit_sl and hit_tp:
|
|
if same_bar_policy == "tp_first":
|
|
hit_sl = False
|
|
else:
|
|
hit_tp = False
|
|
exit_price: Optional[float] = None
|
|
reason = ""
|
|
if hit_tp:
|
|
exit_price, reason = tp, "tp"
|
|
elif hit_sl:
|
|
exit_price, reason = sl, "sl"
|
|
if exit_price is not None:
|
|
_record_trade(
|
|
trades, equity_pct, position, entry_bar, j, exit_price, reason,
|
|
_pnl_pct(position, exit_price, entry_cost, exit_cost),
|
|
_r_multiple(position, exit_price),
|
|
)
|
|
position = None
|
|
continue
|
|
|
|
# (b) strategy management at bar close (no SL/TP hit this bar)
|
|
ctx = _make_context(frame, config, j, position, symbol_spec)
|
|
before = position
|
|
action = strategy.manage_position(ctx, position)
|
|
position, realized = apply_position_action(before, action, last_close, entry_cost, exit_cost)
|
|
if realized is not None:
|
|
_record_trade(
|
|
trades, equity_pct, before, entry_bar, j,
|
|
realized["exit"], realized["reason"],
|
|
realized["pnl_pct"], realized["r_multiple"],
|
|
close_volume=realized.get("close_volume"),
|
|
)
|
|
continue
|
|
|
|
if position is None and j < n_bars - 1:
|
|
# entry evaluation at bar j (close); trades only act from bar j+1 on
|
|
ctx = _make_context(frame, config, j, None, symbol_spec)
|
|
signal = strategy.signal(ctx)
|
|
if signal is None:
|
|
continue
|
|
entry = last_close
|
|
direction = 1 if signal.side.upper() in ("BUY", "LONG", "BUY_LIMIT") else -1
|
|
sl = signal.stop_loss
|
|
tp = signal.take_profit
|
|
if not sl or not tp or sl <= 0 or tp <= 0:
|
|
continue # levels missing (e.g. ATR warmup) — treat as no trade
|
|
volume = float(signal.volume) if signal.volume and signal.volume > 0 else 1.0
|
|
position = _sim_position(direction, entry, sl, tp, volume, signal.reason)
|
|
entry_bar = j
|
|
|
|
# timeout: any position still open at the end closes at the last bar's close
|
|
if position is not None:
|
|
_record_trade(
|
|
trades, equity_pct, position, entry_bar, n_bars - 1, last_close, "timeout",
|
|
_pnl_pct(position, last_close, entry_cost, exit_cost),
|
|
_r_multiple(position, last_close),
|
|
)
|
|
|
|
if not trades:
|
|
return {"method": "strategy_backtest", "trades": [], "stats": {"n_trades": 0}}
|
|
|
|
pnls = [t["pnl_pct"] for t in trades]
|
|
wins = [p for p in pnls if p > 0]
|
|
losses = [p for p in pnls if p <= 0]
|
|
gross_win = sum(wins)
|
|
gross_loss = abs(sum(losses))
|
|
peak = -math.inf
|
|
max_dd = 0.0
|
|
for eq in equity_pct:
|
|
peak = max(peak, eq)
|
|
max_dd = max(max_dd, peak - eq)
|
|
|
|
stats: dict[str, Any] = {
|
|
"n_trades": len(trades),
|
|
"win_rate": round(len(wins) / len(trades), 4),
|
|
"avg_r": round(sum(t["r_multiple"] for t in trades) / len(trades), 4),
|
|
"total_return_pct": round(equity_pct[-1], 4),
|
|
"max_drawdown_pct": round(max_dd, 4),
|
|
"profit_factor": round(gross_win / gross_loss, 4) if gross_loss > 0 else None,
|
|
"avg_bars_held": round(sum(t["bars_held"] for t in trades) / len(trades), 2),
|
|
"cost_model": {
|
|
"spread_bps": config.spread_bps,
|
|
"commission_bps_per_side": config.commission_bps_per_side,
|
|
"slippage_bps": config.slippage_bps,
|
|
},
|
|
}
|
|
return {"method": "strategy_backtest", "trades": trades, "stats": stats, "equity_pct": [round(e, 4) for e in equity_pct]}
|
|
|
|
|
|
def backtest_registered(
|
|
spec: str,
|
|
frame: pd.DataFrame,
|
|
config: ResearchConfig,
|
|
*,
|
|
params: Optional[dict[str, Any]] = None,
|
|
symbol_spec: Optional[dict] = None,
|
|
) -> dict[str, Any]:
|
|
"""Backtest a registry-resolved strategy by name or module path.
|
|
|
|
Convenience for the research CLI and for scripted sweeps: resolve
|
|
``"ma_cross"`` / ``"trading.strategies.ma_cross"`` / any module path
|
|
through the same loader the live shell uses, then run
|
|
:func:`backtest_strategy`.
|
|
"""
|
|
from trading.strategies import load
|
|
|
|
strategy = load(spec, params or {})
|
|
return backtest_strategy(strategy, frame, config, symbol_spec=symbol_spec)
|