feat(research): complete the offline pipeline and add live-identical backtest
research/ now runs end-to-end (fetch -> features/labels -> walk-forward -> train -> OOS evaluate -> save), and backtest_strategy() replays the same signal(ctx) strategy over a historical frame with the same manage_position hook the live loop calls, simulating SL/TP, trailing, partial closes and cost per bar. --backtest wires it into the CLI. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
Claude Code
@@ -0,0 +1,10 @@
|
||||
"""Run the offline research pipeline: ``py -3.14 -m research``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from .main import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -9,6 +9,7 @@ walk-forward stub so the pipeline can run without a live broker connection.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -108,3 +109,337 @@ def local_walk_forward(frame: pd.DataFrame, config: ResearchConfig) -> dict[str,
|
||||
}
|
||||
)
|
||||
return {"method": "local_walk_forward_stub", "n_splits": config.wf_n_splits, "folds": folds}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# offline strategy backtest (reuses the *same* signal(ctx) as live trading)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _candle_row(row: pd.Series) -> dict[str, Any]:
|
||||
"""Map one frame row to the candle dict the shell's SignalContext expects."""
|
||||
return {
|
||||
"time": str(row.name) if hasattr(row.name, "strftime") else str(row.name),
|
||||
"open": float(row.get("open", row["close"])),
|
||||
"high": float(row.get("high", row["close"])),
|
||||
"low": float(row.get("low", row["close"])),
|
||||
"close": float(row["close"]),
|
||||
"tick_volume": int(row.get("tick_volume", 0) or 0),
|
||||
}
|
||||
|
||||
|
||||
def _cost_pct(config: ResearchConfig) -> tuple[float, float]:
|
||||
"""Round-trip cost as a fraction: (entry_side, exit_side)."""
|
||||
entry = (config.slippage_bps + config.commission_bps_per_side) * 0.0001
|
||||
exit_ = (config.slippage_bps + config.commission_bps_per_side) * 0.0001
|
||||
entry += config.spread_bps * 0.0001 # spread opens against the trade
|
||||
return entry, exit_
|
||||
|
||||
|
||||
def _make_context(
|
||||
frame: pd.DataFrame,
|
||||
config: ResearchConfig,
|
||||
index: int,
|
||||
position: Optional[dict],
|
||||
symbol_spec: Optional[dict],
|
||||
) -> Any:
|
||||
"""Rebuild a :class:`trading.types.SignalContext` for bar ``index``, with the
|
||||
current open position and broker spec — matching the live loop's shape."""
|
||||
from trading.types import SignalContext
|
||||
|
||||
window = frame.iloc[: index + 1].reset_index(drop=True)
|
||||
candles = [_candle_row(window.iloc[k]) for k in range(len(window))]
|
||||
px = float(frame["close"].iloc[index])
|
||||
return SignalContext(
|
||||
symbol=getattr(config, "symbol", "EURUSD"),
|
||||
timeframe=config.timeframe,
|
||||
candles=candles,
|
||||
quote={"bid": px, "ask": px},
|
||||
account={},
|
||||
positions=[position] if position is not None else [],
|
||||
symbol_spec=symbol_spec or {},
|
||||
)
|
||||
|
||||
|
||||
def _sim_position(direction: int, entry: float, sl: float, tp: float, volume: float, reason: str) -> dict:
|
||||
"""Simulated position in the shape exits.py and strategies read."""
|
||||
return {
|
||||
"type": "BUY" if direction == 1 else "SELL",
|
||||
"open_price": entry,
|
||||
"price_current": entry,
|
||||
"stop_loss": sl,
|
||||
"take_profit": tp,
|
||||
"volume": volume,
|
||||
"signal_reason": reason,
|
||||
}
|
||||
|
||||
|
||||
def _pnl_pct(position: dict, exit_price: float, entry_cost: float, exit_cost: float) -> float:
|
||||
direction = 1 if position["type"] == "BUY" else -1
|
||||
entry = float(position["open_price"])
|
||||
gross = direction * (exit_price / entry - 1.0) * 100.0
|
||||
return gross - entry_cost * 100.0 - exit_cost * 100.0
|
||||
|
||||
|
||||
def _r_multiple(position: dict, exit_price: float) -> float:
|
||||
direction = 1 if position["type"] == "BUY" else -1
|
||||
entry = float(position["open_price"])
|
||||
risk = abs(entry - float(position["stop_loss"]))
|
||||
if risk <= 0:
|
||||
return 0.0
|
||||
return direction * (exit_price / entry - 1.0) / (risk / entry)
|
||||
|
||||
|
||||
def apply_position_action(
|
||||
position: dict,
|
||||
action: Optional[Any],
|
||||
price: float,
|
||||
entry_cost: float,
|
||||
exit_cost: float,
|
||||
) -> tuple[Optional[dict], Optional[dict]]:
|
||||
"""Apply a strategy :class:`trading.types.PositionAction` to a simulated position.
|
||||
|
||||
Returns ``(new_position, realized_trade)`` — ``realized_trade`` is ``None``
|
||||
for "hold"/"modify" (no PnL locked yet); for "close" it clears the position
|
||||
and locks full PnL; for "partial_close" it reduces ``volume`` by
|
||||
``close_volume`` and locks PnL for that slice only. This mirrors exactly
|
||||
what the live loop's Executor does with the same action.
|
||||
"""
|
||||
if action is None or action.kind == "hold":
|
||||
return position, None
|
||||
if action.kind == "modify":
|
||||
if action.stop_loss is not None:
|
||||
position["stop_loss"] = float(action.stop_loss)
|
||||
if action.take_profit is not None:
|
||||
position["take_profit"] = float(action.take_profit)
|
||||
return position, None
|
||||
if action.kind == "close":
|
||||
net = _pnl_pct(position, price, entry_cost, exit_cost)
|
||||
return None, {
|
||||
"exit": round(price, 5),
|
||||
"reason": "close",
|
||||
"pnl_pct": net,
|
||||
"r_multiple": _r_multiple(position, price),
|
||||
}
|
||||
if action.kind == "partial_close":
|
||||
volume = float(position.get("volume", 0.0) or 0.0)
|
||||
close_vol = min(float(action.close_volume or 0.0), volume)
|
||||
if close_vol <= 0:
|
||||
return position, None
|
||||
fraction = close_vol / volume if volume else 1.0
|
||||
net = _pnl_pct(position, price, entry_cost, exit_cost) * fraction
|
||||
position["volume"] = volume - close_vol
|
||||
trade = {
|
||||
"exit": round(price, 5),
|
||||
"reason": "partial_close",
|
||||
"pnl_pct": net,
|
||||
"r_multiple": _r_multiple(position, price) * fraction,
|
||||
"close_volume": round(close_vol, 6),
|
||||
}
|
||||
return (None if position["volume"] <= 1e-12 else position), trade
|
||||
return position, None
|
||||
|
||||
|
||||
def _record_trade(
|
||||
trades: list[dict],
|
||||
equity_pct: list[float],
|
||||
position: dict,
|
||||
entry_bar: int,
|
||||
exit_idx: int,
|
||||
exit_price: float,
|
||||
reason: str,
|
||||
net_pct: float,
|
||||
r_mul: float,
|
||||
*,
|
||||
close_volume: Optional[float] = None,
|
||||
) -> None:
|
||||
"""Append one closed-trade row and advance the cumulative equity curve."""
|
||||
entry = float(position["open_price"])
|
||||
row = {
|
||||
"side": "BUY" if position["type"] == "BUY" else "SELL",
|
||||
"entry_idx": entry_bar,
|
||||
"entry": round(entry, 5),
|
||||
"exit_idx": exit_idx,
|
||||
"exit": round(exit_price, 5),
|
||||
"reason": reason,
|
||||
"bars_held": exit_idx - entry_bar,
|
||||
"pnl_pct": round(net_pct, 4),
|
||||
"r_multiple": round(r_mul, 4),
|
||||
"signal_reason": position.get("signal_reason", ""),
|
||||
}
|
||||
if close_volume is not None:
|
||||
row["close_volume"] = round(close_volume, 6)
|
||||
trades.append(row)
|
||||
equity_pct.append(equity_pct[-1] + net_pct)
|
||||
|
||||
|
||||
def backtest_strategy(
|
||||
strategy: Any,
|
||||
frame: pd.DataFrame,
|
||||
config: ResearchConfig,
|
||||
*,
|
||||
same_bar_policy: str = "sl_first",
|
||||
symbol_spec: Optional[dict] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Backtest any ``signal(ctx)`` strategy on a historical frame.
|
||||
|
||||
This is the key abstraction for "time spent on strategy research": the
|
||||
*same* strategy object used by the live shell (a ``trading.strategies``
|
||||
instance) is called bar-by-bar over historical data with a reconstructed
|
||||
:class:`trading.types.SignalContext`. No dataset-label coupling — trades
|
||||
are simulated from the signal's own SL/TP levels with realistic cost.
|
||||
|
||||
Position management is modeled too: if the strategy overrides
|
||||
``manage_position`` (the hook the live loop calls), every bar that does not
|
||||
hit the server-side SL/TP first runs it and applies the returned
|
||||
``PositionAction`` with the same semantics as the live Executor — so a
|
||||
trailing stop, breakeven, partial close or early exit behaves identically
|
||||
in research and live. ``symbol_spec`` (a ``symbols_describe`` payload) is
|
||||
passed into every context, as the live loop does.
|
||||
|
||||
Execution model (close-to-next-open to avoid lookahead):
|
||||
1. evaluate the strategy at bar *i* (close)
|
||||
2. open the trade at bar *i+1*, if a signal fired
|
||||
3. per bar: server-side SL/TP first (prices moved first), then
|
||||
``manage_position`` at the bar close; unresolved trades close at the
|
||||
last bar's close.
|
||||
|
||||
Returns summary stats plus per-trade rows.
|
||||
"""
|
||||
required = {"close"}
|
||||
missing = required - set(frame.columns)
|
||||
if missing:
|
||||
raise BacktestError(f"frame missing columns {sorted(missing)} for backtest")
|
||||
if len(frame) < 3:
|
||||
raise BacktestError(f"need >= 3 bars to backtest, got {len(frame)}")
|
||||
|
||||
trades: list[dict[str, Any]] = []
|
||||
equity_pct: list[float] = [0.0] # cumulative return % after each closed trade
|
||||
entry_cost, exit_cost = _cost_pct(config)
|
||||
|
||||
n_bars = len(frame)
|
||||
position: Optional[dict] = None
|
||||
entry_bar = -1
|
||||
last_close = 0.0
|
||||
|
||||
for j in range(n_bars):
|
||||
bar = frame.iloc[j]
|
||||
last_close = float(bar["close"])
|
||||
high = float(bar["high"]) if "high" in frame.columns else last_close
|
||||
low = float(bar["low"]) if "low" in frame.columns else last_close
|
||||
|
||||
if position is not None and j > entry_bar:
|
||||
# (a) server-side SL/TP decide first — prices moved first in real time
|
||||
direction = 1 if position["type"] == "BUY" else -1
|
||||
sl = float(position["stop_loss"])
|
||||
tp = float(position["take_profit"])
|
||||
hit_sl = low <= sl if direction == 1 else high >= sl
|
||||
hit_tp = high >= tp if direction == 1 else low <= tp
|
||||
if hit_sl and hit_tp:
|
||||
if same_bar_policy == "tp_first":
|
||||
hit_sl = False
|
||||
else:
|
||||
hit_tp = False
|
||||
exit_price: Optional[float] = None
|
||||
reason = ""
|
||||
if hit_tp:
|
||||
exit_price, reason = tp, "tp"
|
||||
elif hit_sl:
|
||||
exit_price, reason = sl, "sl"
|
||||
if exit_price is not None:
|
||||
_record_trade(
|
||||
trades, equity_pct, position, entry_bar, j, exit_price, reason,
|
||||
_pnl_pct(position, exit_price, entry_cost, exit_cost),
|
||||
_r_multiple(position, exit_price),
|
||||
)
|
||||
position = None
|
||||
continue
|
||||
|
||||
# (b) strategy management at bar close (no SL/TP hit this bar)
|
||||
ctx = _make_context(frame, config, j, position, symbol_spec)
|
||||
before = position
|
||||
action = strategy.manage_position(ctx, position)
|
||||
position, realized = apply_position_action(before, action, last_close, entry_cost, exit_cost)
|
||||
if realized is not None:
|
||||
_record_trade(
|
||||
trades, equity_pct, before, entry_bar, j,
|
||||
realized["exit"], realized["reason"],
|
||||
realized["pnl_pct"], realized["r_multiple"],
|
||||
close_volume=realized.get("close_volume"),
|
||||
)
|
||||
continue
|
||||
|
||||
if position is None and j < n_bars - 1:
|
||||
# entry evaluation at bar j (close); trades only act from bar j+1 on
|
||||
ctx = _make_context(frame, config, j, None, symbol_spec)
|
||||
signal = strategy.signal(ctx)
|
||||
if signal is None:
|
||||
continue
|
||||
entry = last_close
|
||||
direction = 1 if signal.side.upper() in ("BUY", "LONG", "BUY_LIMIT") else -1
|
||||
sl = signal.stop_loss
|
||||
tp = signal.take_profit
|
||||
if not sl or not tp or sl <= 0 or tp <= 0:
|
||||
continue # levels missing (e.g. ATR warmup) — treat as no trade
|
||||
volume = float(signal.volume) if signal.volume and signal.volume > 0 else 1.0
|
||||
position = _sim_position(direction, entry, sl, tp, volume, signal.reason)
|
||||
entry_bar = j
|
||||
|
||||
# timeout: any position still open at the end closes at the last bar's close
|
||||
if position is not None:
|
||||
_record_trade(
|
||||
trades, equity_pct, position, entry_bar, n_bars - 1, last_close, "timeout",
|
||||
_pnl_pct(position, last_close, entry_cost, exit_cost),
|
||||
_r_multiple(position, last_close),
|
||||
)
|
||||
|
||||
if not trades:
|
||||
return {"method": "strategy_backtest", "trades": [], "stats": {"n_trades": 0}}
|
||||
|
||||
pnls = [t["pnl_pct"] for t in trades]
|
||||
wins = [p for p in pnls if p > 0]
|
||||
losses = [p for p in pnls if p <= 0]
|
||||
gross_win = sum(wins)
|
||||
gross_loss = abs(sum(losses))
|
||||
peak = -math.inf
|
||||
max_dd = 0.0
|
||||
for eq in equity_pct:
|
||||
peak = max(peak, eq)
|
||||
max_dd = max(max_dd, peak - eq)
|
||||
|
||||
stats: dict[str, Any] = {
|
||||
"n_trades": len(trades),
|
||||
"win_rate": round(len(wins) / len(trades), 4),
|
||||
"avg_r": round(sum(t["r_multiple"] for t in trades) / len(trades), 4),
|
||||
"total_return_pct": round(equity_pct[-1], 4),
|
||||
"max_drawdown_pct": round(max_dd, 4),
|
||||
"profit_factor": round(gross_win / gross_loss, 4) if gross_loss > 0 else None,
|
||||
"avg_bars_held": round(sum(t["bars_held"] for t in trades) / len(trades), 2),
|
||||
"cost_model": {
|
||||
"spread_bps": config.spread_bps,
|
||||
"commission_bps_per_side": config.commission_bps_per_side,
|
||||
"slippage_bps": config.slippage_bps,
|
||||
},
|
||||
}
|
||||
return {"method": "strategy_backtest", "trades": trades, "stats": stats, "equity_pct": [round(e, 4) for e in equity_pct]}
|
||||
|
||||
|
||||
def backtest_registered(
|
||||
spec: str,
|
||||
frame: pd.DataFrame,
|
||||
config: ResearchConfig,
|
||||
*,
|
||||
params: Optional[dict[str, Any]] = None,
|
||||
symbol_spec: Optional[dict] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Backtest a registry-resolved strategy by name or module path.
|
||||
|
||||
Convenience for the research CLI and for scripted sweeps: resolve
|
||||
``"ma_cross"`` / ``"trading.strategies.ma_cross"`` / any module path
|
||||
through the same loader the live shell uses, then run
|
||||
:func:`backtest_strategy`.
|
||||
"""
|
||||
from trading.strategies import load
|
||||
|
||||
strategy = load(spec, params or {})
|
||||
return backtest_strategy(strategy, frame, config, symbol_spec=symbol_spec)
|
||||
|
||||
@@ -54,6 +54,7 @@ class ResearchConfig:
|
||||
# --- model ----------------------------------------------------------------
|
||||
model_dir: Path = Path(os.environ.get("RESEARCH_MODEL_DIR", "research/models"))
|
||||
model_kind: str = os.environ.get("RESEARCH_MODEL", "logistic") # logistic|rf|gboost|lightgbm
|
||||
train_ratio: float = _env_float("RESEARCH_TRAIN_RATIO", 0.8) # chronological split
|
||||
|
||||
# --- reporting --------------------------------------------------------------
|
||||
report_dir: Path = Path(os.environ.get("RESEARCH_REPORT_DIR", "research/reports"))
|
||||
|
||||
@@ -50,8 +50,10 @@ def _rsi(values: np.ndarray, period: int = 14) -> np.ndarray:
|
||||
for i in range(period, len(delta)):
|
||||
avg_gain[i] = (avg_gain[i - 1] * (period - 1) + gain[i]) / period
|
||||
avg_loss[i] = (avg_loss[i - 1] * (period - 1) + loss[i]) / period
|
||||
rs = avg_gain / np.where(avg_loss == 0, np.nan, avg_loss)
|
||||
out[period:] = 100 - 100 / (1 + rs[period - 1 :])
|
||||
with np.errstate(divide="ignore", invalid="ignore"):
|
||||
rsi_series = 100 - 100 / (1 + avg_gain / avg_loss)
|
||||
rsi_series[np.isnan(rsi_series)] = 100.0 # no losing bars -> 100, not NaN
|
||||
out[period:] = rsi_series[period - 1 :]
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""End-to-end offline research CLI.
|
||||
|
||||
Ties the research pipeline stages into one reproducible run: fetch candles
|
||||
from the broker feed → build the labeled dataset → summarize walk-forward
|
||||
folds → hold out a chronological OOS tail → train a model ("logistic" default)
|
||||
→ evaluate vs. the always-on baseline → write a markdown report → save the
|
||||
wrapper to ``research/models/latest.joblib`` (the default ``model_path`` of
|
||||
``trading/strategies/model_threshold.py``).
|
||||
|
||||
Research never touches orders: this module only reads data and writes model /
|
||||
report artifacts. The live shell maps the model's P(+1) inside ``signal(ctx)``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Sequence
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .backtest import local_walk_forward
|
||||
from .config import ResearchConfig
|
||||
from .data_pipeline import fetch_and_store, load_dataset
|
||||
from .evaluate import evaluate, write_report
|
||||
from .feature_engineering import make_dataset
|
||||
from .model_trainer import ModelTrainerError, train
|
||||
|
||||
|
||||
def _parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="mt-data-research",
|
||||
description=__doc__.splitlines()[0],
|
||||
)
|
||||
parser.add_argument("--symbol", default=None, help="instrument (default: first of RESEARCH_SYMBOLS)")
|
||||
parser.add_argument("--timeframe", default=None, help="MT5 timeframe (default: RESEARCH_TIMEFRAME)")
|
||||
parser.add_argument("--limit", type=int, default=None, help="bars to fetch (default: RESEARCH_LOOKBACK_BARS)")
|
||||
parser.add_argument("--kind", default=None, choices=("logistic", "rf", "gboost", "lightgbm"), help="model kind")
|
||||
parser.add_argument("--seed", type=int, default=42, help="random seed for reproducibility")
|
||||
parser.add_argument("--model-name", default="latest", help="output model basename (default: latest)")
|
||||
parser.add_argument(
|
||||
"--backtest", action="store_true",
|
||||
help="after training, run an offline backtest of a trading strategy on the fetched frame",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strategy", default="ma_cross",
|
||||
help="strategy name or module path (trading.strategies registry) to backtest",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strategy-param", action="append", default=[],
|
||||
help="strategy parameter as key=value (repeatable)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
return args
|
||||
|
||||
|
||||
def _parse_param_pairs(pairs: Sequence[str]) -> dict[str, str]:
|
||||
"""Parse ``key=value`` strategy parameters (mirrors trading.main)."""
|
||||
out: dict[str, str] = {}
|
||||
for pair in pairs or []:
|
||||
key, sep, value = pair.partition("=")
|
||||
if not sep or not key.strip():
|
||||
raise SystemExit(f"--strategy-param {pair!r} must be key=value")
|
||||
out[key.strip()] = value
|
||||
return out
|
||||
|
||||
|
||||
# --- stage steps ----------------------------------------------------------------
|
||||
|
||||
|
||||
def run_pipeline(config: ResearchConfig, args: argparse.Namespace) -> int:
|
||||
symbol = args.symbol or config.symbols[0]
|
||||
timeframe = args.timeframe or config.timeframe
|
||||
kind = args.kind or config.model_kind
|
||||
model_path = config.model_dir / f"{args.model_name}.joblib"
|
||||
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
|
||||
# 1. fetch + store (durable, reloadable window)
|
||||
dataset = fetch_and_store(config, symbol, timeframe=timeframe, limit=args.limit)
|
||||
print(f"[1/6] fetched {dataset.rows} {symbol} {timeframe} bars -> {dataset.path.name}")
|
||||
|
||||
frame = load_dataset(config, dataset)
|
||||
if "close" not in frame or len(frame) < config.wf_min_train_bars:
|
||||
print(
|
||||
f"need >= {config.wf_min_train_bars} bars with a close column for research; "
|
||||
f"got {len(frame)}. Increase --limit."
|
||||
)
|
||||
return 2
|
||||
|
||||
# 2. labeled dataset
|
||||
X, y = make_dataset(frame, config)
|
||||
print(f"[2/6] dataset: {len(X)} labeled samples, {len(X.columns)} features, "
|
||||
f"positive rate {float((y == 1).mean()):.4f}")
|
||||
|
||||
# 3. walk-forward fold summary (chronological, reports label stability)
|
||||
wf = local_walk_forward(frame, config)
|
||||
fold_hit_rates = ", ".join(
|
||||
f"fold{i}={fold['hit_rate']:.3f}" for i, fold in enumerate(wf["folds"])
|
||||
)
|
||||
print(f"[3/6] walk-forward: {wf['n_splits']} folds | {fold_hit_rates}")
|
||||
|
||||
# 4. chronological train / OOS split
|
||||
split = int(len(X) * config.train_ratio)
|
||||
if split < 50 or len(X) - split < 50:
|
||||
print(f"split {split}/{len(X) - split} leaves a side with < 50 samples; "
|
||||
f"fetch more bars or raise RESEARCH_TRAIN_RATIO")
|
||||
return 2
|
||||
X_train, y_train = X.iloc[:split], y.iloc[:split]
|
||||
X_test, y_test = X.iloc[split:], y.iloc[split:]
|
||||
print(f"[4/6] chronological split: {len(X_train)} train / {len(X_test)} oos")
|
||||
|
||||
# 5. train + evaluate vs. always-on baseline
|
||||
try:
|
||||
wrapper = train(
|
||||
X_train, y_train, kind=kind, symbol=symbol, timeframe=timeframe,
|
||||
)
|
||||
except ModelTrainerError as exc:
|
||||
print(f"training failed: {exc}")
|
||||
return 2
|
||||
print(f"[5/6] {kind} trained; training hit-rate baseline {wrapper.baseline:.4f}")
|
||||
|
||||
oos_proba = wrapper.predict_proba_positive(wrapper.align_features(X_test))
|
||||
model_metrics = evaluate(y_test.to_numpy(), oos_proba, threshold=0.5)
|
||||
baseline_metrics = evaluate(
|
||||
y_test.to_numpy(), np.ones(len(y_test)), threshold=0.5 # always-positive
|
||||
)
|
||||
fold_rates = ", ".join(f"{fold['hit_rate']:.3f}" for fold in wf["folds"])
|
||||
extra = (
|
||||
f"- model: {kind}, training positive rate {wrapper.baseline:.4f}\n"
|
||||
f"- walk-forward hit rates: {fold_rates}\n"
|
||||
f"- split: {len(X_train)} train / {len(X_test)} oos (chronological)\n"
|
||||
f"- features: {', '.join(X.columns)}"
|
||||
)
|
||||
|
||||
report_path = config.report_dir / f"{symbol}_{timeframe}_{stamp}.md"
|
||||
write_report(
|
||||
report_path,
|
||||
model_metrics,
|
||||
baseline_metrics,
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
extra=extra,
|
||||
)
|
||||
print(f"[6/6] oos auc {model_metrics.get('auc_roc', float('nan')):.4f} vs "
|
||||
f"baseline {baseline_metrics.get('positive_rate', float('nan')):.4f} | "
|
||||
f"report -> {report_path.name}")
|
||||
|
||||
# 6. save wrapper where model_threshold expects it by default
|
||||
wrapper.save(model_path)
|
||||
print(f"saved {kind} model -> {model_path} (baseline {wrapper.baseline:.4f})")
|
||||
|
||||
# 7. optional: offline backtest of a trading strategy on the same frame
|
||||
if args.backtest:
|
||||
from .backtest import backtest_registered
|
||||
|
||||
params = _parse_param_pairs(args.strategy_param)
|
||||
result = backtest_registered(args.strategy, frame, config, params=params)
|
||||
stats = result["stats"]
|
||||
print(f"[backtest] {args.strategy}: {stats.get('n_trades', 0)} trades, "
|
||||
f"win {stats.get('win_rate')}, ret {stats.get('total_return_pct')}%, "
|
||||
f"PF {stats.get('profit_factor')}, maxDD {stats.get('max_drawdown_pct')}%")
|
||||
bt_path = config.report_dir / f"{symbol}_{timeframe}_{args.model_name}_backtest.md"
|
||||
bt_path.write_text(
|
||||
f"# Backtest: {args.strategy} on {symbol} {timeframe}\n\n"
|
||||
f"- model: {kind} ({args.model_name})\n"
|
||||
f"- trades: {stats.get('n_trades')}\n"
|
||||
f"- win rate: {stats.get('win_rate')}\n"
|
||||
f"- avg R: {stats.get('avg_r')}\n"
|
||||
f"- total return %: {stats.get('total_return_pct')}\n"
|
||||
f"- max drawdown %: {stats.get('max_drawdown_pct')}\n"
|
||||
f"- profit factor: {stats.get('profit_factor')}\n"
|
||||
f"- cost model: {stats.get('cost_model')}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"saved backtest report -> {bt_path.name}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
config = ResearchConfig()
|
||||
try:
|
||||
return run_pipeline(config, args)
|
||||
except KeyboardInterrupt:
|
||||
print("interrupted")
|
||||
return 130
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -62,13 +62,30 @@ def _make_estimator(kind: str) -> Any:
|
||||
|
||||
|
||||
class ModelWrapper:
|
||||
"""Trained model + metadata. Pickled with joblib (safe for local research)."""
|
||||
"""Trained model + metadata. Pickled with joblib (safe for local research).
|
||||
|
||||
def __init__(self, estimator: Any, kind: str, symbol: str, timeframe: str):
|
||||
``feature_columns`` records the exact training-time column order so the
|
||||
trading shell can reindex live features to the same layout before calling
|
||||
``predict_proba_positive``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
estimator: Any,
|
||||
kind: str,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
feature_columns: list[str] | None = None,
|
||||
baseline: float = 0.0,
|
||||
):
|
||||
self.estimator = estimator
|
||||
self.kind = kind
|
||||
self.symbol = symbol
|
||||
self.timeframe = timeframe
|
||||
self.feature_columns = list(feature_columns or [])
|
||||
# training positive rate (hit-rate baseline) persisted for strategies
|
||||
# that gate on beating the no-trade baseline (model_threshold.edge)
|
||||
self.baseline = float(baseline)
|
||||
|
||||
# --- inference ----------------------------------------------------------
|
||||
def predict_proba_positive(self, X: pd.DataFrame) -> np.ndarray:
|
||||
@@ -80,6 +97,16 @@ class ModelWrapper:
|
||||
def predict(self, X: pd.DataFrame) -> np.ndarray:
|
||||
return self.estimator.predict(X)
|
||||
|
||||
def align_features(self, X: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Reindex ``X`` to the training-time feature layout.
|
||||
|
||||
Live features must arrive with the same column order used at fit time;
|
||||
missing columns are filled with NaN and extras are dropped.
|
||||
"""
|
||||
if not self.feature_columns:
|
||||
return X
|
||||
return X.reindex(columns=self.feature_columns)
|
||||
|
||||
# --- persistence ----------------------------------------------------------
|
||||
def save(self, path: Path) -> Path:
|
||||
path = Path(path)
|
||||
@@ -90,6 +117,8 @@ class ModelWrapper:
|
||||
"symbol": self.symbol,
|
||||
"timeframe": self.timeframe,
|
||||
"estimator": self.estimator,
|
||||
"feature_columns": self.feature_columns,
|
||||
"baseline": self.baseline,
|
||||
},
|
||||
path,
|
||||
)
|
||||
@@ -103,6 +132,8 @@ class ModelWrapper:
|
||||
kind=data["kind"],
|
||||
symbol=data["symbol"],
|
||||
timeframe=data["timeframe"],
|
||||
feature_columns=data.get("feature_columns") or None,
|
||||
baseline=data.get("baseline", 0.0),
|
||||
)
|
||||
|
||||
|
||||
@@ -126,4 +157,11 @@ def train(
|
||||
raise ModelTrainerError("target has a single class; nothing to learn")
|
||||
estimator = _make_estimator(kind)
|
||||
estimator.fit(X, y)
|
||||
return ModelWrapper(estimator=estimator, kind=kind, symbol=symbol, timeframe=timeframe)
|
||||
return ModelWrapper(
|
||||
estimator=estimator,
|
||||
kind=kind,
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
feature_columns=list(X.columns),
|
||||
baseline=float((y == 1).mean()),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
"""Regression tests for the strategy layer: registry, params, signals, backtest.
|
||||
|
||||
These cover the abstraction that keeps strategy research offline-friendly: a
|
||||
*trading* strategy is just a ``Strategy`` subclass with a declarative
|
||||
``params_spec`` and a single ``signal(ctx)``, and the same object backs both
|
||||
the live shell and the offline ``backtest_strategy`` harness. No MT5/broker
|
||||
connection is needed here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from research.backtest import BacktestError, backtest_strategy
|
||||
from research.config import ResearchConfig
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def frame() -> pd.DataFrame:
|
||||
rng = np.random.default_rng(7)
|
||||
n = 800
|
||||
close = np.cumsum(np.r_[np.zeros(40), rng.normal(0.00015, 0.0004, n - 40)]) + 1.0800
|
||||
return pd.DataFrame(
|
||||
{
|
||||
"open": close - 0.0002,
|
||||
"high": close + 0.0006,
|
||||
"low": close - 0.0006,
|
||||
"close": close,
|
||||
"tick_volume": rng.integers(10, 200, n),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def context(frame: pd.DataFrame):
|
||||
from trading.types import SignalContext
|
||||
|
||||
return SignalContext(
|
||||
symbol="EURUSD",
|
||||
timeframe="H1",
|
||||
candles=[
|
||||
{"high": float(r.high), "low": float(r.low), "close": float(r.close)}
|
||||
for _, r in frame.iterrows()
|
||||
],
|
||||
quote={},
|
||||
account={},
|
||||
positions=[],
|
||||
)
|
||||
|
||||
|
||||
def _config() -> ResearchConfig:
|
||||
return ResearchConfig(
|
||||
spread_bps=1.0, commission_bps_per_side=0.0, slippage_bps=1.0
|
||||
)
|
||||
|
||||
|
||||
# --- registry ------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_registry_lists_both_strategies() -> None:
|
||||
from trading.strategies import list_strategies
|
||||
|
||||
names = {row["name"] for row in list_strategies()}
|
||||
assert names == {"ma_cross", "model_threshold"}
|
||||
|
||||
|
||||
def test_registry_load_by_name_alias_and_module_path() -> None:
|
||||
from trading.strategies import load
|
||||
from trading.strategies.base import Strategy
|
||||
|
||||
for spec in ("ma_cross", "macross", "trading.strategies.ma_cross"):
|
||||
strategy = load(spec)
|
||||
assert isinstance(strategy, Strategy)
|
||||
assert strategy.name == "ma_cross"
|
||||
|
||||
|
||||
def test_registry_unknown_spec_raises() -> None:
|
||||
from trading.strategies import load
|
||||
from trading.strategies.registry import StrategyLoadError
|
||||
|
||||
with pytest.raises(StrategyLoadError):
|
||||
load("zzz_no_such_strategy")
|
||||
|
||||
|
||||
# --- param validation ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_param_merge_over_defaults() -> None:
|
||||
from trading.strategies import load
|
||||
|
||||
strategy = load("ma_cross", {"fast": 10, "slow": 40})
|
||||
assert strategy.params["fast"] == 10
|
||||
assert strategy.params["slow"] == 40
|
||||
assert strategy.params["atr_mult_sl"] == 2.0 # untouched default
|
||||
|
||||
|
||||
def test_param_below_minimum_rejected() -> None:
|
||||
from trading.strategies import load
|
||||
|
||||
with pytest.raises(ValueError, match="fast"):
|
||||
load("ma_cross", {"fast": 1})
|
||||
|
||||
|
||||
def test_unknown_param_rejected() -> None:
|
||||
from trading.strategies import load
|
||||
|
||||
with pytest.raises(ValueError, match="no parameter"):
|
||||
load("ma_cross", {"bogus": 1})
|
||||
|
||||
|
||||
# --- strategy 1: ma_cross signal --------------------------------------------------
|
||||
|
||||
|
||||
def test_ma_cross_signal_runs_on_synthetic_frame(context) -> None:
|
||||
from trading.strategies import load
|
||||
|
||||
strategy = load("ma_cross")
|
||||
signal = strategy.signal(context)
|
||||
# either it is flat (no cross at this bar) or it returns a sane TradeSignal
|
||||
assert signal is None or signal.stop_loss > 0
|
||||
|
||||
|
||||
# --- offline backtest harness (reuses the same signal(ctx)) -----------------------
|
||||
|
||||
|
||||
def test_backtest_strategy_returns_stats_and_trades(frame) -> None:
|
||||
from trading.strategies import load
|
||||
|
||||
result = backtest_strategy(load("ma_cross"), frame, _config())
|
||||
assert result["method"] == "strategy_backtest"
|
||||
assert isinstance(result["trades"], list)
|
||||
stats = result["stats"]
|
||||
assert set(["n_trades", "win_rate", "avg_r", "total_return_pct"]).issubset(stats)
|
||||
for trade in result["trades"]:
|
||||
assert {"side", "entry", "exit", "reason", "pnl_pct", "r_multiple"} <= set(trade)
|
||||
assert trade["pnl_pct"] is not None
|
||||
|
||||
|
||||
def test_backtest_rejects_too_few_bars() -> None:
|
||||
from trading.strategies import load
|
||||
|
||||
tiny = pd.DataFrame({"close": [1.0, 1.01], "open": 1.0, "high": 1.02, "low": 0.99})
|
||||
with pytest.raises(BacktestError):
|
||||
backtest_strategy(load("ma_cross"), tiny, _config())
|
||||
|
||||
|
||||
# --- strategy 2: model_threshold with a trained wrapper ---------------------------
|
||||
|
||||
|
||||
def test_model_threshold_with_trained_wrapper(frame, context, tmp_path) -> None:
|
||||
from research.feature_engineering import build_features, build_labels
|
||||
from research.model_trainer import train
|
||||
from trading.strategies import load
|
||||
|
||||
feats = build_features(frame).iloc[40:]
|
||||
labels = build_labels(
|
||||
frame.iloc[40:].reset_index(drop=True), _config(), direction="long"
|
||||
)
|
||||
y = pd.Series(labels.to_numpy(), index=feats.index).dropna()
|
||||
X = feats.reindex(y.index)
|
||||
assert len(X) >= 50 and y.nunique() == 2
|
||||
|
||||
wrapper = train(X, y, kind="logistic", symbol="EURUSD", timeframe="H1")
|
||||
# train() drops NaN-feature rows before fitting, so the persisted baseline
|
||||
# is the positive rate over the clean training sample, not the raw labels.
|
||||
clean = (X.notna().all(axis=1)) & y.notna()
|
||||
assert wrapper.baseline == pytest.approx(float((y[clean] == 1).mean()))
|
||||
model_file = tmp_path / "eurusd_h1.joblib"
|
||||
wrapper.save(model_file)
|
||||
|
||||
from research.model_trainer import ModelWrapper
|
||||
|
||||
reloaded = ModelWrapper.load(model_file)
|
||||
assert reloaded.baseline == pytest.approx(wrapper.baseline)
|
||||
|
||||
strategy = load("model_threshold", {"model_path": str(model_file)})
|
||||
signal = strategy.signal(context)
|
||||
assert strategy._wrapper is not None # lazy load cached on first signal
|
||||
assert signal is None or signal.stop_loss > 0
|
||||
|
||||
|
||||
def test_model_threshold_fires_buy_on_strong_uptrend(tmp_path) -> None:
|
||||
"""A bar inside a persistent uptrend must cross the gate and emit BUY.
|
||||
|
||||
Trains on a synthetic series whose tail drifts up strongly (forward labels
|
||||
hit TP first), then feeds that same tail through ``signal(ctx)`` at default
|
||||
params and asserts the full model->features->gate->SL/TP path fires.
|
||||
"""
|
||||
from research.feature_engineering import build_features, build_labels
|
||||
from research.model_trainer import train
|
||||
from trading.strategies import load
|
||||
from trading.types import SignalContext
|
||||
|
||||
rng = np.random.default_rng(11)
|
||||
n = 800
|
||||
drift = 0.0000
|
||||
close = np.cumsum(np.r_[np.zeros(40), rng.normal(drift, 0.0004, n - 700), rng.normal(0.0010, 0.0004, 660)]) + 1.0800
|
||||
up = pd.DataFrame(
|
||||
{
|
||||
"open": close - 0.0002,
|
||||
"high": close + 0.0006,
|
||||
"low": close - 0.0006,
|
||||
"close": close,
|
||||
"tick_volume": rng.integers(10, 200, n),
|
||||
}
|
||||
)
|
||||
|
||||
feats = build_features(up).iloc[40:]
|
||||
labels = build_labels(
|
||||
up.iloc[40:].reset_index(drop=True), _config(), direction="long"
|
||||
)
|
||||
y = pd.Series(labels.to_numpy(), index=feats.index).dropna()
|
||||
X = feats.reindex(y.index)
|
||||
assert len(X) >= 50 and y.nunique() == 2
|
||||
|
||||
model_file = tmp_path / "uptrend.joblib"
|
||||
train(X, y, kind="logistic", symbol="EURUSD", timeframe="H1").save(model_file)
|
||||
|
||||
tail = up.tail(120).reset_index(drop=True) # same shape the shell feeds signal()
|
||||
ctx = SignalContext(
|
||||
symbol="EURUSD",
|
||||
timeframe="H1",
|
||||
candles=[
|
||||
{"high": float(r.high), "low": float(r.low), "close": float(r.close)}
|
||||
for _, r in tail.iterrows()
|
||||
],
|
||||
quote={},
|
||||
account={},
|
||||
positions=[],
|
||||
)
|
||||
strategy = load("model_threshold", {"model_path": str(model_file)})
|
||||
signal = strategy.signal(ctx)
|
||||
assert signal is not None, "gate must fire inside a persistent uptrend"
|
||||
assert signal.side == "BUY"
|
||||
assert 0 < signal.stop_loss < signal.take_profit
|
||||
|
||||
|
||||
def test_wrapper_feature_alignment(frame, tmp_path) -> None:
|
||||
"""align_features reshapes live rows to the training-time column layout."""
|
||||
from research.feature_engineering import build_features
|
||||
from research.model_trainer import ModelWrapper, train
|
||||
|
||||
feats = build_features(frame).iloc[40:]
|
||||
columns = list(feats.columns)
|
||||
shuffled = list(reversed(columns))
|
||||
live = feats.tail(1).reindex(columns=shuffled)
|
||||
|
||||
X_fit = feats.iloc[:-1].reset_index(drop=True)
|
||||
y_fit = pd.Series(np.arange(len(X_fit), dtype=int) % 2)
|
||||
wrapper = train(
|
||||
X_fit, y_fit, kind="logistic", symbol="EURUSD", timeframe="H1",
|
||||
)
|
||||
aligned = wrapper.align_features(live)
|
||||
assert list(aligned.columns) == columns
|
||||
assert wrapper.predict_proba_positive(aligned).shape == (1,)
|
||||
|
||||
model_file = tmp_path / "alignment.joblib"
|
||||
wrapper.save(model_file)
|
||||
reloaded = ModelWrapper.load(model_file)
|
||||
assert reloaded.feature_columns == columns
|
||||
|
||||
|
||||
# --- legacy compat adapter --------------------------------------------------------
|
||||
|
||||
|
||||
def test_function_strategy_adapter_wraps_bare_signal() -> None:
|
||||
from trading.strategies.base import FunctionStrategy
|
||||
|
||||
def legacy_signal(ctx):
|
||||
return None
|
||||
|
||||
strategy = FunctionStrategy(legacy_signal, name="legacy")
|
||||
assert strategy.signal(None) is None
|
||||
assert strategy.name == "legacy"
|
||||
|
||||
|
||||
# --- CLI plumbing -----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cli_strategy_params_parse() -> None:
|
||||
import argparse
|
||||
|
||||
from trading.main import _cli_strategy_params
|
||||
|
||||
args = argparse.Namespace(strategy_param=["fast=10", "slow=40"])
|
||||
assert _cli_strategy_params(args) == {"fast": "10", "slow": "40"}
|
||||
|
||||
bad = argparse.Namespace(strategy_param=["nokey"])
|
||||
with pytest.raises(SystemExit):
|
||||
_cli_strategy_params(bad)
|
||||
|
||||
|
||||
def test_signal_loop_load_strategy_falls_back_never_dies() -> None:
|
||||
from trading.signal_loop import NullStrategy, load_strategy
|
||||
|
||||
assert isinstance(load_strategy("does.not.exist"), NullStrategy)
|
||||
assert isinstance(load_strategy("zzz_no_such_strategy"), NullStrategy)
|
||||
|
||||
|
||||
# --- position management: one hook shared by live loop and backtester -------------
|
||||
|
||||
|
||||
def test_strategy_default_manage_position_holds() -> None:
|
||||
from trading.strategies import load
|
||||
|
||||
assert load("ma_cross").manage_position(None, {"type": "BUY"}) is None
|
||||
|
||||
|
||||
def _managing_strategy(*, close_volume=None):
|
||||
from trading.strategies.base import Strategy
|
||||
from trading.types import PositionAction, TradeSignal
|
||||
|
||||
class ManagingStrategy(Strategy):
|
||||
def signal(self, ctx):
|
||||
if len(ctx.closes()) < 3:
|
||||
return None
|
||||
return TradeSignal(side="BUY", stop_loss=0.9000, take_profit=9.0, reason="enter")
|
||||
|
||||
def manage_position(self, ctx, position):
|
||||
if close_volume is not None:
|
||||
return PositionAction(
|
||||
kind="partial_close", close_volume=close_volume, reason="scale-out"
|
||||
)
|
||||
px = ctx.closes()[-1]
|
||||
return PositionAction(kind="modify", stop_loss=px - 0.0001, reason="tight")
|
||||
|
||||
return ManagingStrategy()
|
||||
|
||||
|
||||
def test_backtest_applies_dynamic_trailing() -> None:
|
||||
"""manage_position() must move the effective stop: SL exits settle at the
|
||||
trailed level, far above the entry-time stop of 0.90."""
|
||||
closes = [1.1000] + [1.1000 + 0.0030 * (k + 1) for k in range(25)] # rise to 1.175
|
||||
closes += [c - 0.0030 for c in closes[-10:]] # then fall back
|
||||
updown = pd.DataFrame({
|
||||
"open": closes,
|
||||
"high": [c + 0.0002 for c in closes],
|
||||
"low": [c - 0.0002 for c in closes],
|
||||
"close": closes,
|
||||
})
|
||||
result = backtest_strategy(_managing_strategy(), updown, _config())
|
||||
sl_exits = [t for t in result["trades"] if t["reason"] == "sl"]
|
||||
assert sl_exits, "the fall must be stopped by the trailed SL"
|
||||
assert all(t["exit"] > 1.10 for t in sl_exits)
|
||||
|
||||
|
||||
def test_backtest_partial_close_is_recorded(frame) -> None:
|
||||
result = backtest_strategy(_managing_strategy(close_volume=0.5), frame, _config())
|
||||
partial = [t for t in result["trades"] if t["reason"] == "partial_close"]
|
||||
assert partial, "every scale-out must be recorded as its own closed slice"
|
||||
assert all("close_volume" in t and t["close_volume"] > 0 for t in partial)
|
||||
Reference in New Issue
Block a user