""" Unified, verifiable backtesting engine. Single entry point (`backtest_signal`) used by: - scripts/nexquant_gen_strategies_real_bt.py - rdagent/scenarios/qlib/local/strategy_orchestrator.py - rdagent/scenarios/qlib/local/optuna_optimizer.py - rdagent/components/backtesting/backtest_engine.py Design goals ------------ 1. One formula for every metric, used everywhere. 2. Annualization uses 252 * 1440 = 362,880 bars/year (1-min EUR/USD convention). 3. Transaction cost applied on every position change; default 1.5 bps. 4. Position is signal.shift(1) (no look-ahead). 5. No silent return clipping; extreme bars are flagged in ``data_quality_flag``. 6. n_trades = actual roundtrips (entry→exit), not position-diff count. 7. Returns are cross-checked against vectorbt; mismatch raises in dev mode. """ from __future__ import annotations from typing import Any import numpy as np import pandas as pd try: import vectorbt as vbt # noqa: F401 VBT_AVAILABLE = True except ImportError: VBT_AVAILABLE = False # 2.35 pip realistic EUR/USD cost: 1.5 spread + 0.5 slippage + 0.35 commission # At EUR/USD ≈ 1.10: 2.35 pip * (0.0001/1.10) ≈ 2.14 bps of notional. DEFAULT_TXN_COST_BPS = 2.14 DEFAULT_BARS_PER_YEAR = 252 * 1440 # 252 trading days * 1440 min/day = 362,880 EXTREME_BAR_THRESHOLD = 0.05 # |ret| > 5% on a single 1-min bar → suspicious # RiskMgmt 100k account rules (enforced in backtest_signal when riskmgmt=True) INITIAL_CAPITAL = 100_000.0 MAX_DAILY_LOSS = 0.05 # 5% of initial → block new trades rest of day MAX_TOTAL_LOSS = 0.10 # 10% of initial → simulation ends # Risk-based position sizing: 1.5% equity risk per trade, 10-pip stop, max 1:30 leverage RISK_PER_TRADE = 0.015 STOP_PIPS = 10 PIP_SIZE = 0.0001 MAX_LEVERAGE = 30 def _compute_trade_pnl(position: pd.Series, strategy_returns: pd.Series) -> pd.Series: """ Group strategy returns into trade epochs (runs of same-sign position). Each non-flat epoch = one trade roundtrip; its P&L is the sum of strategy_returns within that epoch. """ position_sign = np.sign(position).astype(int) epoch = (position_sign != position_sign.shift(1)).cumsum() epoch_sign = position_sign.groupby(epoch).first() pnl_per_epoch = strategy_returns.groupby(epoch).sum() return pnl_per_epoch[epoch_sign != 0] def _cross_check_with_vbt( close: pd.Series, position: pd.Series, txn_cost: float, freq: str, ) -> float | None: """Run a vectorbt simulation and return its total_return for comparison.""" if not VBT_AVAILABLE: return None try: import vectorbt as vbt pf = vbt.Portfolio.from_orders( close=close, size=position, size_type="targetpercent", fees=txn_cost, init_cash=10_000.0, freq=freq, ) tr = float(pf.total_return()) return tr if np.isfinite(tr) else None except Exception: return None def backtest_signal( close: pd.Series, signal: pd.Series, txn_cost_bps: float = DEFAULT_TXN_COST_BPS, freq: str = "1min", bars_per_year: int = DEFAULT_BARS_PER_YEAR, forward_returns: pd.Series | None = None, cross_check: bool = False, ) -> dict[str, Any]: """ Run a single-asset backtest from a position signal. Parameters ---------- close : pd.Series Close-price series indexed by datetime. signal : pd.Series Target position as fraction of equity, in [-1, +1]. {-1, 0, 1} or continuous both supported. Missing bars → 0 (flat). txn_cost_bps : float One-sided transaction cost in basis points, charged on every position change in proportion to |Δposition|. freq : str Pandas frequency string for vectorbt cross-check. Does NOT affect manual metric formulas — those use ``bars_per_year``. bars_per_year : int Used only for Sharpe / Sortino / volatility / arithmetic annualized return. Default 252 * 1440. forward_returns : pd.Series, optional If given, IC (correlation of raw signal with forward returns) is computed and returned. cross_check : bool If True, also run vectorbt and include its total_return in the result dict as ``vbt_total_return`` for verification. Returns ------- dict with keys: status, sharpe, sortino, calmar, max_drawdown, win_rate, profit_factor, total_return, annualized_return, annual_return_cagr, monthly_return, monthly_return_pct, annual_return_pct, volatility, n_trades, n_position_changes, n_bars, n_months, signal_long, signal_short, signal_neutral, ic, txn_cost_bps, bars_per_year, data_quality_flag (optional), vbt_total_return (if cross_check) """ if not isinstance(close, pd.Series): raise TypeError(f"close must be a pd.Series, got {type(close)}") if not isinstance(signal, pd.Series): raise TypeError(f"signal must be a pd.Series, got {type(signal)}") close = pd.to_numeric(close, errors="coerce").dropna().astype(float) if len(close) < 2: return {"status": "failed", "reason": f"insufficient close data ({len(close)} bars)"} signal = pd.to_numeric(signal, errors="coerce") signal = signal.reindex(close.index).fillna(0).clip(-1, 1).astype(float) # Position is lagged by one bar: signal generated at t executes at t+1. position = signal.shift(1).fillna(0) # Bar returns from close prices, aligned to position index. bar_ret = close.pct_change().fillna(0) # Strategy returns = position * bar_ret - turnover cost. txn_cost = txn_cost_bps / 10_000.0 position_change = position.diff().abs().fillna(position.abs()) gross_ret = position * bar_ret strategy_returns = gross_ret - position_change * txn_cost # Data quality flag: single-bar moves over 5% are almost certainly # data spikes, strategy bugs, or an unrealistic leverage setting. extreme_bars = int((strategy_returns.abs() > EXTREME_BAR_THRESHOLD).sum()) if strategy_returns.std() > 0: sharpe = float(strategy_returns.mean() / strategy_returns.std() * np.sqrt(bars_per_year)) else: sharpe = 0.0 downside = strategy_returns[strategy_returns < 0] if len(downside) > 1 and downside.std() > 0: sortino = float(strategy_returns.mean() / downside.std() * np.sqrt(bars_per_year)) else: sortino = 0.0 total_return = float((1 + strategy_returns).prod() - 1) ann_return_arith = float(strategy_returns.mean() * bars_per_year) volatility = float(strategy_returns.std() * np.sqrt(bars_per_year)) equity = (1 + strategy_returns).cumprod() running_max = equity.cummax() # equity is strictly positive unless a bar return <= -100%, which we don't clip. # If that happens we propagate NaN rather than silently clip. running_max_safe = running_max.where(running_max > 0, np.nan) drawdown = (equity - running_max) / running_max_safe drawdown = drawdown.replace([np.inf, -np.inf], np.nan).fillna(0) max_dd = float(drawdown.min()) if len(drawdown) > 0 else 0.0 # Time span — always derived from the actual DatetimeIndex, never from # n_bars / (bars_per_year / 12) which silently fails on gapped data. if isinstance(close.index, pd.DatetimeIndex) and len(close.index) > 1: span_days = (close.index[-1] - close.index[0]).total_seconds() / 86400.0 n_months = max(1.0, span_days / 30.4375) else: n_months = max(1.0, len(strategy_returns) / (bars_per_year / 12)) if n_months > 0 and (1 + total_return) > 0: monthly_return = (1 + total_return) ** (1 / n_months) - 1 annual_return_cagr = (1 + total_return) ** (12 / n_months) - 1 else: monthly_return = total_return / n_months annual_return_cagr = total_return * 12 / n_months calmar = ann_return_arith / abs(max_dd) if max_dd < 0 else 0.0 trade_pnl = _compute_trade_pnl(position, strategy_returns) n_trades = len(trade_pnl) n_position_changes = int((position.diff().fillna(0) != 0).sum()) if n_trades > 0: win_rate = float((trade_pnl > 0).mean()) wins = trade_pnl[trade_pnl > 0].sum() losses = -trade_pnl[trade_pnl < 0].sum() profit_factor = float(wins / losses) if losses > 0 else float("inf") if wins > 0 else 0.0 else: win_rate = 0.0 profit_factor = 0.0 ic: float | None = None if forward_returns is not None: fwd = pd.to_numeric(forward_returns, errors="coerce") common = signal.index.intersection(fwd.dropna().index) if len(common) > 10: s = signal.loc[common] f = fwd.loc[common] if s.std() > 0 and f.std() > 0: ic_val = float(s.corr(f)) ic = ic_val if np.isfinite(ic_val) else None result: dict[str, Any] = { "status": "success", "sharpe": sharpe, "sortino": sortino, "calmar": calmar, "max_drawdown": max_dd, "win_rate": win_rate, "profit_factor": profit_factor, "total_return": total_return, "annualized_return": ann_return_arith, "annual_return_cagr": annual_return_cagr, "monthly_return": monthly_return, "monthly_return_pct": monthly_return * 100, "annual_return_pct": annual_return_cagr * 100, "volatility": volatility, "n_trades": n_trades, "n_position_changes": n_position_changes, "n_bars": len(strategy_returns), "n_months": float(n_months), "signal_long": int((signal > 0).sum()), "signal_short": int((signal < 0).sum()), "signal_neutral": int((signal == 0).sum()), "ic": ic, "txn_cost_bps": txn_cost_bps, "bars_per_year": bars_per_year, } if extreme_bars > 0: result["data_quality_flag"] = ( f"extreme_returns: {extreme_bars} bars with |ret|>{EXTREME_BAR_THRESHOLD:.0%}" ) if cross_check: result["vbt_total_return"] = _cross_check_with_vbt( close=close, position=position, txn_cost=txn_cost, freq=freq, ) from rdagent.components.backtesting.verify import verify_and_log verify_and_log(result, factor_name="backtest_signal") return result def _apply_risk_mask( signal: pd.Series, close: pd.Series, leverage: float, txn_cost_bps: float, ) -> tuple[pd.Series, dict]: """ Apply RiskMgmt daily/total loss rules to a signal series. Returns a masked signal (positions zeroed after each limit breach) and a dict of RiskMgmt compliance metrics. """ txn_cost = txn_cost_bps / 10_000.0 position = signal.shift(1).fillna(0) * leverage bar_ret = close.pct_change().fillna(0) equity = INITIAL_CAPITAL peak_day = INITIAL_CAPITAL masked = signal.copy() daily_breaches = 0 total_breached = False total_breach_ts: pd.Timestamp | None = None current_day = None day_start_eq = INITIAL_CAPITAL pos_prev = 0.0 for ts, sig_i in signal.items(): day = ts.date() if hasattr(ts, "date") else ts if day != current_day: current_day = day day_start_eq = equity pos_i = float(signal.at[ts]) * leverage ret_i = float(bar_ret.get(ts, 0.0)) cost_i = abs(pos_i - pos_prev) * txn_cost ret_frac = pos_prev * ret_i - cost_i equity *= 1.0 + ret_frac if equity > 0 else 1.0 pos_prev = pos_i if total_breached: masked.at[ts] = 0 continue daily_loss = (equity - day_start_eq) / INITIAL_CAPITAL total_loss = (equity - INITIAL_CAPITAL) / INITIAL_CAPITAL if daily_loss < -MAX_DAILY_LOSS: daily_breaches += 1 day_start_eq = -999 # block rest of day masked.at[ts] = 0 if total_loss < -MAX_TOTAL_LOSS: total_breached = True total_breach_ts = ts masked.at[ts] = 0 return masked, { "riskmgmt_daily_breaches": daily_breaches, "riskmgmt_total_breached": total_breached, "riskmgmt_total_breach_ts": str(total_breach_ts) if total_breach_ts else None, "riskmgmt_compliant": not total_breached and daily_breaches == 0, } OOS_START_DEFAULT = "2024-01-01" # Rolling walk-forward default windows (IS years, OOS years, step years) WF_IS_YEARS = 1 WF_OOS_YEARS = 1 WF_STEP_YEARS = 1 def monte_carlo_trade_pvalue( trade_pnl: pd.Series, n_permutations: int = 1000, seed: int = 0, ) -> float: """ Monte Carlo permutation test on trade-level P&L. Runs a one-sided binomial test on trade-level win rate. Tests H0: win_rate = 0.5 (random trading) against H1: win_rate > 0.5. The ``n_permutations`` parameter is kept for API compatibility but is unused. p < 0.05 → win rate is significantly above 50%, indicating a genuine per-trade edge. Parameters ---------- trade_pnl : pd.Series Per-trade net returns (output of ``_compute_trade_pnl``). n_permutations : int Number of random permutations (default 1000). seed : int RNG seed for reproducibility. Returns ------- float p-value in [0, 1]. Lower is better. """ if len(trade_pnl) < 2: return 1.0 trades = trade_pnl.values.copy() # Binomial test: is the win rate significantly above 50%? # p = probability of observing >= n_wins out of n_trades under null (win_rate=0.5). # Low p → strategy has a significant positive edge per trade. from scipy.stats import binomtest n_wins = int((trades > 0).sum()) n_total = len(trades) result = binomtest(n_wins, n_total, p=0.5, alternative="greater") return float(result.pvalue) def walk_forward_rolling( close: pd.Series, signal: pd.Series, leverage: float, txn_cost_bps: float = DEFAULT_TXN_COST_BPS, bars_per_year: int = DEFAULT_BARS_PER_YEAR, is_years: int = WF_IS_YEARS, oos_years: int = WF_OOS_YEARS, step_years: int = WF_STEP_YEARS, ) -> dict[str, Any]: """ Rolling walk-forward validation: multiple IS/OOS windows shifted by ``step_years``. Each window runs an independent RiskMgmt simulation on the IS and OOS slices. Produces aggregate OOS statistics to measure cross-time consistency. Returns ------- dict with keys: wf_n_windows, wf_oos_sharpe_mean, wf_oos_sharpe_std, wf_oos_monthly_return_mean, wf_oos_consistency (fraction of windows with OOS Sharpe > 0), wf_windows (list of per-window dicts) """ if not isinstance(close.index, pd.DatetimeIndex): return {"wf_n_windows": 0} start_year = close.index[0].year end_year = close.index[-1].year windows = [] yr = start_year while True: is_start = pd.Timestamp(f"{yr}-01-01") is_end = pd.Timestamp(f"{yr + is_years}-01-01") oos_end = pd.Timestamp(f"{yr + is_years + oos_years}-01-01") if oos_end.year > end_year + 1: break is_mask = (close.index >= is_start) & (close.index < is_end) oos_mask = (close.index >= is_end) & (close.index < oos_end) if is_mask.sum() < 1000 or oos_mask.sum() < 1000: yr += step_years continue window: dict[str, Any] = { "is_start": str(is_start.date()), "is_end": str(is_end.date()), "oos_start": str(is_end.date()), "oos_end": str(oos_end.date()), } for mask, prefix in [(is_mask, "is"), (oos_mask, "oos")]: close_s = close.loc[mask] signal_s = signal.loc[mask] masked_s, _ = _apply_risk_mask(signal_s, close_s, leverage, txn_cost_bps) r = backtest_signal(close=close_s, signal=masked_s, txn_cost_bps=txn_cost_bps, bars_per_year=bars_per_year) window[f"{prefix}_sharpe"] = r.get("sharpe", 0.0) window[f"{prefix}_monthly_return_pct"] = r.get("monthly_return_pct", 0.0) window[f"{prefix}_n_trades"] = r.get("n_trades", 0) windows.append(window) yr += step_years if not windows: return {"wf_n_windows": 0} oos_sharpes = [w["oos_sharpe"] for w in windows] oos_monthly = [w["oos_monthly_return_pct"] for w in windows] return { "wf_n_windows": len(windows), "wf_oos_sharpe_mean": float(np.mean(oos_sharpes)), "wf_oos_sharpe_std": float(np.std(oos_sharpes)), "wf_oos_monthly_return_mean": float(np.mean(oos_monthly)), "wf_oos_consistency": float(np.mean([s > 0 for s in oos_sharpes])), "wf_windows": windows, } def backtest_signal_risk( close: pd.Series, signal: pd.Series, txn_cost_bps: float = DEFAULT_TXN_COST_BPS, eurusd_price: float = 1.10, risk_pct: float = RISK_PER_TRADE, stop_pips: float = STOP_PIPS, max_leverage: float = MAX_LEVERAGE, bars_per_year: int = DEFAULT_BARS_PER_YEAR, forward_returns: pd.Series | None = None, oos_start: str | None = OOS_START_DEFAULT, wf_rolling: bool = True, mc_n_permutations: int = 0, ) -> dict[str, Any]: """ RiskMgmt-compliant backtest of a strategy signal on EUR/USD. Applies on top of ``backtest_signal``: - Realistic costs: default 2.14 bps (≈ 2.35 pip spread+slippage+commission) - Risk-based position sizing: risk_pct equity per trade, stop_pips hard stop - Max leverage cap: max_leverage (default 1:30, RiskMgmt standard) - RiskMgmt daily loss limit (5%): positions zeroed rest of day after breach - RiskMgmt total loss limit (10%): all positions zeroed after breach - RiskMgmt-specific metrics added to result dict - Walk-forward OOS split: IS metrics (before oos_start) + OOS metrics (after) Parameters ---------- close : pd.Series 1-min EUR/USD close prices. signal : pd.Series Raw strategy signal in {-1, 0, +1}. txn_cost_bps : float Transaction cost in bps (default 2.14 ≈ 2.35 pip on EUR/USD). eurusd_price : float Representative EUR/USD price for pip→bps conversion (default 1.10). risk_pct : float Fraction of equity risked per trade (default 0.005 = 0.5%). stop_pips : float Hard stop-loss distance in pips (default 10). max_leverage : float Maximum leverage (default 30 = RiskMgmt 1:30). oos_start : str or None Start of out-of-sample period (ISO date). None disables OOS split. wf_rolling : bool If True, run rolling walk-forward validation (multiple IS/OOS windows). Results are stored under ``wf_*`` keys. Default False. mc_n_permutations : int Number of Monte Carlo trade permutations. 0 = disabled (default). When > 0, computes ``mc_pvalue``: fraction of permuted sequences whose total return >= real total return. p < 0.05 indicates a genuine edge. """ stop_price = stop_pips * PIP_SIZE leverage_by_risk = risk_pct / (stop_price / eurusd_price) leverage = min(leverage_by_risk, max_leverage) masked_signal, risk_metrics = _apply_risk_mask(signal, close, leverage, txn_cost_bps) result = backtest_signal( close=close, signal=masked_signal, txn_cost_bps=txn_cost_bps, bars_per_year=bars_per_year, forward_returns=forward_returns, ) result.update(risk_metrics) result["riskmgmt_leverage"] = round(leverage, 2) result["riskmgmt_risk_pct"] = risk_pct result["riskmgmt_stop_pips"] = stop_pips # Re-scale reported equity metrics to INITIAL_CAPITAL result["riskmgmt_end_equity"] = INITIAL_CAPITAL * (1 + result.get("total_return", 0)) result["riskmgmt_monthly_profit"] = INITIAL_CAPITAL * result.get("monthly_return", 0) # Walk-forward OOS split if oos_start is not None: oos_ts = pd.Timestamp(oos_start) is_mask = close.index < oos_ts oos_mask = close.index >= oos_ts def _split_bt(mask: pd.Series[bool], prefix: str) -> None: if mask.sum() < 100: return close_s = close.loc[mask] signal_s = signal.loc[mask] # raw signal, not masked — fresh RiskMgmt sim per period fwd_split = forward_returns.loc[mask] if forward_returns is not None else None masked_s, _ = _apply_risk_mask(signal_s, close_s, leverage, txn_cost_bps) split_result = backtest_signal( close=close_s, signal=masked_s, txn_cost_bps=txn_cost_bps, bars_per_year=bars_per_year, forward_returns=fwd_split, ) for k, v in split_result.items(): if k not in ("equity_curve", "status"): result[f"{prefix}_{k}"] = v _split_bt(is_mask, "is") _split_bt(oos_mask, "oos") result["oos_start"] = oos_start result["is_n_bars"] = int(is_mask.sum()) result["oos_n_bars"] = int(oos_mask.sum()) # Rolling walk-forward validation if wf_rolling: wf = walk_forward_rolling( close=close, signal=signal, leverage=leverage, txn_cost_bps=txn_cost_bps, bars_per_year=bars_per_year, ) result.update(wf) # Monte Carlo trade permutation test if mc_n_permutations > 0: position = masked_signal.shift(1).fillna(0) bar_ret = close.pct_change().fillna(0) txn_cost = txn_cost_bps / 10_000.0 position_change = position.diff().abs().fillna(position.abs()) strat_ret = position * bar_ret - position_change * txn_cost trade_pnl = _compute_trade_pnl(position, strat_ret) result["mc_pvalue"] = monte_carlo_trade_pvalue(trade_pnl, mc_n_permutations) result["mc_n_permutations"] = mc_n_permutations from rdagent.components.backtesting.verify import verify_and_log verify_and_log(result, factor_name="backtest_from_forward_returns") return result def backtest_from_forward_returns( factor_values: pd.Series, forward_returns: pd.Series, txn_cost_bps: float = DEFAULT_TXN_COST_BPS, bars_per_year: int = DEFAULT_BARS_PER_YEAR, ) -> dict[str, Any]: """ Backtest a factor using sign(factor) as signal against forward returns. This is the legacy FactorBacktester mode: no close series available, just (factor, forward_return) pairs. All time-based metrics degrade gracefully (n_months approximated from n_bars). """ factor_values = pd.to_numeric(factor_values, errors="coerce") forward_returns = pd.to_numeric(forward_returns, errors="coerce") common = factor_values.dropna().index.intersection(forward_returns.dropna().index) if len(common) < 10: return {"status": "failed", "reason": f"insufficient aligned data ({len(common)} rows)"} f = factor_values.loc[common] r = forward_returns.loc[common] signal = np.sign(f).astype(float) position = signal.shift(1).fillna(0) txn_cost = txn_cost_bps / 10_000.0 position_change = position.diff().abs().fillna(position.abs()) strategy_returns = position * r - position_change * txn_cost if strategy_returns.std() > 0: sharpe = float(strategy_returns.mean() / strategy_returns.std() * np.sqrt(bars_per_year)) else: sharpe = 0.0 total_return = float((1 + strategy_returns).prod() - 1) equity = (1 + strategy_returns).cumprod() max_dd = float(((equity - equity.cummax()) / equity.cummax().replace(0, np.nan)).min() or 0.0) ic_val = float(f.corr(r)) if f.std() > 0 and r.std() > 0 else 0.0 ic = ic_val if np.isfinite(ic_val) else 0.0 trade_pnl = _compute_trade_pnl(position, strategy_returns) n_trades = len(trade_pnl) win_rate = float((trade_pnl > 0).mean()) if n_trades > 0 else 0.0 ann_return = float(strategy_returns.mean() * bars_per_year) volatility = float(strategy_returns.std() * np.sqrt(bars_per_year)) return { "status": "success", "sharpe": sharpe, "max_drawdown": max_dd, "total_return": total_return, "annualized_return": ann_return, "volatility": volatility, "win_rate": win_rate, "n_trades": n_trades, "ic": ic, "n_bars": len(strategy_returns), "txn_cost_bps": txn_cost_bps, "bars_per_year": bars_per_year, }