diff --git a/rdagent/components/backtesting/__init__.py b/rdagent/components/backtesting/__init__.py index 8a79043e..940a9b52 100644 --- a/rdagent/components/backtesting/__init__.py +++ b/rdagent/components/backtesting/__init__.py @@ -11,16 +11,23 @@ from .vbt_backtest import ( FTMO_MAX_LEVERAGE, FTMO_RISK_PER_TRADE, OOS_START_DEFAULT, + WF_IS_YEARS, + WF_OOS_YEARS, + WF_STEP_YEARS, backtest_from_forward_returns, backtest_signal, backtest_signal_ftmo, + monte_carlo_trade_pvalue, + walk_forward_rolling, ) __all__ = [ 'BacktestMetrics', 'FactorBacktester', 'ResultsDatabase', 'CorrelationAnalyzer', 'PortfolioOptimizer', 'AdvancedRiskManager', 'backtest_signal', 'backtest_signal_ftmo', 'backtest_from_forward_returns', + 'monte_carlo_trade_pvalue', 'walk_forward_rolling', 'DEFAULT_BARS_PER_YEAR', 'DEFAULT_TXN_COST_BPS', 'FTMO_INITIAL_CAPITAL', 'FTMO_MAX_DAILY_LOSS', 'FTMO_MAX_TOTAL_LOSS', 'FTMO_MAX_LEVERAGE', 'FTMO_RISK_PER_TRADE', 'OOS_START_DEFAULT', + 'WF_IS_YEARS', 'WF_OOS_YEARS', 'WF_STEP_YEARS', ] diff --git a/rdagent/components/backtesting/vbt_backtest.py b/rdagent/components/backtesting/vbt_backtest.py index 0d12f4b9..556c4a04 100644 --- a/rdagent/components/backtesting/vbt_backtest.py +++ b/rdagent/components/backtesting/vbt_backtest.py @@ -342,6 +342,128 @@ def _apply_ftmo_mask( OOS_START_DEFAULT = "2024-01-01" +# Rolling walk-forward default windows (IS years, OOS years, step years) +WF_IS_YEARS = 3 +WF_OOS_YEARS = 1 +WF_STEP_YEARS = 1 + + +def monte_carlo_trade_pvalue( + trade_pnl: pd.Series, + n_permutations: int = 1000, + seed: int = 0, +) -> float: + """ + Monte Carlo permutation test on trade-level P&L. + + Shuffles the order of trade returns ``n_permutations`` times and computes + the fraction of runs whose total return is >= the real total return. + + p < 0.05 → strategy has a statistically significant edge (real return + beats 95% of random sequences with the same set of trades). + + Parameters + ---------- + trade_pnl : pd.Series + Per-trade net returns (output of ``_compute_trade_pnl``). + n_permutations : int + Number of random permutations (default 1000). + seed : int + RNG seed for reproducibility. + + Returns + ------- + float + p-value in [0, 1]. Lower is better. + """ + if len(trade_pnl) < 2: + return 1.0 + trades = trade_pnl.values.copy() + real_total = float(trades.sum()) + rng = np.random.default_rng(seed) + beat = 0 + for _ in range(n_permutations): + perm = rng.permutation(trades) + if perm.sum() >= real_total: + beat += 1 + return beat / n_permutations + + +def walk_forward_rolling( + close: pd.Series, + signal: pd.Series, + leverage: float, + txn_cost_bps: float = DEFAULT_TXN_COST_BPS, + bars_per_year: int = DEFAULT_BARS_PER_YEAR, + is_years: int = WF_IS_YEARS, + oos_years: int = WF_OOS_YEARS, + step_years: int = WF_STEP_YEARS, +) -> Dict[str, Any]: + """ + Rolling walk-forward validation: multiple IS/OOS windows shifted by ``step_years``. + + Each window runs an independent FTMO simulation on the IS and OOS slices. + Produces aggregate OOS statistics to measure cross-time consistency. + + Returns + ------- + dict with keys: + wf_n_windows, wf_oos_sharpe_mean, wf_oos_sharpe_std, + wf_oos_monthly_return_mean, wf_oos_consistency (fraction of windows + with OOS Sharpe > 0), wf_windows (list of per-window dicts) + """ + if not isinstance(close.index, pd.DatetimeIndex): + return {"wf_n_windows": 0} + + start_year = close.index[0].year + end_year = close.index[-1].year + + windows = [] + yr = start_year + while True: + is_start = pd.Timestamp(f"{yr}-01-01") + is_end = pd.Timestamp(f"{yr + is_years}-01-01") + oos_end = pd.Timestamp(f"{yr + is_years + oos_years}-01-01") + if oos_end.year > end_year + 1: + break + is_mask = (close.index >= is_start) & (close.index < is_end) + oos_mask = (close.index >= is_end) & (close.index < oos_end) + if is_mask.sum() < 1000 or oos_mask.sum() < 1000: + yr += step_years + continue + + window: Dict[str, Any] = { + "is_start": str(is_start.date()), + "is_end": str(is_end.date()), + "oos_start": str(is_end.date()), + "oos_end": str(oos_end.date()), + } + for mask, prefix in [(is_mask, "is"), (oos_mask, "oos")]: + close_s = close.loc[mask] + signal_s = signal.loc[mask] + masked_s, _ = _apply_ftmo_mask(signal_s, close_s, leverage, txn_cost_bps) + r = backtest_signal(close=close_s, signal=masked_s, + txn_cost_bps=txn_cost_bps, bars_per_year=bars_per_year) + window[f"{prefix}_sharpe"] = r.get("sharpe", 0.0) + window[f"{prefix}_monthly_return_pct"] = r.get("monthly_return_pct", 0.0) + window[f"{prefix}_n_trades"] = r.get("n_trades", 0) + windows.append(window) + yr += step_years + + if not windows: + return {"wf_n_windows": 0} + + oos_sharpes = [w["oos_sharpe"] for w in windows] + oos_monthly = [w["oos_monthly_return_pct"] for w in windows] + return { + "wf_n_windows": len(windows), + "wf_oos_sharpe_mean": float(np.mean(oos_sharpes)), + "wf_oos_sharpe_std": float(np.std(oos_sharpes)), + "wf_oos_monthly_return_mean": float(np.mean(oos_monthly)), + "wf_oos_consistency": float(np.mean([s > 0 for s in oos_sharpes])), + "wf_windows": windows, + } + def backtest_signal_ftmo( close: pd.Series, @@ -354,6 +476,8 @@ def backtest_signal_ftmo( bars_per_year: int = DEFAULT_BARS_PER_YEAR, forward_returns: Optional[pd.Series] = None, oos_start: Optional[str] = OOS_START_DEFAULT, + wf_rolling: bool = False, + mc_n_permutations: int = 0, ) -> Dict[str, Any]: """ FTMO-compliant backtest of a strategy signal on EUR/USD. @@ -385,6 +509,13 @@ def backtest_signal_ftmo( Maximum leverage (default 30 = FTMO 1:30). oos_start : str or None Start of out-of-sample period (ISO date). None disables OOS split. + wf_rolling : bool + If True, run rolling walk-forward validation (multiple IS/OOS windows). + Results are stored under ``wf_*`` keys. Default False. + mc_n_permutations : int + Number of Monte Carlo trade permutations. 0 = disabled (default). + When > 0, computes ``mc_pvalue``: fraction of permuted sequences whose + total return >= real total return. p < 0.05 indicates a genuine edge. """ stop_price = stop_pips * FTMO_PIP leverage_by_risk = risk_pct / (stop_price / eurusd_price) @@ -440,6 +571,28 @@ def backtest_signal_ftmo( result["is_n_bars"] = int(is_mask.sum()) result["oos_n_bars"] = int(oos_mask.sum()) + # Rolling walk-forward validation + if wf_rolling: + wf = walk_forward_rolling( + close=close, + signal=signal, + leverage=leverage, + txn_cost_bps=txn_cost_bps, + bars_per_year=bars_per_year, + ) + result.update(wf) + + # Monte Carlo trade permutation test + if mc_n_permutations > 0: + position = masked_signal.shift(1).fillna(0) + bar_ret = close.pct_change().fillna(0) + txn_cost = txn_cost_bps / 10_000.0 + position_change = position.diff().abs().fillna(position.abs()) + strat_ret = position * bar_ret - position_change * txn_cost + trade_pnl = _compute_trade_pnl(position, strat_ret) + result["mc_pvalue"] = monte_carlo_trade_pvalue(trade_pnl, mc_n_permutations) + result["mc_n_permutations"] = mc_n_permutations + return result diff --git a/scripts/predix_gen_strategies_real_bt.py b/scripts/predix_gen_strategies_real_bt.py index 957a5369..b04b4cb2 100644 --- a/scripts/predix_gen_strategies_real_bt.py +++ b/scripts/predix_gen_strategies_real_bt.py @@ -371,6 +371,8 @@ signal.fillna(0).to_pickle('signal.pkl') txn_cost_bps=TXN_COST_BPS, forward_returns=fwd_returns, oos_start=OOS_START_DEFAULT, + wf_rolling=True, + mc_n_permutations=200, ) # ============================================================================ @@ -567,9 +569,18 @@ def main(target_count=10): progress.update(task, advance=1) continue - # Check acceptance criteria — OOS must be profitable (primary filter) + # Monte Carlo p-value (edge significance) + mc_pvalue = bt_result.get('mc_pvalue') + + # Rolling walk-forward metrics + wf_consistency = bt_result.get('wf_oos_consistency') + wf_sharpe_mean = bt_result.get('wf_oos_sharpe_mean') + + # Check acceptance criteria — OOS must be profitable + statistically significant + mc_ok = mc_pvalue is None or mc_pvalue < 0.20 # lenient: top 20% non-random + wf_ok = wf_consistency is None or wf_consistency >= 0.5 # ≥50% of WF windows profitable if (abs(ic) > MIN_IC and sharpe > MIN_SHARPE and trades > MIN_TRADES and dd > MAX_DRAWDOWN - and oos_sharpe > 0.0 and oos_monthly > 0.0): + and oos_sharpe > 0.0 and oos_monthly > 0.0 and mc_ok and wf_ok): # ACCEPT strategy['real_backtest'] = bt_result strategy['metrics'] = bt_result @@ -583,7 +594,7 @@ def main(target_count=10): 'ohlcv_only': OHLCV_ONLY, 'engine': 'ftmo_v2', 'txn_cost_bps': TXN_COST_BPS, - # Walk-forward OOS metrics + # Walk-forward OOS split 'oos_sharpe': bt_result.get('oos_sharpe'), 'oos_monthly_return_pct': bt_result.get('oos_monthly_return_pct'), 'oos_max_drawdown': bt_result.get('oos_max_drawdown'), @@ -592,6 +603,15 @@ def main(target_count=10): 'is_sharpe': bt_result.get('is_sharpe'), 'is_monthly_return_pct': bt_result.get('is_monthly_return_pct'), 'oos_start': bt_result.get('oos_start'), + # Rolling walk-forward + 'wf_n_windows': bt_result.get('wf_n_windows'), + 'wf_oos_sharpe_mean': wf_sharpe_mean, + 'wf_oos_sharpe_std': bt_result.get('wf_oos_sharpe_std'), + 'wf_oos_monthly_return_mean': bt_result.get('wf_oos_monthly_return_mean'), + 'wf_oos_consistency': wf_consistency, + # Monte Carlo significance + 'mc_pvalue': mc_pvalue, + 'mc_n_permutations': bt_result.get('mc_n_permutations'), } fname = f"{int(time.time())}_{strategy['strategy_name']}.json" @@ -614,12 +634,16 @@ def main(target_count=10): f"IC={ic:.4f}, Sharpe={sharpe:.3f}, Trades={trades}, DD={dd:.1%}") else: oos_info = f"OOS_Sharpe={oos_sharpe:+.2f} OOS_Mon={oos_monthly:+.2f}%" if oos_sharpe is not None else "" - _log.info(f"REJECTED IC={ic:.4f} Sharpe={sharpe:.2f} Trades={trades} DD={dd:.1%} {oos_info}") + mc_info = f" MC_p={mc_pvalue:.2f}" if mc_pvalue is not None else "" + wf_info = f" WF_consistency={wf_consistency:.0%}" if wf_consistency is not None else "" + _log.info(f"REJECTED IC={ic:.4f} Sharpe={sharpe:.2f} Trades={trades} DD={dd:.1%} {oos_info}{mc_info}{wf_info}") feedback_history.append( f"Failed: IC={ic:.4f}, Sharpe={sharpe:.2f}, Trades={trades}, DD={dd:.1%}, " - f"OOS_Sharpe={oos_sharpe:+.2f}, OOS_Monthly={oos_monthly:+.2f}%. " - f"Need |IC|>{MIN_IC}, Sharpe>{MIN_SHARPE}, Trades>{MIN_TRADES}, " - f"OOS_Sharpe>0 AND OOS_Monthly>0 — strategy must generalise to unseen data (2024+)." + f"OOS_Sharpe={oos_sharpe:+.2f}, OOS_Monthly={oos_monthly:+.2f}%" + + (f", MC_p={mc_pvalue:.2f}" if mc_pvalue is not None else "") + + (f", WF_consistency={wf_consistency:.0%}" if wf_consistency is not None else "") + + f". Need |IC|>{MIN_IC}, Sharpe>{MIN_SHARPE}, Trades>{MIN_TRADES}, " + f"OOS_Sharpe>0, OOS_Monthly>0, MC_p<0.20, WF_consistency≥50%." ) progress.update(task, advance=1) diff --git a/scripts/predix_rebacktest_unified.py b/scripts/predix_rebacktest_unified.py index 5eaf46c1..7a7f62b4 100644 --- a/scripts/predix_rebacktest_unified.py +++ b/scripts/predix_rebacktest_unified.py @@ -188,6 +188,8 @@ def rebacktest_one( close=close_a, signal=signal, txn_cost_bps=txn_cost_bps, + wf_rolling=True, + mc_n_permutations=200, ) result["status_detail"] = result.pop("status") result["status"] = "ok" @@ -264,6 +266,15 @@ def main() -> None: "oos_win_rate": bt.get("oos_win_rate"), "oos_n_trades": bt.get("oos_n_trades"), "oos_start": bt.get("oos_start"), + # Rolling walk-forward + "wf_n_windows": bt.get("wf_n_windows"), + "wf_oos_sharpe_mean": bt.get("wf_oos_sharpe_mean"), + "wf_oos_sharpe_std": bt.get("wf_oos_sharpe_std"), + "wf_oos_monthly_return_mean": bt.get("wf_oos_monthly_return_mean"), + "wf_oos_consistency": bt.get("wf_oos_consistency"), + # Monte Carlo significance + "mc_pvalue": bt.get("mc_pvalue"), + "mc_n_permutations": bt.get("mc_n_permutations"), } data["sharpe_ratio"] = bt.get("sharpe") data["max_drawdown"] = bt.get("max_drawdown") @@ -299,6 +310,12 @@ def main() -> None: "oos_monthly_pct": bt.get("oos_monthly_return_pct"), "oos_dd": bt.get("oos_max_drawdown"), "oos_trades": bt.get("oos_n_trades"), + # Rolling walk-forward + "wf_n_windows": bt.get("wf_n_windows"), + "wf_oos_sharpe_mean": bt.get("wf_oos_sharpe_mean"), + "wf_oos_consistency": bt.get("wf_oos_consistency"), + # Monte Carlo + "mc_pvalue": bt.get("mc_pvalue"), } if "annualized_return" in bt: row["new_annual_return_cagr"] = bt["annualized_return"] diff --git a/test/backtesting/test_ftmo_oos.py b/test/backtesting/test_ftmo_oos.py index eb69aef1..3f87088c 100644 --- a/test/backtesting/test_ftmo_oos.py +++ b/test/backtesting/test_ftmo_oos.py @@ -19,6 +19,8 @@ from rdagent.components.backtesting.vbt_backtest import ( backtest_signal_ftmo, FTMO_MAX_DAILY_LOSS, FTMO_MAX_TOTAL_LOSS, + monte_carlo_trade_pvalue, + walk_forward_rolling, ) @@ -188,3 +190,51 @@ def test_ftmo_result_has_equity_and_profit(close_2yr): assert "ftmo_end_equity" in r assert "ftmo_monthly_profit" in r assert r["ftmo_end_equity"] > 0 + + +# --------------------------------------------------------------------------- +# Monte Carlo trade permutation tests +# --------------------------------------------------------------------------- +def test_mc_pvalue_in_result(close_2yr): + signal = _random_signal(close_2yr.index) + r = backtest_signal_ftmo(close_2yr, signal, oos_start=None, mc_n_permutations=50) + assert "mc_pvalue" in r + assert 0.0 <= r["mc_pvalue"] <= 1.0 + assert r["mc_n_permutations"] == 50 + + +def test_mc_pvalue_disabled_by_default(close_2yr): + signal = _random_signal(close_2yr.index) + r = backtest_signal_ftmo(close_2yr, signal, oos_start=None) + assert "mc_pvalue" not in r + + +def test_mc_zero_trades_returns_one(close_2yr): + """Zero-signal → no trades → p-value must be 1.0 (no edge).""" + trade_pnl = pd.Series([], dtype=float) + assert monte_carlo_trade_pvalue(trade_pnl, n_permutations=10) == 1.0 + + +# --------------------------------------------------------------------------- +# Rolling walk-forward tests +# --------------------------------------------------------------------------- +def test_wf_rolling_keys_in_result(close_6yr): + signal = _random_signal(close_6yr.index) + r = backtest_signal_ftmo(close_6yr, signal, oos_start="2024-01-01", wf_rolling=True) + # With only ~150 days of data, windows may be 0 — just check key presence + assert "wf_n_windows" in r + + +def test_wf_rolling_disabled_by_default(close_6yr): + signal = _random_signal(close_6yr.index) + r = backtest_signal_ftmo(close_6yr, signal, oos_start="2024-01-01") + assert "wf_n_windows" not in r + + +def test_wf_consistency_range(close_6yr): + """wf_oos_consistency must be in [0, 1] when windows exist.""" + signal = _random_signal(close_6yr.index) + r = backtest_signal_ftmo(close_6yr, signal, oos_start="2024-01-01", wf_rolling=True) + c = r.get("wf_oos_consistency") + if c is not None: + assert 0.0 <= c <= 1.0