gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 15:37:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(backtest): add rolling walk-forward validation and Monte Carlo trade permutation test

Browse Source 
- monte_carlo_trade_pvalue(): shuffles trade P&L N times, returns fraction of
  permuted sequences that beat real total return (p<0.05 = genuine edge)
- walk_forward_rolling(): multiple IS/OOS windows (IS=3yr, OOS=1yr, step=1yr),
  computes wf_oos_sharpe_mean, wf_oos_consistency (% profitable windows)
- backtest_signal_riskmgmt(): new wf_rolling and mc_n_permutations params
- Strategy generator: enables both (200 MC permutations), adds mc_ok and wf_ok
  to acceptance filter (mc_p<0.20, wf_consistency>=50%)
- Rebacktest script: enables both, stores all wf_*/mc_* fields in write-back
- 6 new tests covering MC pvalue, disabled-by-default, zero-trades edge case,
  rolling WF key presence and consistency range

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
TPTBusiness
2026-04-19 18:59:00 +02:00
parent 6d514d3ed5
commit b0490c5f0d
5 changed files with 258 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/backtesting/__init__.py
+7
View File
@@ -11,16 +11,23 @@ from .vbt_backtest import (
FTMO_MAX_LEVERAGE,
FTMO_RISK_PER_TRADE,
OOS_START_DEFAULT,
WF_IS_YEARS,
WF_OOS_YEARS,
WF_STEP_YEARS,
backtest_from_forward_returns,
backtest_signal,
backtest_signal_ftmo,
monte_carlo_trade_pvalue,
walk_forward_rolling,
)
__all__ = [
'BacktestMetrics', 'FactorBacktester', 'ResultsDatabase',
'CorrelationAnalyzer', 'PortfolioOptimizer', 'AdvancedRiskManager',
'backtest_signal', 'backtest_signal_ftmo', 'backtest_from_forward_returns',
'monte_carlo_trade_pvalue', 'walk_forward_rolling',
'DEFAULT_BARS_PER_YEAR', 'DEFAULT_TXN_COST_BPS',
'FTMO_INITIAL_CAPITAL', 'FTMO_MAX_DAILY_LOSS', 'FTMO_MAX_TOTAL_LOSS',
'FTMO_MAX_LEVERAGE', 'FTMO_RISK_PER_TRADE', 'OOS_START_DEFAULT',
'WF_IS_YEARS', 'WF_OOS_YEARS', 'WF_STEP_YEARS',
]
rdagent/components/backtesting/vbt_backtest.py
+153
View File
@@ -342,6 +342,128 @@ def _apply_ftmo_mask(
OOS_START_DEFAULT = "2024-01-01"
# Rolling walk-forward default windows (IS years, OOS years, step years)
WF_IS_YEARS = 3
WF_OOS_YEARS = 1
WF_STEP_YEARS = 1
def monte_carlo_trade_pvalue(
trade_pnl: pd.Series,
n_permutations: int = 1000,
seed: int = 0,
) -> float:
"""
Monte Carlo permutation test on trade-level P&L.
Shuffles the order of trade returns ``n_permutations`` times and computes
the fraction of runs whose total return is >= the real total return.
p < 0.05 → strategy has a statistically significant edge (real return
beats 95% of random sequences with the same set of trades).
Parameters
----------
trade_pnl : pd.Series
Per-trade net returns (output of ``_compute_trade_pnl``).
n_permutations : int
Number of random permutations (default 1000).
seed : int
RNG seed for reproducibility.
Returns
-------
float
p-value in [0, 1]. Lower is better.
"""
if len(trade_pnl) < 2:
return 1.0
trades = trade_pnl.values.copy()
real_total = float(trades.sum())
rng = np.random.default_rng(seed)
beat = 0
for _ in range(n_permutations):
perm = rng.permutation(trades)
if perm.sum() >= real_total:
beat += 1
return beat / n_permutations
def walk_forward_rolling(
close: pd.Series,
signal: pd.Series,
leverage: float,
txn_cost_bps: float = DEFAULT_TXN_COST_BPS,
bars_per_year: int = DEFAULT_BARS_PER_YEAR,
is_years: int = WF_IS_YEARS,
oos_years: int = WF_OOS_YEARS,
step_years: int = WF_STEP_YEARS,
) -> Dict[str, Any]:
"""
Rolling walk-forward validation: multiple IS/OOS windows shifted by ``step_years``.
Each window runs an independent FTMO simulation on the IS and OOS slices.
Produces aggregate OOS statistics to measure cross-time consistency.
Returns
-------
dict with keys:
wf_n_windows, wf_oos_sharpe_mean, wf_oos_sharpe_std,
wf_oos_monthly_return_mean, wf_oos_consistency (fraction of windows
with OOS Sharpe > 0), wf_windows (list of per-window dicts)
"""
if not isinstance(close.index, pd.DatetimeIndex):
return {"wf_n_windows": 0}
start_year = close.index[0].year
end_year = close.index[-1].year
windows = []
yr = start_year
while True:
is_start = pd.Timestamp(f"{yr}-01-01")
is_end = pd.Timestamp(f"{yr + is_years}-01-01")
oos_end = pd.Timestamp(f"{yr + is_years + oos_years}-01-01")
if oos_end.year > end_year + 1:
break
is_mask = (close.index >= is_start) & (close.index < is_end)
oos_mask = (close.index >= is_end) & (close.index < oos_end)
if is_mask.sum() < 1000 or oos_mask.sum() < 1000:
yr += step_years
continue
window: Dict[str, Any] = {
"is_start": str(is_start.date()),
"is_end": str(is_end.date()),
"oos_start": str(is_end.date()),
"oos_end": str(oos_end.date()),
}
for mask, prefix in [(is_mask, "is"), (oos_mask, "oos")]:
close_s = close.loc[mask]
signal_s = signal.loc[mask]
masked_s, _ = _apply_ftmo_mask(signal_s, close_s, leverage, txn_cost_bps)
r = backtest_signal(close=close_s, signal=masked_s,
txn_cost_bps=txn_cost_bps, bars_per_year=bars_per_year)
window[f"{prefix}_sharpe"] = r.get("sharpe", 0.0)
window[f"{prefix}_monthly_return_pct"] = r.get("monthly_return_pct", 0.0)
window[f"{prefix}_n_trades"] = r.get("n_trades", 0)
windows.append(window)
yr += step_years
if not windows:
return {"wf_n_windows": 0}
oos_sharpes = [w["oos_sharpe"] for w in windows]
oos_monthly = [w["oos_monthly_return_pct"] for w in windows]
return {
"wf_n_windows": len(windows),
"wf_oos_sharpe_mean": float(np.mean(oos_sharpes)),
"wf_oos_sharpe_std": float(np.std(oos_sharpes)),
"wf_oos_monthly_return_mean": float(np.mean(oos_monthly)),
"wf_oos_consistency": float(np.mean([s > 0 for s in oos_sharpes])),
"wf_windows": windows,
}
def backtest_signal_ftmo(
close: pd.Series,
@@ -354,6 +476,8 @@ def backtest_signal_ftmo(
bars_per_year: int = DEFAULT_BARS_PER_YEAR,
forward_returns: Optional[pd.Series] = None,
oos_start: Optional[str] = OOS_START_DEFAULT,
wf_rolling: bool = False,
mc_n_permutations: int = 0,
) -> Dict[str, Any]:
"""
FTMO-compliant backtest of a strategy signal on EUR/USD.
@@ -385,6 +509,13 @@ def backtest_signal_ftmo(
Maximum leverage (default 30 = FTMO 1:30).
oos_start : str or None
Start of out-of-sample period (ISO date). None disables OOS split.
wf_rolling : bool
If True, run rolling walk-forward validation (multiple IS/OOS windows).
Results are stored under ``wf_*`` keys. Default False.
mc_n_permutations : int
Number of Monte Carlo trade permutations. 0 = disabled (default).
When > 0, computes ``mc_pvalue``: fraction of permuted sequences whose
total return >= real total return. p < 0.05 indicates a genuine edge.
"""
stop_price = stop_pips * FTMO_PIP
leverage_by_risk = risk_pct / (stop_price / eurusd_price)
@@ -440,6 +571,28 @@ def backtest_signal_ftmo(
result["is_n_bars"] = int(is_mask.sum())
result["oos_n_bars"] = int(oos_mask.sum())
# Rolling walk-forward validation
if wf_rolling:
wf = walk_forward_rolling(
close=close,
signal=signal,
leverage=leverage,
txn_cost_bps=txn_cost_bps,
bars_per_year=bars_per_year,
)
result.update(wf)
# Monte Carlo trade permutation test
if mc_n_permutations > 0:
position = masked_signal.shift(1).fillna(0)
bar_ret = close.pct_change().fillna(0)
txn_cost = txn_cost_bps / 10_000.0
position_change = position.diff().abs().fillna(position.abs())
strat_ret = position * bar_ret - position_change * txn_cost
trade_pnl = _compute_trade_pnl(position, strat_ret)
result["mc_pvalue"] = monte_carlo_trade_pvalue(trade_pnl, mc_n_permutations)
result["mc_n_permutations"] = mc_n_permutations
return result
scripts/predix_gen_strategies_real_bt.py
+31 -7
View File
@@ -371,6 +371,8 @@ signal.fillna(0).to_pickle('signal.pkl')
txn_cost_bps=TXN_COST_BPS,
forward_returns=fwd_returns,
oos_start=OOS_START_DEFAULT,
wf_rolling=True,
mc_n_permutations=200,
)
# ============================================================================
@@ -567,9 +569,18 @@ def main(target_count=10):
progress.update(task, advance=1)
continue
# Check acceptance criteria — OOS must be profitable (primary filter)
# Monte Carlo p-value (edge significance)
mc_pvalue = bt_result.get('mc_pvalue')
# Rolling walk-forward metrics
wf_consistency = bt_result.get('wf_oos_consistency')
wf_sharpe_mean = bt_result.get('wf_oos_sharpe_mean')
# Check acceptance criteria — OOS must be profitable + statistically significant
mc_ok = mc_pvalue is None or mc_pvalue < 0.20 # lenient: top 20% non-random
wf_ok = wf_consistency is None or wf_consistency >= 0.5 # ≥50% of WF windows profitable
if (abs(ic) > MIN_IC and sharpe > MIN_SHARPE and trades > MIN_TRADES and dd > MAX_DRAWDOWN
and oos_sharpe > 0.0 and oos_monthly > 0.0):
and oos_sharpe > 0.0 and oos_monthly > 0.0 and mc_ok and wf_ok):
# ACCEPT
strategy['real_backtest'] = bt_result
strategy['metrics'] = bt_result
@@ -583,7 +594,7 @@ def main(target_count=10):
'ohlcv_only': OHLCV_ONLY,
'engine': 'ftmo_v2',
'txn_cost_bps': TXN_COST_BPS,
# Walk-forward OOS metrics
# Walk-forward OOS split
'oos_sharpe': bt_result.get('oos_sharpe'),
'oos_monthly_return_pct': bt_result.get('oos_monthly_return_pct'),
'oos_max_drawdown': bt_result.get('oos_max_drawdown'),
@@ -592,6 +603,15 @@ def main(target_count=10):
'is_sharpe': bt_result.get('is_sharpe'),
'is_monthly_return_pct': bt_result.get('is_monthly_return_pct'),
'oos_start': bt_result.get('oos_start'),
# Rolling walk-forward
'wf_n_windows': bt_result.get('wf_n_windows'),
'wf_oos_sharpe_mean': wf_sharpe_mean,
'wf_oos_sharpe_std': bt_result.get('wf_oos_sharpe_std'),
'wf_oos_monthly_return_mean': bt_result.get('wf_oos_monthly_return_mean'),
'wf_oos_consistency': wf_consistency,
# Monte Carlo significance
'mc_pvalue': mc_pvalue,
'mc_n_permutations': bt_result.get('mc_n_permutations'),
}
fname = f"{int(time.time())}_{strategy['strategy_name']}.json"
@@ -614,12 +634,16 @@ def main(target_count=10):
f"IC={ic:.4f}, Sharpe={sharpe:.3f}, Trades={trades}, DD={dd:.1%}")
else:
oos_info = f"OOS_Sharpe={oos_sharpe:+.2f} OOS_Mon={oos_monthly:+.2f}%" if oos_sharpe is not None else ""
_log.info(f"REJECTED IC={ic:.4f} Sharpe={sharpe:.2f} Trades={trades} DD={dd:.1%} {oos_info}")
mc_info = f" MC_p={mc_pvalue:.2f}" if mc_pvalue is not None else ""
wf_info = f" WF_consistency={wf_consistency:.0%}" if wf_consistency is not None else ""
_log.info(f"REJECTED IC={ic:.4f} Sharpe={sharpe:.2f} Trades={trades} DD={dd:.1%} {oos_info}{mc_info}{wf_info}")
feedback_history.append(
f"Failed: IC={ic:.4f}, Sharpe={sharpe:.2f}, Trades={trades}, DD={dd:.1%}, "
f"OOS_Sharpe={oos_sharpe:+.2f}, OOS_Monthly={oos_monthly:+.2f}%. "
f"Need |IC|>{MIN_IC}, Sharpe>{MIN_SHARPE}, Trades>{MIN_TRADES}, "
f"OOS_Sharpe>0 AND OOS_Monthly>0 — strategy must generalise to unseen data (2024+)."
f"OOS_Sharpe={oos_sharpe:+.2f}, OOS_Monthly={oos_monthly:+.2f}%"
+ (f", MC_p={mc_pvalue:.2f}" if mc_pvalue is not None else "")
+ (f", WF_consistency={wf_consistency:.0%}" if wf_consistency is not None else "")
+ f". Need |IC|>{MIN_IC}, Sharpe>{MIN_SHARPE}, Trades>{MIN_TRADES}, "
f"OOS_Sharpe>0, OOS_Monthly>0, MC_p<0.20, WF_consistency≥50%."
)
progress.update(task, advance=1)
scripts/predix_rebacktest_unified.py
+17
View File
@@ -188,6 +188,8 @@ def rebacktest_one(
close=close_a,
signal=signal,
txn_cost_bps=txn_cost_bps,
wf_rolling=True,
mc_n_permutations=200,
)
result["status_detail"] = result.pop("status")
result["status"] = "ok"
@@ -264,6 +266,15 @@ def main() -> None:
"oos_win_rate": bt.get("oos_win_rate"),
"oos_n_trades": bt.get("oos_n_trades"),
"oos_start": bt.get("oos_start"),
# Rolling walk-forward
"wf_n_windows": bt.get("wf_n_windows"),
"wf_oos_sharpe_mean": bt.get("wf_oos_sharpe_mean"),
"wf_oos_sharpe_std": bt.get("wf_oos_sharpe_std"),
"wf_oos_monthly_return_mean": bt.get("wf_oos_monthly_return_mean"),
"wf_oos_consistency": bt.get("wf_oos_consistency"),
# Monte Carlo significance
"mc_pvalue": bt.get("mc_pvalue"),
"mc_n_permutations": bt.get("mc_n_permutations"),
}
data["sharpe_ratio"] = bt.get("sharpe")
data["max_drawdown"] = bt.get("max_drawdown")
@@ -299,6 +310,12 @@ def main() -> None:
"oos_monthly_pct": bt.get("oos_monthly_return_pct"),
"oos_dd": bt.get("oos_max_drawdown"),
"oos_trades": bt.get("oos_n_trades"),
# Rolling walk-forward
"wf_n_windows": bt.get("wf_n_windows"),
"wf_oos_sharpe_mean": bt.get("wf_oos_sharpe_mean"),
"wf_oos_consistency": bt.get("wf_oos_consistency"),
# Monte Carlo
"mc_pvalue": bt.get("mc_pvalue"),
}
if "annualized_return" in bt:
row["new_annual_return_cagr"] = bt["annualized_return"]
test/backtesting/test_ftmo_oos.py
+50
View File
@@ -19,6 +19,8 @@ from rdagent.components.backtesting.vbt_backtest import (
backtest_signal_ftmo,
FTMO_MAX_DAILY_LOSS,
FTMO_MAX_TOTAL_LOSS,
monte_carlo_trade_pvalue,
walk_forward_rolling,
)
@@ -188,3 +190,51 @@ def test_ftmo_result_has_equity_and_profit(close_2yr):
assert "ftmo_end_equity" in r
assert "ftmo_monthly_profit" in r
assert r["ftmo_end_equity"] > 0
# ---------------------------------------------------------------------------
# Monte Carlo trade permutation tests
# ---------------------------------------------------------------------------
def test_mc_pvalue_in_result(close_2yr):
signal = _random_signal(close_2yr.index)
r = backtest_signal_ftmo(close_2yr, signal, oos_start=None, mc_n_permutations=50)
assert "mc_pvalue" in r
assert 0.0 <= r["mc_pvalue"] <= 1.0
assert r["mc_n_permutations"] == 50
def test_mc_pvalue_disabled_by_default(close_2yr):
signal = _random_signal(close_2yr.index)
r = backtest_signal_ftmo(close_2yr, signal, oos_start=None)
assert "mc_pvalue" not in r
def test_mc_zero_trades_returns_one(close_2yr):
"""Zero-signal → no trades → p-value must be 1.0 (no edge)."""
trade_pnl = pd.Series([], dtype=float)
assert monte_carlo_trade_pvalue(trade_pnl, n_permutations=10) == 1.0
# ---------------------------------------------------------------------------
# Rolling walk-forward tests
# ---------------------------------------------------------------------------
def test_wf_rolling_keys_in_result(close_6yr):
signal = _random_signal(close_6yr.index)
r = backtest_signal_ftmo(close_6yr, signal, oos_start="2024-01-01", wf_rolling=True)
# With only ~150 days of data, windows may be 0 — just check key presence
assert "wf_n_windows" in r
def test_wf_rolling_disabled_by_default(close_6yr):
signal = _random_signal(close_6yr.index)
r = backtest_signal_ftmo(close_6yr, signal, oos_start="2024-01-01")
assert "wf_n_windows" not in r
def test_wf_consistency_range(close_6yr):
"""wf_oos_consistency must be in [0, 1] when windows exist."""
signal = _random_signal(close_6yr.index)
r = backtest_signal_ftmo(close_6yr, signal, oos_start="2024-01-01", wf_rolling=True)
c = r.get("wf_oos_consistency")
if c is not None:
assert 0.0 <= c <= 1.0