gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(backtest): add rolling walk-forward validation and Monte Carlo trade permutation test

Browse Source 
- monte_carlo_trade_pvalue(): shuffles trade P&L N times, returns fraction of
  permuted sequences that beat real total return (p<0.05 = genuine edge)
- walk_forward_rolling(): multiple IS/OOS windows (IS=3yr, OOS=1yr, step=1yr),
  computes wf_oos_sharpe_mean, wf_oos_consistency (% profitable windows)
- backtest_signal_riskmgmt(): new wf_rolling and mc_n_permutations params
- Strategy generator: enables both (200 MC permutations), adds mc_ok and wf_ok
  to acceptance filter (mc_p<0.20, wf_consistency>=50%)
- Rebacktest script: enables both, stores all wf_*/mc_* fields in write-back
- 6 new tests covering MC pvalue, disabled-by-default, zero-trades edge case,
  rolling WF key presence and consistency range

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
TPTBusiness
2026-04-19 18:59:00 +02:00
parent 6d514d3ed5
commit b0490c5f0d
5 changed files with 258 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/predix_gen_strategies_real_bt.py
+31 -7
View File
@@ -371,6 +371,8 @@ signal.fillna(0).to_pickle('signal.pkl')
txn_cost_bps=TXN_COST_BPS,
forward_returns=fwd_returns,
oos_start=OOS_START_DEFAULT,
wf_rolling=True,
mc_n_permutations=200,
)
# ============================================================================
@@ -567,9 +569,18 @@ def main(target_count=10):
progress.update(task, advance=1)
continue
# Check acceptance criteria — OOS must be profitable (primary filter)
# Monte Carlo p-value (edge significance)
mc_pvalue = bt_result.get('mc_pvalue')
# Rolling walk-forward metrics
wf_consistency = bt_result.get('wf_oos_consistency')
wf_sharpe_mean = bt_result.get('wf_oos_sharpe_mean')
# Check acceptance criteria — OOS must be profitable + statistically significant
mc_ok = mc_pvalue is None or mc_pvalue < 0.20 # lenient: top 20% non-random
wf_ok = wf_consistency is None or wf_consistency >= 0.5 # ≥50% of WF windows profitable
if (abs(ic) > MIN_IC and sharpe > MIN_SHARPE and trades > MIN_TRADES and dd > MAX_DRAWDOWN
and oos_sharpe > 0.0 and oos_monthly > 0.0):
and oos_sharpe > 0.0 and oos_monthly > 0.0 and mc_ok and wf_ok):
# ACCEPT
strategy['real_backtest'] = bt_result
strategy['metrics'] = bt_result
@@ -583,7 +594,7 @@ def main(target_count=10):
'ohlcv_only': OHLCV_ONLY,
'engine': 'ftmo_v2',
'txn_cost_bps': TXN_COST_BPS,
# Walk-forward OOS metrics
# Walk-forward OOS split
'oos_sharpe': bt_result.get('oos_sharpe'),
'oos_monthly_return_pct': bt_result.get('oos_monthly_return_pct'),
'oos_max_drawdown': bt_result.get('oos_max_drawdown'),
@@ -592,6 +603,15 @@ def main(target_count=10):
'is_sharpe': bt_result.get('is_sharpe'),
'is_monthly_return_pct': bt_result.get('is_monthly_return_pct'),
'oos_start': bt_result.get('oos_start'),
# Rolling walk-forward
'wf_n_windows': bt_result.get('wf_n_windows'),
'wf_oos_sharpe_mean': wf_sharpe_mean,
'wf_oos_sharpe_std': bt_result.get('wf_oos_sharpe_std'),
'wf_oos_monthly_return_mean': bt_result.get('wf_oos_monthly_return_mean'),
'wf_oos_consistency': wf_consistency,
# Monte Carlo significance
'mc_pvalue': mc_pvalue,
'mc_n_permutations': bt_result.get('mc_n_permutations'),
}
fname = f"{int(time.time())}_{strategy['strategy_name']}.json"
@@ -614,12 +634,16 @@ def main(target_count=10):
f"IC={ic:.4f}, Sharpe={sharpe:.3f}, Trades={trades}, DD={dd:.1%}")
else:
oos_info = f"OOS_Sharpe={oos_sharpe:+.2f} OOS_Mon={oos_monthly:+.2f}%" if oos_sharpe is not None else ""
_log.info(f"REJECTED IC={ic:.4f} Sharpe={sharpe:.2f} Trades={trades} DD={dd:.1%} {oos_info}")
mc_info = f" MC_p={mc_pvalue:.2f}" if mc_pvalue is not None else ""
wf_info = f" WF_consistency={wf_consistency:.0%}" if wf_consistency is not None else ""
_log.info(f"REJECTED IC={ic:.4f} Sharpe={sharpe:.2f} Trades={trades} DD={dd:.1%} {oos_info}{mc_info}{wf_info}")
feedback_history.append(
f"Failed: IC={ic:.4f}, Sharpe={sharpe:.2f}, Trades={trades}, DD={dd:.1%}, "
f"OOS_Sharpe={oos_sharpe:+.2f}, OOS_Monthly={oos_monthly:+.2f}%. "
f"Need |IC|>{MIN_IC}, Sharpe>{MIN_SHARPE}, Trades>{MIN_TRADES}, "
f"OOS_Sharpe>0 AND OOS_Monthly>0 — strategy must generalise to unseen data (2024+)."
f"OOS_Sharpe={oos_sharpe:+.2f}, OOS_Monthly={oos_monthly:+.2f}%"
+ (f", MC_p={mc_pvalue:.2f}" if mc_pvalue is not None else "")
+ (f", WF_consistency={wf_consistency:.0%}" if wf_consistency is not None else "")
+ f". Need |IC|>{MIN_IC}, Sharpe>{MIN_SHARPE}, Trades>{MIN_TRADES}, "
f"OOS_Sharpe>0, OOS_Monthly>0, MC_p<0.20, WF_consistency≥50%."
)
progress.update(task, advance=1)
scripts/predix_rebacktest_unified.py
+17
View File
@@ -188,6 +188,8 @@ def rebacktest_one(
close=close_a,
signal=signal,
txn_cost_bps=txn_cost_bps,
wf_rolling=True,
mc_n_permutations=200,
)
result["status_detail"] = result.pop("status")
result["status"] = "ok"
@@ -264,6 +266,15 @@ def main() -> None:
"oos_win_rate": bt.get("oos_win_rate"),
"oos_n_trades": bt.get("oos_n_trades"),
"oos_start": bt.get("oos_start"),
# Rolling walk-forward
"wf_n_windows": bt.get("wf_n_windows"),
"wf_oos_sharpe_mean": bt.get("wf_oos_sharpe_mean"),
"wf_oos_sharpe_std": bt.get("wf_oos_sharpe_std"),
"wf_oos_monthly_return_mean": bt.get("wf_oos_monthly_return_mean"),
"wf_oos_consistency": bt.get("wf_oos_consistency"),
# Monte Carlo significance
"mc_pvalue": bt.get("mc_pvalue"),
"mc_n_permutations": bt.get("mc_n_permutations"),
}
data["sharpe_ratio"] = bt.get("sharpe")
data["max_drawdown"] = bt.get("max_drawdown")
@@ -299,6 +310,12 @@ def main() -> None:
"oos_monthly_pct": bt.get("oos_monthly_return_pct"),
"oos_dd": bt.get("oos_max_drawdown"),
"oos_trades": bt.get("oos_n_trades"),
# Rolling walk-forward
"wf_n_windows": bt.get("wf_n_windows"),
"wf_oos_sharpe_mean": bt.get("wf_oos_sharpe_mean"),
"wf_oos_consistency": bt.get("wf_oos_consistency"),
# Monte Carlo
"mc_pvalue": bt.get("mc_pvalue"),
}
if "annualized_return" in bt:
row["new_annual_return_cagr"] = bt["annualized_return"]