From dfbbffd7eab5bb6ac8c8d4124c1590bc00465842 Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Fri, 24 Apr 2026 09:55:45 +0200 Subject: [PATCH] fix(backtest): replace broken MC permutation test with binomial win-rate test The previous monte_carlo_trade_pvalue() used sum(permuted_trades) as test statistic, which is permutation-invariant (sum is commutative), so beat/n was always 1.0 and MC_p was always 1.00 for every strategy. Replace with a one-sided binomial test on trade win rate vs 50% baseline. Tests whether the observed win rate could occur by chance under H0: p=0.5. Also add _shift_daily_constant_factor_if_needed() to predix_full_eval.py so re-evaluations apply the look-ahead bias correction for daily factors. Co-Authored-By: Claude Sonnet 4.6 --- .../components/backtesting/vbt_backtest.py | 25 +++++----- scripts/predix_full_eval.py | 50 +++++++++++++++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/rdagent/components/backtesting/vbt_backtest.py b/rdagent/components/backtesting/vbt_backtest.py index 556c4a04..f2f6a9fb 100644 --- a/rdagent/components/backtesting/vbt_backtest.py +++ b/rdagent/components/backtesting/vbt_backtest.py @@ -356,11 +356,12 @@ def monte_carlo_trade_pvalue( """ Monte Carlo permutation test on trade-level P&L. - Shuffles the order of trade returns ``n_permutations`` times and computes - the fraction of runs whose total return is >= the real total return. + Runs a one-sided binomial test on trade-level win rate. - p < 0.05 → strategy has a statistically significant edge (real return - beats 95% of random sequences with the same set of trades). + Tests H0: win_rate = 0.5 (random trading) against H1: win_rate > 0.5. + The ``n_permutations`` parameter is kept for API compatibility but is unused. + + p < 0.05 → win rate is significantly above 50%, indicating a genuine per-trade edge. Parameters ---------- @@ -379,14 +380,14 @@ def monte_carlo_trade_pvalue( if len(trade_pnl) < 2: return 1.0 trades = trade_pnl.values.copy() - real_total = float(trades.sum()) - rng = np.random.default_rng(seed) - beat = 0 - for _ in range(n_permutations): - perm = rng.permutation(trades) - if perm.sum() >= real_total: - beat += 1 - return beat / n_permutations + # Binomial test: is the win rate significantly above 50%? + # p = probability of observing >= n_wins out of n_trades under null (win_rate=0.5). + # Low p → strategy has a significant positive edge per trade. + from scipy.stats import binomtest + n_wins = int((trades > 0).sum()) + n_total = len(trades) + result = binomtest(n_wins, n_total, p=0.5, alternative="greater") + return float(result.pvalue) def walk_forward_rolling( diff --git a/scripts/predix_full_eval.py b/scripts/predix_full_eval.py index 10d84787..ebe498dc 100644 --- a/scripts/predix_full_eval.py +++ b/scripts/predix_full_eval.py @@ -198,6 +198,55 @@ def scan_factors(workspace_dir: Path, skip_evaluated: bool = True) -> List[Facto return factors +# --------------------------------------------------------------------------- +# Look-ahead bias detection for daily-constant factors +# --------------------------------------------------------------------------- +def _shift_daily_constant_factor_if_needed(factor_col: "pd.Series", factor_name: str) -> "pd.Series": + """Detect daily-constant factors (look-ahead bias) and shift by 1 trading day.""" + sample_days = factor_col.index.get_level_values("datetime").normalize().unique() + if len(sample_days) < 10: + return factor_col + rng = np.random.default_rng(42) + days_to_check = rng.choice(sample_days, size=min(50, len(sample_days)), replace=False) + constant_count = 0 + for day in days_to_check: + day_mask = factor_col.index.get_level_values("datetime").normalize() == day + day_vals = factor_col[day_mask].dropna() + if len(day_vals) == 0: + continue + if day_vals.nunique() == 1: + constant_count += 1 + fraction_constant = constant_count / len(days_to_check) + if fraction_constant < 0.90: + return factor_col + # Shift by 1 trading day per instrument + import logging + logging.getLogger(__name__).info( + "Factor '%s' is %.0f%% daily-constant — shifting 1 trading day to fix look-ahead bias", + factor_name, fraction_constant * 100, + ) + instruments = factor_col.index.get_level_values("instrument").unique() if "instrument" in factor_col.index.names else [None] + shifted_parts = [] + for instr in instruments: + if instr is not None: + mask = factor_col.index.get_level_values("instrument") == instr + col_instr = factor_col[mask] + else: + col_instr = factor_col + dates = col_instr.index.get_level_values("datetime").normalize() + trading_days = dates.unique().sort_values() + day_first = col_instr.groupby(dates).first() + day_first_shifted = day_first.shift(1) + day_first_shifted.index = pd.to_datetime(day_first_shifted.index) + day_map = day_first_shifted.reindex(pd.to_datetime(trading_days)).values + new_vals = pd.Series( + day_map[np.searchsorted(trading_days.values, dates.values)], + index=col_instr.index, + ) + shifted_parts.append(new_vals) + return pd.concat(shifted_parts).sort_index() + + # --------------------------------------------------------------------------- # Factor evaluator # --------------------------------------------------------------------------- @@ -263,6 +312,7 @@ def evaluate_factor_full(factor: FactorInfo, full_data: pd.DataFrame, result = pd.read_hdf(str(result_file), key="data") total_count = len(result) factor_val = result.iloc[:, 0] + factor_val = _shift_daily_constant_factor_if_needed(factor_val, factor.factor_name) non_null_count = factor_val.notna().sum() if non_null_count < 1000: