gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(backtest): replace broken MC permutation test with binomial win-rate test

Browse Source 
The previous monte_carlo_trade_pvalue() used sum(permuted_trades) as test
statistic, which is permutation-invariant (sum is commutative), so beat/n
was always 1.0 and MC_p was always 1.00 for every strategy.

Replace with a one-sided binomial test on trade win rate vs 50% baseline.
Tests whether the observed win rate could occur by chance under H0: p=0.5.

Also add _shift_daily_constant_factor_if_needed() to predix_full_eval.py
so re-evaluations apply the look-ahead bias correction for daily factors.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
TPTBusiness
2026-04-24 09:55:45 +02:00
parent 78fb607dbe
commit dfbbffd7ea
2 changed files with 63 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/predix_full_eval.py
+50
View File
@@ -198,6 +198,55 @@ def scan_factors(workspace_dir: Path, skip_evaluated: bool = True) -> List[Facto
return factors
# ---------------------------------------------------------------------------
# Look-ahead bias detection for daily-constant factors
# ---------------------------------------------------------------------------
def _shift_daily_constant_factor_if_needed(factor_col: "pd.Series", factor_name: str) -> "pd.Series":
"""Detect daily-constant factors (look-ahead bias) and shift by 1 trading day."""
sample_days = factor_col.index.get_level_values("datetime").normalize().unique()
if len(sample_days) < 10:
return factor_col
rng = np.random.default_rng(42)
days_to_check = rng.choice(sample_days, size=min(50, len(sample_days)), replace=False)
constant_count = 0
for day in days_to_check:
day_mask = factor_col.index.get_level_values("datetime").normalize() == day
day_vals = factor_col[day_mask].dropna()
if len(day_vals) == 0:
continue
if day_vals.nunique() == 1:
constant_count += 1
fraction_constant = constant_count / len(days_to_check)
if fraction_constant < 0.90:
return factor_col
# Shift by 1 trading day per instrument
import logging
logging.getLogger(__name__).info(
"Factor '%s' is %.0f%% daily-constant — shifting 1 trading day to fix look-ahead bias",
factor_name, fraction_constant * 100,
)
instruments = factor_col.index.get_level_values("instrument").unique() if "instrument" in factor_col.index.names else [None]
shifted_parts = []
for instr in instruments:
if instr is not None:
mask = factor_col.index.get_level_values("instrument") == instr
col_instr = factor_col[mask]
else:
col_instr = factor_col
dates = col_instr.index.get_level_values("datetime").normalize()
trading_days = dates.unique().sort_values()
day_first = col_instr.groupby(dates).first()
day_first_shifted = day_first.shift(1)
day_first_shifted.index = pd.to_datetime(day_first_shifted.index)
day_map = day_first_shifted.reindex(pd.to_datetime(trading_days)).values
new_vals = pd.Series(
day_map[np.searchsorted(trading_days.values, dates.values)],
index=col_instr.index,
)
shifted_parts.append(new_vals)
return pd.concat(shifted_parts).sort_index()
# ---------------------------------------------------------------------------
# Factor evaluator
# ---------------------------------------------------------------------------
@@ -263,6 +312,7 @@ def evaluate_factor_full(factor: FactorInfo, full_data: pd.DataFrame,
result = pd.read_hdf(str(result_file), key="data")
total_count = len(result)
factor_val = result.iloc[:, 0]
factor_val = _shift_daily_constant_factor_if_needed(factor_val, factor.factor_name)
non_null_count = factor_val.notna().sum()
if non_null_count < 1000: