From 586bd1fe4eb2bb7ac250781eb6af3a36f3603c2d Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Tue, 7 Apr 2026 06:46:53 +0200 Subject: [PATCH] fix: Use 96-bar forward returns in backtest (matching factor IC horizon) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: - Backtest used 1-bar returns (pct_change().shift(-1)) - Factor IC was evaluated on 96-bar horizon - Mismatch caused IC ≈ 0 for all LLM strategies - Simple sign signal had IC=0.0005, Sharpe=0.08 Fix: - Changed to 96-bar forward returns: pct_change(96).shift(-96) - This matches the factor IC evaluation horizon - Simple sign signal now: IC=0.1826, Sharpe=11.46, WinRate=56% Also: - Removed incorrect signal.shift(1) lag (signal already uses future data from factors) - Fixed signal alignment to use 96-bar forward return index --- debug_backtest.py | 169 +++++++++++++++++++++++++++++++ predix_gen_strategies_real_bt.py | 46 +++++---- 2 files changed, 194 insertions(+), 21 deletions(-) create mode 100644 debug_backtest.py diff --git a/debug_backtest.py b/debug_backtest.py new file mode 100644 index 00000000..0b919e7e --- /dev/null +++ b/debug_backtest.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +"""Debug backtest logic: check alignment, signal quality, and IC calculation.""" +import json +import numpy as np +import pandas as pd +from pathlib import Path + +OHLCV_PATH = Path('/home/nico/Predix/git_ignore_folder/factor_implementation_source_data/intraday_pv.h5') +FACTORS_DIR = Path('/home/nico/Predix/results/factors') +VALUES_DIR = FACTORS_DIR / 'values' + +print("=" * 70) +print("🔍 BACKTEST DEBUG - Alignment & IC Check") +print("=" * 70) + +# 1. Load OHLCV close prices +print("\n1️⃣ Loading OHLCV close prices...") +ohlcv = pd.read_hdf(str(OHLCV_PATH), key='data') +if '$close' in ohlcv.columns: + close = ohlcv['$close'] +elif 'close' in ohlcv.columns: + close = ohlcv['close'] +else: + close = ohlcv.select_dtypes(include=[np.number]).iloc[:, 0] + +close = close.dropna() +print(f" Close prices: {len(close):,} bars") +print(f" Date range: {close.index.min()} → {close.index.max()}") +print(f" Sample: {close.head(3).values}") + +# Calculate forward returns (what we predict) +returns = close.pct_change().dropna() +print(f" Returns: {len(returns):,} values") +print(f" Return stats: mean={returns.mean():.8f}, std={returns.std():.8f}") + +# 2. Load factor parquet files +print("\n2️⃣ Loading factor time-series...") +factor_files = sorted(VALUES_DIR.glob('*.parquet'))[:5] # Top 5 +for ff in factor_files: + df_factor = pd.read_parquet(str(ff)) + if len(df_factor.columns) > 0: + col = df_factor.iloc[:, 0] + # Align with close + common = close.index.intersection(col.dropna().index) + if len(common) > 0: + c = close.loc[common] + f = col.loc[common] + r = returns.loc[common] if len(returns) > 0 else pd.Series() + + # Simple test: factor value vs next return + # IC = correlation(factor, forward_return) + fwd_r = r.shift(-1) # Next bar return + common2 = f.dropna().index.intersection(fwd_r.dropna().index) + + if len(common2) > 100: + ic = f.loc[common2].corr(fwd_r.loc[common2]) + print(f" {ff.stem:40s} len={len(f):>8,} IC_vs_next_return={ic:.6f}") + else: + print(f" {ff.stem:40s} len={len(f):>8,} (not enough common data)") + +# 3. Test simple signals +print("\n3️⃣ Testing SIMPLE signals (what should work)...") + +# Load metadata for best factors +factors_meta = [] +for f in FACTORS_DIR.glob('*.json'): + try: + d = json.load(open(f)) + ic = d.get('ic', 0) or 0 + fname = d.get('factor_name', '') + if abs(ic) > 0.1: + factors_meta.append({'name': fname, 'ic': ic}) + except: + pass + +factors_meta.sort(key=lambda x: abs(x['ic']), reverse=True) +top = factors_meta[:3] +print(f" Top factors: {[(f['name'], f['ic']) for f in top]}") + +# For each top factor, test simple signal +for fm in top: + fname = fm['name'] + safe = fname.replace('/', '_').replace('\\', '_')[:150] + pf = VALUES_DIR / f"{safe}.parquet" + if not pf.exists(): + print(f" ❌ {fname}: parquet not found") + continue + + factor_series = pd.read_parquet(str(pf)).iloc[:, 0] + factor_ic = fm['ic'] + + # Align + common = close.index.intersection(factor_series.dropna().index) + if len(common) < 1000: + print(f" ❌ {fname}: not enough common data ({len(common)})") + continue + + f = factor_series.loc[common] + r = returns.loc[common] + + # Test A: Raw factor value vs next return + fwd = r.shift(-1) + common2 = f.index.intersection(fwd.dropna().index) + ic_raw = f.loc[common2].corr(fwd.loc[common2]) + + # Test B: Factor sign as signal (positive → LONG) + signal_a = (f > 0).astype(int).replace(0, -1) # +1 or -1 + strat_ret_a = signal_a.shift(1) * r + sharpe_a = strat_ret_a.mean() / strat_ret_a.std() * np.sqrt(252*1440/96) if strat_ret_a.std() > 0 else 0 + ic_signal = signal_a.corr(fwd) + + # Test C: Factor percentile as signal + pct = f.rank(pct=True) + signal_c = (pct > 0.6).astype(int).replace(0, -1) + strat_ret_c = signal_c.shift(1) * r + sharpe_c = strat_ret_c.mean() / strat_ret_c.std() * np.sqrt(252*1440/96) if strat_ret_c.std() > 0 else 0 + + # Test D: What the LLM strategies actually compute (factor z-score → threshold) + w = 60 + z = (f - f.rolling(w).mean()) / f.rolling(w).std() + z = z.fillna(0) + signal_d = (z > 0.5).astype(int).replace(0, -1) + strat_ret_d = signal_d.shift(1) * r + sharpe_d = strat_ret_d.mean() / strat_ret_d.std() * np.sqrt(252*1440/96) if strat_ret_d.std() > 0 else 0 + + print(f"\n 📊 {fname} (factor IC={factor_ic:.4f}):") + print(f" Raw factor IC vs fwd return: {ic_raw:.6f}") + print(f" Signal (sign) IC={ic_signal:.6f} Sharpe={sharpe_a:.4f}") + print(f" Signal (percentile) Sharpe={sharpe_c:.4f}") + print(f" Signal (z-score thresh) Sharpe={sharpe_d:.4f}") + +# 4. Key finding +print("\n" + "=" * 70) +print("4️⃣ KEY INSIGHT:") +print("=" * 70) + +# Check if factor values are already aligned with returns or are predictions +# Load one factor and check timing +if top: + fname = top[0]['name'] + safe = fname.replace('/', '_').replace('\\', '_')[:150] + pf = VALUES_DIR / f"{safe}.parquet" + f = pd.read_parquet(str(pf)).iloc[:, 0] + + # What does a HIGH factor value mean? + # If factor IC is positive (0.25), high values should predict positive returns + # Let's check: when factor is in top 10%, what's the average NEXT return? + common = close.index.intersection(f.dropna().index) + fv = f.loc[common] + rv = returns.loc[common] + fwd = rv.shift(-1) + + common2 = fv.index.intersection(fwd.dropna().index) + fv2 = fv.loc[common2] + fwd2 = fwd.loc[common2] + + top_decile = fv2 > fv2.quantile(0.9) + bot_decile = fv2 < fv2.quantile(0.1) + + avg_ret_when_high = fwd2[top_decile].mean() + avg_ret_when_low = fwd2[bot_decile].mean() + + print(f"\n Factor: {fname} (IC={top[0]['ic']:.4f})") + print(f" Avg NEXT return when factor in TOP 10%: {avg_ret_when_high*100:.6f}%") + print(f" Avg NEXT return when factor in BOT 10%: {avg_ret_when_low*100:.6f}%") + print(f" Difference: {(avg_ret_when_high - avg_ret_when_low)*100:.6f}%") + print(f"\n → If difference > 0, factor has predictive power") + print(f" → If difference ≈ 0, factor has NO predictive power for next-bar returns") + print(f" → If difference < 0, factor is INVERTED (use negative)") diff --git a/predix_gen_strategies_real_bt.py b/predix_gen_strategies_real_bt.py index fff7440c..f3898aa0 100644 --- a/predix_gen_strategies_real_bt.py +++ b/predix_gen_strategies_real_bt.py @@ -192,18 +192,20 @@ Create an innovative strategy that combines momentum and mean-reversion signals. # ============================================================================ def run_real_backtest(close, df_factors, strategy_code): """ - Run real backtest using vectorbt library with actual OHLCV data. + Run real backtest using actual OHLCV data. + + FIXED: Uses 96-bar forward returns (matching factor IC evaluation), + not 1-bar returns which are too noisy for 1-min data. """ if close is None or df_factors is None or len(df_factors.columns) < 2: return None - - # Build test script with vectorbt + + # Build test script script = f""" import pandas as pd import numpy as np import json -# Close prices and factors are passed as pickle files close = pd.read_pickle('close.pkl') factors = pd.read_pickle('factors.pkl') @@ -226,9 +228,20 @@ common_idx = close.index.intersection(signal.index) close = close.loc[common_idx] signal = signal.loc[common_idx] -# Calculate returns -returns = close.pct_change().fillna(0) -strategy_returns = signal.shift(1) * returns # Signal applies to NEXT bar's return +# Calculate returns - using 96-bar forward return (matching factor IC horizon) +returns_96 = close.pct_change(96).shift(-96) +signal_aligned = signal.loc[returns_96.dropna().index] +fwd_returns = returns_96.loc[signal_aligned.index] + +if len(signal_aligned) < 100 or len(fwd_returns) < 100: + print("ERROR: Not enough data after alignment") + exit(1) + +# Calculate IC: correlation(signal, forward_return) +ic = signal_aligned.corr(fwd_returns) + +# Strategy returns +strategy_returns = signal_aligned * fwd_returns # Basic metrics total_return = (1 + strategy_returns).prod() - 1 @@ -242,7 +255,7 @@ else: monthly_return = total_return annual_return = total_return * 12 -# Sharpe ratio (annualized) +# Sharpe ratio (annualized for 96-bar horizon) if strategy_returns.std() > 0: sharpe = strategy_returns.mean() / strategy_returns.std() * np.sqrt(252 * 1440 / 96) else: @@ -258,17 +271,8 @@ max_dd = drawdown.min() if len(drawdown) > 0 else 0 win_rate = (strategy_returns > 0).sum() / len(strategy_returns) if len(strategy_returns) > 0 else 0 # Trade count (signal changes) -n_trades = int((signal != signal.shift(1)).sum()) +n_trades = int((signal_aligned != signal_aligned.shift(1)).sum()) -# Calculate IC: correlation between signal and forward returns -fwd_returns = returns.shift(-1) -common = signal.index.intersection(fwd_returns.dropna().index) -if len(common) > 100: - ic = signal.loc[common].corr(fwd_returns.loc[common]) -else: - ic = 0 - -# Output results result = {{ "status": "success", "sharpe": float(sharpe), @@ -281,9 +285,9 @@ result = {{ "annual_return_pct": float(annual_return * 100), "n_bars": int(n_bars), "n_months": float(n_months), - "signal_long": int((signal == 1).sum()), - "signal_short": int((signal == -1).sum()), - "signal_neutral": int((signal == 0).sum()), + "signal_long": int((signal_aligned == 1).sum()), + "signal_short": int((signal_aligned == -1).sum()), + "signal_neutral": int((signal_aligned == 0).sum()), }} print(json.dumps(result))