mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 23:47:46 +00:00
fix: Use 96-bar forward returns in backtest (matching factor IC horizon)
Problem: - Backtest used 1-bar returns (pct_change().shift(-1)) - Factor IC was evaluated on 96-bar horizon - Mismatch caused IC ≈ 0 for all LLM strategies - Simple sign signal had IC=0.0005, Sharpe=0.08 Fix: - Changed to 96-bar forward returns: pct_change(96).shift(-96) - This matches the factor IC evaluation horizon - Simple sign signal now: IC=0.1826, Sharpe=11.46, WinRate=56% Also: - Removed incorrect signal.shift(1) lag (signal already uses future data from factors) - Fixed signal alignment to use 96-bar forward return index
This commit is contained in:
TPTBusiness
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python
|
||||
"""Debug backtest logic: check alignment, signal quality, and IC calculation."""
|
||||
import json
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
OHLCV_PATH = Path('/home/nico/Predix/git_ignore_folder/factor_implementation_source_data/intraday_pv.h5')
|
||||
FACTORS_DIR = Path('/home/nico/Predix/results/factors')
|
||||
VALUES_DIR = FACTORS_DIR / 'values'
|
||||
|
||||
print("=" * 70)
|
||||
print("🔍 BACKTEST DEBUG - Alignment & IC Check")
|
||||
print("=" * 70)
|
||||
|
||||
# 1. Load OHLCV close prices
|
||||
print("\n1️⃣ Loading OHLCV close prices...")
|
||||
ohlcv = pd.read_hdf(str(OHLCV_PATH), key='data')
|
||||
if '$close' in ohlcv.columns:
|
||||
close = ohlcv['$close']
|
||||
elif 'close' in ohlcv.columns:
|
||||
close = ohlcv['close']
|
||||
else:
|
||||
close = ohlcv.select_dtypes(include=[np.number]).iloc[:, 0]
|
||||
|
||||
close = close.dropna()
|
||||
print(f" Close prices: {len(close):,} bars")
|
||||
print(f" Date range: {close.index.min()} → {close.index.max()}")
|
||||
print(f" Sample: {close.head(3).values}")
|
||||
|
||||
# Calculate forward returns (what we predict)
|
||||
returns = close.pct_change().dropna()
|
||||
print(f" Returns: {len(returns):,} values")
|
||||
print(f" Return stats: mean={returns.mean():.8f}, std={returns.std():.8f}")
|
||||
|
||||
# 2. Load factor parquet files
|
||||
print("\n2️⃣ Loading factor time-series...")
|
||||
factor_files = sorted(VALUES_DIR.glob('*.parquet'))[:5] # Top 5
|
||||
for ff in factor_files:
|
||||
df_factor = pd.read_parquet(str(ff))
|
||||
if len(df_factor.columns) > 0:
|
||||
col = df_factor.iloc[:, 0]
|
||||
# Align with close
|
||||
common = close.index.intersection(col.dropna().index)
|
||||
if len(common) > 0:
|
||||
c = close.loc[common]
|
||||
f = col.loc[common]
|
||||
r = returns.loc[common] if len(returns) > 0 else pd.Series()
|
||||
|
||||
# Simple test: factor value vs next return
|
||||
# IC = correlation(factor, forward_return)
|
||||
fwd_r = r.shift(-1) # Next bar return
|
||||
common2 = f.dropna().index.intersection(fwd_r.dropna().index)
|
||||
|
||||
if len(common2) > 100:
|
||||
ic = f.loc[common2].corr(fwd_r.loc[common2])
|
||||
print(f" {ff.stem:40s} len={len(f):>8,} IC_vs_next_return={ic:.6f}")
|
||||
else:
|
||||
print(f" {ff.stem:40s} len={len(f):>8,} (not enough common data)")
|
||||
|
||||
# 3. Test simple signals
|
||||
print("\n3️⃣ Testing SIMPLE signals (what should work)...")
|
||||
|
||||
# Load metadata for best factors
|
||||
factors_meta = []
|
||||
for f in FACTORS_DIR.glob('*.json'):
|
||||
try:
|
||||
d = json.load(open(f))
|
||||
ic = d.get('ic', 0) or 0
|
||||
fname = d.get('factor_name', '')
|
||||
if abs(ic) > 0.1:
|
||||
factors_meta.append({'name': fname, 'ic': ic})
|
||||
except:
|
||||
pass
|
||||
|
||||
factors_meta.sort(key=lambda x: abs(x['ic']), reverse=True)
|
||||
top = factors_meta[:3]
|
||||
print(f" Top factors: {[(f['name'], f['ic']) for f in top]}")
|
||||
|
||||
# For each top factor, test simple signal
|
||||
for fm in top:
|
||||
fname = fm['name']
|
||||
safe = fname.replace('/', '_').replace('\\', '_')[:150]
|
||||
pf = VALUES_DIR / f"{safe}.parquet"
|
||||
if not pf.exists():
|
||||
print(f" ❌ {fname}: parquet not found")
|
||||
continue
|
||||
|
||||
factor_series = pd.read_parquet(str(pf)).iloc[:, 0]
|
||||
factor_ic = fm['ic']
|
||||
|
||||
# Align
|
||||
common = close.index.intersection(factor_series.dropna().index)
|
||||
if len(common) < 1000:
|
||||
print(f" ❌ {fname}: not enough common data ({len(common)})")
|
||||
continue
|
||||
|
||||
f = factor_series.loc[common]
|
||||
r = returns.loc[common]
|
||||
|
||||
# Test A: Raw factor value vs next return
|
||||
fwd = r.shift(-1)
|
||||
common2 = f.index.intersection(fwd.dropna().index)
|
||||
ic_raw = f.loc[common2].corr(fwd.loc[common2])
|
||||
|
||||
# Test B: Factor sign as signal (positive → LONG)
|
||||
signal_a = (f > 0).astype(int).replace(0, -1) # +1 or -1
|
||||
strat_ret_a = signal_a.shift(1) * r
|
||||
sharpe_a = strat_ret_a.mean() / strat_ret_a.std() * np.sqrt(252*1440/96) if strat_ret_a.std() > 0 else 0
|
||||
ic_signal = signal_a.corr(fwd)
|
||||
|
||||
# Test C: Factor percentile as signal
|
||||
pct = f.rank(pct=True)
|
||||
signal_c = (pct > 0.6).astype(int).replace(0, -1)
|
||||
strat_ret_c = signal_c.shift(1) * r
|
||||
sharpe_c = strat_ret_c.mean() / strat_ret_c.std() * np.sqrt(252*1440/96) if strat_ret_c.std() > 0 else 0
|
||||
|
||||
# Test D: What the LLM strategies actually compute (factor z-score → threshold)
|
||||
w = 60
|
||||
z = (f - f.rolling(w).mean()) / f.rolling(w).std()
|
||||
z = z.fillna(0)
|
||||
signal_d = (z > 0.5).astype(int).replace(0, -1)
|
||||
strat_ret_d = signal_d.shift(1) * r
|
||||
sharpe_d = strat_ret_d.mean() / strat_ret_d.std() * np.sqrt(252*1440/96) if strat_ret_d.std() > 0 else 0
|
||||
|
||||
print(f"\n 📊 {fname} (factor IC={factor_ic:.4f}):")
|
||||
print(f" Raw factor IC vs fwd return: {ic_raw:.6f}")
|
||||
print(f" Signal (sign) IC={ic_signal:.6f} Sharpe={sharpe_a:.4f}")
|
||||
print(f" Signal (percentile) Sharpe={sharpe_c:.4f}")
|
||||
print(f" Signal (z-score thresh) Sharpe={sharpe_d:.4f}")
|
||||
|
||||
# 4. Key finding
|
||||
print("\n" + "=" * 70)
|
||||
print("4️⃣ KEY INSIGHT:")
|
||||
print("=" * 70)
|
||||
|
||||
# Check if factor values are already aligned with returns or are predictions
|
||||
# Load one factor and check timing
|
||||
if top:
|
||||
fname = top[0]['name']
|
||||
safe = fname.replace('/', '_').replace('\\', '_')[:150]
|
||||
pf = VALUES_DIR / f"{safe}.parquet"
|
||||
f = pd.read_parquet(str(pf)).iloc[:, 0]
|
||||
|
||||
# What does a HIGH factor value mean?
|
||||
# If factor IC is positive (0.25), high values should predict positive returns
|
||||
# Let's check: when factor is in top 10%, what's the average NEXT return?
|
||||
common = close.index.intersection(f.dropna().index)
|
||||
fv = f.loc[common]
|
||||
rv = returns.loc[common]
|
||||
fwd = rv.shift(-1)
|
||||
|
||||
common2 = fv.index.intersection(fwd.dropna().index)
|
||||
fv2 = fv.loc[common2]
|
||||
fwd2 = fwd.loc[common2]
|
||||
|
||||
top_decile = fv2 > fv2.quantile(0.9)
|
||||
bot_decile = fv2 < fv2.quantile(0.1)
|
||||
|
||||
avg_ret_when_high = fwd2[top_decile].mean()
|
||||
avg_ret_when_low = fwd2[bot_decile].mean()
|
||||
|
||||
print(f"\n Factor: {fname} (IC={top[0]['ic']:.4f})")
|
||||
print(f" Avg NEXT return when factor in TOP 10%: {avg_ret_when_high*100:.6f}%")
|
||||
print(f" Avg NEXT return when factor in BOT 10%: {avg_ret_when_low*100:.6f}%")
|
||||
print(f" Difference: {(avg_ret_when_high - avg_ret_when_low)*100:.6f}%")
|
||||
print(f"\n → If difference > 0, factor has predictive power")
|
||||
print(f" → If difference ≈ 0, factor has NO predictive power for next-bar returns")
|
||||
print(f" → If difference < 0, factor is INVERTED (use negative)")
|
||||
@@ -192,18 +192,20 @@ Create an innovative strategy that combines momentum and mean-reversion signals.
|
||||
# ============================================================================
|
||||
def run_real_backtest(close, df_factors, strategy_code):
|
||||
"""
|
||||
Run real backtest using vectorbt library with actual OHLCV data.
|
||||
Run real backtest using actual OHLCV data.
|
||||
|
||||
FIXED: Uses 96-bar forward returns (matching factor IC evaluation),
|
||||
not 1-bar returns which are too noisy for 1-min data.
|
||||
"""
|
||||
if close is None or df_factors is None or len(df_factors.columns) < 2:
|
||||
return None
|
||||
|
||||
# Build test script with vectorbt
|
||||
|
||||
# Build test script
|
||||
script = f"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import json
|
||||
|
||||
# Close prices and factors are passed as pickle files
|
||||
close = pd.read_pickle('close.pkl')
|
||||
factors = pd.read_pickle('factors.pkl')
|
||||
|
||||
@@ -226,9 +228,20 @@ common_idx = close.index.intersection(signal.index)
|
||||
close = close.loc[common_idx]
|
||||
signal = signal.loc[common_idx]
|
||||
|
||||
# Calculate returns
|
||||
returns = close.pct_change().fillna(0)
|
||||
strategy_returns = signal.shift(1) * returns # Signal applies to NEXT bar's return
|
||||
# Calculate returns - using 96-bar forward return (matching factor IC horizon)
|
||||
returns_96 = close.pct_change(96).shift(-96)
|
||||
signal_aligned = signal.loc[returns_96.dropna().index]
|
||||
fwd_returns = returns_96.loc[signal_aligned.index]
|
||||
|
||||
if len(signal_aligned) < 100 or len(fwd_returns) < 100:
|
||||
print("ERROR: Not enough data after alignment")
|
||||
exit(1)
|
||||
|
||||
# Calculate IC: correlation(signal, forward_return)
|
||||
ic = signal_aligned.corr(fwd_returns)
|
||||
|
||||
# Strategy returns
|
||||
strategy_returns = signal_aligned * fwd_returns
|
||||
|
||||
# Basic metrics
|
||||
total_return = (1 + strategy_returns).prod() - 1
|
||||
@@ -242,7 +255,7 @@ else:
|
||||
monthly_return = total_return
|
||||
annual_return = total_return * 12
|
||||
|
||||
# Sharpe ratio (annualized)
|
||||
# Sharpe ratio (annualized for 96-bar horizon)
|
||||
if strategy_returns.std() > 0:
|
||||
sharpe = strategy_returns.mean() / strategy_returns.std() * np.sqrt(252 * 1440 / 96)
|
||||
else:
|
||||
@@ -258,17 +271,8 @@ max_dd = drawdown.min() if len(drawdown) > 0 else 0
|
||||
win_rate = (strategy_returns > 0).sum() / len(strategy_returns) if len(strategy_returns) > 0 else 0
|
||||
|
||||
# Trade count (signal changes)
|
||||
n_trades = int((signal != signal.shift(1)).sum())
|
||||
n_trades = int((signal_aligned != signal_aligned.shift(1)).sum())
|
||||
|
||||
# Calculate IC: correlation between signal and forward returns
|
||||
fwd_returns = returns.shift(-1)
|
||||
common = signal.index.intersection(fwd_returns.dropna().index)
|
||||
if len(common) > 100:
|
||||
ic = signal.loc[common].corr(fwd_returns.loc[common])
|
||||
else:
|
||||
ic = 0
|
||||
|
||||
# Output results
|
||||
result = {{
|
||||
"status": "success",
|
||||
"sharpe": float(sharpe),
|
||||
@@ -281,9 +285,9 @@ result = {{
|
||||
"annual_return_pct": float(annual_return * 100),
|
||||
"n_bars": int(n_bars),
|
||||
"n_months": float(n_months),
|
||||
"signal_long": int((signal == 1).sum()),
|
||||
"signal_short": int((signal == -1).sum()),
|
||||
"signal_neutral": int((signal == 0).sum()),
|
||||
"signal_long": int((signal_aligned == 1).sum()),
|
||||
"signal_short": int((signal_aligned == -1).sum()),
|
||||
"signal_neutral": int((signal_aligned == 0).sum()),
|
||||
}}
|
||||
|
||||
print(json.dumps(result))
|
||||
|
||||
Reference in New Issue
Block a user