#!/usr/bin/env python
"""Debug backtest logic: check alignment, signal quality, and IC calculation."""
import json
import numpy as np
import pandas as pd
from pathlib import Path

OHLCV_PATH = Path('/home/nico/NexQuant/git_ignore_folder/factor_implementation_source_data/intraday_pv.h5')
FACTORS_DIR = Path('/home/nico/NexQuant/results/factors')
VALUES_DIR = FACTORS_DIR / 'values'

print("=" * 70)
print("🔍 BACKTEST DEBUG - Alignment & IC Check")
print("=" * 70)

# 1. Load OHLCV close prices
print("\n1️⃣ Loading OHLCV close prices...")
ohlcv = pd.read_hdf(str(OHLCV_PATH), key='data')
if '$close' in ohlcv.columns:
    close = ohlcv['$close']
elif 'close' in ohlcv.columns:
    close = ohlcv['close']
else:
    close = ohlcv.select_dtypes(include=[np.number]).iloc[:, 0]

close = close.dropna()
print(f"   Close prices: {len(close):,} bars")
print(f"   Date range: {close.index.min()} → {close.index.max()}")
print(f"   Sample: {close.head(3).values}")

# Calculate forward returns (what we predict)
returns = close.pct_change().dropna()
print(f"   Returns: {len(returns):,} values")
print(f"   Return stats: mean={returns.mean():.8f}, std={returns.std():.8f}")

# 2. Load factor parquet files
print("\n2️⃣ Loading factor time-series...")
factor_files = sorted(VALUES_DIR.glob('*.parquet'))[:5]  # Top 5
for ff in factor_files:
    df_factor = pd.read_parquet(str(ff))
    if len(df_factor.columns) > 0:
        col = df_factor.iloc[:, 0]
        # Align with close
        common = close.index.intersection(col.dropna().index)
        if len(common) > 0:
            c = close.loc[common]
            f = col.loc[common]
            r = returns.loc[common] if len(returns) > 0 else pd.Series()
            
            # Simple test: factor value vs next return
            # IC = correlation(factor, forward_return)
            fwd_r = r.shift(-1)  # Next bar return
            common2 = f.dropna().index.intersection(fwd_r.dropna().index)
            
            if len(common2) > 100:
                ic = f.loc[common2].corr(fwd_r.loc[common2])
                print(f"   {ff.stem:40s} len={len(f):>8,} IC_vs_next_return={ic:.6f}")
            else:
                print(f"   {ff.stem:40s} len={len(f):>8,} (not enough common data)")

# 3. Test simple signals
print("\n3️⃣ Testing SIMPLE signals (what should work)...")

# Load metadata for best factors
factors_meta = []
for f in FACTORS_DIR.glob('*.json'):
    try:
        d = json.load(open(f))
        ic = d.get('ic', 0) or 0
        fname = d.get('factor_name', '')
        if abs(ic) > 0.1:
            factors_meta.append({'name': fname, 'ic': ic})
    except:
        pass

factors_meta.sort(key=lambda x: abs(x['ic']), reverse=True)
top = factors_meta[:3]
print(f"   Top factors: {[(f['name'], f['ic']) for f in top]}")

# For each top factor, test simple signal
for fm in top:
    fname = fm['name']
    safe = fname.replace('/', '_').replace('\\', '_')[:150]
    pf = VALUES_DIR / f"{safe}.parquet"
    if not pf.exists():
        print(f"   ❌ {fname}: parquet not found")
        continue
    
    factor_series = pd.read_parquet(str(pf)).iloc[:, 0]
    factor_ic = fm['ic']
    
    # Align
    common = close.index.intersection(factor_series.dropna().index)
    if len(common) < 1000:
        print(f"   ❌ {fname}: not enough common data ({len(common)})")
        continue
    
    f = factor_series.loc[common]
    r = returns.loc[common]
    
    # Test A: Raw factor value vs next return
    fwd = r.shift(-1)
    common2 = f.index.intersection(fwd.dropna().index)
    ic_raw = f.loc[common2].corr(fwd.loc[common2])
    
    # Test B: Factor sign as signal (positive → LONG)
    signal_a = (f > 0).astype(int).replace(0, -1)  # +1 or -1
    strat_ret_a = signal_a.shift(1) * r
    sharpe_a = strat_ret_a.mean() / strat_ret_a.std() * np.sqrt(252*1440/96) if strat_ret_a.std() > 0 else 0
    ic_signal = signal_a.corr(fwd)
    
    # Test C: Factor percentile as signal
    pct = f.rank(pct=True)
    signal_c = (pct > 0.6).astype(int).replace(0, -1)
    strat_ret_c = signal_c.shift(1) * r
    sharpe_c = strat_ret_c.mean() / strat_ret_c.std() * np.sqrt(252*1440/96) if strat_ret_c.std() > 0 else 0
    
    # Test D: What the LLM strategies actually compute (factor z-score → threshold)
    w = 60
    z = (f - f.rolling(w).mean()) / f.rolling(w).std()
    z = z.fillna(0)
    signal_d = (z > 0.5).astype(int).replace(0, -1)
    strat_ret_d = signal_d.shift(1) * r
    sharpe_d = strat_ret_d.mean() / strat_ret_d.std() * np.sqrt(252*1440/96) if strat_ret_d.std() > 0 else 0
    
    print(f"\n   📊 {fname} (factor IC={factor_ic:.4f}):")
    print(f"      Raw factor IC vs fwd return: {ic_raw:.6f}")
    print(f"      Signal (sign) IC={ic_signal:.6f}  Sharpe={sharpe_a:.4f}")
    print(f"      Signal (percentile) Sharpe={sharpe_c:.4f}")
    print(f"      Signal (z-score thresh) Sharpe={sharpe_d:.4f}")

# 4. Key finding
print("\n" + "=" * 70)
print("4️⃣ KEY INSIGHT:")
print("=" * 70)

# Check if factor values are already aligned with returns or are predictions
# Load one factor and check timing
if top:
    fname = top[0]['name']
    safe = fname.replace('/', '_').replace('\\', '_')[:150]
    pf = VALUES_DIR / f"{safe}.parquet"
    f = pd.read_parquet(str(pf)).iloc[:, 0]
    
    # What does a HIGH factor value mean?
    # If factor IC is positive (0.25), high values should predict positive returns
    # Let's check: when factor is in top 10%, what's the average NEXT return?
    common = close.index.intersection(f.dropna().index)
    fv = f.loc[common]
    rv = returns.loc[common]
    fwd = rv.shift(-1)
    
    common2 = fv.index.intersection(fwd.dropna().index)
    fv2 = fv.loc[common2]
    fwd2 = fwd.loc[common2]
    
    top_decile = fv2 > fv2.quantile(0.9)
    bot_decile = fv2 < fv2.quantile(0.1)
    
    avg_ret_when_high = fwd2[top_decile].mean()
    avg_ret_when_low = fwd2[bot_decile].mean()
    
    print(f"\n   Factor: {fname} (IC={top[0]['ic']:.4f})")
    print(f"   Avg NEXT return when factor in TOP 10%: {avg_ret_when_high*100:.6f}%")
    print(f"   Avg NEXT return when factor in BOT 10%: {avg_ret_when_low*100:.6f}%")
    print(f"   Difference: {(avg_ret_when_high - avg_ret_when_low)*100:.6f}%")
    print(f"\n   → If difference > 0, factor has predictive power")
    print(f"   → If difference ≈ 0, factor has NO predictive power for next-bar returns")
    print(f"   → If difference < 0, factor is INVERTED (use negative)")
