Vanszs
a4619dd005
- Remove tracked generated artifacts: backtest logs (52), xlsx (43), experiment model pkls (7), ml_v3 training logs (11), result csv/txt - Remove junk files: stray =1.4.5, training_output.log, *_analysis_output.txt, dead api.log, runtime bot.lock - Remove throwaway scripts: analyze_performance, test_trajectory_bug, verify_settings - Move reusable analysis scripts to scripts/analysis/ - Move status/report docs to docs/reports/ - Tighten .gitignore to prevent re-adding generated artifacts; ignore .kiro/
424 lines
16 KiB
Python
424 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Deep ML Model Analysis Script
|
||
Analyzes xgboost_model_v2d.pkl for performance, feature importance, and limitations
|
||
"""
|
||
|
||
import sys
|
||
import pickle
|
||
import json
|
||
from pathlib import Path
|
||
import numpy as np
|
||
import polars as pl
|
||
from collections import defaultdict
|
||
|
||
# Add src to path
|
||
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
||
|
||
print("=" * 80)
|
||
print("ML MODEL DEEP DIVE ANALYSIS")
|
||
print("=" * 80)
|
||
|
||
# ============================================================================
|
||
# 1. LOAD AND INSPECT V2D MODEL
|
||
# ============================================================================
|
||
print("\n" + "=" * 80)
|
||
print("1. MODEL INSPECTION: xgboost_model_v2d.pkl")
|
||
print("=" * 80)
|
||
|
||
model_path = Path("models/xgboost_model_v2d.pkl")
|
||
if not model_path.exists():
|
||
print(f"ERROR: Model not found at {model_path}")
|
||
sys.exit(1)
|
||
|
||
with open(model_path, "rb") as f:
|
||
model_data = pickle.load(f)
|
||
|
||
print(f"\nModel pickle keys: {list(model_data.keys())}")
|
||
|
||
# Extract model and metadata
|
||
model = model_data.get("model")
|
||
metadata = model_data.get("metadata", {})
|
||
feature_names = model_data.get("feature_names", [])
|
||
|
||
print(f"\nModel type: {type(model)}")
|
||
print(f"Number of features: {len(feature_names)}")
|
||
print(f"\nMetadata keys: {list(metadata.keys())}")
|
||
|
||
# Display all metadata
|
||
print("\n--- MODEL METADATA ---")
|
||
for key, value in metadata.items():
|
||
if isinstance(value, (int, float, str, bool)):
|
||
print(f"{key}: {value}")
|
||
elif isinstance(value, dict):
|
||
print(f"{key}:")
|
||
for k, v in value.items():
|
||
print(f" {k}: {v}")
|
||
elif isinstance(value, (list, tuple)) and len(value) < 10:
|
||
print(f"{key}: {value}")
|
||
else:
|
||
print(f"{key}: {type(value)} (length={len(value) if hasattr(value, '__len__') else 'N/A'})")
|
||
|
||
# Extract key metrics
|
||
train_auc = metadata.get("train_auc", "N/A")
|
||
test_auc = metadata.get("test_auc", "N/A")
|
||
train_samples = metadata.get("train_samples", "N/A")
|
||
test_samples = metadata.get("test_samples", "N/A")
|
||
class_distribution = metadata.get("class_distribution", {})
|
||
|
||
print("\n--- KEY METRICS ---")
|
||
print(f"Training AUC: {train_auc}")
|
||
print(f"Test AUC: {test_auc}")
|
||
if isinstance(train_auc, float) and isinstance(test_auc, float):
|
||
overfitting_gap = train_auc - test_auc
|
||
print(f"Overfitting gap: {overfitting_gap:.4f} ({'HIGH' if overfitting_gap > 0.05 else 'NORMAL'})")
|
||
|
||
print(f"\nTraining samples: {train_samples}")
|
||
print(f"Test samples: {test_samples}")
|
||
|
||
print("\n--- CLASS DISTRIBUTION ---")
|
||
for class_name, count in class_distribution.items():
|
||
print(f"{class_name}: {count}")
|
||
|
||
# ============================================================================
|
||
# 2. FEATURE IMPORTANCE ANALYSIS
|
||
# ============================================================================
|
||
print("\n" + "=" * 80)
|
||
print("2. FEATURE IMPORTANCE RANKING (ALL FEATURES)")
|
||
print("=" * 80)
|
||
|
||
# Get feature importance from XGBoost
|
||
if hasattr(model, 'feature_importances_'):
|
||
importance_scores = model.feature_importances_
|
||
elif hasattr(model, 'get_score'):
|
||
# For XGBoost Booster
|
||
importance_dict = model.get_score(importance_type='gain')
|
||
importance_scores = [importance_dict.get(f"f{i}", 0) for i in range(len(feature_names))]
|
||
else:
|
||
print("WARNING: Could not extract feature importance from model")
|
||
importance_scores = [0] * len(feature_names)
|
||
|
||
# Create ranking
|
||
feature_importance = list(zip(feature_names, importance_scores))
|
||
feature_importance.sort(key=lambda x: x[1], reverse=True)
|
||
|
||
print(f"\nTotal features: {len(feature_importance)}")
|
||
|
||
# Categorize features
|
||
h1_features = []
|
||
m15_features = []
|
||
smc_features = []
|
||
other_features = []
|
||
|
||
for feat, score in feature_importance:
|
||
if "_h1" in feat.lower():
|
||
h1_features.append((feat, score))
|
||
elif "ob_" in feat or "fvg_" in feat or "bos" in feat or "choch" in feat:
|
||
smc_features.append((feat, score))
|
||
elif any(x in feat.lower() for x in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"]):
|
||
m15_features.append((feat, score))
|
||
else:
|
||
other_features.append((feat, score))
|
||
|
||
print("\n--- TOP 20 FEATURES (BY IMPORTANCE) ---")
|
||
for i, (feat, score) in enumerate(feature_importance[:20], 1):
|
||
category = "H1" if "_h1" in feat.lower() else "SMC" if any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]) else "M15"
|
||
print(f"{i:2d}. {feat:40s} {score:10.4f} [{category}]")
|
||
|
||
print("\n--- H1 FEATURES IN TOP 10 ---")
|
||
h1_in_top10 = [feat for feat, score in feature_importance[:10] if "_h1" in feat.lower()]
|
||
print(f"Count: {len(h1_in_top10)}")
|
||
for feat in h1_in_top10:
|
||
rank = [f for f, s in feature_importance].index(feat) + 1
|
||
score = [s for f, s in feature_importance if f == feat][0]
|
||
print(f" Rank #{rank}: {feat} (importance: {score:.4f})")
|
||
|
||
print("\n--- FEATURE CATEGORY SUMMARY ---")
|
||
print(f"H1 features: {len(h1_features)} total")
|
||
print(f"M15 technical features: {len(m15_features)} total")
|
||
print(f"SMC features: {len(smc_features)} total")
|
||
print(f"Other features: {len(other_features)} total")
|
||
|
||
# Calculate average importance by category
|
||
if h1_features:
|
||
avg_h1 = np.mean([s for f, s in h1_features])
|
||
print(f"\nAverage H1 importance: {avg_h1:.4f}")
|
||
if m15_features:
|
||
avg_m15 = np.mean([s for f, s in m15_features])
|
||
print(f"Average M15 importance: {avg_m15:.4f}")
|
||
if smc_features:
|
||
avg_smc = np.mean([s for f, s in smc_features])
|
||
print(f"Average SMC importance: {avg_smc:.4f}")
|
||
|
||
# ============================================================================
|
||
# 3. TRAINING LOGS ANALYSIS
|
||
# ============================================================================
|
||
print("\n" + "=" * 80)
|
||
print("3. TRAINING LOGS ANALYSIS")
|
||
print("=" * 80)
|
||
|
||
log_file = Path("logs/training_2026-02-04.log")
|
||
if log_file.exists():
|
||
print(f"\nReading: {log_file}")
|
||
with open(log_file, "r") as f:
|
||
log_content = f.read()
|
||
|
||
# Extract key training info
|
||
lines = log_content.split("\n")
|
||
|
||
# Look for training metrics
|
||
print("\n--- TRAINING METRICS FROM LOG ---")
|
||
for line in lines:
|
||
if any(kw in line.lower() for kw in ["auc", "accuracy", "precision", "recall", "f1", "samples", "features", "hyperparameter"]):
|
||
print(line.strip())
|
||
else:
|
||
print(f"\nNo training log found at {log_file}")
|
||
|
||
# ============================================================================
|
||
# 4. TARGET VARIABLE STATISTICS
|
||
# ============================================================================
|
||
print("\n" + "=" * 80)
|
||
print("4. TARGET VARIABLE STATISTICS")
|
||
print("=" * 80)
|
||
|
||
data_file = Path("data/training_data.parquet")
|
||
if data_file.exists():
|
||
print(f"\nLoading training data from {data_file}...")
|
||
df = pl.read_parquet(data_file)
|
||
|
||
print(f"Dataset shape: {df.shape}")
|
||
print(f"Columns: {df.columns}")
|
||
|
||
# Check if we have necessary columns
|
||
has_target = "target_signal" in df.columns or "target" in df.columns
|
||
has_atr = "atr" in df.columns
|
||
has_returns = any("return" in col.lower() for col in df.columns)
|
||
|
||
print(f"\nHas target column: {has_target}")
|
||
print(f"Has ATR column: {has_atr}")
|
||
print(f"Has return columns: {has_returns}")
|
||
|
||
if has_target:
|
||
target_col = "target_signal" if "target_signal" in df.columns else "target"
|
||
target_dist = df.group_by(target_col).agg(pl.count()).sort(target_col)
|
||
print(f"\n--- TARGET DISTRIBUTION ---")
|
||
print(target_dist)
|
||
|
||
# Calculate percentages
|
||
total = df.shape[0]
|
||
for row in target_dist.iter_rows(named=True):
|
||
pct = (row['count'] / total) * 100
|
||
print(f"{row[target_col]}: {row['count']} ({pct:.2f}%)")
|
||
|
||
# Analyze returns if available
|
||
return_cols = [col for col in df.columns if "return" in col.lower()]
|
||
if return_cols and has_atr:
|
||
print(f"\n--- RETURN ANALYSIS ---")
|
||
print(f"Available return columns: {return_cols}")
|
||
|
||
for ret_col in return_cols[:5]: # First 5 return columns
|
||
if ret_col in df.columns:
|
||
# Calculate stats
|
||
ret_mean = df[ret_col].mean()
|
||
ret_std = df[ret_col].std()
|
||
ret_positive = (df[ret_col] > 0).sum()
|
||
ret_negative = (df[ret_col] < 0).sum()
|
||
|
||
print(f"\n{ret_col}:")
|
||
print(f" Mean: {ret_mean:.6f}")
|
||
print(f" Std: {ret_std:.6f}")
|
||
print(f" Positive: {ret_positive} ({(ret_positive/total)*100:.2f}%)")
|
||
print(f" Negative: {ret_negative} ({(ret_negative/total)*100:.2f}%)")
|
||
|
||
# Calculate ATR-normalized returns
|
||
if "atr" in df.columns:
|
||
atr_mean = df["atr"].mean()
|
||
print(f" Mean ATR: {atr_mean:.4f}")
|
||
|
||
# Check different thresholds
|
||
thresh_03 = ((df[ret_col] / df["atr"]) > 0.3).sum()
|
||
thresh_05 = ((df[ret_col] / df["atr"]) > 0.5).sum()
|
||
thresh_10 = ((df[ret_col] / df["atr"]) > 1.0).sum()
|
||
|
||
print(f" Returns > 0.3×ATR: {thresh_03} ({(thresh_03/total)*100:.2f}%)")
|
||
print(f" Returns > 0.5×ATR: {thresh_05} ({(thresh_05/total)*100:.2f}%)")
|
||
print(f" Returns > 1.0×ATR: {thresh_10} ({(thresh_10/total)*100:.2f}%)")
|
||
else:
|
||
print(f"\nNo training data found at {data_file}")
|
||
|
||
# ============================================================================
|
||
# 5. FEATURE CORRELATION ANALYSIS
|
||
# ============================================================================
|
||
print("\n" + "=" * 80)
|
||
print("5. FEATURE CORRELATION ANALYSIS")
|
||
print("=" * 80)
|
||
|
||
if data_file.exists() and df is not None:
|
||
# Extract H1 and M15 features
|
||
h1_cols = [col for col in df.columns if "_h1" in col.lower()]
|
||
m15_cols = [col for col in df.columns if any(ind in col.lower() for ind in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"])]
|
||
|
||
print(f"\nH1 columns found: {len(h1_cols)}")
|
||
print(f"M15 columns found: {len(m15_cols)}")
|
||
|
||
if h1_cols and m15_cols:
|
||
# Select numeric columns only
|
||
numeric_h1 = [col for col in h1_cols if df[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]]
|
||
numeric_m15 = [col for col in m15_cols if df[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]]
|
||
|
||
print(f"Numeric H1 columns: {len(numeric_h1)}")
|
||
print(f"Numeric M15 columns: {len(numeric_m15)}")
|
||
|
||
if numeric_h1 and numeric_m15:
|
||
# Calculate correlations between H1 and M15 features
|
||
print("\n--- HIGH CORRELATIONS BETWEEN H1 AND M15 FEATURES ---")
|
||
print("(Correlation > 0.7 suggests redundancy)")
|
||
|
||
high_corr_count = 0
|
||
for h1_col in numeric_h1[:10]: # Check first 10 H1 features
|
||
for m15_col in numeric_m15[:10]: # Against first 10 M15 features
|
||
try:
|
||
corr_df = df.select([h1_col, m15_col]).drop_nulls()
|
||
if corr_df.shape[0] > 0:
|
||
corr = corr_df.corr()[h1_col, m15_col]
|
||
if abs(corr) > 0.7:
|
||
print(f" {h1_col} <-> {m15_col}: {corr:.3f}")
|
||
high_corr_count += 1
|
||
except:
|
||
pass
|
||
|
||
if high_corr_count == 0:
|
||
print(" No high correlations found (good - features are independent)")
|
||
else:
|
||
print(f"\n Total high correlations: {high_corr_count}")
|
||
|
||
# Check H1 feature autocorrelation
|
||
print("\n--- H1 FEATURE INTERNAL CORRELATIONS ---")
|
||
if len(numeric_h1) >= 2:
|
||
high_h1_corr = 0
|
||
for i, col1 in enumerate(numeric_h1[:10]):
|
||
for col2 in numeric_h1[i+1:10]:
|
||
try:
|
||
corr_df = df.select([col1, col2]).drop_nulls()
|
||
if corr_df.shape[0] > 0:
|
||
corr = corr_df.corr()[col1, col2]
|
||
if abs(corr) > 0.8:
|
||
print(f" {col1} <-> {col2}: {corr:.3f}")
|
||
high_h1_corr += 1
|
||
except:
|
||
pass
|
||
|
||
if high_h1_corr == 0:
|
||
print(" No high internal correlations (good)")
|
||
else:
|
||
print("\nCannot perform correlation analysis - data not available")
|
||
|
||
# ============================================================================
|
||
# 6. PREDICTION CONSISTENCY CHECK
|
||
# ============================================================================
|
||
print("\n" + "=" * 80)
|
||
print("6. PREDICTION CONSISTENCY ANALYSIS")
|
||
print("=" * 80)
|
||
|
||
persistence_file = Path("data/signal_persistence.json")
|
||
if persistence_file.exists():
|
||
print(f"\nReading: {persistence_file}")
|
||
with open(persistence_file, "r") as f:
|
||
persistence_data = json.load(f)
|
||
|
||
print(f"Persistence data: {json.dumps(persistence_data, indent=2)}")
|
||
else:
|
||
print(f"\nNo persistence data found at {persistence_file}")
|
||
|
||
# Analyze recent logs for signal flipping
|
||
recent_log = Path("logs/trading_bot_2026-02-09.log")
|
||
if recent_log.exists():
|
||
print(f"\n--- ANALYZING RECENT SIGNALS FROM LOG ---")
|
||
print(f"Reading: {recent_log}")
|
||
|
||
signal_history = []
|
||
with open(recent_log, "r") as f:
|
||
for line in f:
|
||
if "ML Signal:" in line or "prediction:" in line.lower() or "signal=" in line.lower():
|
||
signal_history.append(line.strip())
|
||
|
||
print(f"\nFound {len(signal_history)} signal-related log entries")
|
||
|
||
if signal_history:
|
||
print("\n--- RECENT SIGNAL SAMPLES (Last 20) ---")
|
||
for entry in signal_history[-20:]:
|
||
print(f" {entry}")
|
||
|
||
# Count signal changes
|
||
signals = []
|
||
for entry in signal_history:
|
||
if "BUY" in entry.upper():
|
||
signals.append("BUY")
|
||
elif "SELL" in entry.upper():
|
||
signals.append("SELL")
|
||
elif "HOLD" in entry.upper():
|
||
signals.append("HOLD")
|
||
|
||
if len(signals) > 1:
|
||
changes = sum(1 for i in range(1, len(signals)) if signals[i] != signals[i-1])
|
||
print(f"\n--- SIGNAL STABILITY ---")
|
||
print(f"Total signals tracked: {len(signals)}")
|
||
print(f"Signal changes: {changes}")
|
||
print(f"Change rate: {(changes/len(signals))*100:.2f}%")
|
||
|
||
# Count by type
|
||
from collections import Counter
|
||
signal_counts = Counter(signals)
|
||
print(f"\nSignal distribution:")
|
||
for sig, count in signal_counts.items():
|
||
print(f" {sig}: {count} ({(count/len(signals))*100:.2f}%)")
|
||
else:
|
||
print(f"\nNo recent log found at {recent_log}")
|
||
|
||
# ============================================================================
|
||
# 7. MODEL METRICS FROM DATA
|
||
# ============================================================================
|
||
print("\n" + "=" * 80)
|
||
print("7. MODEL METRICS (from data/model_metrics.json)")
|
||
print("=" * 80)
|
||
|
||
metrics_file = Path("data/model_metrics.json")
|
||
if metrics_file.exists():
|
||
with open(metrics_file, "r") as f:
|
||
metrics = json.load(f)
|
||
|
||
print(json.dumps(metrics, indent=2))
|
||
else:
|
||
print(f"\nNo metrics file found at {metrics_file}")
|
||
|
||
# ============================================================================
|
||
# SUMMARY
|
||
# ============================================================================
|
||
print("\n" + "=" * 80)
|
||
print("ANALYSIS SUMMARY")
|
||
print("=" * 80)
|
||
|
||
print("\n1. MODEL PERFORMANCE:")
|
||
print(f" - Test AUC: {test_auc}")
|
||
print(f" - Overfitting: {'YES' if isinstance(train_auc, float) and isinstance(test_auc, float) and (train_auc - test_auc) > 0.05 else 'NO'}")
|
||
|
||
print("\n2. FEATURE IMPORTANCE:")
|
||
print(f" - H1 features in top 10: {len(h1_in_top10)}")
|
||
print(f" - Total H1 features: {len(h1_features)}")
|
||
|
||
if h1_in_top10:
|
||
print(f" - Highest ranked H1: {h1_in_top10[0]} (rank #{[f for f, s in feature_importance].index(h1_in_top10[0]) + 1})")
|
||
else:
|
||
print(" - No H1 features in top 10")
|
||
|
||
print("\n3. DATA QUALITY:")
|
||
if data_file.exists() and has_target:
|
||
print(f" - Training samples: {total}")
|
||
print(f" - Target balance: See distribution above")
|
||
else:
|
||
print(" - Could not analyze training data")
|
||
|
||
print("\n" + "=" * 80)
|
||
print("ANALYSIS COMPLETE")
|
||
print("=" * 80)
|