chore: clean up workspace for production
- Remove tracked generated artifacts: backtest logs (52), xlsx (43), experiment model pkls (7), ml_v3 training logs (11), result csv/txt - Remove junk files: stray =1.4.5, training_output.log, *_analysis_output.txt, dead api.log, runtime bot.lock - Remove throwaway scripts: analyze_performance, test_trajectory_bug, verify_settings - Move reusable analysis scripts to scripts/analysis/ - Move status/report docs to docs/reports/ - Tighten .gitignore to prevent re-adding generated artifacts; ignore .kiro/
This commit is contained in:
Vanszs
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
H1 Feature Analysis - Check if H1 features actually exist and their correlation
|
||||
"""
|
||||
|
||||
import sys
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
||||
|
||||
print("=" * 80)
|
||||
print("H1 FEATURE DEEP ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
# Load model
|
||||
model_path = Path("models/xgboost_model_v2d.pkl")
|
||||
with open(model_path, "rb") as f:
|
||||
model_data = pickle.load(f)
|
||||
|
||||
feature_names = model_data.get("feature_names", [])
|
||||
feature_importance = model_data.get("feature_importance", {})
|
||||
|
||||
print(f"\nTotal features in model: {len(feature_names)}")
|
||||
|
||||
# Find all H1-related features
|
||||
h1_features = [f for f in feature_names if "h1" in f.lower() or "H1" in f]
|
||||
print(f"\nH1 features found: {len(h1_features)}")
|
||||
|
||||
if h1_features:
|
||||
print("\n--- ALL H1 FEATURES ---")
|
||||
for feat in sorted(h1_features):
|
||||
importance = feature_importance.get(feat, 0)
|
||||
# Find rank
|
||||
sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True)
|
||||
rank = [f for f, s in sorted_features].index(feat) + 1 if feat in dict(sorted_features) else 999
|
||||
print(f" Rank #{rank:2d}: {feat:40s} importance={importance:10.4f}")
|
||||
|
||||
# Top H1 features
|
||||
h1_with_importance = [(f, feature_importance.get(f, 0)) for f in h1_features]
|
||||
h1_with_importance.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
print("\n--- TOP 10 H1 FEATURES (by importance) ---")
|
||||
for i, (feat, imp) in enumerate(h1_with_importance[:10], 1):
|
||||
sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True)
|
||||
rank = [f for f, s in sorted_features].index(feat) + 1
|
||||
print(f"{i:2d}. Rank #{rank:3d}: {feat:40s} {imp:10.4f}")
|
||||
|
||||
# Summary stats
|
||||
importances = [imp for f, imp in h1_with_importance]
|
||||
print(f"\n--- H1 FEATURE STATISTICS ---")
|
||||
print(f"Total H1 features: {len(h1_features)}")
|
||||
print(f"Mean importance: {np.mean(importances):.4f}")
|
||||
print(f"Median importance: {np.median(importances):.4f}")
|
||||
print(f"Max importance: {np.max(importances):.4f}")
|
||||
print(f"Min importance: {np.min(importances):.4f}")
|
||||
|
||||
# Check how many in top N
|
||||
sorted_all = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True)
|
||||
top10_features = [f for f, s in sorted_all[:10]]
|
||||
top20_features = [f for f, s in sorted_all[:20]]
|
||||
top30_features = [f for f, s in sorted_all[:30]]
|
||||
|
||||
h1_in_top10 = [f for f in top10_features if "h1" in f.lower()]
|
||||
h1_in_top20 = [f for f in top20_features if "h1" in f.lower()]
|
||||
h1_in_top30 = [f for f in top30_features if "h1" in f.lower()]
|
||||
|
||||
print(f"\nH1 features in top 10: {len(h1_in_top10)}")
|
||||
print(f"H1 features in top 20: {len(h1_in_top20)}")
|
||||
print(f"H1 features in top 30: {len(h1_in_top30)}")
|
||||
|
||||
else:
|
||||
print("\nNO H1 FEATURES FOUND IN MODEL!")
|
||||
|
||||
# Check all feature names
|
||||
print("\n" + "=" * 80)
|
||||
print("ALL FEATURE NAMES IN MODEL")
|
||||
print("=" * 80)
|
||||
|
||||
for i, feat in enumerate(feature_names, 1):
|
||||
importance = feature_importance.get(feat, 0)
|
||||
print(f"{i:2d}. {feat:50s} {importance:10.4f}")
|
||||
|
||||
# Load training data and check for H1 columns
|
||||
print("\n" + "=" * 80)
|
||||
print("CHECKING TRAINING DATA FOR H1 FEATURES")
|
||||
print("=" * 80)
|
||||
|
||||
data_file = Path("data/training_data.parquet")
|
||||
if data_file.exists():
|
||||
df = pl.read_parquet(data_file)
|
||||
print(f"\nDataset columns: {len(df.columns)}")
|
||||
|
||||
# Find H1 columns
|
||||
h1_cols = [col for col in df.columns if "h1" in col.lower() or "H1" in col]
|
||||
print(f"H1 columns in dataset: {len(h1_cols)}")
|
||||
|
||||
if h1_cols:
|
||||
print("\n--- H1 COLUMNS IN DATASET ---")
|
||||
for col in sorted(h1_cols):
|
||||
# Check if in model features
|
||||
in_model = "YES" if col in feature_names else "NO"
|
||||
print(f" {col:50s} in_model={in_model}")
|
||||
else:
|
||||
print("\nNO H1 COLUMNS IN TRAINING DATA!")
|
||||
|
||||
# Check if there are any columns that might be H1-related
|
||||
print("\nLooking for potential H1-related columns:")
|
||||
potential = [col for col in df.columns if any(x in col.lower() for x in ["hour", "h4", "d1", "timeframe"])]
|
||||
if potential:
|
||||
for col in potential:
|
||||
print(f" {col}")
|
||||
else:
|
||||
print(" None found")
|
||||
|
||||
# Check if feature engineering creates H1 features
|
||||
print("\n" + "=" * 80)
|
||||
print("CHECKING FEATURE ENGINEERING CODE")
|
||||
print("=" * 80)
|
||||
|
||||
feature_eng_file = Path("src/feature_eng.py")
|
||||
if feature_eng_file.exists():
|
||||
with open(feature_eng_file, "r", encoding="utf-8", errors="ignore") as f:
|
||||
content = f.read()
|
||||
|
||||
# Search for H1 references
|
||||
if "h1" in content.lower() or "H1" in content:
|
||||
print("\nH1 references found in feature_eng.py:")
|
||||
lines = content.split("\n")
|
||||
for i, line in enumerate(lines, 1):
|
||||
if "h1" in line.lower() or "H1" in line:
|
||||
print(f" Line {i}: {line.strip()}")
|
||||
else:
|
||||
print("\nNO H1 references found in feature_eng.py")
|
||||
|
||||
# Check for multi-timeframe
|
||||
if "timeframe" in content.lower() or "TIMEFRAME_H1" in content or "mt5.TIMEFRAME_H1" in content:
|
||||
print("\nMulti-timeframe references found:")
|
||||
lines = content.split("\n")
|
||||
for i, line in enumerate(lines, 1):
|
||||
if "timeframe" in line.lower():
|
||||
print(f" Line {i}: {line.strip()}")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("CONCLUSION")
|
||||
print("=" * 80)
|
||||
|
||||
if h1_features:
|
||||
print(f"\n✓ Model HAS {len(h1_features)} H1 features")
|
||||
print(f"✓ Highest ranked H1 feature: {h1_with_importance[0][0]} at rank #{[f for f, s in sorted_all].index(h1_with_importance[0][0]) + 1}")
|
||||
print(f"✓ Average H1 importance: {np.mean(importances):.4f}")
|
||||
else:
|
||||
print("\n✗ Model has NO H1 features!")
|
||||
print("✗ The 'V2D' model does not include H1 timeframe data")
|
||||
print("✗ Need to retrain with H1 features to test hypothesis")
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Deep ML Model Analysis Script - Fixed version
|
||||
"""
|
||||
|
||||
import sys
|
||||
import pickle
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
||||
|
||||
def print_section(title):
|
||||
print("\n" + "=" * 80)
|
||||
print(title)
|
||||
print("=" * 80)
|
||||
|
||||
print_section("ML MODEL DEEP DIVE ANALYSIS")
|
||||
|
||||
# ============================================================================
|
||||
# 1. LOAD AND INSPECT V2D MODEL
|
||||
# ============================================================================
|
||||
print_section("1. MODEL INSPECTION: xgboost_model_v2d.pkl")
|
||||
|
||||
model_path = Path("models/xgboost_model_v2d.pkl")
|
||||
if not model_path.exists():
|
||||
print(f"ERROR: Model not found at {model_path}")
|
||||
sys.exit(1)
|
||||
|
||||
with open(model_path, "rb") as f:
|
||||
model_data = pickle.load(f)
|
||||
|
||||
print(f"\nModel pickle structure:")
|
||||
for key in model_data.keys():
|
||||
value = model_data[key]
|
||||
if isinstance(value, (list, dict)):
|
||||
print(f" {key}: {type(value).__name__} (length={len(value)})")
|
||||
else:
|
||||
print(f" {key}: {type(value).__name__}")
|
||||
|
||||
# Extract components
|
||||
xgb_model = model_data.get("xgb_model")
|
||||
lgb_model = model_data.get("lgb_model")
|
||||
feature_names = model_data.get("feature_names", [])
|
||||
feature_importance = model_data.get("feature_importance", {})
|
||||
train_metrics = model_data.get("train_metrics", {})
|
||||
xgb_params = model_data.get("xgb_params", {})
|
||||
|
||||
print(f"\nXGBoost model: {type(xgb_model)}")
|
||||
print(f"LightGBM model: {type(lgb_model)}")
|
||||
print(f"Total features: {len(feature_names)}")
|
||||
|
||||
# Display training metrics
|
||||
print("\n--- TRAINING METRICS ---")
|
||||
for key, value in train_metrics.items():
|
||||
if isinstance(value, (int, float)):
|
||||
print(f"{key}: {value}")
|
||||
elif isinstance(value, dict):
|
||||
print(f"{key}:")
|
||||
for k, v in value.items():
|
||||
print(f" {k}: {v}")
|
||||
|
||||
# XGBoost parameters
|
||||
print("\n--- XGBOOST PARAMETERS ---")
|
||||
for key, value in xgb_params.items():
|
||||
print(f"{key}: {value}")
|
||||
|
||||
# ============================================================================
|
||||
# 2. FEATURE IMPORTANCE ANALYSIS
|
||||
# ============================================================================
|
||||
print_section("2. FEATURE IMPORTANCE RANKING")
|
||||
|
||||
if feature_importance:
|
||||
# Sort by importance
|
||||
sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True)
|
||||
|
||||
print(f"\nTotal features with importance: {len(sorted_features)}")
|
||||
|
||||
# Categorize
|
||||
h1_features = []
|
||||
m15_features = []
|
||||
smc_features = []
|
||||
|
||||
for feat, score in sorted_features:
|
||||
if "_h1" in feat.lower():
|
||||
h1_features.append((feat, score))
|
||||
elif any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]):
|
||||
smc_features.append((feat, score))
|
||||
elif any(x in feat.lower() for x in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"]):
|
||||
m15_features.append((feat, score))
|
||||
|
||||
print("\n--- TOP 30 FEATURES ---")
|
||||
for i, (feat, score) in enumerate(sorted_features[:30], 1):
|
||||
category = "H1" if "_h1" in feat.lower() else "SMC" if any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]) else "M15"
|
||||
print(f"{i:2d}. {feat:40s} {score:12.6f} [{category}]")
|
||||
|
||||
# H1 features in top 10
|
||||
h1_in_top10 = [feat for feat, score in sorted_features[:10] if "_h1" in feat.lower()]
|
||||
print(f"\n--- H1 FEATURES IN TOP 10 ---")
|
||||
print(f"Count: {len(h1_in_top10)}")
|
||||
for feat in h1_in_top10:
|
||||
rank = [f for f, s in sorted_features].index(feat) + 1
|
||||
score = dict(sorted_features)[feat]
|
||||
print(f" Rank #{rank}: {feat} (importance: {score:.6f})")
|
||||
|
||||
# Category summary
|
||||
print("\n--- FEATURE CATEGORY SUMMARY ---")
|
||||
print(f"H1 features: {len(h1_features)}")
|
||||
print(f"M15 technical features: {len(m15_features)}")
|
||||
print(f"SMC features: {len(smc_features)}")
|
||||
|
||||
if h1_features:
|
||||
avg_h1 = np.mean([s for f, s in h1_features])
|
||||
print(f"\nAverage H1 importance: {avg_h1:.6f}")
|
||||
if m15_features:
|
||||
avg_m15 = np.mean([s for f, s in m15_features])
|
||||
print(f"Average M15 importance: {avg_m15:.6f}")
|
||||
if smc_features:
|
||||
avg_smc = np.mean([s for f, s in smc_features])
|
||||
print(f"Average SMC importance: {avg_smc:.6f}")
|
||||
|
||||
# Top H1 features
|
||||
if h1_features:
|
||||
print("\n--- ALL H1 FEATURES (sorted by importance) ---")
|
||||
for i, (feat, score) in enumerate(h1_features, 1):
|
||||
rank = [f for f, s in sorted_features].index(feat) + 1
|
||||
print(f"{i:2d}. Rank #{rank:2d}: {feat:40s} {score:12.6f}")
|
||||
else:
|
||||
print("\nNo feature importance data found in model")
|
||||
|
||||
# ============================================================================
|
||||
# 3. TARGET VARIABLE STATISTICS
|
||||
# ============================================================================
|
||||
print_section("3. TARGET VARIABLE STATISTICS")
|
||||
|
||||
data_file = Path("data/training_data.parquet")
|
||||
if data_file.exists():
|
||||
print(f"\nLoading: {data_file}")
|
||||
df = pl.read_parquet(data_file)
|
||||
|
||||
print(f"Dataset shape: {df.shape}")
|
||||
|
||||
# Target distribution
|
||||
if "target" in df.columns:
|
||||
target_counts = df.group_by("target").agg(pl.len().alias("count")).sort("target")
|
||||
|
||||
print("\n--- TARGET DISTRIBUTION ---")
|
||||
total = df.shape[0]
|
||||
for row in target_counts.iter_rows(named=True):
|
||||
pct = (row['count'] / total) * 100
|
||||
target_label = {0: "SELL", 1: "HOLD", 2: "BUY"}.get(row['target'], row['target'])
|
||||
print(f"{target_label}: {row['count']:6d} ({pct:5.2f}%)")
|
||||
|
||||
# ATR-normalized return analysis
|
||||
if "target_return" in df.columns and "atr" in df.columns:
|
||||
print("\n--- RETURN ANALYSIS (M15 bars) ---")
|
||||
|
||||
# Calculate normalized returns
|
||||
df_analysis = df.with_columns([
|
||||
(pl.col("target_return") / pl.col("atr")).alias("norm_return")
|
||||
])
|
||||
|
||||
total = df_analysis.shape[0]
|
||||
|
||||
# Different threshold analysis
|
||||
thresholds = [0.1, 0.2, 0.3, 0.5, 0.7, 1.0]
|
||||
print("\nBars with 3-bar returns > X*ATR:")
|
||||
for thresh in thresholds:
|
||||
count = (df_analysis["norm_return"] > thresh).sum()
|
||||
pct = (count / total) * 100
|
||||
print(f" > {thresh:.1f}*ATR: {count:5d} ({pct:5.2f}%)")
|
||||
|
||||
# Mean and median
|
||||
mean_norm = df_analysis["norm_return"].mean()
|
||||
median_norm = df_analysis["norm_return"].median()
|
||||
print(f"\nMean normalized return: {mean_norm:.4f}")
|
||||
print(f"Median normalized return: {median_norm:.4f}")
|
||||
|
||||
# Positive vs negative
|
||||
positive = (df_analysis["target_return"] > 0).sum()
|
||||
negative = (df_analysis["target_return"] < 0).sum()
|
||||
print(f"\nPositive returns: {positive} ({(positive/total)*100:.2f}%)")
|
||||
print(f"Negative returns: {negative} ({(negative/total)*100:.2f}%)")
|
||||
|
||||
# Check if H1 data exists
|
||||
h1_cols = [col for col in df.columns if "_h1" in col.lower()]
|
||||
print(f"\n--- H1 FEATURES IN DATASET ---")
|
||||
print(f"H1 columns found: {len(h1_cols)}")
|
||||
if h1_cols:
|
||||
print("Sample H1 columns:")
|
||||
for col in h1_cols[:10]:
|
||||
print(f" {col}")
|
||||
else:
|
||||
print(f"\nData file not found at {data_file}")
|
||||
|
||||
# ============================================================================
|
||||
# 4. PREDICTION CONSISTENCY
|
||||
# ============================================================================
|
||||
print_section("4. PREDICTION CONSISTENCY ANALYSIS")
|
||||
|
||||
# Check recent logs
|
||||
recent_log = Path("logs/trading_bot_2026-02-09.log")
|
||||
if recent_log.exists():
|
||||
print(f"\nAnalyzing: {recent_log}")
|
||||
|
||||
signals = []
|
||||
timestamps = []
|
||||
|
||||
with open(recent_log, "r", encoding="utf-8", errors="ignore") as f:
|
||||
for line in f:
|
||||
if "ML Signal:" in line or "ML prediction:" in line:
|
||||
# Extract signal
|
||||
if "BUY" in line.upper():
|
||||
signals.append("BUY")
|
||||
elif "SELL" in line.upper():
|
||||
signals.append("SELL")
|
||||
elif "HOLD" in line.upper():
|
||||
signals.append("HOLD")
|
||||
|
||||
# Try to extract timestamp
|
||||
if "|" in line:
|
||||
parts = line.split("|")
|
||||
if len(parts) > 0:
|
||||
timestamps.append(parts[0].strip())
|
||||
|
||||
if signals:
|
||||
print(f"\n--- SIGNAL TRACKING ---")
|
||||
print(f"Total signals logged: {len(signals)}")
|
||||
|
||||
# Count changes
|
||||
changes = sum(1 for i in range(1, len(signals)) if signals[i] != signals[i-1])
|
||||
print(f"Signal changes: {changes}")
|
||||
print(f"Change rate: {(changes/len(signals))*100:.2f}%")
|
||||
|
||||
# Distribution
|
||||
signal_counts = Counter(signals)
|
||||
print(f"\nSignal distribution:")
|
||||
for sig in ["BUY", "SELL", "HOLD"]:
|
||||
count = signal_counts.get(sig, 0)
|
||||
pct = (count / len(signals)) * 100
|
||||
print(f" {sig}: {count} ({pct:.2f}%)")
|
||||
|
||||
# Recent signals
|
||||
print(f"\n--- LAST 10 SIGNALS ---")
|
||||
for i, sig in enumerate(signals[-10:], 1):
|
||||
print(f"{i:2d}. {sig}")
|
||||
else:
|
||||
print(f"\nNo recent log at {recent_log}")
|
||||
|
||||
# ============================================================================
|
||||
# 5. MODEL METRICS
|
||||
# ============================================================================
|
||||
print_section("5. CURRENT MODEL METRICS")
|
||||
|
||||
metrics_file = Path("data/model_metrics.json")
|
||||
if metrics_file.exists():
|
||||
with open(metrics_file, "r") as f:
|
||||
metrics = json.load(f)
|
||||
|
||||
print("\n--- MODEL METRICS (from data/model_metrics.json) ---")
|
||||
print(json.dumps(metrics, indent=2))
|
||||
|
||||
# ============================================================================
|
||||
# 6. OVERFITTING ANALYSIS
|
||||
# ============================================================================
|
||||
print_section("6. OVERFITTING ANALYSIS")
|
||||
|
||||
# From training logs
|
||||
print("\nFrom training_2026-02-04.log:")
|
||||
print(" Initial training: Train AUC=0.8106, Test AUC=0.6553")
|
||||
print(" Overfitting gap: 0.1553 (HIGH)")
|
||||
print("\n Walk-forward average: Train AUC=0.8107, Test AUC=0.5722")
|
||||
print(" Overfitting gap: 0.2385 (VERY HIGH)")
|
||||
|
||||
print("\nConclusion:")
|
||||
print(" - Model shows significant overfitting")
|
||||
print(" - Test AUC of 0.57-0.66 is barely better than random (0.50)")
|
||||
print(" - High train AUC (0.81) but poor generalization")
|
||||
|
||||
# ============================================================================
|
||||
# SUMMARY
|
||||
# ============================================================================
|
||||
print_section("CRITICAL FINDINGS")
|
||||
|
||||
print("\n1. MODEL PERFORMANCE:")
|
||||
print(" - Test AUC: 0.5722 (walk-forward) - POOR")
|
||||
print(" - Overfitting gap: 0.2385 - VERY HIGH")
|
||||
print(" - Model barely better than random guessing")
|
||||
|
||||
print("\n2. FEATURE IMPORTANCE:")
|
||||
if h1_in_top10:
|
||||
print(f" - H1 features in top 10: {len(h1_in_top10)}")
|
||||
else:
|
||||
print(" - H1 features NOT in top 10 - Low predictive value")
|
||||
|
||||
print("\n3. DATA QUALITY:")
|
||||
if data_file.exists():
|
||||
print(f" - Training samples: {df.shape[0]}")
|
||||
print(" - Target imbalance likely causing issues")
|
||||
print(" - Most returns < 0.3*ATR (target too weak)")
|
||||
|
||||
print("\n4. RECOMMENDATIONS:")
|
||||
print(" a. Current V2D model has POOR performance - needs replacement")
|
||||
print(" b. H1 features show low importance - may not help")
|
||||
print(" c. Consider new target variable (stronger signal)")
|
||||
print(" d. Address class imbalance in training")
|
||||
print(" e. Reduce model complexity to prevent overfitting")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
@@ -0,0 +1,423 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Deep ML Model Analysis Script
|
||||
Analyzes xgboost_model_v2d.pkl for performance, feature importance, and limitations
|
||||
"""
|
||||
|
||||
import sys
|
||||
import pickle
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from collections import defaultdict
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
||||
|
||||
print("=" * 80)
|
||||
print("ML MODEL DEEP DIVE ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
# ============================================================================
|
||||
# 1. LOAD AND INSPECT V2D MODEL
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("1. MODEL INSPECTION: xgboost_model_v2d.pkl")
|
||||
print("=" * 80)
|
||||
|
||||
model_path = Path("models/xgboost_model_v2d.pkl")
|
||||
if not model_path.exists():
|
||||
print(f"ERROR: Model not found at {model_path}")
|
||||
sys.exit(1)
|
||||
|
||||
with open(model_path, "rb") as f:
|
||||
model_data = pickle.load(f)
|
||||
|
||||
print(f"\nModel pickle keys: {list(model_data.keys())}")
|
||||
|
||||
# Extract model and metadata
|
||||
model = model_data.get("model")
|
||||
metadata = model_data.get("metadata", {})
|
||||
feature_names = model_data.get("feature_names", [])
|
||||
|
||||
print(f"\nModel type: {type(model)}")
|
||||
print(f"Number of features: {len(feature_names)}")
|
||||
print(f"\nMetadata keys: {list(metadata.keys())}")
|
||||
|
||||
# Display all metadata
|
||||
print("\n--- MODEL METADATA ---")
|
||||
for key, value in metadata.items():
|
||||
if isinstance(value, (int, float, str, bool)):
|
||||
print(f"{key}: {value}")
|
||||
elif isinstance(value, dict):
|
||||
print(f"{key}:")
|
||||
for k, v in value.items():
|
||||
print(f" {k}: {v}")
|
||||
elif isinstance(value, (list, tuple)) and len(value) < 10:
|
||||
print(f"{key}: {value}")
|
||||
else:
|
||||
print(f"{key}: {type(value)} (length={len(value) if hasattr(value, '__len__') else 'N/A'})")
|
||||
|
||||
# Extract key metrics
|
||||
train_auc = metadata.get("train_auc", "N/A")
|
||||
test_auc = metadata.get("test_auc", "N/A")
|
||||
train_samples = metadata.get("train_samples", "N/A")
|
||||
test_samples = metadata.get("test_samples", "N/A")
|
||||
class_distribution = metadata.get("class_distribution", {})
|
||||
|
||||
print("\n--- KEY METRICS ---")
|
||||
print(f"Training AUC: {train_auc}")
|
||||
print(f"Test AUC: {test_auc}")
|
||||
if isinstance(train_auc, float) and isinstance(test_auc, float):
|
||||
overfitting_gap = train_auc - test_auc
|
||||
print(f"Overfitting gap: {overfitting_gap:.4f} ({'HIGH' if overfitting_gap > 0.05 else 'NORMAL'})")
|
||||
|
||||
print(f"\nTraining samples: {train_samples}")
|
||||
print(f"Test samples: {test_samples}")
|
||||
|
||||
print("\n--- CLASS DISTRIBUTION ---")
|
||||
for class_name, count in class_distribution.items():
|
||||
print(f"{class_name}: {count}")
|
||||
|
||||
# ============================================================================
|
||||
# 2. FEATURE IMPORTANCE ANALYSIS
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("2. FEATURE IMPORTANCE RANKING (ALL FEATURES)")
|
||||
print("=" * 80)
|
||||
|
||||
# Get feature importance from XGBoost
|
||||
if hasattr(model, 'feature_importances_'):
|
||||
importance_scores = model.feature_importances_
|
||||
elif hasattr(model, 'get_score'):
|
||||
# For XGBoost Booster
|
||||
importance_dict = model.get_score(importance_type='gain')
|
||||
importance_scores = [importance_dict.get(f"f{i}", 0) for i in range(len(feature_names))]
|
||||
else:
|
||||
print("WARNING: Could not extract feature importance from model")
|
||||
importance_scores = [0] * len(feature_names)
|
||||
|
||||
# Create ranking
|
||||
feature_importance = list(zip(feature_names, importance_scores))
|
||||
feature_importance.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
print(f"\nTotal features: {len(feature_importance)}")
|
||||
|
||||
# Categorize features
|
||||
h1_features = []
|
||||
m15_features = []
|
||||
smc_features = []
|
||||
other_features = []
|
||||
|
||||
for feat, score in feature_importance:
|
||||
if "_h1" in feat.lower():
|
||||
h1_features.append((feat, score))
|
||||
elif "ob_" in feat or "fvg_" in feat or "bos" in feat or "choch" in feat:
|
||||
smc_features.append((feat, score))
|
||||
elif any(x in feat.lower() for x in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"]):
|
||||
m15_features.append((feat, score))
|
||||
else:
|
||||
other_features.append((feat, score))
|
||||
|
||||
print("\n--- TOP 20 FEATURES (BY IMPORTANCE) ---")
|
||||
for i, (feat, score) in enumerate(feature_importance[:20], 1):
|
||||
category = "H1" if "_h1" in feat.lower() else "SMC" if any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]) else "M15"
|
||||
print(f"{i:2d}. {feat:40s} {score:10.4f} [{category}]")
|
||||
|
||||
print("\n--- H1 FEATURES IN TOP 10 ---")
|
||||
h1_in_top10 = [feat for feat, score in feature_importance[:10] if "_h1" in feat.lower()]
|
||||
print(f"Count: {len(h1_in_top10)}")
|
||||
for feat in h1_in_top10:
|
||||
rank = [f for f, s in feature_importance].index(feat) + 1
|
||||
score = [s for f, s in feature_importance if f == feat][0]
|
||||
print(f" Rank #{rank}: {feat} (importance: {score:.4f})")
|
||||
|
||||
print("\n--- FEATURE CATEGORY SUMMARY ---")
|
||||
print(f"H1 features: {len(h1_features)} total")
|
||||
print(f"M15 technical features: {len(m15_features)} total")
|
||||
print(f"SMC features: {len(smc_features)} total")
|
||||
print(f"Other features: {len(other_features)} total")
|
||||
|
||||
# Calculate average importance by category
|
||||
if h1_features:
|
||||
avg_h1 = np.mean([s for f, s in h1_features])
|
||||
print(f"\nAverage H1 importance: {avg_h1:.4f}")
|
||||
if m15_features:
|
||||
avg_m15 = np.mean([s for f, s in m15_features])
|
||||
print(f"Average M15 importance: {avg_m15:.4f}")
|
||||
if smc_features:
|
||||
avg_smc = np.mean([s for f, s in smc_features])
|
||||
print(f"Average SMC importance: {avg_smc:.4f}")
|
||||
|
||||
# ============================================================================
|
||||
# 3. TRAINING LOGS ANALYSIS
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("3. TRAINING LOGS ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
log_file = Path("logs/training_2026-02-04.log")
|
||||
if log_file.exists():
|
||||
print(f"\nReading: {log_file}")
|
||||
with open(log_file, "r") as f:
|
||||
log_content = f.read()
|
||||
|
||||
# Extract key training info
|
||||
lines = log_content.split("\n")
|
||||
|
||||
# Look for training metrics
|
||||
print("\n--- TRAINING METRICS FROM LOG ---")
|
||||
for line in lines:
|
||||
if any(kw in line.lower() for kw in ["auc", "accuracy", "precision", "recall", "f1", "samples", "features", "hyperparameter"]):
|
||||
print(line.strip())
|
||||
else:
|
||||
print(f"\nNo training log found at {log_file}")
|
||||
|
||||
# ============================================================================
|
||||
# 4. TARGET VARIABLE STATISTICS
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("4. TARGET VARIABLE STATISTICS")
|
||||
print("=" * 80)
|
||||
|
||||
data_file = Path("data/training_data.parquet")
|
||||
if data_file.exists():
|
||||
print(f"\nLoading training data from {data_file}...")
|
||||
df = pl.read_parquet(data_file)
|
||||
|
||||
print(f"Dataset shape: {df.shape}")
|
||||
print(f"Columns: {df.columns}")
|
||||
|
||||
# Check if we have necessary columns
|
||||
has_target = "target_signal" in df.columns or "target" in df.columns
|
||||
has_atr = "atr" in df.columns
|
||||
has_returns = any("return" in col.lower() for col in df.columns)
|
||||
|
||||
print(f"\nHas target column: {has_target}")
|
||||
print(f"Has ATR column: {has_atr}")
|
||||
print(f"Has return columns: {has_returns}")
|
||||
|
||||
if has_target:
|
||||
target_col = "target_signal" if "target_signal" in df.columns else "target"
|
||||
target_dist = df.group_by(target_col).agg(pl.count()).sort(target_col)
|
||||
print(f"\n--- TARGET DISTRIBUTION ---")
|
||||
print(target_dist)
|
||||
|
||||
# Calculate percentages
|
||||
total = df.shape[0]
|
||||
for row in target_dist.iter_rows(named=True):
|
||||
pct = (row['count'] / total) * 100
|
||||
print(f"{row[target_col]}: {row['count']} ({pct:.2f}%)")
|
||||
|
||||
# Analyze returns if available
|
||||
return_cols = [col for col in df.columns if "return" in col.lower()]
|
||||
if return_cols and has_atr:
|
||||
print(f"\n--- RETURN ANALYSIS ---")
|
||||
print(f"Available return columns: {return_cols}")
|
||||
|
||||
for ret_col in return_cols[:5]: # First 5 return columns
|
||||
if ret_col in df.columns:
|
||||
# Calculate stats
|
||||
ret_mean = df[ret_col].mean()
|
||||
ret_std = df[ret_col].std()
|
||||
ret_positive = (df[ret_col] > 0).sum()
|
||||
ret_negative = (df[ret_col] < 0).sum()
|
||||
|
||||
print(f"\n{ret_col}:")
|
||||
print(f" Mean: {ret_mean:.6f}")
|
||||
print(f" Std: {ret_std:.6f}")
|
||||
print(f" Positive: {ret_positive} ({(ret_positive/total)*100:.2f}%)")
|
||||
print(f" Negative: {ret_negative} ({(ret_negative/total)*100:.2f}%)")
|
||||
|
||||
# Calculate ATR-normalized returns
|
||||
if "atr" in df.columns:
|
||||
atr_mean = df["atr"].mean()
|
||||
print(f" Mean ATR: {atr_mean:.4f}")
|
||||
|
||||
# Check different thresholds
|
||||
thresh_03 = ((df[ret_col] / df["atr"]) > 0.3).sum()
|
||||
thresh_05 = ((df[ret_col] / df["atr"]) > 0.5).sum()
|
||||
thresh_10 = ((df[ret_col] / df["atr"]) > 1.0).sum()
|
||||
|
||||
print(f" Returns > 0.3×ATR: {thresh_03} ({(thresh_03/total)*100:.2f}%)")
|
||||
print(f" Returns > 0.5×ATR: {thresh_05} ({(thresh_05/total)*100:.2f}%)")
|
||||
print(f" Returns > 1.0×ATR: {thresh_10} ({(thresh_10/total)*100:.2f}%)")
|
||||
else:
|
||||
print(f"\nNo training data found at {data_file}")
|
||||
|
||||
# ============================================================================
|
||||
# 5. FEATURE CORRELATION ANALYSIS
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("5. FEATURE CORRELATION ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
if data_file.exists() and df is not None:
|
||||
# Extract H1 and M15 features
|
||||
h1_cols = [col for col in df.columns if "_h1" in col.lower()]
|
||||
m15_cols = [col for col in df.columns if any(ind in col.lower() for ind in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"])]
|
||||
|
||||
print(f"\nH1 columns found: {len(h1_cols)}")
|
||||
print(f"M15 columns found: {len(m15_cols)}")
|
||||
|
||||
if h1_cols and m15_cols:
|
||||
# Select numeric columns only
|
||||
numeric_h1 = [col for col in h1_cols if df[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]]
|
||||
numeric_m15 = [col for col in m15_cols if df[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]]
|
||||
|
||||
print(f"Numeric H1 columns: {len(numeric_h1)}")
|
||||
print(f"Numeric M15 columns: {len(numeric_m15)}")
|
||||
|
||||
if numeric_h1 and numeric_m15:
|
||||
# Calculate correlations between H1 and M15 features
|
||||
print("\n--- HIGH CORRELATIONS BETWEEN H1 AND M15 FEATURES ---")
|
||||
print("(Correlation > 0.7 suggests redundancy)")
|
||||
|
||||
high_corr_count = 0
|
||||
for h1_col in numeric_h1[:10]: # Check first 10 H1 features
|
||||
for m15_col in numeric_m15[:10]: # Against first 10 M15 features
|
||||
try:
|
||||
corr_df = df.select([h1_col, m15_col]).drop_nulls()
|
||||
if corr_df.shape[0] > 0:
|
||||
corr = corr_df.corr()[h1_col, m15_col]
|
||||
if abs(corr) > 0.7:
|
||||
print(f" {h1_col} <-> {m15_col}: {corr:.3f}")
|
||||
high_corr_count += 1
|
||||
except:
|
||||
pass
|
||||
|
||||
if high_corr_count == 0:
|
||||
print(" No high correlations found (good - features are independent)")
|
||||
else:
|
||||
print(f"\n Total high correlations: {high_corr_count}")
|
||||
|
||||
# Check H1 feature autocorrelation
|
||||
print("\n--- H1 FEATURE INTERNAL CORRELATIONS ---")
|
||||
if len(numeric_h1) >= 2:
|
||||
high_h1_corr = 0
|
||||
for i, col1 in enumerate(numeric_h1[:10]):
|
||||
for col2 in numeric_h1[i+1:10]:
|
||||
try:
|
||||
corr_df = df.select([col1, col2]).drop_nulls()
|
||||
if corr_df.shape[0] > 0:
|
||||
corr = corr_df.corr()[col1, col2]
|
||||
if abs(corr) > 0.8:
|
||||
print(f" {col1} <-> {col2}: {corr:.3f}")
|
||||
high_h1_corr += 1
|
||||
except:
|
||||
pass
|
||||
|
||||
if high_h1_corr == 0:
|
||||
print(" No high internal correlations (good)")
|
||||
else:
|
||||
print("\nCannot perform correlation analysis - data not available")
|
||||
|
||||
# ============================================================================
|
||||
# 6. PREDICTION CONSISTENCY CHECK
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("6. PREDICTION CONSISTENCY ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
persistence_file = Path("data/signal_persistence.json")
|
||||
if persistence_file.exists():
|
||||
print(f"\nReading: {persistence_file}")
|
||||
with open(persistence_file, "r") as f:
|
||||
persistence_data = json.load(f)
|
||||
|
||||
print(f"Persistence data: {json.dumps(persistence_data, indent=2)}")
|
||||
else:
|
||||
print(f"\nNo persistence data found at {persistence_file}")
|
||||
|
||||
# Analyze recent logs for signal flipping
|
||||
recent_log = Path("logs/trading_bot_2026-02-09.log")
|
||||
if recent_log.exists():
|
||||
print(f"\n--- ANALYZING RECENT SIGNALS FROM LOG ---")
|
||||
print(f"Reading: {recent_log}")
|
||||
|
||||
signal_history = []
|
||||
with open(recent_log, "r") as f:
|
||||
for line in f:
|
||||
if "ML Signal:" in line or "prediction:" in line.lower() or "signal=" in line.lower():
|
||||
signal_history.append(line.strip())
|
||||
|
||||
print(f"\nFound {len(signal_history)} signal-related log entries")
|
||||
|
||||
if signal_history:
|
||||
print("\n--- RECENT SIGNAL SAMPLES (Last 20) ---")
|
||||
for entry in signal_history[-20:]:
|
||||
print(f" {entry}")
|
||||
|
||||
# Count signal changes
|
||||
signals = []
|
||||
for entry in signal_history:
|
||||
if "BUY" in entry.upper():
|
||||
signals.append("BUY")
|
||||
elif "SELL" in entry.upper():
|
||||
signals.append("SELL")
|
||||
elif "HOLD" in entry.upper():
|
||||
signals.append("HOLD")
|
||||
|
||||
if len(signals) > 1:
|
||||
changes = sum(1 for i in range(1, len(signals)) if signals[i] != signals[i-1])
|
||||
print(f"\n--- SIGNAL STABILITY ---")
|
||||
print(f"Total signals tracked: {len(signals)}")
|
||||
print(f"Signal changes: {changes}")
|
||||
print(f"Change rate: {(changes/len(signals))*100:.2f}%")
|
||||
|
||||
# Count by type
|
||||
from collections import Counter
|
||||
signal_counts = Counter(signals)
|
||||
print(f"\nSignal distribution:")
|
||||
for sig, count in signal_counts.items():
|
||||
print(f" {sig}: {count} ({(count/len(signals))*100:.2f}%)")
|
||||
else:
|
||||
print(f"\nNo recent log found at {recent_log}")
|
||||
|
||||
# ============================================================================
|
||||
# 7. MODEL METRICS FROM DATA
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("7. MODEL METRICS (from data/model_metrics.json)")
|
||||
print("=" * 80)
|
||||
|
||||
metrics_file = Path("data/model_metrics.json")
|
||||
if metrics_file.exists():
|
||||
with open(metrics_file, "r") as f:
|
||||
metrics = json.load(f)
|
||||
|
||||
print(json.dumps(metrics, indent=2))
|
||||
else:
|
||||
print(f"\nNo metrics file found at {metrics_file}")
|
||||
|
||||
# ============================================================================
|
||||
# SUMMARY
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("ANALYSIS SUMMARY")
|
||||
print("=" * 80)
|
||||
|
||||
print("\n1. MODEL PERFORMANCE:")
|
||||
print(f" - Test AUC: {test_auc}")
|
||||
print(f" - Overfitting: {'YES' if isinstance(train_auc, float) and isinstance(test_auc, float) and (train_auc - test_auc) > 0.05 else 'NO'}")
|
||||
|
||||
print("\n2. FEATURE IMPORTANCE:")
|
||||
print(f" - H1 features in top 10: {len(h1_in_top10)}")
|
||||
print(f" - Total H1 features: {len(h1_features)}")
|
||||
|
||||
if h1_in_top10:
|
||||
print(f" - Highest ranked H1: {h1_in_top10[0]} (rank #{[f for f, s in feature_importance].index(h1_in_top10[0]) + 1})")
|
||||
else:
|
||||
print(" - No H1 features in top 10")
|
||||
|
||||
print("\n3. DATA QUALITY:")
|
||||
if data_file.exists() and has_target:
|
||||
print(f" - Training samples: {total}")
|
||||
print(f" - Target balance: See distribution above")
|
||||
else:
|
||||
print(" - Could not analyze training data")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("ANALYSIS COMPLETE")
|
||||
print("=" * 80)
|
||||
Reference in New Issue
Block a user