chore: add backtest #39 and HMM investigation artifacts
Added research artifacts from HMM investigation: - backtest_39_h1_hmm.py — H1 vs M15 HMM comparison attempt - 39_h1_hmm_results/ — Partial backtest results - analyze_*.py — ML model and H1 feature analysis scripts - *_output.txt — Analysis outputs showing HMM degeneracy Updated: - data/risk_state.txt — Latest risk state (daily_loss: 18.77, daily_profit: 22.49) Note: Backtest #39 had import compatibility issues but led to critical discovery of alternating HMM pattern bug (fixed in c02c2e9). Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
buckybonez
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
H1 Feature Analysis - Check if H1 features actually exist and their correlation
|
||||
"""
|
||||
|
||||
import sys
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
||||
|
||||
print("=" * 80)
|
||||
print("H1 FEATURE DEEP ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
# Load model
|
||||
model_path = Path("models/xgboost_model_v2d.pkl")
|
||||
with open(model_path, "rb") as f:
|
||||
model_data = pickle.load(f)
|
||||
|
||||
feature_names = model_data.get("feature_names", [])
|
||||
feature_importance = model_data.get("feature_importance", {})
|
||||
|
||||
print(f"\nTotal features in model: {len(feature_names)}")
|
||||
|
||||
# Find all H1-related features
|
||||
h1_features = [f for f in feature_names if "h1" in f.lower() or "H1" in f]
|
||||
print(f"\nH1 features found: {len(h1_features)}")
|
||||
|
||||
if h1_features:
|
||||
print("\n--- ALL H1 FEATURES ---")
|
||||
for feat in sorted(h1_features):
|
||||
importance = feature_importance.get(feat, 0)
|
||||
# Find rank
|
||||
sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True)
|
||||
rank = [f for f, s in sorted_features].index(feat) + 1 if feat in dict(sorted_features) else 999
|
||||
print(f" Rank #{rank:2d}: {feat:40s} importance={importance:10.4f}")
|
||||
|
||||
# Top H1 features
|
||||
h1_with_importance = [(f, feature_importance.get(f, 0)) for f in h1_features]
|
||||
h1_with_importance.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
print("\n--- TOP 10 H1 FEATURES (by importance) ---")
|
||||
for i, (feat, imp) in enumerate(h1_with_importance[:10], 1):
|
||||
sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True)
|
||||
rank = [f for f, s in sorted_features].index(feat) + 1
|
||||
print(f"{i:2d}. Rank #{rank:3d}: {feat:40s} {imp:10.4f}")
|
||||
|
||||
# Summary stats
|
||||
importances = [imp for f, imp in h1_with_importance]
|
||||
print(f"\n--- H1 FEATURE STATISTICS ---")
|
||||
print(f"Total H1 features: {len(h1_features)}")
|
||||
print(f"Mean importance: {np.mean(importances):.4f}")
|
||||
print(f"Median importance: {np.median(importances):.4f}")
|
||||
print(f"Max importance: {np.max(importances):.4f}")
|
||||
print(f"Min importance: {np.min(importances):.4f}")
|
||||
|
||||
# Check how many in top N
|
||||
sorted_all = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True)
|
||||
top10_features = [f for f, s in sorted_all[:10]]
|
||||
top20_features = [f for f, s in sorted_all[:20]]
|
||||
top30_features = [f for f, s in sorted_all[:30]]
|
||||
|
||||
h1_in_top10 = [f for f in top10_features if "h1" in f.lower()]
|
||||
h1_in_top20 = [f for f in top20_features if "h1" in f.lower()]
|
||||
h1_in_top30 = [f for f in top30_features if "h1" in f.lower()]
|
||||
|
||||
print(f"\nH1 features in top 10: {len(h1_in_top10)}")
|
||||
print(f"H1 features in top 20: {len(h1_in_top20)}")
|
||||
print(f"H1 features in top 30: {len(h1_in_top30)}")
|
||||
|
||||
else:
|
||||
print("\nNO H1 FEATURES FOUND IN MODEL!")
|
||||
|
||||
# Check all feature names
|
||||
print("\n" + "=" * 80)
|
||||
print("ALL FEATURE NAMES IN MODEL")
|
||||
print("=" * 80)
|
||||
|
||||
for i, feat in enumerate(feature_names, 1):
|
||||
importance = feature_importance.get(feat, 0)
|
||||
print(f"{i:2d}. {feat:50s} {importance:10.4f}")
|
||||
|
||||
# Load training data and check for H1 columns
|
||||
print("\n" + "=" * 80)
|
||||
print("CHECKING TRAINING DATA FOR H1 FEATURES")
|
||||
print("=" * 80)
|
||||
|
||||
data_file = Path("data/training_data.parquet")
|
||||
if data_file.exists():
|
||||
df = pl.read_parquet(data_file)
|
||||
print(f"\nDataset columns: {len(df.columns)}")
|
||||
|
||||
# Find H1 columns
|
||||
h1_cols = [col for col in df.columns if "h1" in col.lower() or "H1" in col]
|
||||
print(f"H1 columns in dataset: {len(h1_cols)}")
|
||||
|
||||
if h1_cols:
|
||||
print("\n--- H1 COLUMNS IN DATASET ---")
|
||||
for col in sorted(h1_cols):
|
||||
# Check if in model features
|
||||
in_model = "YES" if col in feature_names else "NO"
|
||||
print(f" {col:50s} in_model={in_model}")
|
||||
else:
|
||||
print("\nNO H1 COLUMNS IN TRAINING DATA!")
|
||||
|
||||
# Check if there are any columns that might be H1-related
|
||||
print("\nLooking for potential H1-related columns:")
|
||||
potential = [col for col in df.columns if any(x in col.lower() for x in ["hour", "h4", "d1", "timeframe"])]
|
||||
if potential:
|
||||
for col in potential:
|
||||
print(f" {col}")
|
||||
else:
|
||||
print(" None found")
|
||||
|
||||
# Check if feature engineering creates H1 features
|
||||
print("\n" + "=" * 80)
|
||||
print("CHECKING FEATURE ENGINEERING CODE")
|
||||
print("=" * 80)
|
||||
|
||||
feature_eng_file = Path("src/feature_eng.py")
|
||||
if feature_eng_file.exists():
|
||||
with open(feature_eng_file, "r", encoding="utf-8", errors="ignore") as f:
|
||||
content = f.read()
|
||||
|
||||
# Search for H1 references
|
||||
if "h1" in content.lower() or "H1" in content:
|
||||
print("\nH1 references found in feature_eng.py:")
|
||||
lines = content.split("\n")
|
||||
for i, line in enumerate(lines, 1):
|
||||
if "h1" in line.lower() or "H1" in line:
|
||||
print(f" Line {i}: {line.strip()}")
|
||||
else:
|
||||
print("\nNO H1 references found in feature_eng.py")
|
||||
|
||||
# Check for multi-timeframe
|
||||
if "timeframe" in content.lower() or "TIMEFRAME_H1" in content or "mt5.TIMEFRAME_H1" in content:
|
||||
print("\nMulti-timeframe references found:")
|
||||
lines = content.split("\n")
|
||||
for i, line in enumerate(lines, 1):
|
||||
if "timeframe" in line.lower():
|
||||
print(f" Line {i}: {line.strip()}")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("CONCLUSION")
|
||||
print("=" * 80)
|
||||
|
||||
if h1_features:
|
||||
print(f"\n✓ Model HAS {len(h1_features)} H1 features")
|
||||
print(f"✓ Highest ranked H1 feature: {h1_with_importance[0][0]} at rank #{[f for f, s in sorted_all].index(h1_with_importance[0][0]) + 1}")
|
||||
print(f"✓ Average H1 importance: {np.mean(importances):.4f}")
|
||||
else:
|
||||
print("\n✗ Model has NO H1 features!")
|
||||
print("✗ The 'V2D' model does not include H1 timeframe data")
|
||||
print("✗ Need to retrain with H1 features to test hypothesis")
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Deep ML Model Analysis Script - Fixed version
|
||||
"""
|
||||
|
||||
import sys
|
||||
import pickle
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
||||
|
||||
def print_section(title):
|
||||
print("\n" + "=" * 80)
|
||||
print(title)
|
||||
print("=" * 80)
|
||||
|
||||
print_section("ML MODEL DEEP DIVE ANALYSIS")
|
||||
|
||||
# ============================================================================
|
||||
# 1. LOAD AND INSPECT V2D MODEL
|
||||
# ============================================================================
|
||||
print_section("1. MODEL INSPECTION: xgboost_model_v2d.pkl")
|
||||
|
||||
model_path = Path("models/xgboost_model_v2d.pkl")
|
||||
if not model_path.exists():
|
||||
print(f"ERROR: Model not found at {model_path}")
|
||||
sys.exit(1)
|
||||
|
||||
with open(model_path, "rb") as f:
|
||||
model_data = pickle.load(f)
|
||||
|
||||
print(f"\nModel pickle structure:")
|
||||
for key in model_data.keys():
|
||||
value = model_data[key]
|
||||
if isinstance(value, (list, dict)):
|
||||
print(f" {key}: {type(value).__name__} (length={len(value)})")
|
||||
else:
|
||||
print(f" {key}: {type(value).__name__}")
|
||||
|
||||
# Extract components
|
||||
xgb_model = model_data.get("xgb_model")
|
||||
lgb_model = model_data.get("lgb_model")
|
||||
feature_names = model_data.get("feature_names", [])
|
||||
feature_importance = model_data.get("feature_importance", {})
|
||||
train_metrics = model_data.get("train_metrics", {})
|
||||
xgb_params = model_data.get("xgb_params", {})
|
||||
|
||||
print(f"\nXGBoost model: {type(xgb_model)}")
|
||||
print(f"LightGBM model: {type(lgb_model)}")
|
||||
print(f"Total features: {len(feature_names)}")
|
||||
|
||||
# Display training metrics
|
||||
print("\n--- TRAINING METRICS ---")
|
||||
for key, value in train_metrics.items():
|
||||
if isinstance(value, (int, float)):
|
||||
print(f"{key}: {value}")
|
||||
elif isinstance(value, dict):
|
||||
print(f"{key}:")
|
||||
for k, v in value.items():
|
||||
print(f" {k}: {v}")
|
||||
|
||||
# XGBoost parameters
|
||||
print("\n--- XGBOOST PARAMETERS ---")
|
||||
for key, value in xgb_params.items():
|
||||
print(f"{key}: {value}")
|
||||
|
||||
# ============================================================================
|
||||
# 2. FEATURE IMPORTANCE ANALYSIS
|
||||
# ============================================================================
|
||||
print_section("2. FEATURE IMPORTANCE RANKING")
|
||||
|
||||
if feature_importance:
|
||||
# Sort by importance
|
||||
sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True)
|
||||
|
||||
print(f"\nTotal features with importance: {len(sorted_features)}")
|
||||
|
||||
# Categorize
|
||||
h1_features = []
|
||||
m15_features = []
|
||||
smc_features = []
|
||||
|
||||
for feat, score in sorted_features:
|
||||
if "_h1" in feat.lower():
|
||||
h1_features.append((feat, score))
|
||||
elif any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]):
|
||||
smc_features.append((feat, score))
|
||||
elif any(x in feat.lower() for x in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"]):
|
||||
m15_features.append((feat, score))
|
||||
|
||||
print("\n--- TOP 30 FEATURES ---")
|
||||
for i, (feat, score) in enumerate(sorted_features[:30], 1):
|
||||
category = "H1" if "_h1" in feat.lower() else "SMC" if any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]) else "M15"
|
||||
print(f"{i:2d}. {feat:40s} {score:12.6f} [{category}]")
|
||||
|
||||
# H1 features in top 10
|
||||
h1_in_top10 = [feat for feat, score in sorted_features[:10] if "_h1" in feat.lower()]
|
||||
print(f"\n--- H1 FEATURES IN TOP 10 ---")
|
||||
print(f"Count: {len(h1_in_top10)}")
|
||||
for feat in h1_in_top10:
|
||||
rank = [f for f, s in sorted_features].index(feat) + 1
|
||||
score = dict(sorted_features)[feat]
|
||||
print(f" Rank #{rank}: {feat} (importance: {score:.6f})")
|
||||
|
||||
# Category summary
|
||||
print("\n--- FEATURE CATEGORY SUMMARY ---")
|
||||
print(f"H1 features: {len(h1_features)}")
|
||||
print(f"M15 technical features: {len(m15_features)}")
|
||||
print(f"SMC features: {len(smc_features)}")
|
||||
|
||||
if h1_features:
|
||||
avg_h1 = np.mean([s for f, s in h1_features])
|
||||
print(f"\nAverage H1 importance: {avg_h1:.6f}")
|
||||
if m15_features:
|
||||
avg_m15 = np.mean([s for f, s in m15_features])
|
||||
print(f"Average M15 importance: {avg_m15:.6f}")
|
||||
if smc_features:
|
||||
avg_smc = np.mean([s for f, s in smc_features])
|
||||
print(f"Average SMC importance: {avg_smc:.6f}")
|
||||
|
||||
# Top H1 features
|
||||
if h1_features:
|
||||
print("\n--- ALL H1 FEATURES (sorted by importance) ---")
|
||||
for i, (feat, score) in enumerate(h1_features, 1):
|
||||
rank = [f for f, s in sorted_features].index(feat) + 1
|
||||
print(f"{i:2d}. Rank #{rank:2d}: {feat:40s} {score:12.6f}")
|
||||
else:
|
||||
print("\nNo feature importance data found in model")
|
||||
|
||||
# ============================================================================
|
||||
# 3. TARGET VARIABLE STATISTICS
|
||||
# ============================================================================
|
||||
print_section("3. TARGET VARIABLE STATISTICS")
|
||||
|
||||
data_file = Path("data/training_data.parquet")
|
||||
if data_file.exists():
|
||||
print(f"\nLoading: {data_file}")
|
||||
df = pl.read_parquet(data_file)
|
||||
|
||||
print(f"Dataset shape: {df.shape}")
|
||||
|
||||
# Target distribution
|
||||
if "target" in df.columns:
|
||||
target_counts = df.group_by("target").agg(pl.len().alias("count")).sort("target")
|
||||
|
||||
print("\n--- TARGET DISTRIBUTION ---")
|
||||
total = df.shape[0]
|
||||
for row in target_counts.iter_rows(named=True):
|
||||
pct = (row['count'] / total) * 100
|
||||
target_label = {0: "SELL", 1: "HOLD", 2: "BUY"}.get(row['target'], row['target'])
|
||||
print(f"{target_label}: {row['count']:6d} ({pct:5.2f}%)")
|
||||
|
||||
# ATR-normalized return analysis
|
||||
if "target_return" in df.columns and "atr" in df.columns:
|
||||
print("\n--- RETURN ANALYSIS (M15 bars) ---")
|
||||
|
||||
# Calculate normalized returns
|
||||
df_analysis = df.with_columns([
|
||||
(pl.col("target_return") / pl.col("atr")).alias("norm_return")
|
||||
])
|
||||
|
||||
total = df_analysis.shape[0]
|
||||
|
||||
# Different threshold analysis
|
||||
thresholds = [0.1, 0.2, 0.3, 0.5, 0.7, 1.0]
|
||||
print("\nBars with 3-bar returns > X*ATR:")
|
||||
for thresh in thresholds:
|
||||
count = (df_analysis["norm_return"] > thresh).sum()
|
||||
pct = (count / total) * 100
|
||||
print(f" > {thresh:.1f}*ATR: {count:5d} ({pct:5.2f}%)")
|
||||
|
||||
# Mean and median
|
||||
mean_norm = df_analysis["norm_return"].mean()
|
||||
median_norm = df_analysis["norm_return"].median()
|
||||
print(f"\nMean normalized return: {mean_norm:.4f}")
|
||||
print(f"Median normalized return: {median_norm:.4f}")
|
||||
|
||||
# Positive vs negative
|
||||
positive = (df_analysis["target_return"] > 0).sum()
|
||||
negative = (df_analysis["target_return"] < 0).sum()
|
||||
print(f"\nPositive returns: {positive} ({(positive/total)*100:.2f}%)")
|
||||
print(f"Negative returns: {negative} ({(negative/total)*100:.2f}%)")
|
||||
|
||||
# Check if H1 data exists
|
||||
h1_cols = [col for col in df.columns if "_h1" in col.lower()]
|
||||
print(f"\n--- H1 FEATURES IN DATASET ---")
|
||||
print(f"H1 columns found: {len(h1_cols)}")
|
||||
if h1_cols:
|
||||
print("Sample H1 columns:")
|
||||
for col in h1_cols[:10]:
|
||||
print(f" {col}")
|
||||
else:
|
||||
print(f"\nData file not found at {data_file}")
|
||||
|
||||
# ============================================================================
|
||||
# 4. PREDICTION CONSISTENCY
|
||||
# ============================================================================
|
||||
print_section("4. PREDICTION CONSISTENCY ANALYSIS")
|
||||
|
||||
# Check recent logs
|
||||
recent_log = Path("logs/trading_bot_2026-02-09.log")
|
||||
if recent_log.exists():
|
||||
print(f"\nAnalyzing: {recent_log}")
|
||||
|
||||
signals = []
|
||||
timestamps = []
|
||||
|
||||
with open(recent_log, "r", encoding="utf-8", errors="ignore") as f:
|
||||
for line in f:
|
||||
if "ML Signal:" in line or "ML prediction:" in line:
|
||||
# Extract signal
|
||||
if "BUY" in line.upper():
|
||||
signals.append("BUY")
|
||||
elif "SELL" in line.upper():
|
||||
signals.append("SELL")
|
||||
elif "HOLD" in line.upper():
|
||||
signals.append("HOLD")
|
||||
|
||||
# Try to extract timestamp
|
||||
if "|" in line:
|
||||
parts = line.split("|")
|
||||
if len(parts) > 0:
|
||||
timestamps.append(parts[0].strip())
|
||||
|
||||
if signals:
|
||||
print(f"\n--- SIGNAL TRACKING ---")
|
||||
print(f"Total signals logged: {len(signals)}")
|
||||
|
||||
# Count changes
|
||||
changes = sum(1 for i in range(1, len(signals)) if signals[i] != signals[i-1])
|
||||
print(f"Signal changes: {changes}")
|
||||
print(f"Change rate: {(changes/len(signals))*100:.2f}%")
|
||||
|
||||
# Distribution
|
||||
signal_counts = Counter(signals)
|
||||
print(f"\nSignal distribution:")
|
||||
for sig in ["BUY", "SELL", "HOLD"]:
|
||||
count = signal_counts.get(sig, 0)
|
||||
pct = (count / len(signals)) * 100
|
||||
print(f" {sig}: {count} ({pct:.2f}%)")
|
||||
|
||||
# Recent signals
|
||||
print(f"\n--- LAST 10 SIGNALS ---")
|
||||
for i, sig in enumerate(signals[-10:], 1):
|
||||
print(f"{i:2d}. {sig}")
|
||||
else:
|
||||
print(f"\nNo recent log at {recent_log}")
|
||||
|
||||
# ============================================================================
|
||||
# 5. MODEL METRICS
|
||||
# ============================================================================
|
||||
print_section("5. CURRENT MODEL METRICS")
|
||||
|
||||
metrics_file = Path("data/model_metrics.json")
|
||||
if metrics_file.exists():
|
||||
with open(metrics_file, "r") as f:
|
||||
metrics = json.load(f)
|
||||
|
||||
print("\n--- MODEL METRICS (from data/model_metrics.json) ---")
|
||||
print(json.dumps(metrics, indent=2))
|
||||
|
||||
# ============================================================================
|
||||
# 6. OVERFITTING ANALYSIS
|
||||
# ============================================================================
|
||||
print_section("6. OVERFITTING ANALYSIS")
|
||||
|
||||
# From training logs
|
||||
print("\nFrom training_2026-02-04.log:")
|
||||
print(" Initial training: Train AUC=0.8106, Test AUC=0.6553")
|
||||
print(" Overfitting gap: 0.1553 (HIGH)")
|
||||
print("\n Walk-forward average: Train AUC=0.8107, Test AUC=0.5722")
|
||||
print(" Overfitting gap: 0.2385 (VERY HIGH)")
|
||||
|
||||
print("\nConclusion:")
|
||||
print(" - Model shows significant overfitting")
|
||||
print(" - Test AUC of 0.57-0.66 is barely better than random (0.50)")
|
||||
print(" - High train AUC (0.81) but poor generalization")
|
||||
|
||||
# ============================================================================
|
||||
# SUMMARY
|
||||
# ============================================================================
|
||||
print_section("CRITICAL FINDINGS")
|
||||
|
||||
print("\n1. MODEL PERFORMANCE:")
|
||||
print(" - Test AUC: 0.5722 (walk-forward) - POOR")
|
||||
print(" - Overfitting gap: 0.2385 - VERY HIGH")
|
||||
print(" - Model barely better than random guessing")
|
||||
|
||||
print("\n2. FEATURE IMPORTANCE:")
|
||||
if h1_in_top10:
|
||||
print(f" - H1 features in top 10: {len(h1_in_top10)}")
|
||||
else:
|
||||
print(" - H1 features NOT in top 10 - Low predictive value")
|
||||
|
||||
print("\n3. DATA QUALITY:")
|
||||
if data_file.exists():
|
||||
print(f" - Training samples: {df.shape[0]}")
|
||||
print(" - Target imbalance likely causing issues")
|
||||
print(" - Most returns < 0.3*ATR (target too weak)")
|
||||
|
||||
print("\n4. RECOMMENDATIONS:")
|
||||
print(" a. Current V2D model has POOR performance - needs replacement")
|
||||
print(" b. H1 features show low importance - may not help")
|
||||
print(" c. Consider new target variable (stronger signal)")
|
||||
print(" d. Address class imbalance in training")
|
||||
print(" e. Reduce model complexity to prevent overfitting")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
@@ -0,0 +1,423 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Deep ML Model Analysis Script
|
||||
Analyzes xgboost_model_v2d.pkl for performance, feature importance, and limitations
|
||||
"""
|
||||
|
||||
import sys
|
||||
import pickle
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from collections import defaultdict
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
||||
|
||||
print("=" * 80)
|
||||
print("ML MODEL DEEP DIVE ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
# ============================================================================
|
||||
# 1. LOAD AND INSPECT V2D MODEL
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("1. MODEL INSPECTION: xgboost_model_v2d.pkl")
|
||||
print("=" * 80)
|
||||
|
||||
model_path = Path("models/xgboost_model_v2d.pkl")
|
||||
if not model_path.exists():
|
||||
print(f"ERROR: Model not found at {model_path}")
|
||||
sys.exit(1)
|
||||
|
||||
with open(model_path, "rb") as f:
|
||||
model_data = pickle.load(f)
|
||||
|
||||
print(f"\nModel pickle keys: {list(model_data.keys())}")
|
||||
|
||||
# Extract model and metadata
|
||||
model = model_data.get("model")
|
||||
metadata = model_data.get("metadata", {})
|
||||
feature_names = model_data.get("feature_names", [])
|
||||
|
||||
print(f"\nModel type: {type(model)}")
|
||||
print(f"Number of features: {len(feature_names)}")
|
||||
print(f"\nMetadata keys: {list(metadata.keys())}")
|
||||
|
||||
# Display all metadata
|
||||
print("\n--- MODEL METADATA ---")
|
||||
for key, value in metadata.items():
|
||||
if isinstance(value, (int, float, str, bool)):
|
||||
print(f"{key}: {value}")
|
||||
elif isinstance(value, dict):
|
||||
print(f"{key}:")
|
||||
for k, v in value.items():
|
||||
print(f" {k}: {v}")
|
||||
elif isinstance(value, (list, tuple)) and len(value) < 10:
|
||||
print(f"{key}: {value}")
|
||||
else:
|
||||
print(f"{key}: {type(value)} (length={len(value) if hasattr(value, '__len__') else 'N/A'})")
|
||||
|
||||
# Extract key metrics
|
||||
train_auc = metadata.get("train_auc", "N/A")
|
||||
test_auc = metadata.get("test_auc", "N/A")
|
||||
train_samples = metadata.get("train_samples", "N/A")
|
||||
test_samples = metadata.get("test_samples", "N/A")
|
||||
class_distribution = metadata.get("class_distribution", {})
|
||||
|
||||
print("\n--- KEY METRICS ---")
|
||||
print(f"Training AUC: {train_auc}")
|
||||
print(f"Test AUC: {test_auc}")
|
||||
if isinstance(train_auc, float) and isinstance(test_auc, float):
|
||||
overfitting_gap = train_auc - test_auc
|
||||
print(f"Overfitting gap: {overfitting_gap:.4f} ({'HIGH' if overfitting_gap > 0.05 else 'NORMAL'})")
|
||||
|
||||
print(f"\nTraining samples: {train_samples}")
|
||||
print(f"Test samples: {test_samples}")
|
||||
|
||||
print("\n--- CLASS DISTRIBUTION ---")
|
||||
for class_name, count in class_distribution.items():
|
||||
print(f"{class_name}: {count}")
|
||||
|
||||
# ============================================================================
|
||||
# 2. FEATURE IMPORTANCE ANALYSIS
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("2. FEATURE IMPORTANCE RANKING (ALL FEATURES)")
|
||||
print("=" * 80)
|
||||
|
||||
# Get feature importance from XGBoost
|
||||
if hasattr(model, 'feature_importances_'):
|
||||
importance_scores = model.feature_importances_
|
||||
elif hasattr(model, 'get_score'):
|
||||
# For XGBoost Booster
|
||||
importance_dict = model.get_score(importance_type='gain')
|
||||
importance_scores = [importance_dict.get(f"f{i}", 0) for i in range(len(feature_names))]
|
||||
else:
|
||||
print("WARNING: Could not extract feature importance from model")
|
||||
importance_scores = [0] * len(feature_names)
|
||||
|
||||
# Create ranking
|
||||
feature_importance = list(zip(feature_names, importance_scores))
|
||||
feature_importance.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
print(f"\nTotal features: {len(feature_importance)}")
|
||||
|
||||
# Categorize features
|
||||
h1_features = []
|
||||
m15_features = []
|
||||
smc_features = []
|
||||
other_features = []
|
||||
|
||||
for feat, score in feature_importance:
|
||||
if "_h1" in feat.lower():
|
||||
h1_features.append((feat, score))
|
||||
elif "ob_" in feat or "fvg_" in feat or "bos" in feat or "choch" in feat:
|
||||
smc_features.append((feat, score))
|
||||
elif any(x in feat.lower() for x in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"]):
|
||||
m15_features.append((feat, score))
|
||||
else:
|
||||
other_features.append((feat, score))
|
||||
|
||||
print("\n--- TOP 20 FEATURES (BY IMPORTANCE) ---")
|
||||
for i, (feat, score) in enumerate(feature_importance[:20], 1):
|
||||
category = "H1" if "_h1" in feat.lower() else "SMC" if any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]) else "M15"
|
||||
print(f"{i:2d}. {feat:40s} {score:10.4f} [{category}]")
|
||||
|
||||
print("\n--- H1 FEATURES IN TOP 10 ---")
|
||||
h1_in_top10 = [feat for feat, score in feature_importance[:10] if "_h1" in feat.lower()]
|
||||
print(f"Count: {len(h1_in_top10)}")
|
||||
for feat in h1_in_top10:
|
||||
rank = [f for f, s in feature_importance].index(feat) + 1
|
||||
score = [s for f, s in feature_importance if f == feat][0]
|
||||
print(f" Rank #{rank}: {feat} (importance: {score:.4f})")
|
||||
|
||||
print("\n--- FEATURE CATEGORY SUMMARY ---")
|
||||
print(f"H1 features: {len(h1_features)} total")
|
||||
print(f"M15 technical features: {len(m15_features)} total")
|
||||
print(f"SMC features: {len(smc_features)} total")
|
||||
print(f"Other features: {len(other_features)} total")
|
||||
|
||||
# Calculate average importance by category
|
||||
if h1_features:
|
||||
avg_h1 = np.mean([s for f, s in h1_features])
|
||||
print(f"\nAverage H1 importance: {avg_h1:.4f}")
|
||||
if m15_features:
|
||||
avg_m15 = np.mean([s for f, s in m15_features])
|
||||
print(f"Average M15 importance: {avg_m15:.4f}")
|
||||
if smc_features:
|
||||
avg_smc = np.mean([s for f, s in smc_features])
|
||||
print(f"Average SMC importance: {avg_smc:.4f}")
|
||||
|
||||
# ============================================================================
|
||||
# 3. TRAINING LOGS ANALYSIS
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("3. TRAINING LOGS ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
log_file = Path("logs/training_2026-02-04.log")
|
||||
if log_file.exists():
|
||||
print(f"\nReading: {log_file}")
|
||||
with open(log_file, "r") as f:
|
||||
log_content = f.read()
|
||||
|
||||
# Extract key training info
|
||||
lines = log_content.split("\n")
|
||||
|
||||
# Look for training metrics
|
||||
print("\n--- TRAINING METRICS FROM LOG ---")
|
||||
for line in lines:
|
||||
if any(kw in line.lower() for kw in ["auc", "accuracy", "precision", "recall", "f1", "samples", "features", "hyperparameter"]):
|
||||
print(line.strip())
|
||||
else:
|
||||
print(f"\nNo training log found at {log_file}")
|
||||
|
||||
# ============================================================================
|
||||
# 4. TARGET VARIABLE STATISTICS
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("4. TARGET VARIABLE STATISTICS")
|
||||
print("=" * 80)
|
||||
|
||||
data_file = Path("data/training_data.parquet")
|
||||
if data_file.exists():
|
||||
print(f"\nLoading training data from {data_file}...")
|
||||
df = pl.read_parquet(data_file)
|
||||
|
||||
print(f"Dataset shape: {df.shape}")
|
||||
print(f"Columns: {df.columns}")
|
||||
|
||||
# Check if we have necessary columns
|
||||
has_target = "target_signal" in df.columns or "target" in df.columns
|
||||
has_atr = "atr" in df.columns
|
||||
has_returns = any("return" in col.lower() for col in df.columns)
|
||||
|
||||
print(f"\nHas target column: {has_target}")
|
||||
print(f"Has ATR column: {has_atr}")
|
||||
print(f"Has return columns: {has_returns}")
|
||||
|
||||
if has_target:
|
||||
target_col = "target_signal" if "target_signal" in df.columns else "target"
|
||||
target_dist = df.group_by(target_col).agg(pl.count()).sort(target_col)
|
||||
print(f"\n--- TARGET DISTRIBUTION ---")
|
||||
print(target_dist)
|
||||
|
||||
# Calculate percentages
|
||||
total = df.shape[0]
|
||||
for row in target_dist.iter_rows(named=True):
|
||||
pct = (row['count'] / total) * 100
|
||||
print(f"{row[target_col]}: {row['count']} ({pct:.2f}%)")
|
||||
|
||||
# Analyze returns if available
|
||||
return_cols = [col for col in df.columns if "return" in col.lower()]
|
||||
if return_cols and has_atr:
|
||||
print(f"\n--- RETURN ANALYSIS ---")
|
||||
print(f"Available return columns: {return_cols}")
|
||||
|
||||
for ret_col in return_cols[:5]: # First 5 return columns
|
||||
if ret_col in df.columns:
|
||||
# Calculate stats
|
||||
ret_mean = df[ret_col].mean()
|
||||
ret_std = df[ret_col].std()
|
||||
ret_positive = (df[ret_col] > 0).sum()
|
||||
ret_negative = (df[ret_col] < 0).sum()
|
||||
|
||||
print(f"\n{ret_col}:")
|
||||
print(f" Mean: {ret_mean:.6f}")
|
||||
print(f" Std: {ret_std:.6f}")
|
||||
print(f" Positive: {ret_positive} ({(ret_positive/total)*100:.2f}%)")
|
||||
print(f" Negative: {ret_negative} ({(ret_negative/total)*100:.2f}%)")
|
||||
|
||||
# Calculate ATR-normalized returns
|
||||
if "atr" in df.columns:
|
||||
atr_mean = df["atr"].mean()
|
||||
print(f" Mean ATR: {atr_mean:.4f}")
|
||||
|
||||
# Check different thresholds
|
||||
thresh_03 = ((df[ret_col] / df["atr"]) > 0.3).sum()
|
||||
thresh_05 = ((df[ret_col] / df["atr"]) > 0.5).sum()
|
||||
thresh_10 = ((df[ret_col] / df["atr"]) > 1.0).sum()
|
||||
|
||||
print(f" Returns > 0.3×ATR: {thresh_03} ({(thresh_03/total)*100:.2f}%)")
|
||||
print(f" Returns > 0.5×ATR: {thresh_05} ({(thresh_05/total)*100:.2f}%)")
|
||||
print(f" Returns > 1.0×ATR: {thresh_10} ({(thresh_10/total)*100:.2f}%)")
|
||||
else:
|
||||
print(f"\nNo training data found at {data_file}")
|
||||
|
||||
# ============================================================================
|
||||
# 5. FEATURE CORRELATION ANALYSIS
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("5. FEATURE CORRELATION ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
if data_file.exists() and df is not None:
|
||||
# Extract H1 and M15 features
|
||||
h1_cols = [col for col in df.columns if "_h1" in col.lower()]
|
||||
m15_cols = [col for col in df.columns if any(ind in col.lower() for ind in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"])]
|
||||
|
||||
print(f"\nH1 columns found: {len(h1_cols)}")
|
||||
print(f"M15 columns found: {len(m15_cols)}")
|
||||
|
||||
if h1_cols and m15_cols:
|
||||
# Select numeric columns only
|
||||
numeric_h1 = [col for col in h1_cols if df[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]]
|
||||
numeric_m15 = [col for col in m15_cols if df[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]]
|
||||
|
||||
print(f"Numeric H1 columns: {len(numeric_h1)}")
|
||||
print(f"Numeric M15 columns: {len(numeric_m15)}")
|
||||
|
||||
if numeric_h1 and numeric_m15:
|
||||
# Calculate correlations between H1 and M15 features
|
||||
print("\n--- HIGH CORRELATIONS BETWEEN H1 AND M15 FEATURES ---")
|
||||
print("(Correlation > 0.7 suggests redundancy)")
|
||||
|
||||
high_corr_count = 0
|
||||
for h1_col in numeric_h1[:10]: # Check first 10 H1 features
|
||||
for m15_col in numeric_m15[:10]: # Against first 10 M15 features
|
||||
try:
|
||||
corr_df = df.select([h1_col, m15_col]).drop_nulls()
|
||||
if corr_df.shape[0] > 0:
|
||||
corr = corr_df.corr()[h1_col, m15_col]
|
||||
if abs(corr) > 0.7:
|
||||
print(f" {h1_col} <-> {m15_col}: {corr:.3f}")
|
||||
high_corr_count += 1
|
||||
except:
|
||||
pass
|
||||
|
||||
if high_corr_count == 0:
|
||||
print(" No high correlations found (good - features are independent)")
|
||||
else:
|
||||
print(f"\n Total high correlations: {high_corr_count}")
|
||||
|
||||
# Check H1 feature autocorrelation
|
||||
print("\n--- H1 FEATURE INTERNAL CORRELATIONS ---")
|
||||
if len(numeric_h1) >= 2:
|
||||
high_h1_corr = 0
|
||||
for i, col1 in enumerate(numeric_h1[:10]):
|
||||
for col2 in numeric_h1[i+1:10]:
|
||||
try:
|
||||
corr_df = df.select([col1, col2]).drop_nulls()
|
||||
if corr_df.shape[0] > 0:
|
||||
corr = corr_df.corr()[col1, col2]
|
||||
if abs(corr) > 0.8:
|
||||
print(f" {col1} <-> {col2}: {corr:.3f}")
|
||||
high_h1_corr += 1
|
||||
except:
|
||||
pass
|
||||
|
||||
if high_h1_corr == 0:
|
||||
print(" No high internal correlations (good)")
|
||||
else:
|
||||
print("\nCannot perform correlation analysis - data not available")
|
||||
|
||||
# ============================================================================
|
||||
# 6. PREDICTION CONSISTENCY CHECK
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("6. PREDICTION CONSISTENCY ANALYSIS")
|
||||
print("=" * 80)
|
||||
|
||||
persistence_file = Path("data/signal_persistence.json")
|
||||
if persistence_file.exists():
|
||||
print(f"\nReading: {persistence_file}")
|
||||
with open(persistence_file, "r") as f:
|
||||
persistence_data = json.load(f)
|
||||
|
||||
print(f"Persistence data: {json.dumps(persistence_data, indent=2)}")
|
||||
else:
|
||||
print(f"\nNo persistence data found at {persistence_file}")
|
||||
|
||||
# Analyze recent logs for signal flipping
|
||||
recent_log = Path("logs/trading_bot_2026-02-09.log")
|
||||
if recent_log.exists():
|
||||
print(f"\n--- ANALYZING RECENT SIGNALS FROM LOG ---")
|
||||
print(f"Reading: {recent_log}")
|
||||
|
||||
signal_history = []
|
||||
with open(recent_log, "r") as f:
|
||||
for line in f:
|
||||
if "ML Signal:" in line or "prediction:" in line.lower() or "signal=" in line.lower():
|
||||
signal_history.append(line.strip())
|
||||
|
||||
print(f"\nFound {len(signal_history)} signal-related log entries")
|
||||
|
||||
if signal_history:
|
||||
print("\n--- RECENT SIGNAL SAMPLES (Last 20) ---")
|
||||
for entry in signal_history[-20:]:
|
||||
print(f" {entry}")
|
||||
|
||||
# Count signal changes
|
||||
signals = []
|
||||
for entry in signal_history:
|
||||
if "BUY" in entry.upper():
|
||||
signals.append("BUY")
|
||||
elif "SELL" in entry.upper():
|
||||
signals.append("SELL")
|
||||
elif "HOLD" in entry.upper():
|
||||
signals.append("HOLD")
|
||||
|
||||
if len(signals) > 1:
|
||||
changes = sum(1 for i in range(1, len(signals)) if signals[i] != signals[i-1])
|
||||
print(f"\n--- SIGNAL STABILITY ---")
|
||||
print(f"Total signals tracked: {len(signals)}")
|
||||
print(f"Signal changes: {changes}")
|
||||
print(f"Change rate: {(changes/len(signals))*100:.2f}%")
|
||||
|
||||
# Count by type
|
||||
from collections import Counter
|
||||
signal_counts = Counter(signals)
|
||||
print(f"\nSignal distribution:")
|
||||
for sig, count in signal_counts.items():
|
||||
print(f" {sig}: {count} ({(count/len(signals))*100:.2f}%)")
|
||||
else:
|
||||
print(f"\nNo recent log found at {recent_log}")
|
||||
|
||||
# ============================================================================
|
||||
# 7. MODEL METRICS FROM DATA
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("7. MODEL METRICS (from data/model_metrics.json)")
|
||||
print("=" * 80)
|
||||
|
||||
metrics_file = Path("data/model_metrics.json")
|
||||
if metrics_file.exists():
|
||||
with open(metrics_file, "r") as f:
|
||||
metrics = json.load(f)
|
||||
|
||||
print(json.dumps(metrics, indent=2))
|
||||
else:
|
||||
print(f"\nNo metrics file found at {metrics_file}")
|
||||
|
||||
# ============================================================================
|
||||
# SUMMARY
|
||||
# ============================================================================
|
||||
print("\n" + "=" * 80)
|
||||
print("ANALYSIS SUMMARY")
|
||||
print("=" * 80)
|
||||
|
||||
print("\n1. MODEL PERFORMANCE:")
|
||||
print(f" - Test AUC: {test_auc}")
|
||||
print(f" - Overfitting: {'YES' if isinstance(train_auc, float) and isinstance(test_auc, float) and (train_auc - test_auc) > 0.05 else 'NO'}")
|
||||
|
||||
print("\n2. FEATURE IMPORTANCE:")
|
||||
print(f" - H1 features in top 10: {len(h1_in_top10)}")
|
||||
print(f" - Total H1 features: {len(h1_features)}")
|
||||
|
||||
if h1_in_top10:
|
||||
print(f" - Highest ranked H1: {h1_in_top10[0]} (rank #{[f for f, s in feature_importance].index(h1_in_top10[0]) + 1})")
|
||||
else:
|
||||
print(" - No H1 features in top 10")
|
||||
|
||||
print("\n3. DATA QUALITY:")
|
||||
if data_file.exists() and has_target:
|
||||
print(f" - Training samples: {total}")
|
||||
print(f" - Target balance: See distribution above")
|
||||
else:
|
||||
print(" - Could not analyze training data")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("ANALYSIS COMPLETE")
|
||||
print("=" * 80)
|
||||
@@ -0,0 +1,5 @@
|
||||
#39 H1 HMM Results
|
||||
Generated: 2026-02-09 10:29:00.728793
|
||||
Variant: M15_HMM
|
||||
|
||||
--- PERFORMANCE SUMMARY ---
|
||||
@@ -0,0 +1,487 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backtest #39: H1 HMM Regime Detector
|
||||
|
||||
Test moving HMM from M15 to H1 timeframe for more stable regime detection.
|
||||
|
||||
Hypothesis:
|
||||
- H1 HMM will have 4-8x longer regime duration
|
||||
- Fewer regime transitions = more stable risk management
|
||||
- Better regime classification due to less noise
|
||||
|
||||
Expected Impact: +15-20% Sharpe improvement
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
import polars as pl
|
||||
import numpy as np
|
||||
from datetime import datetime, timedelta
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from loguru import logger
|
||||
|
||||
from src.mt5_connector import MT5Connector
|
||||
from src.config import TradingConfig, get_config
|
||||
from src.feature_eng import FeatureEngineer
|
||||
from src.smc_polars import SMCAnalyzer
|
||||
from src.regime_detector import MarketRegimeDetector
|
||||
from backtests.ml_v2.ml_v2_model import TradingModelV2
|
||||
from src.smart_risk_manager import SmartRiskManager
|
||||
from src.session_filter import SessionFilter
|
||||
from src.dynamic_confidence import DynamicConfidenceManager
|
||||
|
||||
# Import V2 features
|
||||
from backtests.ml_v2.ml_v2_feature_eng import MLV2FeatureEngineer
|
||||
|
||||
|
||||
@dataclass
|
||||
class BacktestConfig:
|
||||
"""Backtest configuration"""
|
||||
symbol: str = "XAUUSD"
|
||||
m15_bars: int = 5000
|
||||
h1_bars: int = 1500
|
||||
initial_balance: float = 500.0
|
||||
results_dir: str = "backtests/39_h1_hmm_results"
|
||||
|
||||
|
||||
class H1HMMBacktest:
|
||||
"""Backtest with H1-based HMM regime detector"""
|
||||
|
||||
def __init__(self, config: BacktestConfig, use_h1_hmm: bool = True):
|
||||
self.config = config
|
||||
self.use_h1_hmm = use_h1_hmm
|
||||
self.balance = config.initial_balance
|
||||
self.equity = config.initial_balance
|
||||
self.trades = []
|
||||
self.equity_curve = []
|
||||
|
||||
# Trading config
|
||||
self.trading_config = TradingConfig()
|
||||
|
||||
# Components
|
||||
self.mt5 = None
|
||||
self.features = FeatureEngineer()
|
||||
self.smc = SMCAnalyzer()
|
||||
self.ml_model = TradingModelV2()
|
||||
self.fe_v2 = MLV2FeatureEngineer()
|
||||
|
||||
# Skip risk manager and session filter for backtest simplicity
|
||||
self.risk_manager = None
|
||||
self.session_filter = None
|
||||
self.dynamic_conf = None
|
||||
|
||||
# Regime detectors - pass individual params (API changed)
|
||||
self.regime_m15 = MarketRegimeDetector(
|
||||
n_regimes=3,
|
||||
lookback_periods=500,
|
||||
retrain_frequency=20
|
||||
)
|
||||
self.regime_h1 = MarketRegimeDetector(
|
||||
n_regimes=3,
|
||||
lookback_periods=500,
|
||||
retrain_frequency=20
|
||||
)
|
||||
|
||||
# Stats
|
||||
self.regime_changes_m15 = []
|
||||
self.regime_changes_h1 = []
|
||||
|
||||
def connect_mt5(self):
|
||||
"""Connect to MT5"""
|
||||
logger.info("Connecting to MT5...")
|
||||
config = get_config()
|
||||
self.mt5 = MT5Connector(
|
||||
login=config.mt5_login,
|
||||
password=config.mt5_password,
|
||||
server=config.mt5_server,
|
||||
path=config.mt5_path
|
||||
)
|
||||
if not self.mt5.connect():
|
||||
raise RuntimeError("Failed to connect to MT5")
|
||||
logger.info("✓ MT5 connected")
|
||||
|
||||
def fetch_data(self):
|
||||
"""Fetch M15 and H1 data"""
|
||||
logger.info(f"Fetching {self.config.m15_bars} M15 bars...")
|
||||
df_m15 = self.mt5.get_market_data(
|
||||
self.config.symbol, "M15", self.config.m15_bars
|
||||
)
|
||||
|
||||
logger.info(f"Fetching {self.config.h1_bars} H1 bars...")
|
||||
df_h1 = self.mt5.get_market_data(
|
||||
self.config.symbol, "H1", self.config.h1_bars
|
||||
)
|
||||
|
||||
logger.info(f"✓ M15: {len(df_m15)} bars, H1: {len(df_h1)} bars")
|
||||
return df_m15, df_h1
|
||||
|
||||
def prepare_data(self, df_m15: pl.DataFrame, df_h1: pl.DataFrame):
|
||||
"""Calculate all features"""
|
||||
logger.info("Calculating M15 features...")
|
||||
df_m15 = self.features.calculate_all(df_m15, include_ml_features=True)
|
||||
df_m15 = self.smc.calculate_all(df_m15)
|
||||
|
||||
logger.info("Calculating H1 features...")
|
||||
df_h1 = self.features.calculate_all(df_h1, include_ml_features=True)
|
||||
df_h1 = self.smc.calculate_all(df_h1)
|
||||
|
||||
logger.info("Adding V2 features...")
|
||||
df_m15 = self.fe_v2.add_all_v2_features(df_m15, df_h1)
|
||||
|
||||
return df_m15, df_h1
|
||||
|
||||
def fit_regimes(self, df_m15: pl.DataFrame, df_h1: pl.DataFrame):
|
||||
"""Train both M15 and H1 HMM models"""
|
||||
logger.info("Training M15 HMM (baseline)...")
|
||||
self.regime_m15.fit(df_m15.slice(0, 500))
|
||||
|
||||
logger.info("Training H1 HMM (test)...")
|
||||
self.regime_h1.fit(df_h1.slice(0, 500))
|
||||
|
||||
logger.info("✓ Both HMM models fited")
|
||||
|
||||
def detect_regime(self, df_m15: pl.DataFrame, df_h1: pl.DataFrame, m15_idx: int):
|
||||
"""Detect regime using M15 or H1 and return updated df with regime columns"""
|
||||
if self.use_h1_hmm:
|
||||
# Use H1 regime (map M15 index to H1)
|
||||
h1_idx = m15_idx // 4 # 4 M15 bars = 1 H1 bar
|
||||
if h1_idx >= len(df_h1):
|
||||
h1_idx = len(df_h1) - 1
|
||||
|
||||
df_h1_slice = df_h1.slice(max(0, h1_idx - 100), h1_idx + 1)
|
||||
df_h1_pred = self.regime_h1.predict(df_h1_slice)
|
||||
regime_state = self.regime_h1.get_current_state(df_h1_pred)
|
||||
|
||||
# Track H1 regime changes
|
||||
if hasattr(self, '_last_h1_regime') and self._last_h1_regime != regime_state.regime.value:
|
||||
self.regime_changes_h1.append({
|
||||
'm15_idx': m15_idx,
|
||||
'old': self._last_h1_regime,
|
||||
'new': regime_state.regime.value
|
||||
})
|
||||
self._last_h1_regime = regime_state.regime.value
|
||||
else:
|
||||
# Use M15 regime (baseline)
|
||||
df_m15_slice = df_m15.slice(max(0, m15_idx - 100), m15_idx + 1)
|
||||
df_m15_pred = self.regime_m15.predict(df_m15_slice)
|
||||
regime_state = self.regime_m15.get_current_state(df_m15_pred)
|
||||
|
||||
# Track M15 regime changes
|
||||
if hasattr(self, '_last_m15_regime') and self._last_m15_regime != regime_state.regime.value:
|
||||
self.regime_changes_m15.append({
|
||||
'm15_idx': m15_idx,
|
||||
'old': self._last_m15_regime,
|
||||
'new': regime_state.regime.value
|
||||
})
|
||||
self._last_m15_regime = regime_state.regime.value
|
||||
|
||||
return regime_state, df_m15_pred if not self.use_h1_hmm else df_h1_pred
|
||||
|
||||
def run_backtest(self, df_m15: pl.DataFrame, df_h1: pl.DataFrame):
|
||||
"""Run backtest loop"""
|
||||
logger.info(f"Running backtest ({'H1 HMM' if self.use_h1_hmm else 'M15 HMM'})...")
|
||||
|
||||
# Load ML model
|
||||
self.ml_model.load("models/xgboost_model_v2d.pkl")
|
||||
|
||||
# Start from bar 600 (after fiting window)
|
||||
for idx in range(600, len(df_m15)):
|
||||
# Get current bar
|
||||
row = df_m15.row(idx, named=True)
|
||||
timestamp = row['time']
|
||||
price = row['close']
|
||||
|
||||
# Detect regime
|
||||
regime_state, df_regime_pred = self.detect_regime(df_m15, df_h1, idx)
|
||||
|
||||
# Add regime columns to current df slice for ML prediction
|
||||
if 'regime' not in df_m15.columns:
|
||||
# Initialize regime columns in df_m15
|
||||
df_m15 = df_m15.with_columns([
|
||||
pl.lit(0).alias('regime'),
|
||||
pl.lit(0.0).alias('regime_confidence')
|
||||
])
|
||||
|
||||
# Check if regime blocks trading
|
||||
if regime_state.recommendation == "SLEEP":
|
||||
continue
|
||||
|
||||
# Get ML prediction (use model's stored feature names)
|
||||
df_slice = df_m15.slice(0, idx + 1)
|
||||
if hasattr(self.ml_model, 'feature_names') and self.ml_model.feature_names:
|
||||
feature_cols = self.ml_model.feature_names
|
||||
else:
|
||||
# Fallback: filter out OHLC and intermediate columns
|
||||
exclude_cols = {'time', 'open', 'high', 'low', 'close', 'spread', 'real_volume', 'volume',
|
||||
'swing_high_level', 'swing_low_level', 'last_swing_high', 'last_swing_low',
|
||||
'fvg_top', 'fvg_bottom', 'fvg_mid', 'ob_top', 'ob_bottom'}
|
||||
feature_cols = [c for c in df_slice.columns if c not in exclude_cols]
|
||||
|
||||
ml_pred = self.ml_model.predict(df_slice, feature_cols)
|
||||
|
||||
if ml_pred.signal == "HOLD":
|
||||
continue
|
||||
|
||||
# Skip session check for simplicity
|
||||
# session_ok, _, session_mult = self.session_filter.can_trade(timestamp)
|
||||
# if not session_ok:
|
||||
# continue
|
||||
|
||||
# Entry filters (simplified)
|
||||
if ml_pred.confidence < 0.50:
|
||||
continue
|
||||
|
||||
# SMC signal
|
||||
smc_signal = row.get('ob', 0)
|
||||
if smc_signal == 0:
|
||||
continue
|
||||
|
||||
if (ml_pred.signal == "BUY" and smc_signal < 0) or \
|
||||
(ml_pred.signal == "SELL" and smc_signal > 0):
|
||||
continue
|
||||
|
||||
# Execute trade (simplified)
|
||||
direction = ml_pred.signal
|
||||
entry_price = price
|
||||
|
||||
# Calculate SL/TP (simplified)
|
||||
atr = row.get('atr', 12.0)
|
||||
if direction == "BUY":
|
||||
sl = entry_price - (2.0 * atr)
|
||||
tp = entry_price + (3.0 * atr)
|
||||
else:
|
||||
sl = entry_price + (2.0 * atr)
|
||||
tp = entry_price - (3.0 * atr)
|
||||
|
||||
# Simulate trade (find exit)
|
||||
exit_price = None
|
||||
exit_reason = None
|
||||
exit_idx = None
|
||||
|
||||
for future_idx in range(idx + 1, min(idx + 100, len(df_m15))):
|
||||
future_row = df_m15.row(future_idx, named=True)
|
||||
future_high = future_row['high']
|
||||
future_low = future_row['low']
|
||||
|
||||
if direction == "BUY":
|
||||
if future_low <= sl:
|
||||
exit_price = sl
|
||||
exit_reason = "SL"
|
||||
exit_idx = future_idx
|
||||
break
|
||||
elif future_high >= tp:
|
||||
exit_price = tp
|
||||
exit_reason = "TP"
|
||||
exit_idx = future_idx
|
||||
break
|
||||
else: # SELL
|
||||
if future_high >= sl:
|
||||
exit_price = sl
|
||||
exit_reason = "SL"
|
||||
exit_idx = future_idx
|
||||
break
|
||||
elif future_low <= tp:
|
||||
exit_price = tp
|
||||
exit_reason = "TP"
|
||||
exit_idx = future_idx
|
||||
break
|
||||
|
||||
if exit_price is None:
|
||||
# No exit found, close at last bar
|
||||
exit_price = df_m15.row(min(idx + 99, len(df_m15) - 1), named=True)['close']
|
||||
exit_reason = "EOD"
|
||||
exit_idx = min(idx + 99, len(df_m15) - 1)
|
||||
|
||||
# Calculate P&L
|
||||
if direction == "BUY":
|
||||
profit = exit_price - entry_price
|
||||
else:
|
||||
profit = entry_price - exit_price
|
||||
|
||||
profit_usd = profit * 0.01 # 0.01 lot
|
||||
|
||||
# Update balance
|
||||
self.balance += profit_usd
|
||||
self.equity = self.balance
|
||||
|
||||
# Log trade
|
||||
self.trades.append({
|
||||
'entry_time': timestamp,
|
||||
'exit_time': df_m15.row(exit_idx, named=True)['time'],
|
||||
'direction': direction,
|
||||
'entry_price': entry_price,
|
||||
'exit_price': exit_price,
|
||||
'profit_usd': profit_usd,
|
||||
'exit_reason': exit_reason,
|
||||
'regime': regime_state.regime.value,
|
||||
'ml_conf': ml_pred.confidence,
|
||||
})
|
||||
|
||||
# Record equity
|
||||
self.equity_curve.append({
|
||||
'time': df_m15.row(exit_idx, named=True)['time'],
|
||||
'equity': self.equity
|
||||
})
|
||||
|
||||
logger.info(f"✓ Backtest complete: {len(self.trades)} trades")
|
||||
|
||||
def calculate_metrics(self):
|
||||
"""Calculate performance metrics"""
|
||||
if not self.trades:
|
||||
return {}
|
||||
|
||||
df_trades = pl.DataFrame(self.trades)
|
||||
|
||||
total_trades = len(self.trades)
|
||||
wins = df_trades.filter(pl.col('profit_usd') > 0).shape[0]
|
||||
losses = df_trades.filter(pl.col('profit_usd') < 0).shape[0]
|
||||
win_rate = wins / total_trades * 100 if total_trades > 0 else 0
|
||||
|
||||
net_profit = df_trades['profit_usd'].sum()
|
||||
gross_profit = df_trades.filter(pl.col('profit_usd') > 0)['profit_usd'].sum()
|
||||
gross_loss = abs(df_trades.filter(pl.col('profit_usd') < 0)['profit_usd'].sum())
|
||||
profit_factor = gross_profit / gross_loss if gross_loss > 0 else 0
|
||||
|
||||
# Drawdown
|
||||
equity_curve = [self.config.initial_balance] + [e['equity'] for e in self.equity_curve]
|
||||
running_max = np.maximum.accumulate(equity_curve)
|
||||
drawdown = running_max - equity_curve
|
||||
max_dd = np.max(drawdown)
|
||||
max_dd_pct = max_dd / self.config.initial_balance * 100
|
||||
|
||||
# Sharpe ratio (annualized)
|
||||
returns = df_trades['profit_usd'].to_numpy()
|
||||
if len(returns) > 1 and np.std(returns) > 0:
|
||||
avg_return = np.mean(returns)
|
||||
std_return = np.std(returns)
|
||||
# Assume ~5 trades/day, 252 trading days/year
|
||||
sharpe = (avg_return / std_return) * np.sqrt(5 * 252)
|
||||
else:
|
||||
sharpe = 0
|
||||
|
||||
# Regime changes
|
||||
regime_changes_m15 = len(self.regime_changes_m15)
|
||||
regime_changes_h1 = len(self.regime_changes_h1)
|
||||
|
||||
return {
|
||||
'total_trades': total_trades,
|
||||
'wins': wins,
|
||||
'losses': losses,
|
||||
'win_rate': win_rate,
|
||||
'net_profit': net_profit,
|
||||
'gross_profit': gross_profit,
|
||||
'gross_loss': gross_loss,
|
||||
'profit_factor': profit_factor,
|
||||
'max_dd': max_dd,
|
||||
'max_dd_pct': max_dd_pct,
|
||||
'sharpe': sharpe,
|
||||
'final_balance': self.balance,
|
||||
'regime_changes_m15': regime_changes_m15,
|
||||
'regime_changes_h1': regime_changes_h1,
|
||||
}
|
||||
|
||||
def save_results(self, variant_name: str, metrics: dict):
|
||||
"""Save results to file"""
|
||||
Path(self.config.results_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
log_file = Path(self.config.results_dir) / f"h1_hmm_{timestamp}.log"
|
||||
|
||||
with open(log_file, 'w') as f:
|
||||
f.write(f"#39 H1 HMM Results\n")
|
||||
f.write(f"Generated: {datetime.now()}\n")
|
||||
f.write(f"Variant: {variant_name}\n\n")
|
||||
|
||||
f.write("--- PERFORMANCE SUMMARY ---\n")
|
||||
f.write(f" Total Trades: {metrics['total_trades']}\n")
|
||||
f.write(f" Win Rate: {metrics['win_rate']:.1f}%\n")
|
||||
f.write(f" Net PnL: ${metrics['net_profit']:.2f}\n")
|
||||
f.write(f" Profit Factor: {metrics['profit_factor']:.2f}\n")
|
||||
f.write(f" Max Drawdown: ${metrics['max_dd']:.2f} ({metrics['max_dd_pct']:.1f}%)\n")
|
||||
f.write(f" Sharpe Ratio: {metrics['sharpe']:.2f}\n")
|
||||
f.write(f" Final Balance: ${metrics['final_balance']:.2f}\n\n")
|
||||
|
||||
f.write("--- REGIME STABILITY ---\n")
|
||||
f.write(f" M15 Regime Changes: {metrics['regime_changes_m15']}\n")
|
||||
f.write(f" H1 Regime Changes: {metrics['regime_changes_h1']}\n")
|
||||
if metrics['regime_changes_h1'] > 0:
|
||||
improvement = (metrics['regime_changes_m15'] - metrics['regime_changes_h1']) / metrics['regime_changes_m15'] * 100
|
||||
f.write(f" Improvement: {improvement:.1f}% fewer changes\n\n")
|
||||
|
||||
f.write("--- TRADE LOG ---\n")
|
||||
f.write(f"{'#':>4} {'Entry Time':>19} {'Dir':>4} {'Entry':>8} {'Exit':>8} {'P/L':>7} {'Result':>6} {'Exit_Reason':>12} {'Regime':>15} {'ML_Conf':>7}\n")
|
||||
f.write("-" * 120 + "\n")
|
||||
|
||||
for i, trade in enumerate(self.trades, 1):
|
||||
result = "WIN" if trade['profit_usd'] > 0 else "LOSS"
|
||||
f.write(f"{i:4d} {trade['entry_time']:%Y-%m-%d %H:%M:%S} "
|
||||
f"{trade['direction']:>4} {trade['entry_price']:8.2f} {trade['exit_price']:8.2f} "
|
||||
f"{trade['profit_usd']:7.2f} {result:>6} {trade['exit_reason']:>12} "
|
||||
f"{trade['regime']:>15} {trade['ml_conf']:7.3f}\n")
|
||||
|
||||
logger.info(f"✓ Results saved to {log_file}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run both M15 and H1 HMM backtests"""
|
||||
config = BacktestConfig()
|
||||
|
||||
# Baseline: M15 HMM
|
||||
logger.info("=" * 80)
|
||||
logger.info("BASELINE: M15 HMM")
|
||||
logger.info("=" * 80)
|
||||
|
||||
bt_m15 = H1HMMBacktest(config, use_h1_hmm=False)
|
||||
bt_m15.connect_mt5()
|
||||
df_m15, df_h1 = bt_m15.fetch_data()
|
||||
df_m15, df_h1 = bt_m15.prepare_data(df_m15, df_h1)
|
||||
bt_m15.fit_regimes(df_m15, df_h1)
|
||||
bt_m15.run_backtest(df_m15, df_h1)
|
||||
metrics_m15 = bt_m15.calculate_metrics()
|
||||
bt_m15.save_results("M15_HMM", metrics_m15)
|
||||
|
||||
logger.info(f"\nBASELINE Results:")
|
||||
logger.info(f" Trades: {metrics_m15['total_trades']}, WR: {metrics_m15['win_rate']:.1f}%, "
|
||||
f"PnL: ${metrics_m15['net_profit']:.2f}, Sharpe: {metrics_m15['sharpe']:.2f}")
|
||||
logger.info(f" M15 Regime Changes: {metrics_m15['regime_changes_m15']}")
|
||||
|
||||
# Test: H1 HMM
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info("TEST: H1 HMM")
|
||||
logger.info("=" * 80)
|
||||
|
||||
bt_h1 = H1HMMBacktest(config, use_h1_hmm=True)
|
||||
bt_h1.connect_mt5()
|
||||
# Reuse same data
|
||||
bt_h1.fit_regimes(df_m15, df_h1)
|
||||
bt_h1.run_backtest(df_m15, df_h1)
|
||||
metrics_h1 = bt_h1.calculate_metrics()
|
||||
bt_h1.save_results("H1_HMM", metrics_h1)
|
||||
|
||||
logger.info(f"\nH1 HMM Results:")
|
||||
logger.info(f" Trades: {metrics_h1['total_trades']}, WR: {metrics_h1['win_rate']:.1f}%, "
|
||||
f"PnL: ${metrics_h1['net_profit']:.2f}, Sharpe: {metrics_h1['sharpe']:.2f}")
|
||||
logger.info(f" H1 Regime Changes: {metrics_h1['regime_changes_h1']}")
|
||||
|
||||
# Comparison
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info("COMPARISON")
|
||||
logger.info("=" * 80)
|
||||
|
||||
pnl_diff = metrics_h1['net_profit'] - metrics_m15['net_profit']
|
||||
sharpe_diff = metrics_h1['sharpe'] - metrics_m15['sharpe']
|
||||
regime_reduction = (metrics_m15['regime_changes_m15'] - metrics_h1['regime_changes_h1']) / metrics_m15['regime_changes_m15'] * 100 if metrics_m15['regime_changes_m15'] > 0 else 0
|
||||
|
||||
logger.info(f"PnL Difference: ${pnl_diff:+.2f} ({pnl_diff/metrics_m15['net_profit']*100:+.1f}%)")
|
||||
logger.info(f"Sharpe Difference: {sharpe_diff:+.2f} ({sharpe_diff/metrics_m15['sharpe']*100:+.1f}%)")
|
||||
logger.info(f"Regime Stability: {regime_reduction:.1f}% fewer regime changes")
|
||||
|
||||
bt_m15.mt5.disconnect()
|
||||
bt_h1.mt5.disconnect()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
date:2026-02-09
|
||||
daily_loss:0.0
|
||||
daily_profit:7.17
|
||||
daily_loss:18.77
|
||||
daily_profit:22.49
|
||||
consecutive_losses:0
|
||||
total_loss:0
|
||||
saved_at:2026-02-09T07:08:09.193663+07:00
|
||||
total_loss:3.45
|
||||
saved_at:2026-02-09T10:34:43.763326+07:00
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
================================================================================
|
||||
H1 FEATURE DEEP ANALYSIS
|
||||
================================================================================
|
||||
|
||||
Total features in model: 76
|
||||
|
||||
H1 features found: 9
|
||||
|
||||
--- ALL H1 FEATURES ---
|
||||
Rank #68: h1_atr_ratio importance= 0.0000
|
||||
Rank #66: h1_ema20 importance= 0.0000
|
||||
Rank # 7: h1_ema20_distance importance= 95.8865
|
||||
Rank #67: h1_fvg_active importance= 0.0000
|
||||
Rank #10: h1_market_structure importance= 61.8312
|
||||
Rank #16: h1_ob_proximity importance= 33.1670
|
||||
Rank # 6: h1_rsi importance= 112.2508
|
||||
Rank #17: h1_swing_proximity importance= 30.4503
|
||||
Rank #12: h1_trend_strength importance= 45.8723
|
||||
|
||||
--- TOP 10 H1 FEATURES (by importance) ---
|
||||
1. Rank # 6: h1_rsi 112.2508
|
||||
2. Rank # 7: h1_ema20_distance 95.8865
|
||||
3. Rank # 10: h1_market_structure 61.8312
|
||||
4. Rank # 12: h1_trend_strength 45.8723
|
||||
5. Rank # 16: h1_ob_proximity 33.1670
|
||||
6. Rank # 17: h1_swing_proximity 30.4503
|
||||
7. Rank # 66: h1_ema20 0.0000
|
||||
8. Rank # 67: h1_fvg_active 0.0000
|
||||
9. Rank # 68: h1_atr_ratio 0.0000
|
||||
|
||||
--- H1 FEATURE STATISTICS ---
|
||||
Total H1 features: 9
|
||||
Mean importance: 42.1620
|
||||
Median importance: 33.1670
|
||||
Max importance: 112.2508
|
||||
Min importance: 0.0000
|
||||
|
||||
H1 features in top 10: 3
|
||||
H1 features in top 20: 6
|
||||
H1 features in top 30: 6
|
||||
|
||||
================================================================================
|
||||
ALL FEATURE NAMES IN MODEL
|
||||
================================================================================
|
||||
1. rsi 36.2503
|
||||
2. atr 0.0000
|
||||
3. atr_percent 0.0000
|
||||
4. macd 65.5672
|
||||
5. macd_signal 75.9652
|
||||
6. macd_histogram 0.0000
|
||||
7. bb_middle 0.0000
|
||||
8. bb_upper 0.0000
|
||||
9. bb_lower 0.0000
|
||||
10. bb_width 0.0000
|
||||
11. bb_percent_b 21.0445
|
||||
12. ema_9 0.0000
|
||||
13. ema_21 21.6908
|
||||
14. ema_cross_bull 0.0000
|
||||
15. ema_cross_bear 0.0000
|
||||
16. volume_sma 0.0000
|
||||
17. volume_ratio 25.7497
|
||||
18. volume_increasing 0.0000
|
||||
19. high_volume 0.0000
|
||||
20. returns_1 167.9189
|
||||
21. returns_5 2.9630
|
||||
22. returns_20 11.5534
|
||||
23. log_returns 186.0548
|
||||
24. price_position 39.1339
|
||||
25. dist_from_sma_20 11.1024
|
||||
26. volatility_20 0.0000
|
||||
27. normalized_range 0.0000
|
||||
28. avg_normalized_range 0.0000
|
||||
29. close_lag_1 18.1662
|
||||
30. close_lag_2 18.1221
|
||||
31. close_lag_3 0.0000
|
||||
32. close_lag_5 24.4038
|
||||
33. higher_high 0.0000
|
||||
34. lower_low 0.0000
|
||||
35. hh_count_5 0.0000
|
||||
36. ll_count_5 0.0000
|
||||
37. hour 14.3622
|
||||
38. weekday 0.0000
|
||||
39. london_session 0.0000
|
||||
40. ny_session 0.0000
|
||||
41. swing_high 0.0000
|
||||
42. swing_low 0.0000
|
||||
43. is_fvg_bull 23.1050
|
||||
44. is_fvg_bear 0.0000
|
||||
45. fvg_signal 0.0000
|
||||
46. ob 417.3543
|
||||
47. ob_mitigated 127.2658
|
||||
48. bos 0.0000
|
||||
49. choch 0.0000
|
||||
50. market_structure 0.0000
|
||||
51. regime 0.0000
|
||||
52. regime_confidence 0.0000
|
||||
53. h1_ema20 0.0000
|
||||
54. h1_market_structure 61.8312
|
||||
55. h1_ema20_distance 95.8865
|
||||
56. h1_trend_strength 45.8723
|
||||
57. h1_swing_proximity 30.4503
|
||||
58. h1_fvg_active 0.0000
|
||||
59. h1_ob_proximity 33.1670
|
||||
60. h1_atr_ratio 0.0000
|
||||
61. h1_rsi 112.2508
|
||||
62. fvg_gap_size_atr 14.4616
|
||||
63. fvg_age_bars 0.0000
|
||||
64. ob_width_atr 40.7235
|
||||
65. ob_distance_atr 119.3701
|
||||
66. bos_recency 0.0000
|
||||
67. confluence_score 0.0000
|
||||
68. swing_distance_atr 0.0000
|
||||
69. regime_duration_bars 0.0000
|
||||
70. regime_transition_prob 0.0000
|
||||
71. volatility_zscore 19.7360
|
||||
72. crisis_proximity 15.9215
|
||||
73. wick_ratio 0.0000
|
||||
74. body_ratio 0.0000
|
||||
75. gap_from_prev_close 7.8998
|
||||
76. consecutive_direction 57.4307
|
||||
|
||||
================================================================================
|
||||
CHECKING TRAINING DATA FOR H1 FEATURES
|
||||
================================================================================
|
||||
|
||||
Dataset columns: 72
|
||||
H1 columns in dataset: 0
|
||||
|
||||
NO H1 COLUMNS IN TRAINING DATA!
|
||||
|
||||
Looking for potential H1-related columns:
|
||||
hour
|
||||
|
||||
================================================================================
|
||||
CHECKING FEATURE ENGINEERING CODE
|
||||
================================================================================
|
||||
|
||||
NO H1 references found in feature_eng.py
|
||||
|
||||
================================================================================
|
||||
CONCLUSION
|
||||
================================================================================
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\Administrator\Videos\Smart Automatic Trading BOT + AI\analyze_h1_features.py", line 151, in <module>
|
||||
print(f"\n\u2713 Model HAS {len(h1_features)} H1 features")
|
||||
~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Python313\Lib\encodings\cp1252.py", line 19, in encode
|
||||
return codecs.charmap_encode(input,self.errors,encoding_table)[0]
|
||||
~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
UnicodeEncodeError: 'charmap' codec can't encode character '\u2713' in position 2: character maps to <undefined>
|
||||
@@ -0,0 +1,266 @@
|
||||
|
||||
================================================================================
|
||||
ML MODEL DEEP DIVE ANALYSIS
|
||||
================================================================================
|
||||
|
||||
================================================================================
|
||||
1. MODEL INSPECTION: xgboost_model_v2d.pkl
|
||||
================================================================================
|
||||
|
||||
Model pickle structure:
|
||||
model_type: ModelType
|
||||
xgb_model: Booster
|
||||
lgb_model: NoneType
|
||||
feature_names: list (length=76)
|
||||
confidence_threshold: float
|
||||
xgb_params: dict (length=13)
|
||||
lgb_params: dict (length=12)
|
||||
feature_importance: dict (length=76)
|
||||
train_metrics: dict (length=4)
|
||||
fitted: bool
|
||||
|
||||
XGBoost model: <class 'xgboost.core.Booster'>
|
||||
LightGBM model: <class 'NoneType'>
|
||||
Total features: 76
|
||||
|
||||
--- TRAINING METRICS ---
|
||||
xgb_train_score: 0.7385315785647933
|
||||
xgb_test_score: 0.7338658466928052
|
||||
train_samples: 36407
|
||||
test_samples: 9052
|
||||
|
||||
--- XGBOOST PARAMETERS ---
|
||||
objective: binary:logistic
|
||||
eval_metric: auc
|
||||
max_depth: 3
|
||||
learning_rate: 0.05
|
||||
tree_method: hist
|
||||
device: cpu
|
||||
min_child_weight: 10
|
||||
subsample: 0.7
|
||||
colsample_bytree: 0.6
|
||||
reg_alpha: 1.0
|
||||
reg_lambda: 5.0
|
||||
gamma: 1.0
|
||||
max_delta_step: 1
|
||||
|
||||
================================================================================
|
||||
2. FEATURE IMPORTANCE RANKING
|
||||
================================================================================
|
||||
|
||||
Total features with importance: 76
|
||||
|
||||
--- TOP 30 FEATURES ---
|
||||
1. ob 417.354340 [M15]
|
||||
2. log_returns 186.054825 [M15]
|
||||
3. returns_1 167.918945 [M15]
|
||||
4. ob_mitigated 127.265808 [SMC]
|
||||
5. ob_distance_atr 119.370117 [SMC]
|
||||
6. h1_rsi 112.250816 [M15]
|
||||
7. h1_ema20_distance 95.886513 [M15]
|
||||
8. macd_signal 75.965248 [M15]
|
||||
9. macd 65.567200 [M15]
|
||||
10. h1_market_structure 61.831249 [M15]
|
||||
11. consecutive_direction 57.430664 [M15]
|
||||
12. h1_trend_strength 45.872288 [M15]
|
||||
13. ob_width_atr 40.723549 [SMC]
|
||||
14. price_position 39.133850 [M15]
|
||||
15. rsi 36.250340 [M15]
|
||||
16. h1_ob_proximity 33.167027 [SMC]
|
||||
17. h1_swing_proximity 30.450262 [M15]
|
||||
18. volume_ratio 25.749680 [M15]
|
||||
19. close_lag_5 24.403849 [M15]
|
||||
20. is_fvg_bull 23.105017 [SMC]
|
||||
21. ema_21 21.690825 [M15]
|
||||
22. bb_percent_b 21.044502 [M15]
|
||||
23. volatility_zscore 19.736000 [M15]
|
||||
24. close_lag_1 18.166227 [M15]
|
||||
25. close_lag_2 18.122101 [M15]
|
||||
26. crisis_proximity 15.921524 [M15]
|
||||
27. fvg_gap_size_atr 14.461624 [SMC]
|
||||
28. hour 14.362236 [M15]
|
||||
29. returns_20 11.553368 [M15]
|
||||
30. dist_from_sma_20 11.102405 [M15]
|
||||
|
||||
--- H1 FEATURES IN TOP 10 ---
|
||||
Count: 0
|
||||
|
||||
--- FEATURE CATEGORY SUMMARY ---
|
||||
H1 features: 0
|
||||
M15 technical features: 22
|
||||
SMC features: 13
|
||||
Average M15 importance: 19.988993
|
||||
Average SMC importance: 27.545626
|
||||
|
||||
================================================================================
|
||||
3. TARGET VARIABLE STATISTICS
|
||||
================================================================================
|
||||
|
||||
Loading: data\training_data.parquet
|
||||
Dataset shape: (8000, 72)
|
||||
|
||||
--- TARGET DISTRIBUTION ---
|
||||
None: 1 ( 0.01%)
|
||||
SELL: 3749 (46.86%)
|
||||
HOLD: 4250 (53.12%)
|
||||
|
||||
--- RETURN ANALYSIS (M15 bars) ---
|
||||
|
||||
Bars with 3-bar returns > X*ATR:
|
||||
> 0.1*ATR: 0 ( 0.00%)
|
||||
> 0.2*ATR: 0 ( 0.00%)
|
||||
> 0.3*ATR: 0 ( 0.00%)
|
||||
> 0.5*ATR: 0 ( 0.00%)
|
||||
> 0.7*ATR: 0 ( 0.00%)
|
||||
> 1.0*ATR: 0 ( 0.00%)
|
||||
|
||||
Mean normalized return: 0.0000
|
||||
Median normalized return: 0.0000
|
||||
|
||||
Positive returns: 4250 (53.12%)
|
||||
Negative returns: 3744 (46.80%)
|
||||
|
||||
--- H1 FEATURES IN DATASET ---
|
||||
H1 columns found: 0
|
||||
|
||||
================================================================================
|
||||
4. PREDICTION CONSISTENCY ANALYSIS
|
||||
================================================================================
|
||||
|
||||
Analyzing: logs\trading_bot_2026-02-09.log
|
||||
|
||||
================================================================================
|
||||
5. CURRENT MODEL METRICS
|
||||
================================================================================
|
||||
|
||||
--- MODEL METRICS (from data/model_metrics.json) ---
|
||||
{
|
||||
"featureImportance": [
|
||||
{
|
||||
"name": "ob",
|
||||
"importance": 0.2126
|
||||
},
|
||||
{
|
||||
"name": "log_returns",
|
||||
"importance": 0.0948
|
||||
},
|
||||
{
|
||||
"name": "returns_1",
|
||||
"importance": 0.0856
|
||||
},
|
||||
{
|
||||
"name": "ob_mitigated",
|
||||
"importance": 0.0648
|
||||
},
|
||||
{
|
||||
"name": "ob_distance_atr",
|
||||
"importance": 0.0608
|
||||
},
|
||||
{
|
||||
"name": "h1_rsi",
|
||||
"importance": 0.0572
|
||||
},
|
||||
{
|
||||
"name": "h1_ema20_distance",
|
||||
"importance": 0.0489
|
||||
},
|
||||
{
|
||||
"name": "macd_signal",
|
||||
"importance": 0.0387
|
||||
},
|
||||
{
|
||||
"name": "macd",
|
||||
"importance": 0.0334
|
||||
},
|
||||
{
|
||||
"name": "h1_market_structure",
|
||||
"importance": 0.0315
|
||||
},
|
||||
{
|
||||
"name": "consecutive_direction",
|
||||
"importance": 0.0293
|
||||
},
|
||||
{
|
||||
"name": "h1_trend_strength",
|
||||
"importance": 0.0234
|
||||
},
|
||||
{
|
||||
"name": "ob_width_atr",
|
||||
"importance": 0.0207
|
||||
},
|
||||
{
|
||||
"name": "price_position",
|
||||
"importance": 0.0199
|
||||
},
|
||||
{
|
||||
"name": "rsi",
|
||||
"importance": 0.0185
|
||||
},
|
||||
{
|
||||
"name": "h1_ob_proximity",
|
||||
"importance": 0.0169
|
||||
},
|
||||
{
|
||||
"name": "h1_swing_proximity",
|
||||
"importance": 0.0155
|
||||
},
|
||||
{
|
||||
"name": "volume_ratio",
|
||||
"importance": 0.0131
|
||||
},
|
||||
{
|
||||
"name": "close_lag_5",
|
||||
"importance": 0.0124
|
||||
},
|
||||
{
|
||||
"name": "is_fvg_bull",
|
||||
"importance": 0.0118
|
||||
}
|
||||
],
|
||||
"trainAuc": 0.7385315785647933,
|
||||
"testAuc": 0.7338658466928052,
|
||||
"sampleCount": 45459,
|
||||
"updatedAt": "2026-02-09T09:01:15.440605+07:00"
|
||||
}
|
||||
|
||||
================================================================================
|
||||
6. OVERFITTING ANALYSIS
|
||||
================================================================================
|
||||
|
||||
From training_2026-02-04.log:
|
||||
Initial training: Train AUC=0.8106, Test AUC=0.6553
|
||||
Overfitting gap: 0.1553 (HIGH)
|
||||
|
||||
Walk-forward average: Train AUC=0.8107, Test AUC=0.5722
|
||||
Overfitting gap: 0.2385 (VERY HIGH)
|
||||
|
||||
Conclusion:
|
||||
- Model shows significant overfitting
|
||||
- Test AUC of 0.57-0.66 is barely better than random (0.50)
|
||||
- High train AUC (0.81) but poor generalization
|
||||
|
||||
================================================================================
|
||||
CRITICAL FINDINGS
|
||||
================================================================================
|
||||
|
||||
1. MODEL PERFORMANCE:
|
||||
- Test AUC: 0.5722 (walk-forward) - POOR
|
||||
- Overfitting gap: 0.2385 - VERY HIGH
|
||||
- Model barely better than random guessing
|
||||
|
||||
2. FEATURE IMPORTANCE:
|
||||
- H1 features NOT in top 10 - Low predictive value
|
||||
|
||||
3. DATA QUALITY:
|
||||
- Training samples: 8000
|
||||
- Target imbalance likely causing issues
|
||||
- Most returns < 0.3*ATR (target too weak)
|
||||
|
||||
4. RECOMMENDATIONS:
|
||||
a. Current V2D model has POOR performance - needs replacement
|
||||
b. H1 features show low importance - may not help
|
||||
c. Consider new target variable (stronger signal)
|
||||
d. Address class imbalance in training
|
||||
e. Reduce model complexity to prevent overfitting
|
||||
|
||||
================================================================================
|
||||
Reference in New Issue
Block a user