From 4626f2a295928ba8a10a3f657a8653068f1bfe74 Mon Sep 17 00:00:00 2001 From: GifariKemal Date: Mon, 9 Feb 2026 10:51:49 +0700 Subject: [PATCH] chore: add backtest #39 and HMM investigation artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added research artifacts from HMM investigation: - backtest_39_h1_hmm.py — H1 vs M15 HMM comparison attempt - 39_h1_hmm_results/ — Partial backtest results - analyze_*.py — ML model and H1 feature analysis scripts - *_output.txt — Analysis outputs showing HMM degeneracy Updated: - data/risk_state.txt — Latest risk state (daily_loss: 18.77, daily_profit: 22.49) Note: Backtest #39 had import compatibility issues but led to critical discovery of alternating HMM pattern bug (fixed in c02c2e9). Co-Authored-By: Claude Sonnet 4.5 --- analyze_h1_features.py | 157 ++++++ analyze_ml_detailed.py | 313 +++++++++++ analyze_ml_model.py | 423 +++++++++++++++ .../h1_hmm_20260209_102900.log | 5 + backtests/backtest_39_h1_hmm.py | 487 ++++++++++++++++++ data/risk_state.txt | 8 +- h1_analysis_output.txt | 150 ++++++ ml_analysis_output.txt | 266 ++++++++++ 8 files changed, 1805 insertions(+), 4 deletions(-) create mode 100644 analyze_h1_features.py create mode 100644 analyze_ml_detailed.py create mode 100644 analyze_ml_model.py create mode 100644 backtests/39_h1_hmm_results/h1_hmm_20260209_102900.log create mode 100644 backtests/backtest_39_h1_hmm.py create mode 100644 h1_analysis_output.txt create mode 100644 ml_analysis_output.txt diff --git a/analyze_h1_features.py b/analyze_h1_features.py new file mode 100644 index 0000000..896975f --- /dev/null +++ b/analyze_h1_features.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +""" +H1 Feature Analysis - Check if H1 features actually exist and their correlation +""" + +import sys +import pickle +from pathlib import Path +import numpy as np +import polars as pl + +sys.path.insert(0, str(Path(__file__).parent / "src")) + +print("=" * 80) +print("H1 FEATURE DEEP ANALYSIS") +print("=" * 80) + +# Load model +model_path = Path("models/xgboost_model_v2d.pkl") +with open(model_path, "rb") as f: + model_data = pickle.load(f) + +feature_names = model_data.get("feature_names", []) +feature_importance = model_data.get("feature_importance", {}) + +print(f"\nTotal features in model: {len(feature_names)}") + +# Find all H1-related features +h1_features = [f for f in feature_names if "h1" in f.lower() or "H1" in f] +print(f"\nH1 features found: {len(h1_features)}") + +if h1_features: + print("\n--- ALL H1 FEATURES ---") + for feat in sorted(h1_features): + importance = feature_importance.get(feat, 0) + # Find rank + sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True) + rank = [f for f, s in sorted_features].index(feat) + 1 if feat in dict(sorted_features) else 999 + print(f" Rank #{rank:2d}: {feat:40s} importance={importance:10.4f}") + + # Top H1 features + h1_with_importance = [(f, feature_importance.get(f, 0)) for f in h1_features] + h1_with_importance.sort(key=lambda x: x[1], reverse=True) + + print("\n--- TOP 10 H1 FEATURES (by importance) ---") + for i, (feat, imp) in enumerate(h1_with_importance[:10], 1): + sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True) + rank = [f for f, s in sorted_features].index(feat) + 1 + print(f"{i:2d}. Rank #{rank:3d}: {feat:40s} {imp:10.4f}") + + # Summary stats + importances = [imp for f, imp in h1_with_importance] + print(f"\n--- H1 FEATURE STATISTICS ---") + print(f"Total H1 features: {len(h1_features)}") + print(f"Mean importance: {np.mean(importances):.4f}") + print(f"Median importance: {np.median(importances):.4f}") + print(f"Max importance: {np.max(importances):.4f}") + print(f"Min importance: {np.min(importances):.4f}") + + # Check how many in top N + sorted_all = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True) + top10_features = [f for f, s in sorted_all[:10]] + top20_features = [f for f, s in sorted_all[:20]] + top30_features = [f for f, s in sorted_all[:30]] + + h1_in_top10 = [f for f in top10_features if "h1" in f.lower()] + h1_in_top20 = [f for f in top20_features if "h1" in f.lower()] + h1_in_top30 = [f for f in top30_features if "h1" in f.lower()] + + print(f"\nH1 features in top 10: {len(h1_in_top10)}") + print(f"H1 features in top 20: {len(h1_in_top20)}") + print(f"H1 features in top 30: {len(h1_in_top30)}") + +else: + print("\nNO H1 FEATURES FOUND IN MODEL!") + +# Check all feature names +print("\n" + "=" * 80) +print("ALL FEATURE NAMES IN MODEL") +print("=" * 80) + +for i, feat in enumerate(feature_names, 1): + importance = feature_importance.get(feat, 0) + print(f"{i:2d}. {feat:50s} {importance:10.4f}") + +# Load training data and check for H1 columns +print("\n" + "=" * 80) +print("CHECKING TRAINING DATA FOR H1 FEATURES") +print("=" * 80) + +data_file = Path("data/training_data.parquet") +if data_file.exists(): + df = pl.read_parquet(data_file) + print(f"\nDataset columns: {len(df.columns)}") + + # Find H1 columns + h1_cols = [col for col in df.columns if "h1" in col.lower() or "H1" in col] + print(f"H1 columns in dataset: {len(h1_cols)}") + + if h1_cols: + print("\n--- H1 COLUMNS IN DATASET ---") + for col in sorted(h1_cols): + # Check if in model features + in_model = "YES" if col in feature_names else "NO" + print(f" {col:50s} in_model={in_model}") + else: + print("\nNO H1 COLUMNS IN TRAINING DATA!") + + # Check if there are any columns that might be H1-related + print("\nLooking for potential H1-related columns:") + potential = [col for col in df.columns if any(x in col.lower() for x in ["hour", "h4", "d1", "timeframe"])] + if potential: + for col in potential: + print(f" {col}") + else: + print(" None found") + +# Check if feature engineering creates H1 features +print("\n" + "=" * 80) +print("CHECKING FEATURE ENGINEERING CODE") +print("=" * 80) + +feature_eng_file = Path("src/feature_eng.py") +if feature_eng_file.exists(): + with open(feature_eng_file, "r", encoding="utf-8", errors="ignore") as f: + content = f.read() + + # Search for H1 references + if "h1" in content.lower() or "H1" in content: + print("\nH1 references found in feature_eng.py:") + lines = content.split("\n") + for i, line in enumerate(lines, 1): + if "h1" in line.lower() or "H1" in line: + print(f" Line {i}: {line.strip()}") + else: + print("\nNO H1 references found in feature_eng.py") + + # Check for multi-timeframe + if "timeframe" in content.lower() or "TIMEFRAME_H1" in content or "mt5.TIMEFRAME_H1" in content: + print("\nMulti-timeframe references found:") + lines = content.split("\n") + for i, line in enumerate(lines, 1): + if "timeframe" in line.lower(): + print(f" Line {i}: {line.strip()}") + +print("\n" + "=" * 80) +print("CONCLUSION") +print("=" * 80) + +if h1_features: + print(f"\n✓ Model HAS {len(h1_features)} H1 features") + print(f"✓ Highest ranked H1 feature: {h1_with_importance[0][0]} at rank #{[f for f, s in sorted_all].index(h1_with_importance[0][0]) + 1}") + print(f"✓ Average H1 importance: {np.mean(importances):.4f}") +else: + print("\n✗ Model has NO H1 features!") + print("✗ The 'V2D' model does not include H1 timeframe data") + print("✗ Need to retrain with H1 features to test hypothesis") diff --git a/analyze_ml_detailed.py b/analyze_ml_detailed.py new file mode 100644 index 0000000..29133d7 --- /dev/null +++ b/analyze_ml_detailed.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +""" +Deep ML Model Analysis Script - Fixed version +""" + +import sys +import pickle +import json +from pathlib import Path +import numpy as np +import polars as pl +from collections import defaultdict, Counter + +# Add src to path +sys.path.insert(0, str(Path(__file__).parent / "src")) + +def print_section(title): + print("\n" + "=" * 80) + print(title) + print("=" * 80) + +print_section("ML MODEL DEEP DIVE ANALYSIS") + +# ============================================================================ +# 1. LOAD AND INSPECT V2D MODEL +# ============================================================================ +print_section("1. MODEL INSPECTION: xgboost_model_v2d.pkl") + +model_path = Path("models/xgboost_model_v2d.pkl") +if not model_path.exists(): + print(f"ERROR: Model not found at {model_path}") + sys.exit(1) + +with open(model_path, "rb") as f: + model_data = pickle.load(f) + +print(f"\nModel pickle structure:") +for key in model_data.keys(): + value = model_data[key] + if isinstance(value, (list, dict)): + print(f" {key}: {type(value).__name__} (length={len(value)})") + else: + print(f" {key}: {type(value).__name__}") + +# Extract components +xgb_model = model_data.get("xgb_model") +lgb_model = model_data.get("lgb_model") +feature_names = model_data.get("feature_names", []) +feature_importance = model_data.get("feature_importance", {}) +train_metrics = model_data.get("train_metrics", {}) +xgb_params = model_data.get("xgb_params", {}) + +print(f"\nXGBoost model: {type(xgb_model)}") +print(f"LightGBM model: {type(lgb_model)}") +print(f"Total features: {len(feature_names)}") + +# Display training metrics +print("\n--- TRAINING METRICS ---") +for key, value in train_metrics.items(): + if isinstance(value, (int, float)): + print(f"{key}: {value}") + elif isinstance(value, dict): + print(f"{key}:") + for k, v in value.items(): + print(f" {k}: {v}") + +# XGBoost parameters +print("\n--- XGBOOST PARAMETERS ---") +for key, value in xgb_params.items(): + print(f"{key}: {value}") + +# ============================================================================ +# 2. FEATURE IMPORTANCE ANALYSIS +# ============================================================================ +print_section("2. FEATURE IMPORTANCE RANKING") + +if feature_importance: + # Sort by importance + sorted_features = sorted(feature_importance.items(), key=lambda x: x[1], reverse=True) + + print(f"\nTotal features with importance: {len(sorted_features)}") + + # Categorize + h1_features = [] + m15_features = [] + smc_features = [] + + for feat, score in sorted_features: + if "_h1" in feat.lower(): + h1_features.append((feat, score)) + elif any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]): + smc_features.append((feat, score)) + elif any(x in feat.lower() for x in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"]): + m15_features.append((feat, score)) + + print("\n--- TOP 30 FEATURES ---") + for i, (feat, score) in enumerate(sorted_features[:30], 1): + category = "H1" if "_h1" in feat.lower() else "SMC" if any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]) else "M15" + print(f"{i:2d}. {feat:40s} {score:12.6f} [{category}]") + + # H1 features in top 10 + h1_in_top10 = [feat for feat, score in sorted_features[:10] if "_h1" in feat.lower()] + print(f"\n--- H1 FEATURES IN TOP 10 ---") + print(f"Count: {len(h1_in_top10)}") + for feat in h1_in_top10: + rank = [f for f, s in sorted_features].index(feat) + 1 + score = dict(sorted_features)[feat] + print(f" Rank #{rank}: {feat} (importance: {score:.6f})") + + # Category summary + print("\n--- FEATURE CATEGORY SUMMARY ---") + print(f"H1 features: {len(h1_features)}") + print(f"M15 technical features: {len(m15_features)}") + print(f"SMC features: {len(smc_features)}") + + if h1_features: + avg_h1 = np.mean([s for f, s in h1_features]) + print(f"\nAverage H1 importance: {avg_h1:.6f}") + if m15_features: + avg_m15 = np.mean([s for f, s in m15_features]) + print(f"Average M15 importance: {avg_m15:.6f}") + if smc_features: + avg_smc = np.mean([s for f, s in smc_features]) + print(f"Average SMC importance: {avg_smc:.6f}") + + # Top H1 features + if h1_features: + print("\n--- ALL H1 FEATURES (sorted by importance) ---") + for i, (feat, score) in enumerate(h1_features, 1): + rank = [f for f, s in sorted_features].index(feat) + 1 + print(f"{i:2d}. Rank #{rank:2d}: {feat:40s} {score:12.6f}") +else: + print("\nNo feature importance data found in model") + +# ============================================================================ +# 3. TARGET VARIABLE STATISTICS +# ============================================================================ +print_section("3. TARGET VARIABLE STATISTICS") + +data_file = Path("data/training_data.parquet") +if data_file.exists(): + print(f"\nLoading: {data_file}") + df = pl.read_parquet(data_file) + + print(f"Dataset shape: {df.shape}") + + # Target distribution + if "target" in df.columns: + target_counts = df.group_by("target").agg(pl.len().alias("count")).sort("target") + + print("\n--- TARGET DISTRIBUTION ---") + total = df.shape[0] + for row in target_counts.iter_rows(named=True): + pct = (row['count'] / total) * 100 + target_label = {0: "SELL", 1: "HOLD", 2: "BUY"}.get(row['target'], row['target']) + print(f"{target_label}: {row['count']:6d} ({pct:5.2f}%)") + + # ATR-normalized return analysis + if "target_return" in df.columns and "atr" in df.columns: + print("\n--- RETURN ANALYSIS (M15 bars) ---") + + # Calculate normalized returns + df_analysis = df.with_columns([ + (pl.col("target_return") / pl.col("atr")).alias("norm_return") + ]) + + total = df_analysis.shape[0] + + # Different threshold analysis + thresholds = [0.1, 0.2, 0.3, 0.5, 0.7, 1.0] + print("\nBars with 3-bar returns > X*ATR:") + for thresh in thresholds: + count = (df_analysis["norm_return"] > thresh).sum() + pct = (count / total) * 100 + print(f" > {thresh:.1f}*ATR: {count:5d} ({pct:5.2f}%)") + + # Mean and median + mean_norm = df_analysis["norm_return"].mean() + median_norm = df_analysis["norm_return"].median() + print(f"\nMean normalized return: {mean_norm:.4f}") + print(f"Median normalized return: {median_norm:.4f}") + + # Positive vs negative + positive = (df_analysis["target_return"] > 0).sum() + negative = (df_analysis["target_return"] < 0).sum() + print(f"\nPositive returns: {positive} ({(positive/total)*100:.2f}%)") + print(f"Negative returns: {negative} ({(negative/total)*100:.2f}%)") + + # Check if H1 data exists + h1_cols = [col for col in df.columns if "_h1" in col.lower()] + print(f"\n--- H1 FEATURES IN DATASET ---") + print(f"H1 columns found: {len(h1_cols)}") + if h1_cols: + print("Sample H1 columns:") + for col in h1_cols[:10]: + print(f" {col}") +else: + print(f"\nData file not found at {data_file}") + +# ============================================================================ +# 4. PREDICTION CONSISTENCY +# ============================================================================ +print_section("4. PREDICTION CONSISTENCY ANALYSIS") + +# Check recent logs +recent_log = Path("logs/trading_bot_2026-02-09.log") +if recent_log.exists(): + print(f"\nAnalyzing: {recent_log}") + + signals = [] + timestamps = [] + + with open(recent_log, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + if "ML Signal:" in line or "ML prediction:" in line: + # Extract signal + if "BUY" in line.upper(): + signals.append("BUY") + elif "SELL" in line.upper(): + signals.append("SELL") + elif "HOLD" in line.upper(): + signals.append("HOLD") + + # Try to extract timestamp + if "|" in line: + parts = line.split("|") + if len(parts) > 0: + timestamps.append(parts[0].strip()) + + if signals: + print(f"\n--- SIGNAL TRACKING ---") + print(f"Total signals logged: {len(signals)}") + + # Count changes + changes = sum(1 for i in range(1, len(signals)) if signals[i] != signals[i-1]) + print(f"Signal changes: {changes}") + print(f"Change rate: {(changes/len(signals))*100:.2f}%") + + # Distribution + signal_counts = Counter(signals) + print(f"\nSignal distribution:") + for sig in ["BUY", "SELL", "HOLD"]: + count = signal_counts.get(sig, 0) + pct = (count / len(signals)) * 100 + print(f" {sig}: {count} ({pct:.2f}%)") + + # Recent signals + print(f"\n--- LAST 10 SIGNALS ---") + for i, sig in enumerate(signals[-10:], 1): + print(f"{i:2d}. {sig}") +else: + print(f"\nNo recent log at {recent_log}") + +# ============================================================================ +# 5. MODEL METRICS +# ============================================================================ +print_section("5. CURRENT MODEL METRICS") + +metrics_file = Path("data/model_metrics.json") +if metrics_file.exists(): + with open(metrics_file, "r") as f: + metrics = json.load(f) + + print("\n--- MODEL METRICS (from data/model_metrics.json) ---") + print(json.dumps(metrics, indent=2)) + +# ============================================================================ +# 6. OVERFITTING ANALYSIS +# ============================================================================ +print_section("6. OVERFITTING ANALYSIS") + +# From training logs +print("\nFrom training_2026-02-04.log:") +print(" Initial training: Train AUC=0.8106, Test AUC=0.6553") +print(" Overfitting gap: 0.1553 (HIGH)") +print("\n Walk-forward average: Train AUC=0.8107, Test AUC=0.5722") +print(" Overfitting gap: 0.2385 (VERY HIGH)") + +print("\nConclusion:") +print(" - Model shows significant overfitting") +print(" - Test AUC of 0.57-0.66 is barely better than random (0.50)") +print(" - High train AUC (0.81) but poor generalization") + +# ============================================================================ +# SUMMARY +# ============================================================================ +print_section("CRITICAL FINDINGS") + +print("\n1. MODEL PERFORMANCE:") +print(" - Test AUC: 0.5722 (walk-forward) - POOR") +print(" - Overfitting gap: 0.2385 - VERY HIGH") +print(" - Model barely better than random guessing") + +print("\n2. FEATURE IMPORTANCE:") +if h1_in_top10: + print(f" - H1 features in top 10: {len(h1_in_top10)}") +else: + print(" - H1 features NOT in top 10 - Low predictive value") + +print("\n3. DATA QUALITY:") +if data_file.exists(): + print(f" - Training samples: {df.shape[0]}") + print(" - Target imbalance likely causing issues") + print(" - Most returns < 0.3*ATR (target too weak)") + +print("\n4. RECOMMENDATIONS:") +print(" a. Current V2D model has POOR performance - needs replacement") +print(" b. H1 features show low importance - may not help") +print(" c. Consider new target variable (stronger signal)") +print(" d. Address class imbalance in training") +print(" e. Reduce model complexity to prevent overfitting") + +print("\n" + "=" * 80) diff --git a/analyze_ml_model.py b/analyze_ml_model.py new file mode 100644 index 0000000..0a8505f --- /dev/null +++ b/analyze_ml_model.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +""" +Deep ML Model Analysis Script +Analyzes xgboost_model_v2d.pkl for performance, feature importance, and limitations +""" + +import sys +import pickle +import json +from pathlib import Path +import numpy as np +import polars as pl +from collections import defaultdict + +# Add src to path +sys.path.insert(0, str(Path(__file__).parent / "src")) + +print("=" * 80) +print("ML MODEL DEEP DIVE ANALYSIS") +print("=" * 80) + +# ============================================================================ +# 1. LOAD AND INSPECT V2D MODEL +# ============================================================================ +print("\n" + "=" * 80) +print("1. MODEL INSPECTION: xgboost_model_v2d.pkl") +print("=" * 80) + +model_path = Path("models/xgboost_model_v2d.pkl") +if not model_path.exists(): + print(f"ERROR: Model not found at {model_path}") + sys.exit(1) + +with open(model_path, "rb") as f: + model_data = pickle.load(f) + +print(f"\nModel pickle keys: {list(model_data.keys())}") + +# Extract model and metadata +model = model_data.get("model") +metadata = model_data.get("metadata", {}) +feature_names = model_data.get("feature_names", []) + +print(f"\nModel type: {type(model)}") +print(f"Number of features: {len(feature_names)}") +print(f"\nMetadata keys: {list(metadata.keys())}") + +# Display all metadata +print("\n--- MODEL METADATA ---") +for key, value in metadata.items(): + if isinstance(value, (int, float, str, bool)): + print(f"{key}: {value}") + elif isinstance(value, dict): + print(f"{key}:") + for k, v in value.items(): + print(f" {k}: {v}") + elif isinstance(value, (list, tuple)) and len(value) < 10: + print(f"{key}: {value}") + else: + print(f"{key}: {type(value)} (length={len(value) if hasattr(value, '__len__') else 'N/A'})") + +# Extract key metrics +train_auc = metadata.get("train_auc", "N/A") +test_auc = metadata.get("test_auc", "N/A") +train_samples = metadata.get("train_samples", "N/A") +test_samples = metadata.get("test_samples", "N/A") +class_distribution = metadata.get("class_distribution", {}) + +print("\n--- KEY METRICS ---") +print(f"Training AUC: {train_auc}") +print(f"Test AUC: {test_auc}") +if isinstance(train_auc, float) and isinstance(test_auc, float): + overfitting_gap = train_auc - test_auc + print(f"Overfitting gap: {overfitting_gap:.4f} ({'HIGH' if overfitting_gap > 0.05 else 'NORMAL'})") + +print(f"\nTraining samples: {train_samples}") +print(f"Test samples: {test_samples}") + +print("\n--- CLASS DISTRIBUTION ---") +for class_name, count in class_distribution.items(): + print(f"{class_name}: {count}") + +# ============================================================================ +# 2. FEATURE IMPORTANCE ANALYSIS +# ============================================================================ +print("\n" + "=" * 80) +print("2. FEATURE IMPORTANCE RANKING (ALL FEATURES)") +print("=" * 80) + +# Get feature importance from XGBoost +if hasattr(model, 'feature_importances_'): + importance_scores = model.feature_importances_ +elif hasattr(model, 'get_score'): + # For XGBoost Booster + importance_dict = model.get_score(importance_type='gain') + importance_scores = [importance_dict.get(f"f{i}", 0) for i in range(len(feature_names))] +else: + print("WARNING: Could not extract feature importance from model") + importance_scores = [0] * len(feature_names) + +# Create ranking +feature_importance = list(zip(feature_names, importance_scores)) +feature_importance.sort(key=lambda x: x[1], reverse=True) + +print(f"\nTotal features: {len(feature_importance)}") + +# Categorize features +h1_features = [] +m15_features = [] +smc_features = [] +other_features = [] + +for feat, score in feature_importance: + if "_h1" in feat.lower(): + h1_features.append((feat, score)) + elif "ob_" in feat or "fvg_" in feat or "bos" in feat or "choch" in feat: + smc_features.append((feat, score)) + elif any(x in feat.lower() for x in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"]): + m15_features.append((feat, score)) + else: + other_features.append((feat, score)) + +print("\n--- TOP 20 FEATURES (BY IMPORTANCE) ---") +for i, (feat, score) in enumerate(feature_importance[:20], 1): + category = "H1" if "_h1" in feat.lower() else "SMC" if any(x in feat for x in ["ob_", "fvg_", "bos", "choch"]) else "M15" + print(f"{i:2d}. {feat:40s} {score:10.4f} [{category}]") + +print("\n--- H1 FEATURES IN TOP 10 ---") +h1_in_top10 = [feat for feat, score in feature_importance[:10] if "_h1" in feat.lower()] +print(f"Count: {len(h1_in_top10)}") +for feat in h1_in_top10: + rank = [f for f, s in feature_importance].index(feat) + 1 + score = [s for f, s in feature_importance if f == feat][0] + print(f" Rank #{rank}: {feat} (importance: {score:.4f})") + +print("\n--- FEATURE CATEGORY SUMMARY ---") +print(f"H1 features: {len(h1_features)} total") +print(f"M15 technical features: {len(m15_features)} total") +print(f"SMC features: {len(smc_features)} total") +print(f"Other features: {len(other_features)} total") + +# Calculate average importance by category +if h1_features: + avg_h1 = np.mean([s for f, s in h1_features]) + print(f"\nAverage H1 importance: {avg_h1:.4f}") +if m15_features: + avg_m15 = np.mean([s for f, s in m15_features]) + print(f"Average M15 importance: {avg_m15:.4f}") +if smc_features: + avg_smc = np.mean([s for f, s in smc_features]) + print(f"Average SMC importance: {avg_smc:.4f}") + +# ============================================================================ +# 3. TRAINING LOGS ANALYSIS +# ============================================================================ +print("\n" + "=" * 80) +print("3. TRAINING LOGS ANALYSIS") +print("=" * 80) + +log_file = Path("logs/training_2026-02-04.log") +if log_file.exists(): + print(f"\nReading: {log_file}") + with open(log_file, "r") as f: + log_content = f.read() + + # Extract key training info + lines = log_content.split("\n") + + # Look for training metrics + print("\n--- TRAINING METRICS FROM LOG ---") + for line in lines: + if any(kw in line.lower() for kw in ["auc", "accuracy", "precision", "recall", "f1", "samples", "features", "hyperparameter"]): + print(line.strip()) +else: + print(f"\nNo training log found at {log_file}") + +# ============================================================================ +# 4. TARGET VARIABLE STATISTICS +# ============================================================================ +print("\n" + "=" * 80) +print("4. TARGET VARIABLE STATISTICS") +print("=" * 80) + +data_file = Path("data/training_data.parquet") +if data_file.exists(): + print(f"\nLoading training data from {data_file}...") + df = pl.read_parquet(data_file) + + print(f"Dataset shape: {df.shape}") + print(f"Columns: {df.columns}") + + # Check if we have necessary columns + has_target = "target_signal" in df.columns or "target" in df.columns + has_atr = "atr" in df.columns + has_returns = any("return" in col.lower() for col in df.columns) + + print(f"\nHas target column: {has_target}") + print(f"Has ATR column: {has_atr}") + print(f"Has return columns: {has_returns}") + + if has_target: + target_col = "target_signal" if "target_signal" in df.columns else "target" + target_dist = df.group_by(target_col).agg(pl.count()).sort(target_col) + print(f"\n--- TARGET DISTRIBUTION ---") + print(target_dist) + + # Calculate percentages + total = df.shape[0] + for row in target_dist.iter_rows(named=True): + pct = (row['count'] / total) * 100 + print(f"{row[target_col]}: {row['count']} ({pct:.2f}%)") + + # Analyze returns if available + return_cols = [col for col in df.columns if "return" in col.lower()] + if return_cols and has_atr: + print(f"\n--- RETURN ANALYSIS ---") + print(f"Available return columns: {return_cols}") + + for ret_col in return_cols[:5]: # First 5 return columns + if ret_col in df.columns: + # Calculate stats + ret_mean = df[ret_col].mean() + ret_std = df[ret_col].std() + ret_positive = (df[ret_col] > 0).sum() + ret_negative = (df[ret_col] < 0).sum() + + print(f"\n{ret_col}:") + print(f" Mean: {ret_mean:.6f}") + print(f" Std: {ret_std:.6f}") + print(f" Positive: {ret_positive} ({(ret_positive/total)*100:.2f}%)") + print(f" Negative: {ret_negative} ({(ret_negative/total)*100:.2f}%)") + + # Calculate ATR-normalized returns + if "atr" in df.columns: + atr_mean = df["atr"].mean() + print(f" Mean ATR: {atr_mean:.4f}") + + # Check different thresholds + thresh_03 = ((df[ret_col] / df["atr"]) > 0.3).sum() + thresh_05 = ((df[ret_col] / df["atr"]) > 0.5).sum() + thresh_10 = ((df[ret_col] / df["atr"]) > 1.0).sum() + + print(f" Returns > 0.3×ATR: {thresh_03} ({(thresh_03/total)*100:.2f}%)") + print(f" Returns > 0.5×ATR: {thresh_05} ({(thresh_05/total)*100:.2f}%)") + print(f" Returns > 1.0×ATR: {thresh_10} ({(thresh_10/total)*100:.2f}%)") +else: + print(f"\nNo training data found at {data_file}") + +# ============================================================================ +# 5. FEATURE CORRELATION ANALYSIS +# ============================================================================ +print("\n" + "=" * 80) +print("5. FEATURE CORRELATION ANALYSIS") +print("=" * 80) + +if data_file.exists() and df is not None: + # Extract H1 and M15 features + h1_cols = [col for col in df.columns if "_h1" in col.lower()] + m15_cols = [col for col in df.columns if any(ind in col.lower() for ind in ["rsi", "macd", "bb", "atr", "ema", "sma", "stoch"])] + + print(f"\nH1 columns found: {len(h1_cols)}") + print(f"M15 columns found: {len(m15_cols)}") + + if h1_cols and m15_cols: + # Select numeric columns only + numeric_h1 = [col for col in h1_cols if df[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]] + numeric_m15 = [col for col in m15_cols if df[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]] + + print(f"Numeric H1 columns: {len(numeric_h1)}") + print(f"Numeric M15 columns: {len(numeric_m15)}") + + if numeric_h1 and numeric_m15: + # Calculate correlations between H1 and M15 features + print("\n--- HIGH CORRELATIONS BETWEEN H1 AND M15 FEATURES ---") + print("(Correlation > 0.7 suggests redundancy)") + + high_corr_count = 0 + for h1_col in numeric_h1[:10]: # Check first 10 H1 features + for m15_col in numeric_m15[:10]: # Against first 10 M15 features + try: + corr_df = df.select([h1_col, m15_col]).drop_nulls() + if corr_df.shape[0] > 0: + corr = corr_df.corr()[h1_col, m15_col] + if abs(corr) > 0.7: + print(f" {h1_col} <-> {m15_col}: {corr:.3f}") + high_corr_count += 1 + except: + pass + + if high_corr_count == 0: + print(" No high correlations found (good - features are independent)") + else: + print(f"\n Total high correlations: {high_corr_count}") + + # Check H1 feature autocorrelation + print("\n--- H1 FEATURE INTERNAL CORRELATIONS ---") + if len(numeric_h1) >= 2: + high_h1_corr = 0 + for i, col1 in enumerate(numeric_h1[:10]): + for col2 in numeric_h1[i+1:10]: + try: + corr_df = df.select([col1, col2]).drop_nulls() + if corr_df.shape[0] > 0: + corr = corr_df.corr()[col1, col2] + if abs(corr) > 0.8: + print(f" {col1} <-> {col2}: {corr:.3f}") + high_h1_corr += 1 + except: + pass + + if high_h1_corr == 0: + print(" No high internal correlations (good)") +else: + print("\nCannot perform correlation analysis - data not available") + +# ============================================================================ +# 6. PREDICTION CONSISTENCY CHECK +# ============================================================================ +print("\n" + "=" * 80) +print("6. PREDICTION CONSISTENCY ANALYSIS") +print("=" * 80) + +persistence_file = Path("data/signal_persistence.json") +if persistence_file.exists(): + print(f"\nReading: {persistence_file}") + with open(persistence_file, "r") as f: + persistence_data = json.load(f) + + print(f"Persistence data: {json.dumps(persistence_data, indent=2)}") +else: + print(f"\nNo persistence data found at {persistence_file}") + +# Analyze recent logs for signal flipping +recent_log = Path("logs/trading_bot_2026-02-09.log") +if recent_log.exists(): + print(f"\n--- ANALYZING RECENT SIGNALS FROM LOG ---") + print(f"Reading: {recent_log}") + + signal_history = [] + with open(recent_log, "r") as f: + for line in f: + if "ML Signal:" in line or "prediction:" in line.lower() or "signal=" in line.lower(): + signal_history.append(line.strip()) + + print(f"\nFound {len(signal_history)} signal-related log entries") + + if signal_history: + print("\n--- RECENT SIGNAL SAMPLES (Last 20) ---") + for entry in signal_history[-20:]: + print(f" {entry}") + + # Count signal changes + signals = [] + for entry in signal_history: + if "BUY" in entry.upper(): + signals.append("BUY") + elif "SELL" in entry.upper(): + signals.append("SELL") + elif "HOLD" in entry.upper(): + signals.append("HOLD") + + if len(signals) > 1: + changes = sum(1 for i in range(1, len(signals)) if signals[i] != signals[i-1]) + print(f"\n--- SIGNAL STABILITY ---") + print(f"Total signals tracked: {len(signals)}") + print(f"Signal changes: {changes}") + print(f"Change rate: {(changes/len(signals))*100:.2f}%") + + # Count by type + from collections import Counter + signal_counts = Counter(signals) + print(f"\nSignal distribution:") + for sig, count in signal_counts.items(): + print(f" {sig}: {count} ({(count/len(signals))*100:.2f}%)") +else: + print(f"\nNo recent log found at {recent_log}") + +# ============================================================================ +# 7. MODEL METRICS FROM DATA +# ============================================================================ +print("\n" + "=" * 80) +print("7. MODEL METRICS (from data/model_metrics.json)") +print("=" * 80) + +metrics_file = Path("data/model_metrics.json") +if metrics_file.exists(): + with open(metrics_file, "r") as f: + metrics = json.load(f) + + print(json.dumps(metrics, indent=2)) +else: + print(f"\nNo metrics file found at {metrics_file}") + +# ============================================================================ +# SUMMARY +# ============================================================================ +print("\n" + "=" * 80) +print("ANALYSIS SUMMARY") +print("=" * 80) + +print("\n1. MODEL PERFORMANCE:") +print(f" - Test AUC: {test_auc}") +print(f" - Overfitting: {'YES' if isinstance(train_auc, float) and isinstance(test_auc, float) and (train_auc - test_auc) > 0.05 else 'NO'}") + +print("\n2. FEATURE IMPORTANCE:") +print(f" - H1 features in top 10: {len(h1_in_top10)}") +print(f" - Total H1 features: {len(h1_features)}") + +if h1_in_top10: + print(f" - Highest ranked H1: {h1_in_top10[0]} (rank #{[f for f, s in feature_importance].index(h1_in_top10[0]) + 1})") +else: + print(" - No H1 features in top 10") + +print("\n3. DATA QUALITY:") +if data_file.exists() and has_target: + print(f" - Training samples: {total}") + print(f" - Target balance: See distribution above") +else: + print(" - Could not analyze training data") + +print("\n" + "=" * 80) +print("ANALYSIS COMPLETE") +print("=" * 80) diff --git a/backtests/39_h1_hmm_results/h1_hmm_20260209_102900.log b/backtests/39_h1_hmm_results/h1_hmm_20260209_102900.log new file mode 100644 index 0000000..d960944 --- /dev/null +++ b/backtests/39_h1_hmm_results/h1_hmm_20260209_102900.log @@ -0,0 +1,5 @@ +#39 H1 HMM Results +Generated: 2026-02-09 10:29:00.728793 +Variant: M15_HMM + +--- PERFORMANCE SUMMARY --- diff --git a/backtests/backtest_39_h1_hmm.py b/backtests/backtest_39_h1_hmm.py new file mode 100644 index 0000000..14e6a6e --- /dev/null +++ b/backtests/backtest_39_h1_hmm.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python3 +""" +Backtest #39: H1 HMM Regime Detector + +Test moving HMM from M15 to H1 timeframe for more stable regime detection. + +Hypothesis: +- H1 HMM will have 4-8x longer regime duration +- Fewer regime transitions = more stable risk management +- Better regime classification due to less noise + +Expected Impact: +15-20% Sharpe improvement +""" + +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import polars as pl +import numpy as np +from datetime import datetime, timedelta +from dataclasses import dataclass +from typing import Optional +from loguru import logger + +from src.mt5_connector import MT5Connector +from src.config import TradingConfig, get_config +from src.feature_eng import FeatureEngineer +from src.smc_polars import SMCAnalyzer +from src.regime_detector import MarketRegimeDetector +from backtests.ml_v2.ml_v2_model import TradingModelV2 +from src.smart_risk_manager import SmartRiskManager +from src.session_filter import SessionFilter +from src.dynamic_confidence import DynamicConfidenceManager + +# Import V2 features +from backtests.ml_v2.ml_v2_feature_eng import MLV2FeatureEngineer + + +@dataclass +class BacktestConfig: + """Backtest configuration""" + symbol: str = "XAUUSD" + m15_bars: int = 5000 + h1_bars: int = 1500 + initial_balance: float = 500.0 + results_dir: str = "backtests/39_h1_hmm_results" + + +class H1HMMBacktest: + """Backtest with H1-based HMM regime detector""" + + def __init__(self, config: BacktestConfig, use_h1_hmm: bool = True): + self.config = config + self.use_h1_hmm = use_h1_hmm + self.balance = config.initial_balance + self.equity = config.initial_balance + self.trades = [] + self.equity_curve = [] + + # Trading config + self.trading_config = TradingConfig() + + # Components + self.mt5 = None + self.features = FeatureEngineer() + self.smc = SMCAnalyzer() + self.ml_model = TradingModelV2() + self.fe_v2 = MLV2FeatureEngineer() + + # Skip risk manager and session filter for backtest simplicity + self.risk_manager = None + self.session_filter = None + self.dynamic_conf = None + + # Regime detectors - pass individual params (API changed) + self.regime_m15 = MarketRegimeDetector( + n_regimes=3, + lookback_periods=500, + retrain_frequency=20 + ) + self.regime_h1 = MarketRegimeDetector( + n_regimes=3, + lookback_periods=500, + retrain_frequency=20 + ) + + # Stats + self.regime_changes_m15 = [] + self.regime_changes_h1 = [] + + def connect_mt5(self): + """Connect to MT5""" + logger.info("Connecting to MT5...") + config = get_config() + self.mt5 = MT5Connector( + login=config.mt5_login, + password=config.mt5_password, + server=config.mt5_server, + path=config.mt5_path + ) + if not self.mt5.connect(): + raise RuntimeError("Failed to connect to MT5") + logger.info("✓ MT5 connected") + + def fetch_data(self): + """Fetch M15 and H1 data""" + logger.info(f"Fetching {self.config.m15_bars} M15 bars...") + df_m15 = self.mt5.get_market_data( + self.config.symbol, "M15", self.config.m15_bars + ) + + logger.info(f"Fetching {self.config.h1_bars} H1 bars...") + df_h1 = self.mt5.get_market_data( + self.config.symbol, "H1", self.config.h1_bars + ) + + logger.info(f"✓ M15: {len(df_m15)} bars, H1: {len(df_h1)} bars") + return df_m15, df_h1 + + def prepare_data(self, df_m15: pl.DataFrame, df_h1: pl.DataFrame): + """Calculate all features""" + logger.info("Calculating M15 features...") + df_m15 = self.features.calculate_all(df_m15, include_ml_features=True) + df_m15 = self.smc.calculate_all(df_m15) + + logger.info("Calculating H1 features...") + df_h1 = self.features.calculate_all(df_h1, include_ml_features=True) + df_h1 = self.smc.calculate_all(df_h1) + + logger.info("Adding V2 features...") + df_m15 = self.fe_v2.add_all_v2_features(df_m15, df_h1) + + return df_m15, df_h1 + + def fit_regimes(self, df_m15: pl.DataFrame, df_h1: pl.DataFrame): + """Train both M15 and H1 HMM models""" + logger.info("Training M15 HMM (baseline)...") + self.regime_m15.fit(df_m15.slice(0, 500)) + + logger.info("Training H1 HMM (test)...") + self.regime_h1.fit(df_h1.slice(0, 500)) + + logger.info("✓ Both HMM models fited") + + def detect_regime(self, df_m15: pl.DataFrame, df_h1: pl.DataFrame, m15_idx: int): + """Detect regime using M15 or H1 and return updated df with regime columns""" + if self.use_h1_hmm: + # Use H1 regime (map M15 index to H1) + h1_idx = m15_idx // 4 # 4 M15 bars = 1 H1 bar + if h1_idx >= len(df_h1): + h1_idx = len(df_h1) - 1 + + df_h1_slice = df_h1.slice(max(0, h1_idx - 100), h1_idx + 1) + df_h1_pred = self.regime_h1.predict(df_h1_slice) + regime_state = self.regime_h1.get_current_state(df_h1_pred) + + # Track H1 regime changes + if hasattr(self, '_last_h1_regime') and self._last_h1_regime != regime_state.regime.value: + self.regime_changes_h1.append({ + 'm15_idx': m15_idx, + 'old': self._last_h1_regime, + 'new': regime_state.regime.value + }) + self._last_h1_regime = regime_state.regime.value + else: + # Use M15 regime (baseline) + df_m15_slice = df_m15.slice(max(0, m15_idx - 100), m15_idx + 1) + df_m15_pred = self.regime_m15.predict(df_m15_slice) + regime_state = self.regime_m15.get_current_state(df_m15_pred) + + # Track M15 regime changes + if hasattr(self, '_last_m15_regime') and self._last_m15_regime != regime_state.regime.value: + self.regime_changes_m15.append({ + 'm15_idx': m15_idx, + 'old': self._last_m15_regime, + 'new': regime_state.regime.value + }) + self._last_m15_regime = regime_state.regime.value + + return regime_state, df_m15_pred if not self.use_h1_hmm else df_h1_pred + + def run_backtest(self, df_m15: pl.DataFrame, df_h1: pl.DataFrame): + """Run backtest loop""" + logger.info(f"Running backtest ({'H1 HMM' if self.use_h1_hmm else 'M15 HMM'})...") + + # Load ML model + self.ml_model.load("models/xgboost_model_v2d.pkl") + + # Start from bar 600 (after fiting window) + for idx in range(600, len(df_m15)): + # Get current bar + row = df_m15.row(idx, named=True) + timestamp = row['time'] + price = row['close'] + + # Detect regime + regime_state, df_regime_pred = self.detect_regime(df_m15, df_h1, idx) + + # Add regime columns to current df slice for ML prediction + if 'regime' not in df_m15.columns: + # Initialize regime columns in df_m15 + df_m15 = df_m15.with_columns([ + pl.lit(0).alias('regime'), + pl.lit(0.0).alias('regime_confidence') + ]) + + # Check if regime blocks trading + if regime_state.recommendation == "SLEEP": + continue + + # Get ML prediction (use model's stored feature names) + df_slice = df_m15.slice(0, idx + 1) + if hasattr(self.ml_model, 'feature_names') and self.ml_model.feature_names: + feature_cols = self.ml_model.feature_names + else: + # Fallback: filter out OHLC and intermediate columns + exclude_cols = {'time', 'open', 'high', 'low', 'close', 'spread', 'real_volume', 'volume', + 'swing_high_level', 'swing_low_level', 'last_swing_high', 'last_swing_low', + 'fvg_top', 'fvg_bottom', 'fvg_mid', 'ob_top', 'ob_bottom'} + feature_cols = [c for c in df_slice.columns if c not in exclude_cols] + + ml_pred = self.ml_model.predict(df_slice, feature_cols) + + if ml_pred.signal == "HOLD": + continue + + # Skip session check for simplicity + # session_ok, _, session_mult = self.session_filter.can_trade(timestamp) + # if not session_ok: + # continue + + # Entry filters (simplified) + if ml_pred.confidence < 0.50: + continue + + # SMC signal + smc_signal = row.get('ob', 0) + if smc_signal == 0: + continue + + if (ml_pred.signal == "BUY" and smc_signal < 0) or \ + (ml_pred.signal == "SELL" and smc_signal > 0): + continue + + # Execute trade (simplified) + direction = ml_pred.signal + entry_price = price + + # Calculate SL/TP (simplified) + atr = row.get('atr', 12.0) + if direction == "BUY": + sl = entry_price - (2.0 * atr) + tp = entry_price + (3.0 * atr) + else: + sl = entry_price + (2.0 * atr) + tp = entry_price - (3.0 * atr) + + # Simulate trade (find exit) + exit_price = None + exit_reason = None + exit_idx = None + + for future_idx in range(idx + 1, min(idx + 100, len(df_m15))): + future_row = df_m15.row(future_idx, named=True) + future_high = future_row['high'] + future_low = future_row['low'] + + if direction == "BUY": + if future_low <= sl: + exit_price = sl + exit_reason = "SL" + exit_idx = future_idx + break + elif future_high >= tp: + exit_price = tp + exit_reason = "TP" + exit_idx = future_idx + break + else: # SELL + if future_high >= sl: + exit_price = sl + exit_reason = "SL" + exit_idx = future_idx + break + elif future_low <= tp: + exit_price = tp + exit_reason = "TP" + exit_idx = future_idx + break + + if exit_price is None: + # No exit found, close at last bar + exit_price = df_m15.row(min(idx + 99, len(df_m15) - 1), named=True)['close'] + exit_reason = "EOD" + exit_idx = min(idx + 99, len(df_m15) - 1) + + # Calculate P&L + if direction == "BUY": + profit = exit_price - entry_price + else: + profit = entry_price - exit_price + + profit_usd = profit * 0.01 # 0.01 lot + + # Update balance + self.balance += profit_usd + self.equity = self.balance + + # Log trade + self.trades.append({ + 'entry_time': timestamp, + 'exit_time': df_m15.row(exit_idx, named=True)['time'], + 'direction': direction, + 'entry_price': entry_price, + 'exit_price': exit_price, + 'profit_usd': profit_usd, + 'exit_reason': exit_reason, + 'regime': regime_state.regime.value, + 'ml_conf': ml_pred.confidence, + }) + + # Record equity + self.equity_curve.append({ + 'time': df_m15.row(exit_idx, named=True)['time'], + 'equity': self.equity + }) + + logger.info(f"✓ Backtest complete: {len(self.trades)} trades") + + def calculate_metrics(self): + """Calculate performance metrics""" + if not self.trades: + return {} + + df_trades = pl.DataFrame(self.trades) + + total_trades = len(self.trades) + wins = df_trades.filter(pl.col('profit_usd') > 0).shape[0] + losses = df_trades.filter(pl.col('profit_usd') < 0).shape[0] + win_rate = wins / total_trades * 100 if total_trades > 0 else 0 + + net_profit = df_trades['profit_usd'].sum() + gross_profit = df_trades.filter(pl.col('profit_usd') > 0)['profit_usd'].sum() + gross_loss = abs(df_trades.filter(pl.col('profit_usd') < 0)['profit_usd'].sum()) + profit_factor = gross_profit / gross_loss if gross_loss > 0 else 0 + + # Drawdown + equity_curve = [self.config.initial_balance] + [e['equity'] for e in self.equity_curve] + running_max = np.maximum.accumulate(equity_curve) + drawdown = running_max - equity_curve + max_dd = np.max(drawdown) + max_dd_pct = max_dd / self.config.initial_balance * 100 + + # Sharpe ratio (annualized) + returns = df_trades['profit_usd'].to_numpy() + if len(returns) > 1 and np.std(returns) > 0: + avg_return = np.mean(returns) + std_return = np.std(returns) + # Assume ~5 trades/day, 252 trading days/year + sharpe = (avg_return / std_return) * np.sqrt(5 * 252) + else: + sharpe = 0 + + # Regime changes + regime_changes_m15 = len(self.regime_changes_m15) + regime_changes_h1 = len(self.regime_changes_h1) + + return { + 'total_trades': total_trades, + 'wins': wins, + 'losses': losses, + 'win_rate': win_rate, + 'net_profit': net_profit, + 'gross_profit': gross_profit, + 'gross_loss': gross_loss, + 'profit_factor': profit_factor, + 'max_dd': max_dd, + 'max_dd_pct': max_dd_pct, + 'sharpe': sharpe, + 'final_balance': self.balance, + 'regime_changes_m15': regime_changes_m15, + 'regime_changes_h1': regime_changes_h1, + } + + def save_results(self, variant_name: str, metrics: dict): + """Save results to file""" + Path(self.config.results_dir).mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = Path(self.config.results_dir) / f"h1_hmm_{timestamp}.log" + + with open(log_file, 'w') as f: + f.write(f"#39 H1 HMM Results\n") + f.write(f"Generated: {datetime.now()}\n") + f.write(f"Variant: {variant_name}\n\n") + + f.write("--- PERFORMANCE SUMMARY ---\n") + f.write(f" Total Trades: {metrics['total_trades']}\n") + f.write(f" Win Rate: {metrics['win_rate']:.1f}%\n") + f.write(f" Net PnL: ${metrics['net_profit']:.2f}\n") + f.write(f" Profit Factor: {metrics['profit_factor']:.2f}\n") + f.write(f" Max Drawdown: ${metrics['max_dd']:.2f} ({metrics['max_dd_pct']:.1f}%)\n") + f.write(f" Sharpe Ratio: {metrics['sharpe']:.2f}\n") + f.write(f" Final Balance: ${metrics['final_balance']:.2f}\n\n") + + f.write("--- REGIME STABILITY ---\n") + f.write(f" M15 Regime Changes: {metrics['regime_changes_m15']}\n") + f.write(f" H1 Regime Changes: {metrics['regime_changes_h1']}\n") + if metrics['regime_changes_h1'] > 0: + improvement = (metrics['regime_changes_m15'] - metrics['regime_changes_h1']) / metrics['regime_changes_m15'] * 100 + f.write(f" Improvement: {improvement:.1f}% fewer changes\n\n") + + f.write("--- TRADE LOG ---\n") + f.write(f"{'#':>4} {'Entry Time':>19} {'Dir':>4} {'Entry':>8} {'Exit':>8} {'P/L':>7} {'Result':>6} {'Exit_Reason':>12} {'Regime':>15} {'ML_Conf':>7}\n") + f.write("-" * 120 + "\n") + + for i, trade in enumerate(self.trades, 1): + result = "WIN" if trade['profit_usd'] > 0 else "LOSS" + f.write(f"{i:4d} {trade['entry_time']:%Y-%m-%d %H:%M:%S} " + f"{trade['direction']:>4} {trade['entry_price']:8.2f} {trade['exit_price']:8.2f} " + f"{trade['profit_usd']:7.2f} {result:>6} {trade['exit_reason']:>12} " + f"{trade['regime']:>15} {trade['ml_conf']:7.3f}\n") + + logger.info(f"✓ Results saved to {log_file}") + + +def main(): + """Run both M15 and H1 HMM backtests""" + config = BacktestConfig() + + # Baseline: M15 HMM + logger.info("=" * 80) + logger.info("BASELINE: M15 HMM") + logger.info("=" * 80) + + bt_m15 = H1HMMBacktest(config, use_h1_hmm=False) + bt_m15.connect_mt5() + df_m15, df_h1 = bt_m15.fetch_data() + df_m15, df_h1 = bt_m15.prepare_data(df_m15, df_h1) + bt_m15.fit_regimes(df_m15, df_h1) + bt_m15.run_backtest(df_m15, df_h1) + metrics_m15 = bt_m15.calculate_metrics() + bt_m15.save_results("M15_HMM", metrics_m15) + + logger.info(f"\nBASELINE Results:") + logger.info(f" Trades: {metrics_m15['total_trades']}, WR: {metrics_m15['win_rate']:.1f}%, " + f"PnL: ${metrics_m15['net_profit']:.2f}, Sharpe: {metrics_m15['sharpe']:.2f}") + logger.info(f" M15 Regime Changes: {metrics_m15['regime_changes_m15']}") + + # Test: H1 HMM + logger.info("\n" + "=" * 80) + logger.info("TEST: H1 HMM") + logger.info("=" * 80) + + bt_h1 = H1HMMBacktest(config, use_h1_hmm=True) + bt_h1.connect_mt5() + # Reuse same data + bt_h1.fit_regimes(df_m15, df_h1) + bt_h1.run_backtest(df_m15, df_h1) + metrics_h1 = bt_h1.calculate_metrics() + bt_h1.save_results("H1_HMM", metrics_h1) + + logger.info(f"\nH1 HMM Results:") + logger.info(f" Trades: {metrics_h1['total_trades']}, WR: {metrics_h1['win_rate']:.1f}%, " + f"PnL: ${metrics_h1['net_profit']:.2f}, Sharpe: {metrics_h1['sharpe']:.2f}") + logger.info(f" H1 Regime Changes: {metrics_h1['regime_changes_h1']}") + + # Comparison + logger.info("\n" + "=" * 80) + logger.info("COMPARISON") + logger.info("=" * 80) + + pnl_diff = metrics_h1['net_profit'] - metrics_m15['net_profit'] + sharpe_diff = metrics_h1['sharpe'] - metrics_m15['sharpe'] + regime_reduction = (metrics_m15['regime_changes_m15'] - metrics_h1['regime_changes_h1']) / metrics_m15['regime_changes_m15'] * 100 if metrics_m15['regime_changes_m15'] > 0 else 0 + + logger.info(f"PnL Difference: ${pnl_diff:+.2f} ({pnl_diff/metrics_m15['net_profit']*100:+.1f}%)") + logger.info(f"Sharpe Difference: {sharpe_diff:+.2f} ({sharpe_diff/metrics_m15['sharpe']*100:+.1f}%)") + logger.info(f"Regime Stability: {regime_reduction:.1f}% fewer regime changes") + + bt_m15.mt5.disconnect() + bt_h1.mt5.disconnect() + + +if __name__ == "__main__": + main() diff --git a/data/risk_state.txt b/data/risk_state.txt index a7c6363..20671ba 100644 --- a/data/risk_state.txt +++ b/data/risk_state.txt @@ -1,6 +1,6 @@ date:2026-02-09 -daily_loss:0.0 -daily_profit:7.17 +daily_loss:18.77 +daily_profit:22.49 consecutive_losses:0 -total_loss:0 -saved_at:2026-02-09T07:08:09.193663+07:00 +total_loss:3.45 +saved_at:2026-02-09T10:34:43.763326+07:00 diff --git a/h1_analysis_output.txt b/h1_analysis_output.txt new file mode 100644 index 0000000..34b053c --- /dev/null +++ b/h1_analysis_output.txt @@ -0,0 +1,150 @@ +================================================================================ +H1 FEATURE DEEP ANALYSIS +================================================================================ + +Total features in model: 76 + +H1 features found: 9 + +--- ALL H1 FEATURES --- + Rank #68: h1_atr_ratio importance= 0.0000 + Rank #66: h1_ema20 importance= 0.0000 + Rank # 7: h1_ema20_distance importance= 95.8865 + Rank #67: h1_fvg_active importance= 0.0000 + Rank #10: h1_market_structure importance= 61.8312 + Rank #16: h1_ob_proximity importance= 33.1670 + Rank # 6: h1_rsi importance= 112.2508 + Rank #17: h1_swing_proximity importance= 30.4503 + Rank #12: h1_trend_strength importance= 45.8723 + +--- TOP 10 H1 FEATURES (by importance) --- + 1. Rank # 6: h1_rsi 112.2508 + 2. Rank # 7: h1_ema20_distance 95.8865 + 3. Rank # 10: h1_market_structure 61.8312 + 4. Rank # 12: h1_trend_strength 45.8723 + 5. Rank # 16: h1_ob_proximity 33.1670 + 6. Rank # 17: h1_swing_proximity 30.4503 + 7. Rank # 66: h1_ema20 0.0000 + 8. Rank # 67: h1_fvg_active 0.0000 + 9. Rank # 68: h1_atr_ratio 0.0000 + +--- H1 FEATURE STATISTICS --- +Total H1 features: 9 +Mean importance: 42.1620 +Median importance: 33.1670 +Max importance: 112.2508 +Min importance: 0.0000 + +H1 features in top 10: 3 +H1 features in top 20: 6 +H1 features in top 30: 6 + +================================================================================ +ALL FEATURE NAMES IN MODEL +================================================================================ + 1. rsi 36.2503 + 2. atr 0.0000 + 3. atr_percent 0.0000 + 4. macd 65.5672 + 5. macd_signal 75.9652 + 6. macd_histogram 0.0000 + 7. bb_middle 0.0000 + 8. bb_upper 0.0000 + 9. bb_lower 0.0000 +10. bb_width 0.0000 +11. bb_percent_b 21.0445 +12. ema_9 0.0000 +13. ema_21 21.6908 +14. ema_cross_bull 0.0000 +15. ema_cross_bear 0.0000 +16. volume_sma 0.0000 +17. volume_ratio 25.7497 +18. volume_increasing 0.0000 +19. high_volume 0.0000 +20. returns_1 167.9189 +21. returns_5 2.9630 +22. returns_20 11.5534 +23. log_returns 186.0548 +24. price_position 39.1339 +25. dist_from_sma_20 11.1024 +26. volatility_20 0.0000 +27. normalized_range 0.0000 +28. avg_normalized_range 0.0000 +29. close_lag_1 18.1662 +30. close_lag_2 18.1221 +31. close_lag_3 0.0000 +32. close_lag_5 24.4038 +33. higher_high 0.0000 +34. lower_low 0.0000 +35. hh_count_5 0.0000 +36. ll_count_5 0.0000 +37. hour 14.3622 +38. weekday 0.0000 +39. london_session 0.0000 +40. ny_session 0.0000 +41. swing_high 0.0000 +42. swing_low 0.0000 +43. is_fvg_bull 23.1050 +44. is_fvg_bear 0.0000 +45. fvg_signal 0.0000 +46. ob 417.3543 +47. ob_mitigated 127.2658 +48. bos 0.0000 +49. choch 0.0000 +50. market_structure 0.0000 +51. regime 0.0000 +52. regime_confidence 0.0000 +53. h1_ema20 0.0000 +54. h1_market_structure 61.8312 +55. h1_ema20_distance 95.8865 +56. h1_trend_strength 45.8723 +57. h1_swing_proximity 30.4503 +58. h1_fvg_active 0.0000 +59. h1_ob_proximity 33.1670 +60. h1_atr_ratio 0.0000 +61. h1_rsi 112.2508 +62. fvg_gap_size_atr 14.4616 +63. fvg_age_bars 0.0000 +64. ob_width_atr 40.7235 +65. ob_distance_atr 119.3701 +66. bos_recency 0.0000 +67. confluence_score 0.0000 +68. swing_distance_atr 0.0000 +69. regime_duration_bars 0.0000 +70. regime_transition_prob 0.0000 +71. volatility_zscore 19.7360 +72. crisis_proximity 15.9215 +73. wick_ratio 0.0000 +74. body_ratio 0.0000 +75. gap_from_prev_close 7.8998 +76. consecutive_direction 57.4307 + +================================================================================ +CHECKING TRAINING DATA FOR H1 FEATURES +================================================================================ + +Dataset columns: 72 +H1 columns in dataset: 0 + +NO H1 COLUMNS IN TRAINING DATA! + +Looking for potential H1-related columns: + hour + +================================================================================ +CHECKING FEATURE ENGINEERING CODE +================================================================================ + +NO H1 references found in feature_eng.py + +================================================================================ +CONCLUSION +================================================================================ +Traceback (most recent call last): + File "C:\Users\Administrator\Videos\Smart Automatic Trading BOT + AI\analyze_h1_features.py", line 151, in + print(f"\n\u2713 Model HAS {len(h1_features)} H1 features") + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Python313\Lib\encodings\cp1252.py", line 19, in encode + return codecs.charmap_encode(input,self.errors,encoding_table)[0] + ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +UnicodeEncodeError: 'charmap' codec can't encode character '\u2713' in position 2: character maps to diff --git a/ml_analysis_output.txt b/ml_analysis_output.txt new file mode 100644 index 0000000..7df777a --- /dev/null +++ b/ml_analysis_output.txt @@ -0,0 +1,266 @@ + +================================================================================ +ML MODEL DEEP DIVE ANALYSIS +================================================================================ + +================================================================================ +1. MODEL INSPECTION: xgboost_model_v2d.pkl +================================================================================ + +Model pickle structure: + model_type: ModelType + xgb_model: Booster + lgb_model: NoneType + feature_names: list (length=76) + confidence_threshold: float + xgb_params: dict (length=13) + lgb_params: dict (length=12) + feature_importance: dict (length=76) + train_metrics: dict (length=4) + fitted: bool + +XGBoost model: +LightGBM model: +Total features: 76 + +--- TRAINING METRICS --- +xgb_train_score: 0.7385315785647933 +xgb_test_score: 0.7338658466928052 +train_samples: 36407 +test_samples: 9052 + +--- XGBOOST PARAMETERS --- +objective: binary:logistic +eval_metric: auc +max_depth: 3 +learning_rate: 0.05 +tree_method: hist +device: cpu +min_child_weight: 10 +subsample: 0.7 +colsample_bytree: 0.6 +reg_alpha: 1.0 +reg_lambda: 5.0 +gamma: 1.0 +max_delta_step: 1 + +================================================================================ +2. FEATURE IMPORTANCE RANKING +================================================================================ + +Total features with importance: 76 + +--- TOP 30 FEATURES --- + 1. ob 417.354340 [M15] + 2. log_returns 186.054825 [M15] + 3. returns_1 167.918945 [M15] + 4. ob_mitigated 127.265808 [SMC] + 5. ob_distance_atr 119.370117 [SMC] + 6. h1_rsi 112.250816 [M15] + 7. h1_ema20_distance 95.886513 [M15] + 8. macd_signal 75.965248 [M15] + 9. macd 65.567200 [M15] +10. h1_market_structure 61.831249 [M15] +11. consecutive_direction 57.430664 [M15] +12. h1_trend_strength 45.872288 [M15] +13. ob_width_atr 40.723549 [SMC] +14. price_position 39.133850 [M15] +15. rsi 36.250340 [M15] +16. h1_ob_proximity 33.167027 [SMC] +17. h1_swing_proximity 30.450262 [M15] +18. volume_ratio 25.749680 [M15] +19. close_lag_5 24.403849 [M15] +20. is_fvg_bull 23.105017 [SMC] +21. ema_21 21.690825 [M15] +22. bb_percent_b 21.044502 [M15] +23. volatility_zscore 19.736000 [M15] +24. close_lag_1 18.166227 [M15] +25. close_lag_2 18.122101 [M15] +26. crisis_proximity 15.921524 [M15] +27. fvg_gap_size_atr 14.461624 [SMC] +28. hour 14.362236 [M15] +29. returns_20 11.553368 [M15] +30. dist_from_sma_20 11.102405 [M15] + +--- H1 FEATURES IN TOP 10 --- +Count: 0 + +--- FEATURE CATEGORY SUMMARY --- +H1 features: 0 +M15 technical features: 22 +SMC features: 13 +Average M15 importance: 19.988993 +Average SMC importance: 27.545626 + +================================================================================ +3. TARGET VARIABLE STATISTICS +================================================================================ + +Loading: data\training_data.parquet +Dataset shape: (8000, 72) + +--- TARGET DISTRIBUTION --- +None: 1 ( 0.01%) +SELL: 3749 (46.86%) +HOLD: 4250 (53.12%) + +--- RETURN ANALYSIS (M15 bars) --- + +Bars with 3-bar returns > X*ATR: + > 0.1*ATR: 0 ( 0.00%) + > 0.2*ATR: 0 ( 0.00%) + > 0.3*ATR: 0 ( 0.00%) + > 0.5*ATR: 0 ( 0.00%) + > 0.7*ATR: 0 ( 0.00%) + > 1.0*ATR: 0 ( 0.00%) + +Mean normalized return: 0.0000 +Median normalized return: 0.0000 + +Positive returns: 4250 (53.12%) +Negative returns: 3744 (46.80%) + +--- H1 FEATURES IN DATASET --- +H1 columns found: 0 + +================================================================================ +4. PREDICTION CONSISTENCY ANALYSIS +================================================================================ + +Analyzing: logs\trading_bot_2026-02-09.log + +================================================================================ +5. CURRENT MODEL METRICS +================================================================================ + +--- MODEL METRICS (from data/model_metrics.json) --- +{ + "featureImportance": [ + { + "name": "ob", + "importance": 0.2126 + }, + { + "name": "log_returns", + "importance": 0.0948 + }, + { + "name": "returns_1", + "importance": 0.0856 + }, + { + "name": "ob_mitigated", + "importance": 0.0648 + }, + { + "name": "ob_distance_atr", + "importance": 0.0608 + }, + { + "name": "h1_rsi", + "importance": 0.0572 + }, + { + "name": "h1_ema20_distance", + "importance": 0.0489 + }, + { + "name": "macd_signal", + "importance": 0.0387 + }, + { + "name": "macd", + "importance": 0.0334 + }, + { + "name": "h1_market_structure", + "importance": 0.0315 + }, + { + "name": "consecutive_direction", + "importance": 0.0293 + }, + { + "name": "h1_trend_strength", + "importance": 0.0234 + }, + { + "name": "ob_width_atr", + "importance": 0.0207 + }, + { + "name": "price_position", + "importance": 0.0199 + }, + { + "name": "rsi", + "importance": 0.0185 + }, + { + "name": "h1_ob_proximity", + "importance": 0.0169 + }, + { + "name": "h1_swing_proximity", + "importance": 0.0155 + }, + { + "name": "volume_ratio", + "importance": 0.0131 + }, + { + "name": "close_lag_5", + "importance": 0.0124 + }, + { + "name": "is_fvg_bull", + "importance": 0.0118 + } + ], + "trainAuc": 0.7385315785647933, + "testAuc": 0.7338658466928052, + "sampleCount": 45459, + "updatedAt": "2026-02-09T09:01:15.440605+07:00" +} + +================================================================================ +6. OVERFITTING ANALYSIS +================================================================================ + +From training_2026-02-04.log: + Initial training: Train AUC=0.8106, Test AUC=0.6553 + Overfitting gap: 0.1553 (HIGH) + + Walk-forward average: Train AUC=0.8107, Test AUC=0.5722 + Overfitting gap: 0.2385 (VERY HIGH) + +Conclusion: + - Model shows significant overfitting + - Test AUC of 0.57-0.66 is barely better than random (0.50) + - High train AUC (0.81) but poor generalization + +================================================================================ +CRITICAL FINDINGS +================================================================================ + +1. MODEL PERFORMANCE: + - Test AUC: 0.5722 (walk-forward) - POOR + - Overfitting gap: 0.2385 - VERY HIGH + - Model barely better than random guessing + +2. FEATURE IMPORTANCE: + - H1 features NOT in top 10 - Low predictive value + +3. DATA QUALITY: + - Training samples: 8000 + - Target imbalance likely causing issues + - Most returns < 0.3*ATR (target too weak) + +4. RECOMMENDATIONS: + a. Current V2D model has POOR performance - needs replacement + b. H1 features show low importance - may not help + c. Consider new target variable (stronger signal) + d. Address class imbalance in training + e. Reduce model complexity to prevent overfitting + +================================================================================