From 15b319190155a983a96e0044cb0c7e9c99391445 Mon Sep 17 00:00:00 2001 From: Arian Haghparast Date: Wed, 17 Jun 2026 17:12:19 -0400 Subject: [PATCH] Add project scaffold with config and dependencies --- .gitignore | 49 ++ README.md | 173 ++++ requirements.txt | 35 + tradingbot/__init__.py | 11 + tradingbot/backtest/__init__.py | 4 + tradingbot/backtest/metrics.py | 60 ++ tradingbot/backtest/walk_forward.py | 280 +++++++ tradingbot/config.py | 18 + tradingbot/data/__init__.py | 12 + tradingbot/data/features.py | 485 +++++++++++ tradingbot/data/loader.py | 55 ++ tradingbot/data/oanda_connector.py | 113 +++ tradingbot/data/processor.py | 140 ++++ tradingbot/models/__init__.py | 10 + tradingbot/models/model_manager.py | 1149 ++++++++++++++++++++++++++ tradingbot/models/neural_ensemble.py | 340 ++++++++ tradingbot/models/tree_ensemble.py | 562 +++++++++++++ tradingbot/signals/__init__.py | 3 + tradingbot/signals/generator.py | 383 +++++++++ tradingbot/viz/__init__.py | 3 + tradingbot/viz/visualize.py | 1008 ++++++++++++++++++++++ 21 files changed, 4893 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 tradingbot/__init__.py create mode 100644 tradingbot/backtest/__init__.py create mode 100644 tradingbot/backtest/metrics.py create mode 100644 tradingbot/backtest/walk_forward.py create mode 100644 tradingbot/config.py create mode 100644 tradingbot/data/__init__.py create mode 100644 tradingbot/data/features.py create mode 100644 tradingbot/data/loader.py create mode 100644 tradingbot/data/oanda_connector.py create mode 100644 tradingbot/data/processor.py create mode 100644 tradingbot/models/__init__.py create mode 100644 tradingbot/models/model_manager.py create mode 100644 tradingbot/models/neural_ensemble.py create mode 100644 tradingbot/models/tree_ensemble.py create mode 100644 tradingbot/signals/__init__.py create mode 100644 tradingbot/signals/generator.py create mode 100644 tradingbot/viz/__init__.py create mode 100644 tradingbot/viz/visualize.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b72fa7a --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# Byte-compiled / optimized / cache +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +.venv/ +venv/ +env/ +ENV/ + +# Distribution / packaging +build/ +dist/ +*.egg-info/ +.eggs/ + +# Secrets / environment +.env +*.env + +# Trained model artifacts (root-level only; not tradingbot/models/ package) +/models/ +*.pkl +*.joblib +*.pth +*.cbm + +# Data files (don't commit large market data) +*.csv +data/*.csv +*.parquet + +# Generated plots +*.png +*.jpg +*.svg + +# Notebook checkpoints +.ipynb_checkpoints/ + +# Optuna / experiment DBs +*.db + +# IDE / OS +.idea/ +.vscode/ +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..df34fc5 --- /dev/null +++ b/README.md @@ -0,0 +1,173 @@ +# tradingbot + +A research framework for **5-minute gold / forex machine-learning trading**. + +It covers the full pipeline: downloading and cleaning candle data, engineering +technical features, training several model architectures, turning predictions +into risk-managed signals, and backtesting them with walk-forward optimization. + +--- + +## Project structure + +``` +tradingbot/ +├── tradingbot/ +│ ├── config.py # central config (reads secrets from env) +│ ├── data/ +│ │ ├── oanda_connector.py # OANDA v20 historical candle download +│ │ ├── loader.py # CSV loading + time-gap analysis +│ │ ├── processor.py # GoldDataProcessor: cleaning, gap fill, outliers +│ │ └── features.py # GoldFeatureEngineer + GoldFeaturePipeline +│ ├── models/ +│ │ ├── tree_ensemble.py # TreeEnsemblePredictor (stacked trees + HMM) +│ │ ├── neural_ensemble.py # NeuralEnsemblePredictor (Transformer/LSTM + Bayesian NN) +│ │ └── model_manager.py # GoldModelManager (Optuna, Boruta, SHAP, 5-class) +│ ├── signals/ +│ │ └── generator.py # GoldSignalGenerator (signals + stops/targets) +│ ├── backtest/ +│ │ ├── walk_forward.py # WalkForwardOptimizer +│ │ └── metrics.py # compute_performance_metrics +│ └── viz/ +│ └── visualize.py # visualize_signals + plotting helpers +├── notebooks/ # exploratory research notebook +├── requirements.txt +└── .gitignore +``` + +## Model approaches + +The framework ships three model architectures that can be used independently: + +1. `**TreeEnsemblePredictor**` (`models/tree_ensemble.py`) — a binary up/down + classifier that stacks XGBoost, LightGBM, CatBoost, RandomForest and + ExtraTrees, with two meta-models and an HMM market-regime filter. +2. `**NeuralEnsemblePredictor**` (`models/neural_ensemble.py`) — stacks + gradient-boosted trees with a Transformer + BiLSTM network and a Bayesian + network (Monte-Carlo dropout) for uncertainty-aware filtering. +3. `**GoldModelManager` pipeline** (`models/model_manager.py` + + `signals/generator.py` + `backtest/walk_forward.py`) — the most complete + path: a 5-class classifier (strong/weak up, sideways, weak/strong down) with + Boruta/RFE/PCA feature selection, Optuna tuning, SHAP analysis, volatility + regime-aware ensembling, and walk-forward backtesting. + +--- + +## Installation + +```bash +pip install -r requirements.txt +``` + +**TA-Lib** also requires the underlying C library: + +- macOS: `brew install ta-lib` +- Ubuntu/Debian: install `ta-lib` (package or build from source) +- Windows: install a prebuilt TA-Lib wheel / binaries + +For live data downloads, set your OANDA token (never commit it): + +```bash +export OANDA_ACCESS_TOKEN="your-token-here" +``` + +--- + +## Usage + +### 1. Download data (optional — or bring your own OHLCV CSV) + +```python +from tradingbot.data.oanda_connector import download + +# Saves e.g. EURUSD_M5__to_.csv in the current directory. +download(instrument="EUR_USD", timeframe="M5", lookback_days=1825) +``` + +A CSV is expected with a `timestamp` column plus `open, high, low, close, volume`. + +### 2. Quick look at the data + +```python +from tradingbot.data.loader import load_ohlcv_csv, summarize, analyze_time_gaps + +data = load_ohlcv_csv("EURUSD_M5_20200309_to_20250308.csv") +summarize(data) +analyze_time_gaps(data) +``` + +### 3a. Tree-ensemble approach + +```python +from tradingbot.models.tree_ensemble import run_model + +predictor, metrics, results, returns = run_model( + data_path="XAUUSD_M5_20200222_to_20250220.csv", + forecast_bars=24, # 2 hours of 5-minute bars + confidence_threshold=0.67, +) +``` + +Visualize the result: + +```python +from tradingbot.viz.visualize import visualize_signals + +visualize_signals(data, results, returns, metrics, save_path="signals.png") +``` + +### 3b. Neural-ensemble approach + +```python +import pandas as pd +from tradingbot.models.neural_ensemble import NeuralEnsemblePredictor + +data = pd.read_csv("XAUUSD_H1_...csv", parse_dates=["timestamp"], index_col="timestamp") +train, test = data[:int(len(data) * 0.9)], data[int(len(data) * 0.9):] + +predictor = NeuralEnsemblePredictor(forecast_period=12, confidence_threshold=0.6) +predictor.fit(train) +signals = predictor.predict(test) +``` + +### 3c. Full pipeline + walk-forward backtest + +```python +from tradingbot.data.features import GoldFeaturePipeline +from tradingbot.backtest.walk_forward import WalkForwardOptimizer + +# Engineer features and the 5-class target. +processed = GoldFeaturePipeline(forecast_horizon=6).process( + "XAUUSD_M5_20200222_to_20250220.csv" +) + +# Walk-forward optimization (windows are in days). +wf = WalkForwardOptimizer( + train_window=35, + step_size=5, + test_window=5, + feature_selection_interval=4, +) +results = wf.optimize(processed) +``` + +Generate signals from a trained model: + +```python +from tradingbot.models.model_manager import GoldModelManager +from tradingbot.signals.generator import GoldSignalGenerator + +manager = GoldModelManager(model_path="models/step_1", feature_selection_method="importance") +manager.load_models() + +gen = GoldSignalGenerator(confidence_threshold=0.7, risk_reward_min=1.5, model_manager=manager) +signals = gen.generate_signals(processed.iloc[-2880:]) # last ~10 days +active = signals[signals["signal"] != 0] +``` + +--- + +## Disclaimer + +This is research / educational code for exploring ML trading ideas on historical +data. It is **not** financial advice and makes no guarantee of profitability. Markets carry real risk of loss, validate any strategy thoroughly before risking capital. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4c89e58 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,35 @@ +# Core +numpy +pandas +scipy +scikit-learn +matplotlib +seaborn +joblib + +# Gradient boosting / tree models +xgboost +lightgbm +catboost + +# Regime detection +hmmlearn + +# Feature selection / hyperparameter tuning / explainability +Boruta +optuna +scikit-optimize +shap + +# Deep learning (neural ensemble) +torch + +# Technical indicators +# NOTE: TA-Lib also needs the underlying C library installed on your system. +# macOS: brew install ta-lib +# Ubuntu: apt-get install ta-lib (or build from source) +# Windows: install a prebuilt wheel / the ta-lib binaries +TA-Lib + +# Live data download (optional) +oandapyV20 diff --git a/tradingbot/__init__.py b/tradingbot/__init__.py new file mode 100644 index 0000000..262788c --- /dev/null +++ b/tradingbot/__init__.py @@ -0,0 +1,11 @@ +"""tradingbot: a 5-minute gold/forex ML trading research framework. + +Sub-packages: + data - data download, loading, cleaning and feature engineering + models - the three model approaches (tree ensemble, neural ensemble, manager) + signals - turning model predictions into risk-managed trade signals + backtest - walk-forward optimization and performance metrics + viz - signal / performance visualization +""" + +__version__ = "0.1.0" diff --git a/tradingbot/backtest/__init__.py b/tradingbot/backtest/__init__.py new file mode 100644 index 0000000..252af1a --- /dev/null +++ b/tradingbot/backtest/__init__.py @@ -0,0 +1,4 @@ +from tradingbot.backtest.walk_forward import WalkForwardOptimizer +from tradingbot.backtest.metrics import compute_performance_metrics + +__all__ = ["WalkForwardOptimizer", "compute_performance_metrics"] diff --git a/tradingbot/backtest/metrics.py b/tradingbot/backtest/metrics.py new file mode 100644 index 0000000..bf409a6 --- /dev/null +++ b/tradingbot/backtest/metrics.py @@ -0,0 +1,60 @@ +"""Simple strategy performance metrics. + +Scores a ``signal`` column against forward returns: total return, annualized +Sharpe, win rate and max drawdown. +""" +import numpy as np + + +def compute_performance_metrics(data, signal_col="signal", forecast_periods=12, + annualization=24 * 365, verbose=True): + """Compute strategy performance from a signal column and forward returns. + + Parameters + ---------- + data : pd.DataFrame + Must contain a ``close`` column and ``signal_col`` (values in {-1, 0, 1}). + forecast_periods : int + Holding horizon (in bars) used to compute the forward return. + annualization : float + Factor applied under the square root when annualizing the Sharpe ratio. + + Returns + ------- + dict + total_return, sharpe_ratio, win_rate, max_drawdown and the augmented frame. + """ + df = data.copy() + + # Forward return over the forecast horizon. + df["future_return"] = df["close"].pct_change(periods=forecast_periods).shift(-forecast_periods) + + # Strategy return: signal * future return (long: +return, short: -return). + df["strategy_return"] = df[signal_col] * df["future_return"] + df = df.dropna(subset=["strategy_return"]) + + # Cumulative return. + df["cumulative_return"] = (1 + df["strategy_return"]).cumprod() - 1 + + total_return = df["cumulative_return"].iloc[-1] if len(df) else float("nan") + sharpe_ratio = ( + df["strategy_return"].mean() / df["strategy_return"].std() * np.sqrt(annualization) + if df["strategy_return"].std() else float("nan") + ) + active = df[df["strategy_return"] != 0] + win_rate = len(df[df["strategy_return"] > 0]) / len(active) if len(active) else float("nan") + max_drawdown = (df["cumulative_return"].cummax() - df["cumulative_return"]).max() + + if verbose: + print(f"Total Return: {total_return:.2%}") + print(f"Sharpe Ratio: {sharpe_ratio:.2f}") + print(f"Win Rate: {win_rate:.2%}") + print(f"Max Drawdown: {max_drawdown:.2%}") + + return { + "total_return": total_return, + "sharpe_ratio": sharpe_ratio, + "win_rate": win_rate, + "max_drawdown": max_drawdown, + "data": df, + } diff --git a/tradingbot/backtest/walk_forward.py b/tradingbot/backtest/walk_forward.py new file mode 100644 index 0000000..d9e8791 --- /dev/null +++ b/tradingbot/backtest/walk_forward.py @@ -0,0 +1,280 @@ +"""Walk-forward optimization harness.""" +import numpy as np +import pandas as pd +from pathlib import Path + +from tradingbot.models.model_manager import GoldModelManager +from tradingbot.signals.generator import GoldSignalGenerator + + +class WalkForwardOptimizer: + """ + Implements walk-forward optimization for model training and validation + """ + def __init__( + self, + train_window=120, # Initial training days (approx. 34560 5-min bars) + step_size=20, # Days to move forward in each step (approx. 5760 bars) + feature_selection_interval=3, # How often to perform feature selection + test_window=10, # Days to test on after each training (approx. 2880 bars) + n_jobs=-1, + model_path='models', + random_state=42 + ): + """ + Initialize the walk-forward optimizer + + Parameters: + ----------- + train_window : int + Number of days for initial training window + step_size : int + Number of days to move forward in each step + feature_selection_interval : int + How many steps between feature selection (to save time) + test_window : int + Number of days to test on after each training + n_jobs : int + Number of parallel jobs for training + model_path : str + Directory to save models + """ + self.train_window = train_window + self.step_size = step_size + self.feature_selection_interval = feature_selection_interval + self.test_window = test_window + self.n_jobs = n_jobs + self.model_path = Path(model_path) + self.model_path.mkdir(exist_ok=True) + self.random_state = random_state + + # Initialize metrics storage + self.metrics = [] + self.signals = [] + + def optimize(self, data, min_train_size=8000): + """ + Perform walk-forward optimization with improved error handling + + Parameters: + ----------- + data : pd.DataFrame + DataFrame with features and target + min_train_size : int + Minimum number of samples required for training + + Returns: + -------- + results : dict + Walk-forward optimization results + """ + # Convert time windows from days to number of 5-minute bars + # Assuming 288 5-minute bars per day (24 hours) + bars_per_day = 288 + train_bars = self.train_window * bars_per_day + step_bars = self.step_size * bars_per_day + test_bars = self.test_window * bars_per_day + + # Ensure data has a datetime index + if not isinstance(data.index, pd.DatetimeIndex): + raise ValueError("Data must have a DatetimeIndex") + + # Calculate number of steps + total_bars = len(data) + n_steps = max(1, (total_bars - train_bars) // step_bars) + + print(f"Starting walk-forward optimization with {n_steps} steps") + + # Initialize results storage + all_signals = pd.DataFrame() + all_metrics = [] + + # Track whether to do feature selection in this step + do_feature_selection = True + selected_features = None + + # For each step + for step in range(n_steps): + print(f"\n{'-'*50}") + print(f"Step {step+1}/{n_steps}") + print(f"{'-'*50}") + + try: + # Calculate indices for this step + train_start = step * step_bars + train_end = train_start + train_bars + test_start = train_end + test_end = min(test_start + test_bars, total_bars) + + # Get data for this step + train_data = data.iloc[train_start:train_end] + test_data = data.iloc[test_start:test_end] + + print(f"Train period: {train_data.index[0]} to {train_data.index[-1]}") + print(f"Test period: {test_data.index[0]} to {test_data.index[-1]}") + + # Skip if not enough training data + if len(train_data) < min_train_size: + print(f"Skipping step {step+1} - not enough training data ({len(train_data)} < {min_train_size})") + continue + + # Create model manager for this step + model_dir = self.model_path / f'step_{step+1}' + model_dir.mkdir(exist_ok=True, parents=True) + + model_manager = GoldModelManager( + n_splits=3, + feature_selection_method='boruta' if do_feature_selection else 'importance', + max_features=25, + ensemble_models=3, + volatility_based_models=True, + n_jobs=self.n_jobs, + model_path=model_dir, + random_state=self.random_state + step # Vary random state by step + ) + + # Extract y_train for class distribution check before training + X_train, y_train = model_manager.prepare_data(train_data) + unique_classes = np.unique(y_train) + class_counts = {cls: np.sum(y_train == cls) for cls in unique_classes} + + print("Class distribution in training data:") + expected_classes = set(range(-2, 3)) # -2, -1, 0, 1, 2 + missing_classes = expected_classes - set(unique_classes) + + for cls in sorted(expected_classes): + count = class_counts.get(cls, 0) + percentage = (count / len(y_train)) * 100 if len(y_train) > 0 else 0 + status = "PRESENT" if cls in unique_classes else "MISSING" + print(f" Class {cls}: {count} samples ({percentage:.2f}%) - {status}") + + if missing_classes: + print(f"Warning: Missing classes in training data: {missing_classes}") + print("Continuing anyway with appropriate class weights...") + + # Set selected features if not doing feature selection + if not do_feature_selection and selected_features is not None: + model_manager.selected_features = selected_features + + # Train models + model_manager.fit(train_data) + + # Store selected features for future steps + if do_feature_selection or selected_features is None: + selected_features = model_manager.selected_features + + # Update feature selection flag for next step + do_feature_selection = ((step + 1) % self.feature_selection_interval == 0) + + # Skip signal generation if model training failed + if model_manager.meta_model is None: + print("Warning: Meta-model training failed, skipping signal generation") + continue + + # Create signal generator + signal_generator = GoldSignalGenerator( + confidence_threshold=0.7, + risk_reward_min=1.5, + model_manager=model_manager + ) + + # Generate signals on test data + signals = signal_generator.generate_signals(test_data) + + # Check if signals DataFrame is empty + if signals.empty or 'signal' not in signals.columns: + print("Warning: No signals generated, skipping analysis") + continue + + # Check if any signals were generated + signal_count = (signals['signal'] != 0).sum() if 'signal' in signals.columns else 0 + if signal_count == 0: + print("Warning: No active signals found in test period") + else: + print(f"Generated {signal_count} active signals") + + # Analyze signals + metrics = signal_generator.analyze_signals(signals, test_data) + + # Store results + signals['step'] = step + 1 + all_signals = pd.concat([all_signals, signals]) + + metrics['step'] = step + 1 + metrics['train_start'] = train_data.index[0] + metrics['train_end'] = train_data.index[-1] + metrics['test_start'] = test_data.index[0] + metrics['test_end'] = test_data.index[-1] + all_metrics.append(metrics) + + except Exception as e: + import traceback + print(f"Error in step {step+1}: {str(e)}") + print("Detailed traceback:") + print(traceback.format_exc()) + continue + + # Compile final results + results = { + 'signals': all_signals, + 'metrics': all_metrics + } + + # Print overall performance + self._print_overall_performance(all_metrics) + + return results + + def _print_overall_performance(self, metrics): + """Print overall performance statistics""" + if not metrics: + print("No metrics available for performance analysis") + return + + print("\n=== Overall Walk-Forward Performance ===") + + # Calculate average metrics + total_signals = sum(m['total_signals'] for m in metrics) + avg_win_rate = np.mean([m['overall_win_rate'] for m in metrics if not np.isnan(m['overall_win_rate'])]) + avg_return = np.mean([m['overall_avg_return'] for m in metrics if not np.isnan(m['overall_avg_return'])]) + + print(f"Total Steps: {len(metrics)}") + print(f"Total Signals: {total_signals}") + print(f"Average Win Rate: {avg_win_rate:.2%}") + print(f"Average Return: {avg_return:.2%}") + + # Calculate performance by regime if available + regime_performance = {} + for m in metrics: + if m['regime_stats'] is not None: + for regime, stats in m['regime_stats'].items(): + if regime not in regime_performance: + regime_performance[regime] = { + 'count': 0, + 'win_rate': [], + 'avg_return': [] + } + + regime_performance[regime]['count'] += stats['count'] + + # Long performance + if not np.isnan(stats['long_win_rate']): + regime_performance[regime]['win_rate'].append(stats['long_win_rate']) + + if not np.isnan(stats['long_avg_return']): + regime_performance[regime]['avg_return'].append(stats['long_avg_return']) + + # Short performance + if not np.isnan(stats['short_win_rate']): + regime_performance[regime]['win_rate'].append(stats['short_win_rate']) + + if not np.isnan(stats['short_avg_return']): + regime_performance[regime]['avg_return'].append(stats['short_avg_return']) + + if regime_performance: + print("\nPerformance by Volatility Regime:") + for regime, stats in regime_performance.items(): + avg_win_rate = np.mean(stats['win_rate']) if stats['win_rate'] else np.nan + avg_return = np.mean(stats['avg_return']) if stats['avg_return'] else np.nan + + print(f"Regime {regime}: {stats['count']} signals, Win Rate: {avg_win_rate:.2%}, Avg Return: {avg_return:.2%}") diff --git a/tradingbot/config.py b/tradingbot/config.py new file mode 100644 index 0000000..d1e94e4 --- /dev/null +++ b/tradingbot/config.py @@ -0,0 +1,18 @@ +"""Central configuration. + +Secrets are read from environment variables so they never live in source code. +Set them in your shell or a local ``.env`` file (which is git-ignored): + + export OANDA_ACCESS_TOKEN="your-token-here" +""" +import os + +# OANDA v20 REST access token (required only for live data downloads). +OANDA_ACCESS_TOKEN = os.environ.get("OANDA_ACCESS_TOKEN", "") + +# Default instrument / granularity for downloads. +DEFAULT_INSTRUMENT = os.environ.get("INSTRUMENT", "EUR_USD") +DEFAULT_GRANULARITY = os.environ.get("GRANULARITY", "M5") + +# Where trained models are written / loaded from. +MODEL_DIR = os.environ.get("MODEL_DIR", "models") diff --git a/tradingbot/data/__init__.py b/tradingbot/data/__init__.py new file mode 100644 index 0000000..56dee23 --- /dev/null +++ b/tradingbot/data/__init__.py @@ -0,0 +1,12 @@ +from tradingbot.data.loader import load_ohlcv_csv, summarize, analyze_time_gaps +from tradingbot.data.processor import GoldDataProcessor +from tradingbot.data.features import GoldFeatureEngineer, GoldFeaturePipeline + +__all__ = [ + "load_ohlcv_csv", + "summarize", + "analyze_time_gaps", + "GoldDataProcessor", + "GoldFeatureEngineer", + "GoldFeaturePipeline", +] diff --git a/tradingbot/data/features.py b/tradingbot/data/features.py new file mode 100644 index 0000000..64177cc --- /dev/null +++ b/tradingbot/data/features.py @@ -0,0 +1,485 @@ +"""Feature engineering and the end-to-end feature pipeline.""" +import numpy as np +import pandas as pd +import talib +from sklearn.base import BaseEstimator, TransformerMixin +from hmmlearn import hmm +from scipy import stats +import warnings + +from tradingbot.data.processor import GoldDataProcessor + +warnings.filterwarnings("ignore") + + +class GoldFeatureEngineer(BaseEstimator, TransformerMixin): + """ + Feature engineering for 5-minute gold trading data. + Implements market microstructure, volatility, indicator adjustments, + and time-based features. + """ + def __init__(self, forecast_horizon=6, volatility_regime_states=3, look_back_window=120): + """ + Initialize the feature engineer + + Parameters: + ----------- + forecast_horizon : int + Number of bars to look ahead for target creation (default: 6 bars = 30 minutes) + volatility_regime_states : int + Number of states for HMM volatility regime detection + look_back_window : int + Window size for rolling computations (default: 120 bars = 10 hours) + """ + self.forecast_horizon = forecast_horizon + self.volatility_regime_states = volatility_regime_states + self.look_back_window = look_back_window + self.regime_model = hmm.GaussianHMM( + n_components=volatility_regime_states, + random_state=42, + covariance_type="diag" + ) + + def fit(self, X, y=None): + """ + Fit method to comply with sklearn transformer interface + Learns volatility regimes from the data + """ + # Ensure X is a DataFrame + if not isinstance(X, pd.DataFrame): + X = pd.DataFrame(X) + + # Detect volatility regimes for later transformation + self._fit_volatility_regimes(X) + + return self + + def _clean_features(self, data): + """ + Clean feature data by handling infinities, NaNs, and extreme values + """ + # Replace infinities with NaN + data = data.replace([np.inf, -np.inf], np.nan) + + # For each feature column, handle NaNs and clip extreme values + for col in data.columns: + if col not in ['open', 'high', 'low', 'close', 'volume', 'target', 'is_weekend']: + # Fill NaNs with median (or forward fill if median is NaN) + median_val = data[col].median() + if pd.isna(median_val): + data[col] = data[col].fillna(method='ffill') + if data[col].isna().any(): + data[col] = data[col].fillna(method='bfill') + if data[col].isna().any(): + data[col] = data[col].fillna(0) + else: + data[col] = data[col].fillna(median_val) + + # Clip extreme values to 5 standard deviations from mean + if not data[col].isna().all(): + mean_val = data[col].mean() + std_val = data[col].std() + if std_val > 0: # Only clip if standard deviation is positive + data[col] = data[col].clip( + lower=mean_val - 5*std_val, + upper=mean_val + 5*std_val + ) + + return data + + def transform(self, X): + """ + Transform the data by adding engineered features + """ + # Ensure X is a DataFrame and create a copy to avoid modifying original + if not isinstance(X, pd.DataFrame): + X = pd.DataFrame(X) + + data = X.copy() + + # Add all feature groups + data = self._add_market_microstructure_features(data) + data = self._add_volatility_features(data) + data = self._add_indicator_features(data) + data = self._add_time_based_features(data) + + # Create target variable if close column exists + if 'close' in data.columns: + data = self._create_target_variable(data) + + data = self._clean_features(data) + + # Drop rows with NaN values resulting from indicators that need lookback + data = data.dropna() + + return data + + def _fit_volatility_regimes(self, data): + """ + Fit HMM model to detect volatility regimes + """ + # Calculate log returns + if 'close' in data.columns: + returns = np.log(data['close'] / data['close'].shift(1)).dropna() + + # Calculate rolling volatility features for HMM + rolling_vol = returns.rolling(window=20).std().dropna() + rolling_range = ((data['high'] / data['low'] - 1) + .rolling(window=20).mean().dropna()) + + # Combine features for regime detection + features = pd.DataFrame({ + 'returns_volatility': rolling_vol, + 'price_range': rolling_range + }).dropna() + + if len(features) > self.volatility_regime_states * 5: # Ensure enough data to fit + # Normalize features for HMM + features = (features - features.mean()) / features.std() + + # Fit the HMM model + self.regime_model.fit(features.values) + + def _add_market_microstructure_features(self, data): + """ + Add market microstructure features: + - Order flow imbalance + - Price velocity + - Relative volume analysis + """ + # Order Flow Imbalance (approximated from OHLC) + data['bar_sentiment'] = np.where( + data['close'] > data['open'], + (data['close'] - data['open']) / (data['high'] - data['low'] + 1e-8), + -1 * (data['open'] - data['close']) / (data['high'] - data['low'] + 1e-8) + ) + + # Smooth bar sentiment + data['smooth_sentiment'] = data['bar_sentiment'].rolling(5).mean() + + # Price Velocity - Rate of change over multiple timeframes + for window in [1, 3, 5, 15]: + data[f'price_velocity_{window}'] = data['close'].pct_change(window) / (window * 5) # Normalize by minutes + + # Acceleration (change in velocity) + data['price_acceleration'] = data['price_velocity_3'].diff() + + # Relative Volume Analysis + # Calculate average volume by time of day first + data['hour'] = data.index.hour + data['minute'] = data.index.minute + data['time_of_day'] = data['hour'] * 60 + data['minute'] + + # Group by time of day and calculate average volume + avg_volume_by_time = data.groupby('time_of_day')['volume'].transform('mean') + data['relative_volume'] = data['volume'] / (avg_volume_by_time + 1e-8) # Avoid division by zero + + # Volume momentum + data['volume_momentum'] = data['volume'].pct_change(5) + + # Drop temporary columns + data = data.drop(['hour', 'minute', 'time_of_day'], axis=1) + + return data + + def _add_volatility_features(self, data): + """ + Add volatility-related features: + - Micro-volatility clusters + - Bollinger Band Width + - Volatility regime detection + """ + # ATR with shorter periods for micro-volatility + for period in [5, 10]: + data[f'atr_{period}'] = talib.ATR( + data['high'].values, + data['low'].values, + data['close'].values, + timeperiod=period + ) + # Normalize ATR by price level + data[f'atr_{period}_pct'] = data[f'atr_{period}'] / data['close'] + + # Bollinger Band Width (20 periods, 2 std) + upper, middle, lower = talib.BBANDS( + data['close'].values, + timeperiod=20, + nbdevup=2, + nbdevdn=2 + ) + data['bb_width'] = (upper - lower) / middle + + # Rate of change of BB width + data['bb_width_change'] = data['bb_width'].pct_change(3) + + # Volatility Regime Detection + if hasattr(self, 'regime_model') and hasattr(self.regime_model, 'transmat_'): + # Calculate the same features used during fitting + returns = np.log(data['close'] / data['close'].shift(1)) + rolling_vol = returns.rolling(window=20).std() + rolling_range = (data['high'] / data['low'] - 1).rolling(window=20).mean() + + # Combine and normalize features + features = pd.DataFrame({ + 'returns_volatility': rolling_vol, + 'price_range': rolling_range + }) + + # Handle NaN values + features = features.fillna(method='bfill') + + # Normalize using the same approach as in fitting + features = (features - features.mean()) / features.std() + + # Predict regimes where we have data + valid_idx = ~features.isnull().any(axis=1) + if valid_idx.sum() > 0: + regimes = self.regime_model.predict(features[valid_idx].values) + + # Create a series with index aligned to original data + regime_series = pd.Series(index=data.index, dtype='float64') + regime_series.loc[features[valid_idx].index] = regimes + + # Forward fill regime values + data['volatility_regime'] = regime_series.fillna(method='ffill') + else: + # If we don't have enough data, use a default regime + data['volatility_regime'] = 1 + else: + # If regime model isn't fitted, use a simpler approach + returns = np.log(data['close'] / data['close'].shift(1)) + vol = returns.rolling(window=20).std() * np.sqrt(252 * 288) # Annualized (288 5-min bars/day) + data['volatility_regime'] = pd.qcut( + vol, + q=self.volatility_regime_states, + labels=False, + duplicates='drop' + ).fillna(self.volatility_regime_states // 2) + + return data + + def _add_indicator_features(self, data): + """ + Add adjusted technical indicators specific for 5-minute data: + - Fast & Slow EMAs + - RSI + - Stochastic Oscillator + - MACD + """ + # EMAs with shorter periods + for period in [10, 20, 50]: + data[f'ema_{period}'] = talib.EMA(data['close'].values, timeperiod=period) + + # Add relative position to EMA + data[f'close_to_ema_{period}'] = (data['close'] / data[f'ema_{period}'] - 1) * 100 + + # EMA Cross Features + data['ema_10_20_cross'] = data['ema_10'] - data['ema_20'] + data['ema_10_50_cross'] = data['ema_10'] - data['ema_50'] + + # Shorter-period RSI + data['rsi_7'] = talib.RSI(data['close'].values, timeperiod=7) + data['rsi_14'] = talib.RSI(data['close'].values, timeperiod=14) + + # RSI momentum and mean-reversion features + data['rsi_7_change'] = data['rsi_7'].diff(3) + data['rsi_divergence'] = data['rsi_14'] - data['rsi_7'] + + # Stochastic Oscillator - Fast and Slow + slowk, slowd = talib.STOCH( + data['high'].values, + data['low'].values, + data['close'].values, + fastk_period=5, + slowk_period=3, + slowk_matype=0, + slowd_period=3, + slowd_matype=0 + ) + data['stoch_k_fast'] = slowk + data['stoch_d_fast'] = slowd + + slowk, slowd = talib.STOCH( + data['high'].values, + data['low'].values, + data['close'].values, + fastk_period=14, + slowk_period=3, + slowk_matype=0, + slowd_period=3, + slowd_matype=0 + ) + data['stoch_k_slow'] = slowk + data['stoch_d_slow'] = slowd + + # Stochastic crossover signal + data['stoch_crossover'] = data['stoch_k_fast'] - data['stoch_d_fast'] + + # MACD with faster settings + macd, macdsignal, macdhist = talib.MACD( + data['close'].values, + fastperiod=8, + slowperiod=17, + signalperiod=9 + ) + data['macd'] = macd + data['macd_signal'] = macdsignal + data['macd_hist'] = macdhist + + # Add momentum oscillator + data['mom_10'] = talib.MOM(data['close'].values, timeperiod=10) + + # Add Average Directional Index for trend strength + data['adx_14'] = talib.ADX( + data['high'].values, + data['low'].values, + data['close'].values, + timeperiod=14 + ) + + return data + + def _add_time_based_features(self, data): + """ + Add time-based features: + - Time of day (hour, minute) + - Day of week + - Session (Tokyo, London, New York) + """ + # Extract basic time components + data['hour'] = data.index.hour + data['minute'] = data.index.minute + data['day_of_week'] = data.index.dayofweek + + # Create cyclical time features (circular encoding to handle continuity) + # For hour of day (24 hours) + data['hour_sin'] = np.sin(2 * np.pi * data['hour'] / 24) + data['hour_cos'] = np.cos(2 * np.pi * data['hour'] / 24) + + # For minute within hour + data['minute_sin'] = np.sin(2 * np.pi * data['minute'] / 60) + data['minute_cos'] = np.cos(2 * np.pi * data['minute'] / 60) + + # For day of week (5-day trading week) + data['day_sin'] = np.sin(2 * np.pi * data['day_of_week'] / 7) + data['day_cos'] = np.cos(2 * np.pi * data['day_of_week'] / 7) + + # Create Trading Session markers + # Define trading sessions (UTC times) + # Tokyo: 00:00-09:00 + # London: 08:00-17:00 + # New York: 13:00-22:00 + + # Convert hour to UTC assuming index is in UTC + # If index is not in UTC, this would need to be adjusted + utc_hour = data['hour'] + + # Trading session flags + data['tokyo_session'] = ((utc_hour >= 0) & (utc_hour < 9)).astype(int) + data['london_session'] = ((utc_hour >= 8) & (utc_hour < 17)).astype(int) + data['ny_session'] = ((utc_hour >= 13) & (utc_hour < 22)).astype(int) + + # Session overlap flags + data['tokyo_london_overlap'] = ((utc_hour >= 8) & (utc_hour < 9)).astype(int) + data['london_ny_overlap'] = ((utc_hour >= 13) & (utc_hour < 17)).astype(int) + + # Drop original time columns + data = data.drop(['hour', 'minute', 'day_of_week'], axis=1) + + return data + + def _create_target_variable(self, data): + """ + Create target variable for prediction: + Multi-class classification based on future price movement + """ + # Calculate future return over forecast_horizon + future_close = data['close'].shift(-self.forecast_horizon) + future_return = (future_close / data['close'] - 1) * 100 # Percentage return + + # Create multi-class target + # Calculate dynamic thresholds based on recent volatility + volatility = data['close'].pct_change().rolling(window=20).std() * 100 # Convert to percentage + + # Define thresholds as a function of volatility + # More volatile periods have wider thresholds + strong_threshold = volatility * 0.75 # 75% of recent volatility + weak_threshold = volatility * 0.25 # 25% of recent volatility + + # Ensure minimum thresholds + strong_threshold = np.maximum(strong_threshold, 0.15) # At least 0.15% + weak_threshold = np.maximum(weak_threshold, 0.05) # At least 0.05% + + # Create targets + conditions = [ + future_return > strong_threshold, + (future_return > weak_threshold) & (future_return <= strong_threshold), + (future_return >= -weak_threshold) & (future_return <= weak_threshold), + (future_return < -weak_threshold) & (future_return >= -strong_threshold), + future_return < -strong_threshold + ] + + choices = [2, 1, 0, -1, -2] # Strong Up, Weak Up, Sideways, Weak Down, Strong Down + + data['target'] = np.select(conditions, choices, default=np.nan) + + # Store the thresholds used for reference + data['strong_threshold'] = strong_threshold + data['weak_threshold'] = weak_threshold + + # Also store raw future return for potential regression tasks + data['future_return'] = future_return + + return data + +class GoldFeaturePipeline: + """ + Complete pipeline for gold 5-minute data preprocessing and feature engineering + """ + def __init__(self, forecast_horizon=6, volatility_regime_states=3, remove_outliers=True): + """ + Initialize the complete pipeline + + Parameters: + ----------- + forecast_horizon : int + Number of bars to look ahead for target creation + volatility_regime_states : int + Number of states for HMM volatility regime detection + remove_outliers : bool + Whether to remove price outliers + """ + self.data_processor = GoldDataProcessor( + fill_gaps=True, + remove_outliers=remove_outliers, + handle_after_hours=True + ) + + self.feature_engineer = GoldFeatureEngineer( + forecast_horizon=forecast_horizon, + volatility_regime_states=volatility_regime_states + ) + + def process(self, file_path): + """ + Process data from file through the complete pipeline + + Parameters: + ----------- + file_path : str + Path to the CSV file with 5-minute OHLCV data + + Returns: + -------- + processed_data : pandas.DataFrame + Processed data with all features and target + """ + # Load and preprocess data + data = self.data_processor.load_data(file_path) + + # Apply feature engineering + self.feature_engineer.fit(data) + processed_data = self.feature_engineer.transform(data) + + # Return the processed data + return processed_data diff --git a/tradingbot/data/loader.py b/tradingbot/data/loader.py new file mode 100644 index 0000000..8fef1d3 --- /dev/null +++ b/tradingbot/data/loader.py @@ -0,0 +1,55 @@ +"""Lightweight CSV loading and data-quality helpers. + +Loads raw 5-minute candles and inspects the gaps between consecutive timestamps. +""" +import pandas as pd + + +def load_ohlcv_csv(file_path, timestamp_col="timestamp"): + """Load an OHLCV CSV with a datetime index. + + Parameters + ---------- + file_path : str + Path to a CSV containing a timestamp column plus OHLCV columns. + timestamp_col : str + Name of the timestamp column to parse and use as the index. + """ + data = pd.read_csv(file_path, index_col=timestamp_col, parse_dates=True) + data = data[~data.index.duplicated(keep="first")] + return data.sort_index() + + +def summarize(data): + """Print basic information about a loaded OHLCV frame.""" + print("Data Info:") + print(data.info()) + print("\nFirst few rows:") + print(data.head()) + print("\nLast few rows:") + print(data.tail()) + print(f"\nTotal candles: {len(data)}") + + +def analyze_time_gaps(data, expected_gap=pd.Timedelta(minutes=5), + big_gap=pd.Timedelta(hours=1), verbose=True): + """Report gaps between consecutive timestamps. + + Returns the Series of gaps larger than ``expected_gap``. + """ + time_diffs = data.index.to_series().diff() + gaps = time_diffs[time_diffs > expected_gap] + + if verbose: + print(f"Total gaps larger than {expected_gap}: {len(gaps)}") + print("\nGap statistics:") + print(gaps.describe()) + + big_gaps = gaps[gaps > big_gap] + print(f"\nNumber of gaps > {big_gap}: {len(big_gaps)}") + print(f"Big gaps (> {big_gap}):") + for gap_start, gap_size in big_gaps.items(): + gap_end = gap_start + gap_size + print(f"Gap from {gap_start} to {gap_end} ({gap_size})") + + return gaps diff --git a/tradingbot/data/oanda_connector.py b/tradingbot/data/oanda_connector.py new file mode 100644 index 0000000..e8f211e --- /dev/null +++ b/tradingbot/data/oanda_connector.py @@ -0,0 +1,113 @@ +"""OANDA v20 historical candle downloader. + +The access token is read from configuration (the ``OANDA_ACCESS_TOKEN`` +environment variable) rather than being hard-coded. +""" +import time +from datetime import datetime, timedelta + +import pandas as pd +import oandapyV20 +import oandapyV20.endpoints.instruments as instruments +from oandapyV20.exceptions import V20Error + +from tradingbot.config import OANDA_ACCESS_TOKEN + + +class OandaDataConnector: + def __init__(self, access_token): + self.client = oandapyV20.API(access_token=access_token) + self.last_request_time = 0 + self.request_limit_delay = 0.01 # 100 requests per second max + + def _respect_rate_limit(self): + current_time = time.time() + time_passed = current_time - self.last_request_time + if time_passed < self.request_limit_delay: + time.sleep(self.request_limit_delay - time_passed) + self.last_request_time = time.time() + + def get_historical_data_chunked(self, instrument, timeframe, start_date, end_date=None, chunk_days=5): + if end_date is None: + end_date = datetime.now() + + all_candles = [] + current_date = start_date + + while current_date < end_date: + self._respect_rate_limit() + + chunk_end = min(current_date + timedelta(days=chunk_days), end_date) + + params = { + "from": current_date.strftime('%Y-%m-%dT%H:%M:%SZ'), + "to": chunk_end.strftime('%Y-%m-%dT%H:%M:%SZ'), + "granularity": timeframe, + "price": "MBA" # Mid, Bid, Ask prices + } + + try: + r = instruments.InstrumentsCandles(instrument=instrument, params=params) + self.client.request(r) + + for candle in r.response['candles']: + if candle['complete']: + all_candles.append({ + 'timestamp': candle['time'], + 'open': float(candle['mid']['o']), + 'high': float(candle['mid']['h']), + 'low': float(candle['mid']['l']), + 'close': float(candle['mid']['c']), + 'volume': float(candle['volume']) + }) + + print(f"Downloaded data from {current_date.date()} to {chunk_end.date()}") + current_date = chunk_end + + except V20Error as e: + print(f"Error fetching data: {e}") + return None + + df = pd.DataFrame(all_candles) + if not df.empty: + df['timestamp'] = pd.to_datetime(df['timestamp']) + df.set_index('timestamp', inplace=True) + df = df.sort_index() + return df + + +def download(instrument="EUR_USD", timeframe="M5", lookback_days=1825, + access_token=None, out_dir="."): + """Download ``lookback_days`` of candles and save them to a CSV. + + Returns the downloaded DataFrame (or ``None`` on failure). + """ + token = access_token or OANDA_ACCESS_TOKEN + if not token: + raise ValueError( + "No OANDA access token. Set the OANDA_ACCESS_TOKEN environment " + "variable or pass access_token=..." + ) + + connector = OandaDataConnector(token) + start_date = datetime.now() - timedelta(days=lookback_days) + data = connector.get_historical_data_chunked( + instrument=instrument, + timeframe=timeframe, + start_date=start_date, + ) + + if data is not None: + name = instrument.replace("_", "") + filename = ( + f"{out_dir}/{name}_{timeframe}_" + f"{start_date.strftime('%Y%m%d')}_to_{datetime.now().strftime('%Y%m%d')}.csv" + ) + data.to_csv(filename) + print(f"Data saved to {filename}") + print(f"Total candles downloaded: {len(data)}") + return data + + +if __name__ == "__main__": + download() diff --git a/tradingbot/data/processor.py b/tradingbot/data/processor.py new file mode 100644 index 0000000..83efb49 --- /dev/null +++ b/tradingbot/data/processor.py @@ -0,0 +1,140 @@ +"""Raw 5-minute OHLCV loading and cleaning.""" +import numpy as np +import pandas as pd + + +class GoldDataProcessor: + """ + Data processor for 5-minute gold price data. + Handles data loading, cleaning, and preprocessing. + """ + def __init__(self, fill_gaps=True, remove_outliers=True, handle_after_hours=True): + self.fill_gaps = fill_gaps + self.remove_outliers = remove_outliers + self.handle_after_hours = handle_after_hours + + def load_data(self, file_path): + """ + Load data from CSV file and set appropriate index + """ + # Load data from CSV + data = pd.read_csv(file_path) + + # Ensure timestamp column exists + if 'timestamp' not in data.columns: + raise ValueError("CSV must contain a 'timestamp' column") + + # Convert timestamp to datetime and set as index + data['timestamp'] = pd.to_datetime(data['timestamp']) + data = data.drop_duplicates(subset=['timestamp']) + data.set_index('timestamp', inplace=True) + + # Ensure all necessary columns exist + required_columns = ['open', 'high', 'low', 'close', 'volume'] + missing_columns = [col for col in required_columns if col not in data.columns] + if missing_columns: + raise ValueError(f"Missing required columns: {missing_columns}") + + # Make column names lowercase if they aren't already + data.columns = [col.lower() for col in data.columns] + + return self._preprocess_data(data) + + def _preprocess_data(self, data): + """ + Apply preprocessing steps to clean and prepare the data + """ + # Sort data by timestamp to ensure chronological order + data = data.sort_index() + + # Fill gaps in 5-minute data + if self.fill_gaps: + data = self._fill_time_gaps(data) + + # Remove outliers + if self.remove_outliers: + data = self._remove_price_outliers(data) + + # Handle after-hours and weekend data + if self.handle_after_hours: + data = self._handle_after_hours_data(data) + + return data + + def _fill_time_gaps(self, data): + """ + Fill gaps in 5-minute data by reindexing with complete 5-minute intervals + during trading hours + """ + # Create complete 5-minute intervals + start_date = data.index.min() + end_date = data.index.max() + full_range = pd.date_range(start=start_date, end=end_date, freq='5min') + + # Reindex the data + reindexed_data = data.reindex(full_range) + + # Forward fill price data (OHLC) + for col in ['open', 'high', 'low', 'close']: + reindexed_data[col] = reindexed_data[col].ffill() + + # Fill volume with zeros + reindexed_data['volume'] = reindexed_data['volume'].fillna(0) + + return reindexed_data + + def _remove_price_outliers(self, data, z_threshold=10): + """ + Remove extreme price outliers based on z-score of returns + """ + # Calculate returns + returns = data['close'].pct_change() + + # Calculate rolling z-score (20 periods ~ 100 minutes) + rolling_mean = returns.rolling(window=20).mean() + rolling_std = returns.rolling(window=20).std() + z_scores = (returns - rolling_mean) / rolling_std + + # Identify outliers + outliers = abs(z_scores) > z_threshold + + if outliers.sum() > 0: + print(f"Identified {outliers.sum()} outliers out of {len(data)} records") + + # Replace outlier prices with interpolated values + clean_data = data.copy() + outlier_indices = outliers[outliers].index + + for col in ['open', 'high', 'low', 'close']: + clean_data.loc[outlier_indices, col] = np.nan + clean_data[col] = clean_data[col].interpolate(method='linear') + + return clean_data + + return data + + def _handle_after_hours_data(self, data): + """ + Handle after-hours and weekend data in gold trading + """ + # Create day of week and hour features + data['day_of_week'] = data.index.dayofweek + data['hour'] = data.index.hour + + # Filter out weekends (Saturday and Sunday) + # For gold, market typically closes Friday ~5PM ET and opens Sunday ~6PM ET + weekend_mask = (data['day_of_week'] == 5) & (data['hour'] >= 17) # After Friday 5PM + weekend_mask |= (data['day_of_week'] == 6) # All of Saturday + weekend_mask |= (data['day_of_week'] == 0) & (data['hour'] < 18) # Before Sunday 6PM + + # Mark weekend data + data['is_weekend'] = weekend_mask + + # Forward fill prices for weekend gaps + # We don't remove weekend data to maintain continuity + # but we mark it so we can filter it later if needed + + # Remove temporary columns if needed + data = data.drop(['day_of_week', 'hour'], axis=1) + + return data diff --git a/tradingbot/models/__init__.py b/tradingbot/models/__init__.py new file mode 100644 index 0000000..78417ff --- /dev/null +++ b/tradingbot/models/__init__.py @@ -0,0 +1,10 @@ +from tradingbot.models.tree_ensemble import TreeEnsemblePredictor, run_model +from tradingbot.models.neural_ensemble import NeuralEnsemblePredictor +from tradingbot.models.model_manager import GoldModelManager + +__all__ = [ + "TreeEnsemblePredictor", + "run_model", + "NeuralEnsemblePredictor", + "GoldModelManager", +] diff --git a/tradingbot/models/model_manager.py b/tradingbot/models/model_manager.py new file mode 100644 index 0000000..76b8506 --- /dev/null +++ b/tradingbot/models/model_manager.py @@ -0,0 +1,1149 @@ +"""Model training, feature selection, evaluation and prediction manager.""" +import time +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns +from sklearn.model_selection import TimeSeriesSplit +from sklearn.metrics import accuracy_score, classification_report, confusion_matrix +from sklearn.preprocessing import StandardScaler +from sklearn.decomposition import PCA +from sklearn.feature_selection import RFE, SelectFromModel +import lightgbm as lgb +from boruta import BorutaPy +from joblib import Parallel, delayed, dump, load +import optuna +import shap + +warnings.filterwarnings("ignore") + + +class GoldModelManager: + """ + Manages model training, evaluation, and prediction for 5-minute gold trading. + Implements various feature selection techniques, training strategies, + and signal generation approaches. + """ + def __init__( + self, + n_splits=5, + test_size=0.2, + feature_selection_method='boruta', + max_features=25, + ensemble_models=3, + volatility_based_models=True, + n_jobs=-1, + model_path='models', + random_state=42 + ): + """ + Initialize the model manager + + Parameters: + ----------- + n_splits : int + Number of splits for time series cross-validation + test_size : float + Proportion of data to use for final testing + feature_selection_method : str + Method for feature selection ('boruta', 'rfe', 'importance', 'pca') + max_features : int + Maximum number of features to select + ensemble_models : int + Number of models in the ensemble + volatility_based_models : bool + Whether to train separate models for different volatility regimes + n_jobs : int + Number of parallel jobs for training + model_path : str + Directory to save trained models + random_state : int + Random seed for reproducibility + """ + self.n_splits = n_splits + self.test_size = test_size + self.feature_selection_method = feature_selection_method + self.max_features = max_features + self.ensemble_models = ensemble_models + self.volatility_based_models = volatility_based_models + self.n_jobs = n_jobs + self.random_state = random_state + self.model_path = Path(model_path) + self.model_path.mkdir(exist_ok=True) + + # Initialize components + self.scaler = StandardScaler() + self.models = {} + self.meta_model = None + self.feature_selector = None + self.selected_features = None + self.feature_importance = None + self.pca = None + def validate_data(self, X, y): + """ + Validate data before modeling to ensure all classes are represented + """ + print("Validating data...") + + # Check for missing values + missing_count = X.isna().sum().sum() + if missing_count > 0: + print(f"Warning: {missing_count} missing values detected in features") + + # Check for infinities - only for numeric columns + numeric_cols = X.select_dtypes(include=['number']).columns + if len(numeric_cols) > 0: + inf_count = np.isinf(X[numeric_cols].values).sum() + if inf_count > 0: + print(f"Warning: {inf_count} infinity values detected in features") + + # Check class distribution + unique_classes = np.unique(y) + class_counts = {cls: np.sum(y == cls) for cls in unique_classes} + + print("Class distribution:") + for cls, count in sorted(class_counts.items()): + print(f" Class {cls}: {count} samples ({count/len(y):.2%})") + + # Verify all expected classes are present + expected_classes = set(range(-2, 3)) # -2, -1, 0, 1, 2 + missing_classes = expected_classes - set(unique_classes) + + if missing_classes: + print(f"Warning: Missing classes in data: {missing_classes}") + print("Model will be trained with available classes only.") + + # Check for severe imbalance + min_class_count = min(class_counts.values()) + if min_class_count < 10: + print(f"Warning: Some classes have very few samples (min: {min_class_count})") + + # Check feature values for numeric columns only + for col in numeric_cols: + try: + col_min = X[col].min() + col_max = X[col].max() + col_mean = X[col].mean() + col_std = X[col].std() + + # Check for suspiciously high values + if col_max > 1e6 or col_min < -1e6: + print(f"Warning: Feature '{col}' has extreme values: min={col_min}, max={col_max}") + + # Check for very low variance + if col_std < 1e-6: + print(f"Warning: Feature '{col}' has very low variance: std={col_std}") + except Exception as e: + print(f"Warning: Could not analyze feature '{col}': {str(e)}") + + # For categorical columns, check cardinality + cat_cols = X.select_dtypes(include=['category', 'object']).columns + for col in cat_cols: + try: + n_unique = X[col].nunique() + print(f"Categorical feature '{col}' has {n_unique} unique values") + except Exception as e: + print(f"Warning: Could not analyze categorical feature '{col}': {str(e)}") + + return True + + def prepare_data(self, data, remove_cols=None): + """ + Prepare data for modeling by separating features from target + and removing unwanted columns + + Parameters: + ----------- + data : pd.DataFrame + DataFrame with features and target + remove_cols : list + List of columns to remove (not used as features) + + Returns: + -------- + X : pd.DataFrame + Feature matrix + y : pd.Series + Target variable + """ + if remove_cols is None: + # Default columns to remove (original data and non-feature columns) + remove_cols = [ + 'open', 'high', 'low', 'close', 'volume', + 'future_return', 'strong_threshold', 'weak_threshold' + ] + + # Create copy to avoid modifying original + df = data.copy() + + # Ensure target column exists + if 'target' not in df.columns: + raise ValueError("Target column 'target' not found in data") + + # Extract target + y = df['target'] + + # Remove target and other non-feature columns + X = df.drop(['target'] + remove_cols, axis=1, errors='ignore') + + # Handle remaining non-numeric columns + for col in X.select_dtypes(include=['object', 'category']).columns: + X[col] = X[col].astype('category') + + return X, y + + def _create_time_series_splits(self, X, y, train_ratio=0.8, validation_ratio=0.1): + """ + Create time-series aware data splits for proper backtesting + + Parameters: + ----------- + X : pd.DataFrame + Feature matrix + y : pd.Series + Target variable + train_ratio : float + Ratio of data to use for training + validation_ratio : float + Ratio of data to use for validation (from end of train) + + Returns: + -------- + X_train, X_val, X_test, y_train, y_val, y_test : DataFrames/Series + Split data components + """ + # Ensure indices match between X and y + assert X.index.equals(y.index), "X and y must have the same index" + + # Calculate split points + n = len(X) + train_end = int(n * train_ratio) + val_end = train_end + int(n * validation_ratio) + + # Split data + X_train = X.iloc[:train_end] + X_val = X.iloc[train_end:val_end] + X_test = X.iloc[val_end:] + + y_train = y.iloc[:train_end] + y_val = y.iloc[train_end:val_end] + y_test = y.iloc[val_end:] + + return X_train, X_val, X_test, y_train, y_val, y_test + + def _get_optimal_lightgbm_params(self, X_train, y_train, X_val, y_val): + """ + Optimize LightGBM hyperparameters using Optuna with proper class weight handling + """ + def objective(trial): + param = { + 'boosting_type': 'gbdt', + 'objective': 'multiclass', + 'num_class': 5, + 'metric': 'multi_logloss', + 'num_leaves': trial.suggest_int('num_leaves', 10, 100), + 'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True), + 'feature_fraction': trial.suggest_float('feature_fraction', 0.5, 1.0), + 'bagging_fraction': trial.suggest_float('bagging_fraction', 0.5, 1.0), + 'bagging_freq': trial.suggest_int('bagging_freq', 1, 10), + 'min_child_samples': trial.suggest_int('min_child_samples', 5, 100), + 'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 10.0, log=True), + 'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 10.0, log=True), + 'verbose': -1, + 'random_state': self.random_state + } + + # Get unique classes in the training data + unique_classes = np.unique(y_train) + + # Create weights for each class that exists in the data + class_weights = {} + for cls in range(-2, 3): # -2, -1, 0, 1, 2 + if cls in unique_classes: + # Calculate actual weight based on class frequency + count = np.sum(y_train == cls) + class_weights[cls] = 1.0 / max(count, 1) + else: + # For missing classes, assign a moderate weight + # (average of existing weights would be another option) + class_weights[cls] = 1.0 + + # Normalize weights to sum to number of classes + total_weight = sum(class_weights.values()) + class_weights = {k: (v / total_weight) * 5 for k, v in class_weights.items()} + + # Train model + model = lgb.LGBMClassifier(**param, class_weight=class_weights) + model.fit( + X_train, y_train, + eval_set=[(X_val, y_val)], + eval_metric='multi_logloss', + callbacks=[lgb.early_stopping(50, verbose=False)], + ) + + # Evaluate on validation set + preds = model.predict(X_val) + return accuracy_score(y_val, preds) + + # Create and run the study + study = optuna.create_study(direction='maximize') + study.optimize(objective, n_trials=50) + + return study.best_params + + def select_features(self, X, y, method=None): + """ + Select the most important features using the specified method + + Parameters: + ----------- + X : pd.DataFrame + Feature matrix + y : pd.Series + Target variable + method : str, optional + Feature selection method (if None, use the instance's method) + + Returns: + -------- + selected_features : list + List of selected feature names + """ + if method is None: + method = self.feature_selection_method + + print(f"Selecting features using {method} method...") + if method == 'boruta': + # Boruta feature selection + try: + # Ensure positive value for n_estimators + base_model = lgb.LGBMClassifier( + n_jobs=self.n_jobs, + random_state=self.random_state, + n_estimators=100, # Explicit positive value + verbose=-1 + ) + + # Properly handle class weights for all possible classes + unique_classes = np.unique(y) + class_weights = {} + + for cls in range(-2, 3): # -2, -1, 0, 1, 2 + if cls in unique_classes: + count = np.sum(y == cls) + class_weights[cls] = 1.0 / max(count, 1) + else: + class_weights[cls] = 1.0 # Default weight for missing classes + + # Normalize weights + total_weight = sum(class_weights.values()) + class_weights = {k: (v / total_weight) * 5 for k, v in class_weights.items()} + + base_model.set_params(class_weight=class_weights) + + self.feature_selector = BorutaPy( + estimator=base_model, + n_estimators=100, # Explicit positive value instead of 'auto' + max_iter=100, + verbose=0, + random_state=self.random_state + ) + + # Convert to numpy arrays + X_values = X.values.astype(np.float64) # Ensure float type + y_values = y.values + + # Check for NaNs and infinities + if np.isnan(X_values).any() or np.isinf(X_values).any(): + print("Warning: NaN or Inf values detected in features, cleaning...") + X_values = np.nan_to_num(X_values, nan=0, posinf=0, neginf=0) + + # Fit Boruta + self.feature_selector.fit(X_values, y_values) + selected_mask = self.feature_selector.support_ + self.selected_features = X.columns[selected_mask].tolist() + + except Exception as e: + print(f"Boruta feature selection failed: {str(e)}") + print("Falling back to importance-based feature selection...") + method = 'importance' # Fall back to importance-based selection + + if method == 'rfe': + # Recursive Feature Elimination + base_model = lgb.LGBMClassifier( + n_jobs=self.n_jobs, + random_state=self.random_state + ) + + self.feature_selector = RFE( + estimator=base_model, + n_features_to_select=min(self.max_features, X.shape[1]), + step=1, + verbose=0 + ) + + # Fit RFE + self.feature_selector.fit(X, y) + selected_mask = self.feature_selector.support_ + self.selected_features = X.columns[selected_mask].tolist() + + if method == 'importance': + # Feature importance-based selection + base_model = lgb.LGBMClassifier( + n_estimators=100, + n_jobs=self.n_jobs, + random_state=self.random_state, + verbose=-1 + ) + + # Properly handle class weights + unique_classes = np.unique(y) + class_weights = {} + + for cls in range(-2, 3): # -2, -1, 0, 1, 2 + if cls in unique_classes: + count = np.sum(y == cls) + class_weights[cls] = 1.0 / max(count, 1) + else: + class_weights[cls] = 1.0 + + # Normalize weights + total_weight = sum(class_weights.values()) + class_weights = {k: (v / total_weight) * 5 for k, v in class_weights.items()} + + base_model.set_params(class_weight=class_weights) + + # Train model + base_model.fit(X, y) + + # Get feature importance + importance = base_model.feature_importances_ + self.feature_importance = pd.DataFrame({ + 'feature': X.columns, + 'importance': importance + }).sort_values('importance', ascending=False) + + # Select top features + self.selected_features = self.feature_importance['feature'].iloc[:min(self.max_features, X.shape[1])].tolist() + + # Create a SelectFromModel as feature_selector for consistency + self.feature_selector = SelectFromModel( + base_model, + max_features=min(self.max_features, X.shape[1]) + ) + self.feature_selector.fit(X, y) + + if method == 'pca': + # PCA-based dimensionality reduction + # Scale the data first + X_scaled = self.scaler.fit_transform(X) + + # Create and fit PCA + self.pca = PCA(n_components=min(self.max_features, X.shape[1])) + self.pca.fit(X_scaled) + + # Get explained variance + explained_variance = self.pca.explained_variance_ratio_ + cumulative_variance = np.cumsum(explained_variance) + + # Determine number of components for 95% variance + n_components = np.argmax(cumulative_variance >= 0.95) + 1 + n_components = min(n_components, self.max_features) + + # Create new PCA with optimal components + self.pca = PCA(n_components=n_components) + self.pca.fit(X_scaled) + + # PCA doesn't select features but transforms them + # For API consistency, we'll return feature names + self.selected_features = X.columns.tolist() + + else: + # No feature selection, use all features + self.selected_features = X.columns.tolist() + + print(f"Selected {len(self.selected_features)} features") + return self.selected_features + + def train_base_models(self, X_train, y_train, X_val, y_val): + """ + Train the base models for the ensemble with proper class weight handling + """ + print("Training base models...") + models = {} + + # Optimize hyperparameters once for shared configuration + best_params = self._get_optimal_lightgbm_params(X_train, y_train, X_val, y_val) + print(f"Optimized hyperparameters: {best_params}") + + # Get unique classes in the training data + unique_classes = np.unique(y_train) + print(f"Classes present in training data: {unique_classes}") + + # Create weights for each class that exists in the data + class_weights = {} + for cls in range(-2, 3): # -2, -1, 0, 1, 2 + if cls in unique_classes: + # Calculate actual weight based on class frequency + count = np.sum(y_train == cls) + class_weights[cls] = 1.0 / max(count, 1) + print(f"Class {cls}: {count} samples, weight: {class_weights[cls]:.4f}") + else: + # For missing classes, assign a moderate weight + class_weights[cls] = 1.0 + print(f"Class {cls}: 0 samples (missing), weight: 1.0000") + + # Normalize weights to sum to number of classes + total_weight = sum(class_weights.values()) + class_weights = {k: (v / total_weight) * 5 for k, v in class_weights.items()} + + print("Normalized class weights:") + for k, v in class_weights.items(): + print(f"Class {k}: {v:.4f}") + + # Train ensemble models with different seeds and subset of features + for i in range(self.ensemble_models): + # Create a unique subset of features for diversity + if len(self.selected_features) > 10: + # Select 80-90% of features randomly for each model + sample_size = int(len(self.selected_features) * (0.8 + 0.1 * np.random.random())) + features = np.random.choice(self.selected_features, size=sample_size, replace=False) + else: + features = self.selected_features + + # Adjust random seed for diversity + model_seed = self.random_state + i + + # Create model with optimized parameters + model = lgb.LGBMClassifier( + **best_params, + random_state=model_seed, + n_jobs=self.n_jobs, + class_weight=class_weights + ) + + # Fit model + model.fit( + X_train[features], y_train, + eval_set=[(X_val[features], y_val)], + eval_metric='multi_logloss', + callbacks=[lgb.early_stopping(50, verbose=False)], + ) + + # Store model and its features + models[f'base_model_{i}'] = { + 'model': model, + 'features': features.tolist() if isinstance(features, np.ndarray) else features + } + + print(f" Trained base model {i+1}/{self.ensemble_models}") + + # If using volatility-based models, train separate models for each regime + if self.volatility_based_models and 'volatility_regime' in X_train.columns: + volatility_regimes = X_train['volatility_regime'].unique() + + for regime in volatility_regimes: + # Get data for this regime + regime_mask = X_train['volatility_regime'] == regime + if regime_mask.sum() < 1000: # Skip if too few samples + continue + + X_regime = X_train[regime_mask] + y_regime = y_train[regime_mask] + + # Validation data for this regime + val_regime_mask = X_val['volatility_regime'] == regime + X_val_regime = X_val[val_regime_mask] + y_val_regime = y_val[val_regime_mask] + + if len(X_val_regime) < 100: # Skip if too few validation samples + continue + + # Get unique classes for this regime + regime_unique_classes = np.unique(y_regime) + + # Create weights for each class that exists in this regime + regime_class_weights = {} + for cls in range(-2, 3): # -2, -1, 0, 1, 2 + if cls in regime_unique_classes: + # Calculate actual weight based on class frequency + count = np.sum(y_regime == cls) + regime_class_weights[cls] = 1.0 / max(count, 1) + else: + # For missing classes, assign a default weight + regime_class_weights[cls] = 1.0 + + # Normalize weights + regime_total_weight = sum(regime_class_weights.values()) + regime_class_weights = {k: (v / regime_total_weight) * 5 for k, v in regime_class_weights.items()} + + # Create and train model + model = lgb.LGBMClassifier( + **best_params, + random_state=self.random_state, + n_jobs=self.n_jobs, + class_weight=regime_class_weights + ) + + # Fit model + model.fit( + X_regime[self.selected_features], y_regime, + eval_set=[(X_val_regime[self.selected_features], y_val_regime)], + eval_metric='multi_logloss', + callbacks=[lgb.early_stopping(50, verbose=False)], + ) + + # Store model + models[f'regime_model_{int(regime)}'] = { + 'model': model, + 'features': self.selected_features, + 'regime': regime + } + + print(f" Trained model for volatility regime {int(regime)}") + + self.models = models + return models + + def train_meta_model(self, X_val, y_val, X_test=None, y_test=None): + """ + Train a meta-model on the predictions of base models + + Parameters: + ----------- + X_val, y_val : Validation data for base model predictions + X_test, y_test : Optional test data + + Returns: + -------- + meta_model : trained meta-model + """ + print("Training meta-model...") + + try: + # Generate predictions from all base models + base_predictions = self._get_ensemble_predictions(X_val) + + # Create meta-features for training + meta_features = pd.DataFrame(base_predictions) + + # Add volatility regime if available + if 'volatility_regime' in X_val.columns: + meta_features['volatility_regime'] = X_val['volatility_regime'].values + + # Check for NaN or inf values + nan_count = np.isnan(meta_features.values).sum() + inf_count = np.isinf(meta_features.values).sum() + + if nan_count > 0 or inf_count > 0: + print(f"Warning: Meta-features contain {nan_count} NaN and {inf_count} inf values") + print("Replacing with zeros...") + meta_features = meta_features.fillna(0) + meta_features = meta_features.replace([np.inf, -np.inf], 0) + + # Get unique classes in validation data + unique_classes = np.unique(y_val) + print(f"Classes present in validation data: {unique_classes}") + + # Create and train meta-model + meta_model = lgb.LGBMClassifier( + n_estimators=100, + learning_rate=0.05, + num_leaves=31, + random_state=self.random_state, + n_jobs=self.n_jobs, + objective='multiclass', + num_class=5 # Explicitly set to 5 for all classes (-2 to 2) + ) + + # Create weights for each class + class_weights = {} + for cls in range(-2, 3): # -2, -1, 0, 1, 2 + if cls in unique_classes: + # Calculate actual weight based on class frequency + count = np.sum(y_val == cls) + class_weights[cls] = 1.0 / max(count, 1) + print(f"Class {cls}: {count} samples, weight: {class_weights[cls]:.4f}") + else: + # For missing classes, assign a default weight + class_weights[cls] = 1.0 + print(f"Class {cls}: 0 samples (missing), weight: 1.0000") + + # Convert class weights to the format expected by LightGBM (0-4 index) + indexed_weights = {} + for cls in range(-2, 3): + idx = cls + 2 # Convert class value to index: -2->0, -1->1, 0->2, 1->3, 2->4 + indexed_weights[idx] = class_weights[cls] + + # Normalize weights + total_weight = sum(indexed_weights.values()) + indexed_weights = {k: (v / total_weight) * 5 for k, v in indexed_weights.items()} + + print("Adjusted class weights for meta-model:") + for k, v in indexed_weights.items(): + print(f"Index {k} (Class {k-2}): {v:.4f}") + + # Set class weights + meta_model.set_params(class_weight=indexed_weights) + + # Fit meta-model + meta_model.fit(meta_features, y_val) + + # Evaluate meta-model if test data is provided + if X_test is not None and y_test is not None: + test_predictions = self._get_ensemble_predictions(X_test) + test_meta_features = pd.DataFrame(test_predictions) + + if 'volatility_regime' in X_test.columns: + test_meta_features['volatility_regime'] = X_test['volatility_regime'].values + + # Replace NaN or inf values + test_meta_features = test_meta_features.fillna(0) + test_meta_features = test_meta_features.replace([np.inf, -np.inf], 0) + + y_pred = meta_model.predict(test_meta_features) + accuracy = accuracy_score(y_test, y_pred) + report = classification_report(y_test, y_pred) + + print(f"Meta-model test accuracy: {accuracy:.4f}") + print("Classification report:") + print(report) + + self.meta_model = meta_model + return meta_model + + except Exception as e: + import traceback + print(f"Error training meta-model: {str(e)}") + print(f"Detailed traceback: {traceback.format_exc()}") + return None + + def _get_ensemble_predictions(self, X): + """ + Get predictions from all base models with improved error handling + + Parameters: + ----------- + X : pd.DataFrame + Feature matrix + + Returns: + -------- + predictions : dict + Dictionary with predictions from each model + """ + predictions = {} + + # Get predictions from each base model + for name, model_info in self.models.items(): + try: + model = model_info['model'] + features = model_info['features'] + + # Make sure we're only using features that exist in X + available_features = [f for f in features if f in X.columns] + if len(available_features) != len(features): + missing_features = set(features) - set(available_features) + print(f"Warning: {len(missing_features)} features missing for model {name}") + if len(missing_features) <= 5: + print(f"Missing features: {missing_features}") + + # For regime models, only predict on matching regime data + if 'regime' in model_info and 'volatility_regime' in X.columns: + regime = model_info['regime'] + regime_mask = X['volatility_regime'] == regime + + # Skip if no data for this regime + if not regime_mask.any(): + print(f"Skipping regime model {name} - no matching data") + # Initialize with zeros for this model for all possible classes + for i in range(5): # 5 classes (-2 to 2) + predictions[f"{name}_class_{i-2}"] = np.zeros(len(X)) + continue + + # Initialize predictions with zeros + proba = np.zeros((len(X), 5)) # Always use 5 classes (-2 to 2) + + # Get predictions for matching regime + try: + regime_proba = model.predict_proba(X.loc[regime_mask, available_features]) + + # Handle case where model has fewer than 5 classes + if regime_proba.shape[1] < 5: + temp_proba = np.zeros((regime_proba.shape[0], 5)) + # Map classes to correct positions in the 5-class array + classes = model.classes_ + 2 # Adjust to 0-4 indices + for i, cls_idx in enumerate(classes): + temp_proba[:, cls_idx] = regime_proba[:, i] + regime_proba = temp_proba + + # Place predictions in the correct indices + regime_indices = np.where(regime_mask)[0] + proba[regime_indices] = regime_proba + + except Exception as e: + print(f"Error predicting for regime {regime}: {str(e)}") + # Keep zeros for this regime + else: + # Regular model prediction on all data + try: + raw_proba = model.predict_proba(X[available_features]) + + # Initialize with zeros for all 5 classes + proba = np.zeros((len(X), 5)) + + # Handle case where model has fewer than 5 classes + if raw_proba.shape[1] < 5: + # Map classes to correct positions in the 5-class array + classes = model.classes_ + 2 # Adjust to 0-4 indices + for i, cls_idx in enumerate(classes): + proba[:, cls_idx] = raw_proba[:, i] + else: + proba = raw_proba + + except Exception as e: + print(f"Error predicting with model {name}: {str(e)}") + # Keep zeros for this model + + # Store class probabilities + for i in range(5): # 5 classes (-2 to 2) + predictions[f"{name}_class_{i-2}"] = proba[:, i] + + except Exception as e: + import traceback + print(f"Error processing model {name}: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + + # Initialize with zeros for all classes + for i in range(5): # 5 classes (-2 to 2) + predictions[f"{name}_class_{i-2}"] = np.zeros(len(X)) + + return predictions + + def fit(self, data, remove_cols=None): + """ + Complete model training pipeline with improved error handling + + Parameters: + ----------- + data : pd.DataFrame + DataFrame with features and target + remove_cols : list, optional + List of columns to remove (not used as features) + + Returns: + -------- + self : for method chaining + """ + print("Starting model training pipeline...") + + try: + # Prepare data + X, y = self.prepare_data(data, remove_cols) + + # Basic data validation + self.validate_data(X, y) + + # Train/validation/test split + X_train, X_val, X_test, y_train, y_val, y_test = self._create_time_series_splits(X, y) + print(f"Data split: train={len(X_train)}, validation={len(X_val)}, test={len(X_test)}") + + # Feature selection + try: + selected_features = self.select_features(X_train, y_train) + except Exception as e: + print(f"Feature selection error: {str(e)}") + print("Using all features as fallback") + self.selected_features = X_train.columns.tolist() + + # Train base models + try: + self.train_base_models(X_train, y_train, X_val, y_val) + except Exception as e: + import traceback + print(f"Base model training error: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + raise + + # Train meta-model + try: + self.train_meta_model(X_val, y_val, X_test, y_test) + except Exception as e: + import traceback + print(f"Meta-model training error: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + raise + + # Save models + try: + self.save_models() + except Exception as e: + print(f"Model saving error: {str(e)}") + + # Final evaluation + try: + self.evaluate(X_test, y_test) + except Exception as e: + print(f"Evaluation error: {str(e)}") + + return self + + except Exception as e: + import traceback + print(f"Error in model training pipeline: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + return self + + def predict(self, X): + """ + Generate predictions using the trained ensemble with improved error handling + + Parameters: + ----------- + X : pd.DataFrame + Feature matrix + + Returns: + -------- + predictions : pd.Series + Class predictions + probabilities : pd.DataFrame + Class probabilities + """ + try: + if self.meta_model is None: + raise ValueError("No trained meta-model available for prediction") + + # Get base model predictions + base_predictions = self._get_ensemble_predictions(X) + meta_features = pd.DataFrame(base_predictions) + + # Add volatility regime if available + if 'volatility_regime' in X.columns: + meta_features['volatility_regime'] = X['volatility_regime'].values + + # Handle missing values + meta_features = meta_features.fillna(0) + meta_features = meta_features.replace([np.inf, -np.inf], 0) + + # Generate predictions + class_proba = self.meta_model.predict_proba(meta_features) + + # Convert to DataFrame with appropriate class labels + class_names = [f'class_{i-2}' for i in range(5)] # class_-2 to class_2 + probabilities = pd.DataFrame( + class_proba, + columns=class_names, + index=X.index + ) + + # Get class predictions + raw_predictions = self.meta_model.predict(meta_features) + predictions = pd.Series( + raw_predictions, + index=X.index, + name='prediction' + ) + + return predictions, probabilities + + except Exception as e: + import traceback + print(f"Error in prediction: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + + # Return empty predictions as fallback + empty_predictions = pd.Series(index=X.index, name='prediction') + empty_probabilities = pd.DataFrame(index=X.index) + + return empty_predictions, empty_probabilities + + def evaluate(self, X_test, y_test): + """ + Evaluate the model on test data + + Parameters: + ----------- + X_test : pd.DataFrame + Test feature matrix + y_test : pd.Series + Test target + + Returns: + -------- + metrics : dict + Dictionary of evaluation metrics + """ + print("\nEvaluating model performance...") + + try: + # Generate predictions + y_pred, y_proba = self.predict(X_test) + + # Check for unique classes in predictions and actual + print(f"Unique classes in test data: {np.unique(y_test)}") + print(f"Unique classes in predictions: {np.unique(y_pred)}") + + # Calculate metrics + accuracy = accuracy_score(y_test, y_pred) + + # Handle case where some classes might be missing + # Get all possible classes from both actual and predicted + all_classes = sorted(set(np.unique(y_test)) | set(np.unique(y_pred))) + + # Generate report with all possible classes + report = classification_report(y_test, y_pred, labels=all_classes, output_dict=True) + + # For confusion matrix, we need to use the same classes + conf_matrix = confusion_matrix(y_test, y_pred, labels=all_classes) + + # Print results + print(f"Test accuracy: {accuracy:.4f}") + print("Classification report:") + print(classification_report(y_test, y_pred, labels=all_classes)) + + # Plot confusion matrix + plt.figure(figsize=(10, 8)) + class_labels = [f"Class {cls}" for cls in all_classes] + sns.heatmap( + conf_matrix, + annot=True, + fmt='d', + cmap='Blues', + xticklabels=class_labels, + yticklabels=class_labels + ) + plt.title('Confusion Matrix') + plt.xlabel('Predicted') + plt.ylabel('Actual') + plt.tight_layout() + plt.savefig(self.model_path / 'confusion_matrix.png') + + # Create metrics dictionary + metrics = { + 'accuracy': accuracy, + 'report': report, + 'confusion_matrix': conf_matrix + } + + return metrics + + except Exception as e: + import traceback + print(f"Error in evaluation: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + + # Return basic metrics + return { + 'accuracy': 0.0, + 'report': {}, + 'confusion_matrix': np.array([[0]]) + } + + def save_models(self): + """Save trained models and metadata to disk""" + # Create model directory if it doesn't exist + self.model_path.mkdir(exist_ok=True) + + # Save base models + for name, model_info in self.models.items(): + model = model_info['model'] + features = model_info['features'] + + # Save model + dump(model, self.model_path / f'{name}.joblib') + + # Save feature list + with open(self.model_path / f'{name}_features.txt', 'w') as f: + f.write('\n'.join(features)) + + # Save meta-model + if self.meta_model is not None: + dump(self.meta_model, self.model_path / 'meta_model.joblib') + + # Save feature selector or PCA + if self.feature_selector is not None: + dump(self.feature_selector, self.model_path / 'feature_selector.joblib') + + if self.pca is not None: + dump(self.pca, self.model_path / 'pca.joblib') + + # Save selected features + if self.selected_features is not None: + with open(self.model_path / 'selected_features.txt', 'w') as f: + f.write('\n'.join(self.selected_features)) + + # Save feature importance if available + if self.feature_importance is not None: + self.feature_importance.to_csv(self.model_path / 'feature_importance.csv', index=False) + + print(f"Models and metadata saved to {self.model_path}") + + def load_models(self): + """Load trained models and metadata from disk""" + # Check if model directory exists + if not self.model_path.exists(): + raise FileNotFoundError(f"Model directory {self.model_path} not found") + + # Load base models + self.models = {} + for model_file in self.model_path.glob('base_model_*.joblib'): + name = model_file.stem + features_file = self.model_path / f'{name}_features.txt' + + if features_file.exists(): + with open(features_file, 'r') as f: + features = f.read().splitlines() + + model = load(model_file) + self.models[name] = { + 'model': model, + 'features': features + } + + # Load regime models + for model_file in self.model_path.glob('regime_model_*.joblib'): + name = model_file.stem + features_file = self.model_path / f'{name}_features.txt' + + if features_file.exists(): + with open(features_file, 'r') as f: + features = f.read().splitlines() + + model = load(model_file) + regime = int(name.split('_')[-1]) + self.models[name] = { + 'model': model, + 'features': features, + 'regime': regime + } + + # Load meta-model + meta_model_file = self.model_path / 'meta_model.joblib' + if meta_model_file.exists(): + self.meta_model = load(meta_model_file) + + # Load feature selector or PCA + feature_selector_file = self.model_path / 'feature_selector.joblib' + if feature_selector_file.exists(): + self.feature_selector = load(feature_selector_file) + + pca_file = self.model_path / 'pca.joblib' + if pca_file.exists(): + self.pca = load(pca_file) + + # Load selected features + selected_features_file = self.model_path / 'selected_features.txt' + if selected_features_file.exists(): + with open(selected_features_file, 'r') as f: + self.selected_features = f.read().splitlines() + + # Load feature importance + feature_importance_file = self.model_path / 'feature_importance.csv' + if feature_importance_file.exists(): + self.feature_importance = pd.read_csv(feature_importance_file) + + print(f"Loaded {len(self.models)} models and metadata from {self.model_path}") diff --git a/tradingbot/models/neural_ensemble.py b/tradingbot/models/neural_ensemble.py new file mode 100644 index 0000000..c435805 --- /dev/null +++ b/tradingbot/models/neural_ensemble.py @@ -0,0 +1,340 @@ +"""Neural + tree hybrid ensemble predictor. + +Stacks gradient-boosted trees with a Transformer/BiLSTM network and a Bayesian +network (Monte-Carlo dropout) for uncertainty-aware signal filtering. +""" +import numpy as np +import pandas as pd +from sklearn.preprocessing import RobustScaler +from sklearn.model_selection import TimeSeriesSplit +import xgboost as xgb +import lightgbm as lgb +from catboost import CatBoostClassifier +from hmmlearn import hmm +import talib +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import Dataset, DataLoader + + +class TimeSeriesDataset(Dataset): + def __init__(self, X, y): + self.X = torch.tensor(X, dtype=torch.float32) + self.y = torch.tensor(y, dtype=torch.long) + + def __len__(self): + return len(self.y) + + def __getitem__(self, idx): + return self.X[idx], self.y[idx] + +class TransformerBlock(nn.Module): + def __init__(self, d_model, nhead, dropout=0.3): + super().__init__() + self.attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout, batch_first=True) + self.norm1 = nn.LayerNorm(d_model) + self.ff = nn.Sequential( + nn.Linear(d_model, 64), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(64, d_model) + ) + self.norm2 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + attn_out, _ = self.attn(x, x, x) + x = self.norm1(x + self.dropout(attn_out)) + ff_out = self.ff(x) + x = self.norm2(x + self.dropout(ff_out)) + return x + +class NeuralEnsemblePredictor: + def __init__(self, forecast_period=12, confidence_threshold=0.6, device='cuda' if torch.cuda.is_available() else 'cpu'): + self.forecast_period = forecast_period + self.confidence_threshold = confidence_threshold + self.device = device + self.scaler = RobustScaler() + self.regime_model = hmm.GaussianHMM(n_components=3, random_state=42) + + # Tree-based models with pre-tuned parameters + self.models = { + 'xgboost': xgb.XGBClassifier( + colsample_bytree=0.9821911945239713, + learning_rate=0.07305593001592295, + max_depth=9, + min_child_weight=5, + subsample=0.7524259059189972, + random_state=42, + eval_metric='logloss' + ), + 'lightgbm': lgb.LGBMClassifier( + feature_fraction=0.9249583953429453, + learning_rate=0.025468440525690465, + max_depth=7, + min_child_samples=41, + subsample=0.8092209312217534, + random_state=44 + ), + 'catboost': CatBoostClassifier( + colsample_bylevel=0.7762414086064304, + depth=4, + learning_rate=0.08067885793496102, + subsample=0.858134636983408, + random_state=45, + verbose=0 + ) + } + + # PyTorch neural networks + self.nn_model = self._build_neural_network().to(self.device) + self.bnn_model = self._build_bayesian_network().to(self.device) + + # Non-linear meta-model + self.meta_model = lgb.LGBMClassifier( + n_estimators=100, max_depth=3, learning_rate=0.05, random_state=46 + ) + + def _build_neural_network(self): + """Transformer-based neural network""" + class Net(nn.Module): + def __init__(self): + super().__init__() + self.transformer = nn.ModuleList([ + TransformerBlock(d_model=6, nhead=2, dropout=0.3) + for _ in range(2) + ]) + self.lstm = nn.LSTM(6, 32, batch_first=True, bidirectional=True) + self.attn = nn.Linear(64, 1) # Attention over 64 from bidirectional LSTM + self.fc = nn.Sequential( + nn.Linear(64, 64), + nn.ReLU(), + nn.Dropout(0.4), + nn.Linear(64, 2) + ) + + def forward(self, x): + for t in self.transformer: + x = t(x) + x, _ = self.lstm(x) # Shape: (batch, 20, 64) + attn_weights = torch.softmax(self.attn(x), dim=1) # Shape: (batch, 20, 1) + x = (x * attn_weights).sum(dim=1) # Shape: (batch, 64) + x = self.fc(x) + return x + + return Net() + + def _build_bayesian_network(self): + """Bayesian neural network with Monte Carlo dropout""" + class BNN(nn.Module): + def __init__(self): + super().__init__() + self.flatten = nn.Flatten() + self.fc1 = nn.Linear(20 * 6, 64) + self.fc2 = nn.Linear(64, 32) + self.fc3 = nn.Linear(32, 2) + self.dropout = nn.Dropout(0.3) + self.relu = nn.ReLU() + + def forward(self, x, training=False): + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.dropout(x) if training else x + x = self.relu(self.fc2(x)) + x = self.dropout(x) if training else x + x = self.fc3(x) + return x + + return BNN() + + def detect_market_regime(self, data): + returns = np.log(data['close'] / data['close'].shift(1)) + volatility = returns.rolling(window=20).std() + combined = pd.DataFrame({'returns': returns, 'volatility': volatility}).dropna() + self.regime_model.fit(combined.values) + regimes = self.regime_model.predict(combined.values) + regime_series = pd.Series(index=data.index, dtype='float64') + regime_series.iloc[len(data)-len(regimes):] = regimes + return regime_series + + def create_advanced_features(self, df): + data = df.copy() + data['market_regime'] = self.detect_market_regime(data) + for period in [21, 55]: + data[f'ema_{period}'] = talib.EMA(data['close'], timeperiod=period) + data[f'trend_{period}'] = (data[f'ema_{period}'] - data[f'ema_{period}'].shift(period)) / data[f'ema_{period}'].shift(period) + data['atr_ratio'] = talib.ATR(data['high'], data['low'], data['close'], 14) / data['close'] + data['rsi'] = talib.RSI(data['close'], 14) + data['volume_ma'] = talib.EMA(data['volume'], timeperiod=20) + data['volume_ratio'] = data['volume'] / data['volume_ma'] + + returns = data['close'].shift(-self.forecast_period) / data['close'] - 1 + data['target'] = np.where(returns > 0.005, 1, np.where(returns < -0.005, 0, None)) + return data.dropna() + + def prepare_features(self, data, for_nn=False): + feature_columns = [ + 'market_regime', 'atr_ratio', 'rsi', 'volume_ratio', + 'trend_21', 'trend_55' + ] + X = data[feature_columns] + y = data['target'].astype(int) + + if for_nn: + X_3d = np.array([X.iloc[i-20:i].values for i in range(20, len(X))]) + y_3d = y.iloc[20:].values + return X_3d, y_3d + return X, y + + def train_nn(self, X_nn, y_nn, model, epochs=50, batch_size=32): + dataset = TimeSeriesDataset(X_nn, y_nn) + train_size = int(0.8 * len(dataset)) + val_size = len(dataset) - train_size + train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size]) + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_dataset, batch_size=batch_size) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.0001) + + best_val_loss = float('inf') + patience = 15 + patience_counter = 0 + + for epoch in range(epochs): + model.train() + train_loss = 0 + for X_batch, y_batch in train_loader: + X_batch, y_batch = X_batch.to(self.device), y_batch.to(self.device) + optimizer.zero_grad() + outputs = model(X_batch) + loss = criterion(outputs, y_batch) + loss.backward() + optimizer.step() + train_loss += loss.item() + + model.eval() + val_loss = 0 + with torch.no_grad(): + for X_batch, y_batch in val_loader: + X_batch, y_batch = X_batch.to(self.device), y_batch.to(self.device) + outputs = model(X_batch) + val_loss += criterion(outputs, y_batch).item() + + train_loss /= len(train_loader) + val_loss /= len(val_loader) + print(f"Epoch {epoch+1}/{epochs}, Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}") + + if val_loss < best_val_loss: + best_val_loss = val_loss + patience_counter = 0 + torch.save(model.state_dict(), 'nn_best.pth') + else: + patience_counter += 1 + if patience_counter >= patience: + print("Early stopping") + break + + model.load_state_dict(torch.load('nn_best.pth')) + return model + + def fit(self, train_data): + processed_data = self.create_advanced_features(train_data) + X, y = self.prepare_features(processed_data) + X_nn, y_nn = self.prepare_features(processed_data, for_nn=True) + X_scaled = self.scaler.fit_transform(X) + X_scaled = pd.DataFrame(X_scaled, columns=X.columns) + + # Train tree-based models (no tuning) + tscv = TimeSeriesSplit(n_splits=5) + oof_preds = np.zeros((len(X_scaled), len(self.models) + 2)) + + for fold, (train_idx, val_idx) in enumerate(tscv.split(X_scaled)): + X_train, X_val = X_scaled.iloc[train_idx], X_scaled.iloc[val_idx] + y_train, y_val = y.iloc[train_idx], y.iloc[val_idx] + + for i, (name, model) in enumerate(self.models.items()): + if name == 'xgboost': + model.fit(X_train, y_train, eval_set=[(X_val, y_val)],verbose=False) + elif name == 'lightgbm': + model.fit(X_train, y_train, eval_set=[(X_val, y_val)]) + elif name == 'catboost': + model.fit(X_train, y_train, eval_set=(X_val, y_val)) + oof_preds[val_idx, i] = model.predict_proba(X_val)[:, 1] + + # Train neural network + nn_idx_shift = len(X_scaled) - len(X_nn) + self.nn_model = self.train_nn(X_nn, y_nn, self.nn_model) + self.nn_model.eval() + with torch.no_grad(): + nn_preds = torch.softmax(self.nn_model(torch.tensor(X_nn, dtype=torch.float32).to(self.device)), dim=1)[:, 1].cpu().numpy() + oof_preds[nn_idx_shift:, len(self.models)] = nn_preds + + # Train Bayesian NN with Monte Carlo dropout + self.bnn_model = self.train_nn(X_nn, y_nn, self.bnn_model) + self.bnn_model.train() # Enable dropout for MC estimation + mc_preds = [] + with torch.no_grad(): + for _ in range(10): # 10 Monte Carlo samples + preds = torch.softmax(self.bnn_model(torch.tensor(X_nn, dtype=torch.float32).to(self.device), training=True), dim=1)[:, 1].cpu().numpy() + mc_preds.append(preds) + bnn_mean = np.mean(mc_preds, axis=0) + bnn_std = np.std(mc_preds, axis=0) + oof_preds[nn_idx_shift:, len(self.models) + 1] = bnn_mean + + # Train meta-model + self.meta_model.fit(oof_preds, y, sample_weight=np.exp(np.linspace(-1, 0, len(y)))) + + # Final training of tree models + for name, model in self.models.items(): + if name in ['xgboost']: + model.fit(X_scaled, y, eval_set=[(X_scaled, y)], verbose=False) + elif name in ['lightgbm']: + model.fit(X_scaled, y, eval_set=[(X_scaled, y)]) + else: + model.fit(X_scaled, y, eval_set=(X_scaled, y)) + + return processed_data + + def predict(self, data): + processed_data = self.create_advanced_features(data) + X, _ = self.prepare_features(processed_data) + X_nn, _ = self.prepare_features(processed_data, for_nn=True) + X_scaled = self.scaler.transform(X) + X_scaled = pd.DataFrame(X_scaled, columns=X.columns) + + base_preds = np.zeros((len(X_scaled), len(self.models) + 2)) + for i, (name, model) in enumerate(self.models.items()): + base_preds[:, i] = model.predict_proba(X_scaled)[:, 1] + + nn_idx_shift = len(X_scaled) - len(X_nn) + self.nn_model.eval() + with torch.no_grad(): + nn_preds = torch.softmax(self.nn_model(torch.tensor(X_nn, dtype=torch.float32).to(self.device)), dim=1)[:, 1].cpu().numpy() + base_preds[nn_idx_shift:, len(self.models)] = nn_preds + + # Bayesian NN predictions with uncertainty + self.bnn_model.train() # Enable dropout + mc_preds = [] + with torch.no_grad(): + for _ in range(10): + preds = torch.softmax(self.bnn_model(torch.tensor(X_nn, dtype=torch.float32).to(self.device), training=True), dim=1)[:, 1].cpu().numpy() + mc_preds.append(preds) + bnn_mean = np.mean(mc_preds, axis=0) + bnn_std = np.std(mc_preds, axis=0) + base_preds[nn_idx_shift:, len(self.models) + 1] = bnn_mean + + # Meta-model prediction + meta_proba = self.meta_model.predict_proba(base_preds) + + signals = pd.Series(0, index=processed_data.index) + valid_indices = processed_data.index[20:] # Skip first 20 due to lookback + meta_proba_valid = meta_proba[nn_idx_shift:] # Align with NN predictions + + long_mask = (meta_proba_valid[:, 1] > self.confidence_threshold) & (bnn_std < 0.2) + short_mask = (meta_proba_valid[:, 0] > self.confidence_threshold) & (bnn_std < 0.2) + signals.loc[valid_indices[long_mask]] = 1 + signals.loc[valid_indices[short_mask]] = -1 + + return signals diff --git a/tradingbot/models/tree_ensemble.py b/tradingbot/models/tree_ensemble.py new file mode 100644 index 0000000..4473cd8 --- /dev/null +++ b/tradingbot/models/tree_ensemble.py @@ -0,0 +1,562 @@ +"""Stacked tree-ensemble predictor for the 5-minute timeframe. + +A binary (up/down) classifier that stacks five gradient-boosted / bagged tree +models with two meta-models and an HMM market-regime filter. +""" +import numpy as np +import pandas as pd +from sklearn.preprocessing import RobustScaler +from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import roc_auc_score +from sklearn.model_selection import TimeSeriesSplit +import xgboost as xgb +import lightgbm as lgb +from catboost import CatBoostClassifier +from hmmlearn import hmm +import talib + + +class TreeEnsemblePredictor: + def __init__(self, forecast_bars=24, confidence_threshold=0.65): + """ + Optimized gold price prediction model for 5-minute timeframe + + Parameters: + forecast_bars (int): Number of future 5-min bars to predict (default=24, which is 2 hours) + confidence_threshold (float): Minimum probability threshold for signal generation + """ + self.forecast_bars = forecast_bars + self.confidence_threshold = confidence_threshold + self.scaler = RobustScaler() + self.regime_model = hmm.GaussianHMM(n_components=3, random_state=42) + + # Tree-based models with parameters optimized for 5-min timeframe + self.models = { + 'xgboost': xgb.XGBClassifier( + colsample_bytree=0.85, + learning_rate=0.05, + max_depth=6, # Reduced depth for faster market dynamics + min_child_weight=3, + subsample=0.8, + n_estimators=100, + random_state=42, + eval_metric='logloss', + use_label_encoder=False + ), + 'lightgbm': lgb.LGBMClassifier( + feature_fraction=0.9, + learning_rate=0.02, + max_depth=5, + min_child_samples=20, # Smaller sample for 5-min data + subsample=0.8, + n_estimators=100, + random_state=44, + boosting_type='dart' # More robust to noise in high-frequency data + ), + 'catboost': CatBoostClassifier( + depth=4, + learning_rate=0.07, + subsample=0.85, + n_estimators=100, + random_state=45, + verbose=0 + ), + 'randomforest': RandomForestClassifier( + n_estimators=100, + max_depth=5, + max_features='sqrt', + min_samples_leaf=5, # Captures more granular patterns + random_state=46 + ), + 'extratrees': ExtraTreesClassifier( + n_estimators=100, + max_depth=5, + max_features='sqrt', + min_samples_leaf=5, + random_state=47 + ) + } + + # Primary meta-model + self.meta_model = lgb.LGBMClassifier( + n_estimators=100, + max_depth=3, + learning_rate=0.03, + random_state=48, + boosting_type='gbdt' + ) + + # Secondary meta-model for consensus validation + self.meta_model_backup = LogisticRegression( + C=0.1, + solver='liblinear', + random_state=49 + ) + + # Track feature importance + self.feature_importances = {} + self.feature_names = [] + + def detect_market_regime(self, data): + """ + Detects market regimes using HMM on returns and volatility + Optimized for 5-minute data with shorter lookback windows + + Returns regime classifications (0=low vol, 1=normal, 2=high vol) + """ + # Calculate returns and volatility + returns = np.log(data['close'] / data['close'].shift(1)) + # Shorter window for 5-minute data (60 periods = 5 hours) + volatility = returns.rolling(window=60).std() + combined = pd.DataFrame({'returns': returns, 'volatility': volatility}).dropna() + + # Fit HMM model if we have enough data + if len(combined) > 100: + self.regime_model.fit(combined.values) + regimes = self.regime_model.predict(combined.values) + regime_series = pd.Series(index=data.index, dtype='float64') + regime_series.iloc[len(data)-len(regimes):] = regimes + return regime_series + else: + # Default to moderate regime if not enough data + return pd.Series(1, index=data.index) + + def create_advanced_features(self, df): + """ + Create features optimized for 5-minute gold price prediction + + Features are organized in categories: + 1. Market regime + 2. Time-based features + 3. Price action features + 4. Volatility indicators + 5. Momentum indicators + 6. Volume indicators + 7. Support/Resistance + 8. Pattern recognition + """ + data = df.copy() + + # 1. Market regime detection + data['market_regime'] = self.detect_market_regime(data) + + # 2. Time-based features for intraday seasonality + data['hour'] = data.index.hour + data['minute'] = data.index.minute + data['day_of_week'] = data.index.dayofweek + # Cyclical encoding of time (circular features) + data['hour_sin'] = np.sin(2 * np.pi * data['hour']/24) + data['hour_cos'] = np.cos(2 * np.pi * data['hour']/24) + + # 3. Short-term price action features + # Moving averages adapted for 5-min timeframe + for period in [12, 24, 48, 96, 144]: # 1h, 2h, 4h, 8h, 12h in 5-minute bars + # Exponential moving averages + data[f'ema_{period}'] = talib.EMA(data['close'], timeperiod=period) + + # Price relative to moving average (normalized distance) + data[f'price_to_ema_{period}'] = data['close'] / data[f'ema_{period}'] - 1 + + # Trend strength + data[f'trend_{period}'] = (data[f'ema_{period}'] - data[f'ema_{period}'].shift(period//4)) / data[f'ema_{period}'].shift(period//4) + + # 4. Volatility indicators + # ATR with periods suitable for 5-minute bars + for period in [12, 24, 48, 96]: # 1h, 2h, a + data[f'atr_{period}'] = talib.ATR(data['high'], data['low'], data['close'], timeperiod=period) + data[f'atr_ratio_{period}'] = data[f'atr_{period}'] / data['close'] + + # Bollinger Bands - essential for mean-reversion detection + for period in [24, 48, 96]: + upper, middle, lower = talib.BBANDS(data['close'], timeperiod=period, nbdevup=2, nbdevdn=2) + data[f'bb_width_{period}'] = (upper - lower) / middle + data[f'bb_position_{period}'] = (data['close'] - lower) / (upper - lower) + + # 5. Momentum indicators + # RSI with different lookback periods for 5-min data + for period in [12, 24, 48, 96]: + data[f'rsi_{period}'] = talib.RSI(data['close'], timeperiod=period) + + # MACD for 5-minute data (faster parameters) + macd, macd_signal, macd_hist = talib.MACD( + data['close'], + fastperiod=6, # Faster for 5-min data + slowperiod=19, # Faster for 5-min data + signalperiod=5 # Faster for 5-min data + ) + data['macd'] = macd + data['macd_signal'] = macd_signal + data['macd_hist'] = macd_hist + + # 6. Volume indicators (crucial for 5-minute signals) + # Volume relative to moving average + for period in [12, 24, 48]: + data[f'volume_ma_{period}'] = talib.SMA(data['volume'], timeperiod=period) + data[f'volume_ratio_{period}'] = data['volume'] / data[f'volume_ma_{period}'] + + # On-balance volume - good for measuring buying/selling pressure + data['obv'] = talib.OBV(data['close'], data['volume']) + data['obv_ma'] = talib.SMA(data['obv'], timeperiod=24) + data['obv_ratio'] = data['obv'] / data['obv_ma'] + + # 7. Support/Resistance levels + # Pivot points for 5-min (using 96 periods = 8 hours) + data['pivot'] = (data['high'].rolling(96).max() + data['low'].rolling(96).min() + data['close'].rolling(96).mean()) / 3 + data['dist_to_pivot'] = (data['close'] - data['pivot']) / data['close'] + + # 8. Candlestick pattern features + # Candle size metrics + data['candle_range'] = (data['high'] - data['low']) / data['close'] + data['candle_body'] = abs(data['open'] - data['close']) / data['close'] + + # Rate of change - important for 5-min momentum + for period in [6, 12, 24]: + data[f'roc_{period}'] = talib.ROC(data['close'], timeperiod=period) + + # Target definition for 5-minute timeframe + # Using appropriate thresholds for smaller price moves + future_return = data['close'].shift(-self.forecast_bars) / data['close'] - 1 + # Lower threshold for 5-minute bars (approximately 0.1-0.15% move) + data['target'] = np.where(future_return > 0.0012, 1, np.where(future_return < -0.0012, 0, None)) + + # Drop rows with missing data + return data.dropna() + + def prepare_features(self, data): + """ + Prepare and select optimal features for the model + Features are grouped by category for easier selection + """ + # Most important features for 5-minute gold prediction + feature_columns = [ + # Market regime + 'market_regime', + + # Time features for intraday patterns + 'hour_sin', 'hour_cos', 'day_of_week', + + # Price action features + 'price_to_ema_12', 'price_to_ema_24', 'price_to_ema_48', + 'trend_24', 'trend_48', 'trend_96', + + # Volatility indicators + 'atr_ratio_12', 'atr_ratio_24', + 'bb_width_24', 'bb_width_48', + 'bb_position_24', 'bb_position_48', + + # Momentum indicators + 'rsi_12', 'rsi_24', 'rsi_48', + 'macd', 'macd_hist', + + # Volume indicators + 'volume_ratio_12', 'volume_ratio_24', + 'obv_ratio', + + # Support/Resistance + 'dist_to_pivot', + + # Pattern recognition + 'candle_range', 'candle_body', + 'roc_6', 'roc_12' + ] + + X = data[feature_columns] + + if 'target' in data.columns: + y = data['target'].astype(int) + return X, y + else: + return X, None + + def fit(self, train_data): + """Train the ensemble model on historical data""" + print("Creating features...") + processed_data = self.create_advanced_features(train_data) + X, y = self.prepare_features(processed_data) + + # Store feature names for importance tracking + self.feature_names = X.columns.tolist() + + # Scale features + X_scaled = self.scaler.fit_transform(X) + X_scaled = pd.DataFrame(X_scaled, columns=X.columns) + + # Time series cross-validation + print("Performing time series cross-validation...") + tscv = TimeSeriesSplit(n_splits=5) + oof_preds = np.zeros((len(X_scaled), len(self.models))) + + for fold, (train_idx, val_idx) in enumerate(tscv.split(X_scaled)): + X_train, X_val = X_scaled.iloc[train_idx], X_scaled.iloc[val_idx] + y_train, y_val = y.iloc[train_idx], y.iloc[val_idx] + + for i, (name, model) in enumerate(self.models.items()): + print(f"Training {name} for fold {fold+1}/5...") + + if name == 'xgboost': + model.fit(X_train, y_train, eval_set=[(X_val, y_val)], verbose=False) + elif name == 'lightgbm': + model.fit(X_train, y_train, eval_set=[(X_val, y_val)]) + elif name == 'catboost': + model.fit(X_train, y_train, eval_set=(X_val, y_val)) + else: + model.fit(X_train, y_train) + + # Store out-of-fold predictions + oof_preds[val_idx, i] = model.predict_proba(X_val)[:, 1] + + # Track feature importance + if hasattr(model, 'feature_importances_'): + if name not in self.feature_importances: + self.feature_importances[name] = np.zeros(len(self.feature_names)) + self.feature_importances[name] += model.feature_importances_ + + # Evaluate base models + for i, name in enumerate(self.models.keys()): + auc = roc_auc_score(y, oof_preds[:, i]) + print(f"{name} Out-of-fold AUC: {auc:.4f}") + + # Train meta-models on out-of-fold predictions + print("Training meta-models...") + # Weight recent data more heavily for financial time series + sample_weights = np.exp(np.linspace(-1, 0, len(y))) + + # Train primary meta-model + self.meta_model.fit(oof_preds, y, sample_weight=sample_weights) + + # Train backup meta-model for consensus + self.meta_model_backup.fit(oof_preds, y, sample_weight=sample_weights) + + # Display feature importance summary + self._print_feature_importance() + + # Final training of all models on full dataset + print("Final training on complete dataset...") + for name, model in self.models.items(): + if name in ['xgboost']: + model.fit(X_scaled, y, eval_set=[(X_scaled, y)], verbose=False) + elif name in ['lightgbm']: + model.fit(X_scaled, y, eval_set=[(X_scaled, y)]) + elif name in ['catboost']: + model.fit(X_scaled, y, eval_set=(X_scaled, y)) + else: + model.fit(X_scaled, y) + + return processed_data + + def _print_feature_importance(self): + """Display top features by importance for each model""" + print("\n=== Feature Importance Analysis ===") + for name, importances in self.feature_importances.items(): + # Normalize importances to percentages + importances = importances / np.sum(importances) * 100 + # Sort by importance + sorted_idx = np.argsort(importances)[::-1] + print(f"\n{name.upper()} Top 10 Features:") + for i in range(min(10, len(sorted_idx))): + idx = sorted_idx[i] + print(f" {self.feature_names[idx]}: {importances[idx]:.2f}%") + + def predict(self, data): + """ + Generate trading signals for 5-minute gold price data + + Returns: + DataFrame with columns: + - signal: Trading signal (-1=short, 0=neutral, 1=long) + - strength: Signal strength (0-100%) + - proba_up: Probability of price increase + - proba_down: Probability of price decrease + - model_agreement: Agreement level between meta-models + - market_regime: Detected market regime + """ + processed_data = self.create_advanced_features(data) + X, _ = self.prepare_features(processed_data) + X_scaled = self.scaler.transform(X) + X_scaled = pd.DataFrame(X_scaled, columns=X.columns) + + # Get predictions from base models + base_preds = np.zeros((len(X_scaled), len(self.models))) + for i, (name, model) in enumerate(self.models.items()): + base_preds[:, i] = model.predict_proba(X_scaled)[:, 1] + + # Get predictions from both meta-models + meta_proba = self.meta_model.predict_proba(base_preds) + backup_proba = self.meta_model_backup.predict_proba(base_preds) + + # Calculate consensus level between meta-models (0-1) + models_agreement = 1 - np.abs(meta_proba[:, 1] - backup_proba[:, 1]) + + # Initialize signals + signals = pd.Series(0, index=processed_data.index) + + # Generate signals with sophisticated filtering + # Long signal: high probability of price increase + high model agreement + long_mask = (meta_proba[:, 1] > self.confidence_threshold) & (models_agreement > 0.8) + + # Short signal: high probability of price decrease + high model agreement + short_mask = (meta_proba[:, 0] > self.confidence_threshold) & (models_agreement > 0.8) + + # Add market regime filter - only take signals in appropriate regimes + market_regimes = processed_data['market_regime'] + + # Only generate long signals in trending or normal regimes (0 or 1) + long_regime_mask = (market_regimes == 0) | (market_regimes == 1) + + # Only generate short signals in trending or high volatility regimes (0 or 2) + short_regime_mask = (market_regimes == 0) | (market_regimes == 2) + + # Apply regime filters to signals + signals.loc[long_mask & long_regime_mask] = 1 + signals.loc[short_mask & short_regime_mask] = -1 + + # Calculate signal strength (0-100%) based on prediction confidence + signal_strength = pd.Series(0.0, index=processed_data.index) + signal_strength.loc[long_mask] = (meta_proba[long_mask, 1] - self.confidence_threshold) * (1 / (1 - self.confidence_threshold)) * 100 + signal_strength.loc[short_mask] = (meta_proba[short_mask, 0] - self.confidence_threshold) * (1 / (1 - self.confidence_threshold)) * 100 + + # Risk management: additional signal filters + # 1. Minimum signal duration (prevent rapid flipping) + min_bars = 3 # Minimum 15 minutes + for i in range(min_bars, len(signals)): + if signals.iloc[i] != 0 and signals.iloc[i] == -signals.iloc[i-1]: + # If signal flips too soon, maintain previous signal + if sum(signals.iloc[i-min_bars:i] == signals.iloc[i-1]) >= min_bars-1: + signals.iloc[i] = signals.iloc[i-1] + + # 2. Filter out signals during extreme volatility + high_vol_mask = processed_data['atr_ratio_24'] > processed_data['atr_ratio_24'].quantile(0.95) + signals.loc[high_vol_mask] = 0 + + # Combine results into a DataFrame + results = pd.DataFrame({ + 'signal': signals, + 'strength': signal_strength, + 'proba_up': meta_proba[:, 1], + 'proba_down': meta_proba[:, 0], + 'model_agreement': models_agreement, + 'market_regime': processed_data['market_regime'] + }, index=processed_data.index) + + return results + + def evaluate_performance(self, test_data): + """ + Evaluate model performance with trading simulation - corrected version + """ + results = self.predict(test_data) + + # Calculate forward returns for evaluation period + close_prices = test_data['close'] + forward_returns = close_prices.shift(-self.forecast_bars) / close_prices - 1 + + # Apply signals to returns (long = 1x return, short = -1x return) + strategy_returns = results['signal'] * forward_returns + + # Properly handle NaN values that might appear from shifts + strategy_returns = strategy_returns.dropna() + + # Calculate performance metrics + total_return = strategy_returns.sum() + + # Correct annualization factor: 252 trading days, 12 hours per day, 12 bars per hour + annualization_factor = np.sqrt(252 * 12 * 12) + sharpe_ratio = strategy_returns.mean() / strategy_returns.std() * annualization_factor + + # Correct win rate calculation - account for signal direction + wins = ((strategy_returns > 0) & (results['signal'] != 0)).sum() + total_trades = (results['signal'] != 0).sum() + win_rate = wins / total_trades if total_trades > 0 else 0 + + # Correct drawdown calculation + cumulative_returns = strategy_returns.cumsum() + drawdowns = cumulative_returns - cumulative_returns.cummax() + max_drawdown = drawdowns.min() + + # Signal statistics + signal_count = (results['signal'] != 0).sum() + signal_changes = results['signal'].diff().abs() + signal_changes = signal_changes[signal_changes > 0].sum() / 2 # Each change counts twice in diff + + # Correct trading days calculation (typically 5 days a week for forex) + # Assuming 12 hours of active trading per day and 12 5-min bars per hour + trading_days = len(results) / (12 * 12) + avg_signals_per_day = signal_count / trading_days + + # Calculate profit factor + profitable_trades = strategy_returns[strategy_returns > 0].sum() + losing_trades = abs(strategy_returns[strategy_returns < 0].sum()) + profit_factor = profitable_trades / losing_trades if losing_trades != 0 else float('inf') + + # Calculate average profit per trade + avg_profit_per_trade = total_return / total_trades if total_trades > 0 else 0 + + # Compile metrics + metrics = { + 'total_return': total_return, + 'annualized_return': total_return * (252 / trading_days), + 'sharpe_ratio': sharpe_ratio, + 'win_rate': win_rate, + 'max_drawdown': max_drawdown, + 'profit_factor': profit_factor, + 'signal_count': signal_count, + 'signal_changes': signal_changes, + 'avg_signals_per_day': avg_signals_per_day, + 'avg_profit_per_trade': avg_profit_per_trade + } + + print("\n=== Performance Evaluation ===") + print(f"Total Return: {total_return:.2%}") + print(f"Annualized Return: {metrics['annualized_return']:.2%}") + print(f"Annualized Sharpe Ratio: {sharpe_ratio:.2f}") + print(f"Win Rate: {win_rate:.2%}") + print(f"Maximum Drawdown: {max_drawdown:.2%}") + print(f"Profit Factor: {profit_factor:.2f}") + print(f"Total Signals: {signal_count}") + print(f"Signal Changes: {signal_changes}") + print(f"Average Signals Per Day: {avg_signals_per_day:.2f}") + print(f"Average Profit Per Trade: {avg_profit_per_trade:.4%}") + + return metrics, results, strategy_returns + +# Example usage +def run_model(data_path, forecast_bars=24, confidence_threshold=0.65): + """ + Run the model on input data + + Parameters: + data_path: Path to CSV file with OHLCV data + forecast_bars: Number of 5-min bars to forecast + confidence_threshold: Threshold for signal generation + + Returns: + predictor: Trained model + metrics: Performance metrics + results: Signal results + returns: Strategy returns + """ + # Load and prepare data + data = pd.read_csv(data_path) + data['timestamp'] = pd.to_datetime(data['timestamp']) + data = data.drop_duplicates(subset=['timestamp']) + data.set_index('timestamp', inplace=True) + + # Split into train/test + #pick the last 40% of data before the last 80% from 40% to 80% + train_size = int(len(data) * 0.95) + train_data = data[int(len(data) * 0.80):train_size].copy() + test_data = data[train_size:].copy() + + # Create and train model + predictor = TreeEnsemblePredictor( + forecast_bars=forecast_bars, + confidence_threshold=confidence_threshold + ) + print(f"Training model with forecast_bars={forecast_bars}, confidence_threshold={confidence_threshold}") + predictor.fit(train_data) + + # Evaluate model + metrics, results, returns = predictor.evaluate_performance(test_data) + + return predictor, metrics, results, returns diff --git a/tradingbot/signals/__init__.py b/tradingbot/signals/__init__.py new file mode 100644 index 0000000..7197c7b --- /dev/null +++ b/tradingbot/signals/__init__.py @@ -0,0 +1,3 @@ +from tradingbot.signals.generator import GoldSignalGenerator + +__all__ = ["GoldSignalGenerator"] diff --git a/tradingbot/signals/generator.py b/tradingbot/signals/generator.py new file mode 100644 index 0000000..f355949 --- /dev/null +++ b/tradingbot/signals/generator.py @@ -0,0 +1,383 @@ +"""Convert model predictions into trading signals with risk management.""" +import numpy as np +import pandas as pd + +from tradingbot.models.model_manager import GoldModelManager + + +class GoldSignalGenerator: + """ + Generates trading signals from model predictions with risk management + """ + def __init__( + self, + confidence_threshold=0.7, + risk_reward_min=1.5, + stop_atr_factor=1.5, + target_atr_factor=2.25, + model_manager=None, + model_path='models' + ): + """ + Initialize the signal generator + + Parameters: + ----------- + confidence_threshold : float + Minimum probability threshold for generating signals + risk_reward_min : float + Minimum risk/reward ratio for valid trades + stop_atr_factor : float + Factor to multiply ATR for stop loss calculation + target_atr_factor : float + Factor to multiply ATR for take profit calculation + model_manager : GoldModelManager, optional + Model manager instance (if None, will load from model_path) + model_path : str + Directory to load models from (if model_manager is None) + """ + self.confidence_threshold = confidence_threshold + self.risk_reward_min = risk_reward_min + self.stop_atr_factor = stop_atr_factor + self.target_atr_factor = target_atr_factor + + # Use provided model manager or create a new one + if model_manager is not None: + self.model_manager = model_manager + else: + self.model_manager = GoldModelManager(model_path=model_path) + self.model_manager.load_models() + + def generate_signals(self, data): + """ + Generate trading signals from data with improved error handling + + Parameters: + ----------- + data : pd.DataFrame + Data with features + + Returns: + -------- + signals : pd.DataFrame + DataFrame with trading signals and risk management + """ + try: + # Prepare features + X, _ = self.model_manager.prepare_data(data, remove_cols=None) + + # Check if model manager has a trained meta model + if self.model_manager.meta_model is None: + print("Warning: No trained meta-model available for prediction") + return pd.DataFrame(index=data.index) + + # Get model predictions + predictions, probabilities = self.model_manager.predict(X) + + # Check if predictions or probabilities are empty + if predictions.empty or probabilities.empty: + print("Warning: Empty predictions or probabilities") + return pd.DataFrame(index=data.index) + + # Initialize signals DataFrame + signals = pd.DataFrame(index=data.index) + signals['prediction'] = predictions + + # Add class probabilities + for col in probabilities.columns: + signals[col] = probabilities[col] + + # Calculate signal confidence with NaN handling + if probabilities.values.size > 0: + confidence_values = np.nanmax(probabilities.values, axis=1) + # Replace any NaN confidence values with 0 + confidence_values = np.nan_to_num(confidence_values, nan=0) + signals['confidence'] = confidence_values + else: + signals['confidence'] = 0 + + # Generate directional signals + signals['signal'] = 0 # Default: no signal + + # Long signals (Strong Up or Weak Up with high confidence) + long_mask = ( + ((signals['prediction'] == 2) & (signals['confidence'] > self.confidence_threshold * 1.1)) | # Higher threshold for strong up + ((signals['prediction'] == 1) & (signals['confidence'] > self.confidence_threshold)) + ) + if not long_mask.empty: + signals.loc[long_mask, 'signal'] = 1 + + # Short signals (Strong Down or Weak Down with high confidence) + short_mask = ( + ((signals['prediction'] == -2) & (signals['confidence'] > self.confidence_threshold * 1.1)) | # Higher threshold for strong down + ((signals['prediction'] == -1) & (signals['confidence'] > self.confidence_threshold)) + ) + if not short_mask.empty: + signals.loc[short_mask, 'signal'] = -1 + + # Add risk management + if 'atr_10' in data.columns: + # Use ATR for stop loss and take profit calculation + signals['atr'] = data['atr_10'] + + # Calculate stops and targets + signals['stop_distance'] = signals['atr'] * self.stop_atr_factor + signals['target_distance'] = signals['atr'] * self.target_atr_factor + + # Set specific stop and target levels + signals['stop_price'] = np.where( + signals['signal'] == 1, + data['close'] - signals['stop_distance'], # Long stop + np.where( + signals['signal'] == -1, + data['close'] + signals['stop_distance'], # Short stop + np.nan + ) + ) + + signals['target_price'] = np.where( + signals['signal'] == 1, + data['close'] + signals['target_distance'], # Long target + np.where( + signals['signal'] == -1, + data['close'] - signals['target_distance'], # Short target + np.nan + ) + ) + + # Calculate risk-reward ratio + signals['risk_reward'] = np.where( + signals['signal'] == 1, + signals['target_distance'] / signals['stop_distance'], # Long R:R + np.where( + signals['signal'] == -1, + signals['target_distance'] / signals['stop_distance'], # Short R:R + np.nan + ) + ) + + # Filter signals by risk-reward ratio + poor_rr_mask = (signals['signal'] != 0) & (signals['risk_reward'] < self.risk_reward_min) + if not poor_rr_mask.empty: + signals.loc[poor_rr_mask, 'signal'] = 0 + + # Add signal strength (1-3) + signals['signal_strength'] = 0 + + # Strength 3: Very high confidence predictions + strong_mask = (signals['signal'] != 0) & (signals['confidence'] > 0.85) + if not strong_mask.empty: + signals.loc[strong_mask, 'signal_strength'] = 3 + + # Strength 2: High confidence predictions + medium_mask = (signals['signal'] != 0) & (signals['confidence'] > 0.75) & (signals['confidence'] <= 0.85) + if not medium_mask.empty: + signals.loc[medium_mask, 'signal_strength'] = 2 + + # Strength 1: Moderate confidence predictions + weak_mask = (signals['signal'] != 0) & (signals['confidence'] <= 0.75) + if not weak_mask.empty: + signals.loc[weak_mask, 'signal_strength'] = 1 + + # Add market context + if 'volatility_regime' in data.columns: + signals['volatility_regime'] = data['volatility_regime'] + + # Add key price levels + signals['close'] = data['close'] + + # Add signal label for easier interpretation + signals['signal_label'] = 'NO_SIGNAL' + long_label_mask = signals['signal'] == 1 + short_label_mask = signals['signal'] == -1 + + if not long_label_mask.empty: + signals.loc[long_label_mask, 'signal_label'] = 'LONG' + + if not short_label_mask.empty: + signals.loc[short_label_mask, 'signal_label'] = 'SHORT' + + # Count active signals + signal_count = (signals['signal'] != 0).sum() + print(f"Generated {signal_count} active signals out of {len(signals)} bars") + + return signals + + except Exception as e: + import traceback + print(f"Signal generation error: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + + # Return empty DataFrame with same index as data + return pd.DataFrame(index=data.index) + + def analyze_signals(self, signals, data): + """ + Analyze generated signals performance with improved error handling + + Parameters: + ----------- + signals : pd.DataFrame + DataFrame with trading signals + data : pd.DataFrame + Original data with price information + + Returns: + -------- + analysis : dict + Dictionary with signal statistics + """ + try: + # Ensure we have price data + if 'close' not in data.columns: + raise ValueError("Price data required for signal analysis") + + # Check if signals DataFrame is empty or has no signal column + if signals.empty or 'signal' not in signals.columns: + print("Warning: Empty signals DataFrame or missing 'signal' column") + return { + 'total_signals': 0, + 'signal_frequency': 0, + 'long_count': 0, + 'short_count': 0, + 'overall_win_rate': np.nan, + 'overall_avg_return': np.nan + } + + # Copy signals to avoid modifying the original + signals_copy = signals.copy() + + # Calculate forward returns for performance assessment + for period in [1, 3, 6, 12]: # Multiple forward periods + signals_copy[f'fwd_return_{period}'] = data['close'].pct_change(period).shift(-period) + + # Count signals + total_signals = (signals_copy['signal'] != 0).sum() + + # If no signals were generated, return empty stats + if total_signals == 0: + print("No active signals found for analysis") + return { + 'total_signals': 0, + 'signal_frequency': 0, + 'long_count': 0, + 'short_count': 0, + 'overall_win_rate': np.nan, + 'overall_avg_return': np.nan + } + + # Separate long and short signals + long_signals = signals_copy[signals_copy['signal'] == 1] + short_signals = signals_copy[signals_copy['signal'] == -1] + + # Calculate win rates with error handling + if len(long_signals) > 0 and 'fwd_return_6' in long_signals.columns: + long_win_rate = (long_signals['fwd_return_6'] > 0).mean() + long_avg_return = long_signals['fwd_return_6'].mean() + else: + long_win_rate = np.nan + long_avg_return = np.nan + + if len(short_signals) > 0 and 'fwd_return_6' in short_signals.columns: + short_win_rate = (short_signals['fwd_return_6'] < 0).mean() + short_avg_return = -short_signals['fwd_return_6'].mean() + else: + short_win_rate = np.nan + short_avg_return = np.nan + + # Calculate overall metrics + win_rates = [r for r in [long_win_rate, short_win_rate] if not np.isnan(r)] + returns = [r for r in [long_avg_return, short_avg_return] if not np.isnan(r)] + + overall_win_rate = np.mean(win_rates) if win_rates else np.nan + overall_avg_return = np.mean(returns) if returns else np.nan + + # Signal frequency + signal_frequency = total_signals / len(signals_copy) + + # Analyze by volatility regime if available + regime_stats = None + if 'volatility_regime' in signals_copy.columns: + regime_stats = {} + for regime in signals_copy['volatility_regime'].unique(): + regime_signals = signals_copy[signals_copy['volatility_regime'] == regime] + + # Skip if too few signals + if (regime_signals['signal'] != 0).sum() < 5: + continue + + regime_long = regime_signals[regime_signals['signal'] == 1] + regime_short = regime_signals[regime_signals['signal'] == -1] + + # Calculate regime metrics with error handling + if len(regime_long) > 0 and 'fwd_return_6' in regime_long.columns: + regime_long_win_rate = (regime_long['fwd_return_6'] > 0).mean() + regime_long_avg_return = regime_long['fwd_return_6'].mean() + else: + regime_long_win_rate = np.nan + regime_long_avg_return = np.nan + + if len(regime_short) > 0 and 'fwd_return_6' in regime_short.columns: + regime_short_win_rate = (regime_short['fwd_return_6'] < 0).mean() + regime_short_avg_return = -regime_short['fwd_return_6'].mean() + else: + regime_short_win_rate = np.nan + regime_short_avg_return = np.nan + + regime_stats[int(regime)] = { + 'count': (regime_signals['signal'] != 0).sum(), + 'frequency': (regime_signals['signal'] != 0).sum() / len(regime_signals), + 'long_win_rate': regime_long_win_rate, + 'short_win_rate': regime_short_win_rate, + 'long_avg_return': regime_long_avg_return, + 'short_avg_return': regime_short_avg_return + } + + # Compile analysis results + analysis = { + 'total_signals': total_signals, + 'signal_frequency': signal_frequency, + 'long_count': len(long_signals), + 'short_count': len(short_signals), + 'long_win_rate': long_win_rate, + 'short_win_rate': short_win_rate, + 'overall_win_rate': overall_win_rate, + 'long_avg_return': long_avg_return, + 'short_avg_return': short_avg_return, + 'overall_avg_return': overall_avg_return, + 'regime_stats': regime_stats + } + + # Print summary + print("\nSignal Analysis:") + print(f"Total Signals: {total_signals} ({signal_frequency:.2%} of bars)") + print(f"Long Signals: {len(long_signals)}, Short Signals: {len(short_signals)}") + + win_rate_str = f"{overall_win_rate:.2%}" if not np.isnan(overall_win_rate) else "N/A" + long_win_rate_str = f"{long_win_rate:.2%}" if not np.isnan(long_win_rate) else "N/A" + short_win_rate_str = f"{short_win_rate:.2%}" if not np.isnan(short_win_rate) else "N/A" + + long_return_str = f"{long_avg_return:.2%}" if not np.isnan(long_avg_return) else "N/A" + short_return_str = f"{short_avg_return:.2%}" if not np.isnan(short_avg_return) else "N/A" + overall_return_str = f"{overall_avg_return:.2%}" if not np.isnan(overall_avg_return) else "N/A" + + print(f"Win Rates - Long: {long_win_rate_str}, Short: {short_win_rate_str}, Overall: {win_rate_str}") + print(f"Avg Returns - Long: {long_return_str}, Short: {short_return_str}, Overall: {overall_return_str}") + + return analysis + + except Exception as e: + import traceback + print(f"Signal analysis error: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + + # Return basic metrics + return { + 'total_signals': 0, + 'signal_frequency': 0, + 'long_count': 0, + 'short_count': 0, + 'overall_win_rate': np.nan, + 'overall_avg_return': np.nan, + 'error': str(e) + } diff --git a/tradingbot/viz/__init__.py b/tradingbot/viz/__init__.py new file mode 100644 index 0000000..e8d1643 --- /dev/null +++ b/tradingbot/viz/__init__.py @@ -0,0 +1,3 @@ +from tradingbot.viz.visualize import visualize_signals, analyze_trading_model, plot_price_signals + +__all__ = ["visualize_signals", "analyze_trading_model", "plot_price_signals"] diff --git a/tradingbot/viz/visualize.py b/tradingbot/viz/visualize.py new file mode 100644 index 0000000..6724f9d --- /dev/null +++ b/tradingbot/viz/visualize.py @@ -0,0 +1,1008 @@ +"""Comprehensive signal / performance visualization.""" +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +import matplotlib.ticker as mtick +import seaborn as sns +from matplotlib.gridspec import GridSpec + + +def visualize_signals(data, results, returns, metrics, save_path=None): + """ + Create comprehensive visualizations of trading signals and performance + + Parameters: + data: Original price data DataFrame + results: Signal results DataFrame from predictor.predict() + returns: Strategy returns Series + metrics: Performance metrics dictionary + save_path: Optional path to save the plots + """ + # Set plotting style + plt.style.use('seaborn-v0_8-darkgrid') + sns.set_palette('Set1') + + # Prepare data by aligning timeframes + aligned_data = data.loc[results.index].copy() + + # Create a Figure with multiple subplots + fig = plt.figure(figsize=(20, 16)) + gs = GridSpec(4, 4, figure=fig) + + # 1. Main price chart with signals + ax_price = fig.add_subplot(gs[0:2, 0:3]) + _plot_price_with_signals(ax_price, aligned_data, results) + + # 2. Equity curve + ax_equity = fig.add_subplot(gs[2:3, 0:3]) + _plot_equity_curve(ax_equity, returns) + + # 3. Signal distribution + ax_signal_dist = fig.add_subplot(gs[0, 3]) + _plot_signal_distribution(ax_signal_dist, results) + + # 4. Signal strength heatmap + ax_signal_heatmap = fig.add_subplot(gs[1, 3]) + _plot_signal_strength_heatmap(ax_signal_heatmap, results) + + # 5. Market regime analysis + ax_regime = fig.add_subplot(gs[2, 3]) + _plot_market_regime(ax_regime, results) + + # 6. Performance metrics + ax_metrics = fig.add_subplot(gs[3, 3]) + _plot_performance_metrics(ax_metrics, metrics) + + # 7. Signal frequency over time + ax_frequency = fig.add_subplot(gs[3, 0:3]) + _plot_signal_frequency(ax_frequency, results) + + # Set the layout tight + fig.tight_layout() + fig.suptitle('Gold Price Trading Signal Analysis', fontsize=16, y=1.02) + + # Save if path is provided + if save_path: + plt.savefig(save_path, bbox_inches='tight', dpi=300) + + plt.show() + + # Create a second figure for detailed analysis + fig2 = plt.figure(figsize=(20, 12)) + gs2 = GridSpec(2, 3, figure=fig2) + + # 1. Win/Loss by hour + ax_hour = fig2.add_subplot(gs2[0, 0]) + _plot_win_loss_by_hour(ax_hour, results, returns) + + # 2. Win/Loss by day of week + ax_day = fig2.add_subplot(gs2[0, 1]) + _plot_win_loss_by_day(ax_day, results, returns) + + # 3. Win/Loss by regime + ax_regime_perf = fig2.add_subplot(gs2[0, 2]) + _plot_win_loss_by_regime(ax_regime_perf, results, returns) + + # 4. Signal duration histogram + ax_duration = fig2.add_subplot(gs2[1, 0]) + _plot_signal_duration(ax_duration, results) + + # 5. Return distribution + ax_return_dist = fig2.add_subplot(gs2[1, 1]) + _plot_return_distribution(ax_return_dist, returns, results) + + # 6. Signal consistency + ax_consistency = fig2.add_subplot(gs2[1, 2]) + _plot_signal_consistency(ax_consistency, results) + + fig2.tight_layout() + fig2.suptitle('Detailed Signal Analysis', fontsize=16, y=1.02) + + # Save if path is provided + if save_path: + detail_path = save_path.replace('.png', '_detail.png') + plt.savefig(detail_path, bbox_inches='tight', dpi=300) + + plt.show() + + # Create a third figure for model attribution analysis + fig3 = plt.figure(figsize=(20, 10)) + gs3 = GridSpec(2, 2, figure=fig3) + + # 1. Model agreement analysis + ax_agreement = fig3.add_subplot(gs3[0, 0]) + _plot_model_agreement(ax_agreement, results, returns) + + # 2. Signal probability analysis + ax_proba = fig3.add_subplot(gs3[0, 1]) + _plot_signal_probability(ax_proba, results, returns) + + # 3. Signal direction by strength + ax_strength = fig3.add_subplot(gs3[1, 0]) + _plot_signal_strength_performance(ax_strength, results, returns) + + # 4. Drawdown analysis + ax_drawdown = fig3.add_subplot(gs3[1, 1]) + _plot_drawdown_analysis(ax_drawdown, returns) + + fig3.tight_layout() + fig3.suptitle('Model Behavior Analysis', fontsize=16, y=1.02) + + # Save if path is provided + if save_path: + model_path = save_path.replace('.png', '_model.png') + plt.savefig(model_path, bbox_inches='tight', dpi=300) + + plt.show() + +def _plot_price_with_signals(ax, data, results): + """Plot price chart with buy/sell signals overlay""" + # Plot price + ax.plot(data.index, data['close'], color='#333333', linewidth=1, alpha=0.7, label='Price') + + # Highlight buy/sell signals + buy_signals = results[results['signal'] == 1].index + sell_signals = results[results['signal'] == -1].index + + # Get price values for the signals + buy_prices = data.loc[buy_signals, 'close'] + sell_prices = data.loc[sell_signals, 'close'] + + # Plot signals with varying sizes based on strength + buy_sizes = results.loc[buy_signals, 'strength'].clip(lower=20, upper=100) / 2 + sell_sizes = results.loc[sell_signals, 'strength'].clip(lower=20, upper=100) / 2 + + ax.scatter(buy_signals, buy_prices, color='green', s=buy_sizes, alpha=0.7, marker='^', label='Buy Signal') + ax.scatter(sell_signals, sell_prices, color='red', s=sell_sizes, alpha=0.7, marker='v', label='Sell Signal') + + # Format x-axis for dates + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + ax.xaxis.set_major_locator(mdates.WeekdayLocator(interval=2)) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=45, ha='right') + + # Add labels and legend + ax.set_title('Gold Price with Trading Signals', fontsize=14) + ax.set_ylabel('Price', fontsize=12) + ax.legend(loc='best') + + # Draw grid + ax.grid(True, alpha=0.3) + + # Annotate some significant signals + top_buy = results[results['signal'] == 1].nlargest(3, 'strength') + top_sell = results[results['signal'] == -1].nlargest(3, 'strength') + + for idx, row in pd.concat([top_buy, top_sell]).iterrows(): + price = data.loc[idx, 'close'] + strength = row['strength'] + if row['signal'] == 1: + ax.annotate(f"{strength:.0f}%", (idx, price), + xytext=(0, 15), textcoords='offset points', + ha='center', va='bottom', fontsize=9, + arrowprops=dict(arrowstyle='->', color='green', alpha=0.7)) + else: + ax.annotate(f"{strength:.0f}%", (idx, price), + xytext=(0, -15), textcoords='offset points', + ha='center', va='top', fontsize=9, + arrowprops=dict(arrowstyle='->', color='red', alpha=0.7)) + +def _plot_equity_curve(ax, returns): + """Plot equity curve from strategy returns""" + # Calculate cumulative returns + cumulative_returns = (1 + returns).cumprod() - 1 + + # Plot the equity curve + ax.plot(cumulative_returns.index, cumulative_returns * 100, linewidth=2, color='#1f77b4') + + # Draw the zero line + ax.axhline(y=0, color='black', linestyle='-', alpha=0.3) + + # Format y-axis as percentage + ax.yaxis.set_major_formatter(mtick.PercentFormatter()) + + # Highlight drawdowns + underwater = cumulative_returns - cumulative_returns.cummax() + ax.fill_between(underwater.index, 0, underwater * 100, color='red', alpha=0.3) + + # Add labels + ax.set_title('Strategy Equity Curve', fontsize=14) + ax.set_ylabel('Cumulative Return (%)', fontsize=12) + + # Calculate and annotate key metrics directly on the chart + final_return = cumulative_returns.iloc[-1] * 100 + max_drawdown = underwater.min() * 100 + + # Annotate final return + ax.annotate(f'Final Return: {final_return:.2f}%', + xy=(0.02, 0.85), xycoords='axes fraction', + bbox=dict(boxstyle="round,pad=0.3", fc="white", ec="gray", alpha=0.8)) + + # Annotate max drawdown + ax.annotate(f'Max Drawdown: {max_drawdown:.2f}%', + xy=(0.02, 0.7), xycoords='axes fraction', + bbox=dict(boxstyle="round,pad=0.3", fc="white", ec="gray", alpha=0.8)) + +def _plot_signal_distribution(ax, results): + """Plot distribution of signal types""" + # Count signal types + signal_counts = results['signal'].value_counts() + + # Create labels + labels = ['Buy (Long)', 'Neutral', 'Sell (Short)'] + + # Ensure we have all three categories (even if count is zero) + values = [signal_counts.get(1, 0), signal_counts.get(0, 0), signal_counts.get(-1, 0)] + + # Calculate percentages + total = sum(values) + percentages = [v/total*100 for v in values] + + # Custom color map + colors = ['green', 'gray', 'red'] + + # Create bar plot + bars = ax.bar(labels, values, color=colors, alpha=0.7) + + # Add percentage labels on top of each bar + for bar, percentage in zip(bars, percentages): + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width()/2., height + 0.1, + f'{percentage:.1f}%', ha='center', va='bottom', fontsize=9) + + # Add title and labels + ax.set_title('Signal Distribution', fontsize=14) + ax.set_ylabel('Count', fontsize=12) + + # Rotate x-labels for better readability + plt.setp(ax.get_xticklabels(), rotation=30, ha='right') + +def _plot_signal_strength_heatmap(ax, results): + """Plot heatmap of signal strength by direction""" + # Filter to get only actual signals + signals = results[results['signal'] != 0].copy() + + # Create strength bins + signals['strength_bin'] = pd.cut(signals['strength'], + bins=[0, 20, 40, 60, 80, 100], + labels=['0-20', '20-40', '40-60', '60-80', '80-100']) + + # Create direction labels + signals['direction'] = signals['signal'].map({1: 'Buy', -1: 'Sell'}) + + # Create count matrix + heatmap_data = pd.crosstab(signals['direction'], signals['strength_bin']) + + # Plot heatmap + sns.heatmap(heatmap_data, annot=True, fmt='d', cmap='YlGnBu', ax=ax) + + # Add title + ax.set_title('Signal Strength Distribution', fontsize=14) + ax.set_xlabel('Strength (%)', fontsize=12) + ax.set_ylabel('Signal Direction', fontsize=12) + +def _plot_market_regime(ax, results): + """Plot market regime distribution and signals per regime""" + # Map regime numbers to descriptive names + regime_map = {0: 'Low Vol', 1: 'Normal', 2: 'High Vol'} + + # Create a copy with regime names + regime_data = results.copy() + regime_data['regime_name'] = regime_data['market_regime'].map(regime_map) + + # Group by regime and count signals + regime_signals = pd.crosstab(regime_data['regime_name'], regime_data['signal']) + + # Rename columns + regime_signals.columns = ['Neutral', 'Buy', 'Sell'] + + # Reorder columns + regime_signals = regime_signals[['Buy', 'Neutral', 'Sell']] + + # Plot stacked bar chart + regime_signals.plot(kind='bar', stacked=True, color=['green', 'gray', 'red'], + alpha=0.7, ax=ax) + + # Add title and labels + ax.set_title('Signals by Market Regime', fontsize=14) + ax.set_xlabel('Market Regime', fontsize=12) + ax.set_ylabel('Count', fontsize=12) + + # Add total percentage annotation + for i, regime in enumerate(regime_signals.index): + total = regime_signals.iloc[i].sum() + percentage = total / len(results) * 100 + ax.text(i, total + 5, f'{percentage:.1f}%', ha='center') + + # Adjust legend + ax.legend(title='Signal Type') + +def _plot_performance_metrics(ax, metrics): + """Plot key performance metrics""" + # Remove axes + ax.axis('off') + + # Create text content + metrics_text = ( + f"Performance Metrics\n" + f"-------------------\n" + f"Total Return: {metrics['total_return']:.2%}\n" + f"Sharpe Ratio: {metrics['sharpe_ratio']:.2f}\n" + f"Win Rate: {metrics['win_rate']:.2%}\n" + f"Max Drawdown: {metrics['max_drawdown']:.2%}\n" + f"Signal Count: {metrics['signal_count']}\n" + f"Avg Signals/Day: {metrics['avg_signals_per_day']:.1f}" + ) + + # Add text box + ax.text(0.5, 0.5, metrics_text, + ha='center', va='center', + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8), + fontsize=12, family='monospace') + + ax.set_title('Performance Summary', fontsize=14) + +def _plot_signal_frequency(ax, results): + """Plot signal frequency over time""" + # Create a resampled view of signals per day + daily_signals = results['signal'].resample('D').apply(lambda x: (x != 0).sum()) + + # Plot as bar chart + ax.bar(daily_signals.index, daily_signals, alpha=0.7, color='#1f77b4') + + # Add a trend line + z = np.polyfit(range(len(daily_signals)), daily_signals, 1) + p = np.poly1d(z) + ax.plot(daily_signals.index, p(range(len(daily_signals))), + linestyle='--', color='red', linewidth=2, + label=f'Trend: {"+" if z[0]>0 else ""}{z[0]:.4f}x + {z[1]:.1f}') + + # Format x-axis for dates + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + ax.xaxis.set_major_locator(mdates.WeekdayLocator(interval=7)) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=45, ha='right') + + # Add labels and legend + ax.set_title('Signal Frequency Over Time', fontsize=14) + ax.set_ylabel('Number of Signals per Day', fontsize=12) + ax.legend() + + # Calculate and display average signals per day + avg_signals = daily_signals.mean() + ax.axhline(y=avg_signals, color='gray', linestyle='--', alpha=0.7) + ax.text(daily_signals.index[10], avg_signals + 0.3, + f'Avg: {avg_signals:.2f} signals/day', fontsize=10) + +def _plot_win_loss_by_hour(ax, results, returns): + """Plot win/loss ratio by hour of day""" + # Combine signals and returns + performance = results.copy() + performance['return'] = returns + + # Group by hour + hourly_perf = performance[performance['signal'] != 0].groupby(performance.index.hour) + + # Calculate win rate and average return per hour + win_rates = hourly_perf['return'].apply(lambda x: (x > 0).mean()) + avg_returns = hourly_perf['return'].mean() + + # Create DataFrame for plotting + hourly_data = pd.DataFrame({ + 'Win Rate': win_rates, + 'Avg Return': avg_returns + }) + + # Set up primary axis for win rate + hourly_data['Win Rate'].plot(kind='bar', color='skyblue', ax=ax, alpha=0.7) + ax.set_xlabel('Hour of Day', fontsize=12) + ax.set_ylabel('Win Rate', fontsize=12) + ax.set_ylim(0, 1) + + # Set up secondary axis for average return + ax2 = ax.twinx() + hourly_data['Avg Return'].plot(kind='line', color='red', marker='o', ax=ax2) + ax2.set_ylabel('Average Return', fontsize=12, color='red') + ax2.tick_params(axis='y', colors='red') + + # Add horizontal line at 0.5 for win rate + ax.axhline(y=0.5, color='gray', linestyle='--', alpha=0.5) + + # Add horizontal line at 0 for average return + ax2.axhline(y=0, color='red', linestyle='--', alpha=0.5) + + # Add title + ax.set_title('Performance by Hour of Day', fontsize=14) + + # Add custom legend + from matplotlib.lines import Line2D + legend_elements = [ + Line2D([0], [0], color='skyblue', lw=0, marker='s', markersize=10, label='Win Rate'), + Line2D([0], [0], color='red', marker='o', markersize=6, label='Avg Return') + ] + ax.legend(handles=legend_elements, loc='upper right') + +def _plot_win_loss_by_day(ax, results, returns): + """Plot win/loss ratio by day of week""" + # Combine signals and returns + performance = results.copy() + performance['return'] = returns + + # Convert day numbers to names + day_names = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] + performance['day_name'] = [day_names[d] for d in performance.index.dayofweek] + + # Group by day + daily_perf = performance[performance['signal'] != 0].groupby('day_name') + + # Calculate win rate and average return per day + win_rates = daily_perf['return'].apply(lambda x: (x > 0).mean()) + avg_returns = daily_perf['return'].mean() + counts = daily_perf.size() + + # Reindex to ensure correct order + win_rates = win_rates.reindex(day_names) + avg_returns = avg_returns.reindex(day_names) + counts = counts.reindex(day_names) + + # Create bar chart + bars = ax.bar(win_rates.index, win_rates, color='lightgreen', alpha=0.7) + + # Add count annotations + for i, (bar, count) in enumerate(zip(bars, counts)): + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width()/2., height + 0.02, + f'n={count}', ha='center', va='bottom', fontsize=9) + + # Set up secondary axis for average return + ax2 = ax.twinx() + ax2.plot(avg_returns.index, avg_returns, color='purple', marker='d') + ax2.set_ylabel('Average Return', fontsize=12, color='purple') + + # Add horizontal line at 0.5 for win rate + ax.axhline(y=0.5, color='gray', linestyle='--', alpha=0.5) + + # Add horizontal line at 0 for average return + ax2.axhline(y=0, color='purple', linestyle='--', alpha=0.5) + + # Add labels and title + ax.set_xlabel('Day of Week', fontsize=12) + ax.set_ylabel('Win Rate', fontsize=12) + ax.set_title('Performance by Day of Week', fontsize=14) + ax.set_ylim(0, 1) + + # Rotate x-labels for better readability + plt.setp(ax.get_xticklabels(), rotation=30, ha='right') + + # Add custom legend + from matplotlib.lines import Line2D + legend_elements = [ + Line2D([0], [0], color='lightgreen', lw=0, marker='s', markersize=10, label='Win Rate'), + Line2D([0], [0], color='purple', marker='d', markersize=6, label='Avg Return') + ] + ax.legend(handles=legend_elements, loc='upper right') + +def _plot_win_loss_by_regime(ax, results, returns): + """Plot win/loss by market regime""" + # Combine signals and returns + performance = results.copy() + performance['return'] = returns + + # Map regime numbers to descriptive names + regime_map = {0: 'Low Vol', 1: 'Normal', 2: 'High Vol'} + performance['regime_name'] = performance['market_regime'].map(regime_map) + + # Group by regime + regime_perf = performance[performance['signal'] != 0].groupby('regime_name') + + # Calculate metrics + win_rates = regime_perf['return'].apply(lambda x: (x > 0).mean()) + avg_returns = regime_perf['return'].mean() + sharpe_ratios = regime_perf['return'].apply(lambda x: x.mean() / x.std() if x.std() > 0 else 0) + counts = regime_perf.size() + + # Create index for the bars + x = np.arange(len(win_rates)) + width = 0.25 + + # Create grouped bar chart + ax.bar(x - width, win_rates, width, label='Win Rate', color='green', alpha=0.7) + ax.bar(x, avg_returns * 10, width, label='Avg Ret (×10)', color='blue', alpha=0.7) + ax.bar(x + width, sharpe_ratios, width, label='Sharpe', color='orange', alpha=0.7) + + # Add count annotations + for i, count in enumerate(counts): + ax.text(i, 0.05, f'n={count}', ha='center', va='bottom', fontsize=9) + + # Set x-tick labels + ax.set_xticks(x) + ax.set_xticklabels(win_rates.index) + + # Add labels and title + ax.set_xlabel('Market Regime', fontsize=12) + ax.set_ylabel('Metric Value', fontsize=12) + ax.set_title('Performance by Market Regime', fontsize=14) + + # Add horizontal line at 0.5 for reference + ax.axhline(y=0.5, color='gray', linestyle='--', alpha=0.5) + + # Add legend + ax.legend() + +def _plot_signal_duration(ax, results): + """Plot histogram of signal duration""" + # Calculate signal duration + signal_changes = results['signal'].diff().abs() + signal_changes = signal_changes[signal_changes > 0] + + # Create intervals between signal changes + durations = [] + current_duration = 0 + current_signal = 0 + + for idx, row in results.iterrows(): + if row['signal'] != current_signal: + if current_signal != 0: # Only count actual signal durations + durations.append(current_duration) + current_duration = 1 + current_signal = row['signal'] + else: + current_duration += 1 + + # Add the last duration if it's a signal + if current_signal != 0: + durations.append(current_duration) + + # Convert to 5-minute intervals + durations_minutes = [d * 5 for d in durations] + + # Plot histogram + bins = [0, 15, 30, 60, 120, 240, 480, 720, 1440] + labels = ['0-15m', '15-30m', '30-60m', '1-2h', '2-4h', '4-8h', '8-12h', '12-24h'] + + ax.hist(durations_minutes, bins=bins, alpha=0.7, color='teal', + edgecolor='black', linewidth=1) + + # Add labels and title + ax.set_xlabel('Signal Duration (minutes)', fontsize=12) + ax.set_ylabel('Frequency', fontsize=12) + ax.set_title('Signal Duration Distribution', fontsize=14) + + # Set custom x-ticks + ax.set_xticks([b + (bins[i+1] - b)/2 for i, b in enumerate(bins[:-1])]) + ax.set_xticklabels(labels) + plt.setp(ax.get_xticklabels(), rotation=30, ha='right') + + # Add summary statistics + mean_duration = np.mean(durations_minutes) + median_duration = np.median(durations_minutes) + + stats_text = ( + f"Mean: {mean_duration:.1f} min\n" + f"Median: {median_duration:.1f} min" + ) + + ax.text(0.7, 0.8, stats_text, + transform=ax.transAxes, + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8), + fontsize=10) + +def _plot_return_distribution(ax, returns, results): + """Plot distribution of strategy returns""" + # Separate returns by signal type + long_returns = returns[results['signal'] == 1] + short_returns = returns[results['signal'] == -1] + + # Create histogram + n_bins = 30 + ax.hist(long_returns, bins=n_bins, alpha=0.5, color='green', label='Long') + ax.hist(short_returns, bins=n_bins, alpha=0.5, color='red', label='Short') + + # Add normal distribution for reference + from scipy import stats + x = np.linspace(min(returns), max(returns), 100) + all_returns = returns[results['signal'] != 0] + mu, std = all_returns.mean(), all_returns.std() + pdf = stats.norm.pdf(x, mu, std) + scaled_pdf = pdf * (len(all_returns) * (max(returns) - min(returns)) / n_bins) + ax.plot(x, scaled_pdf, 'k--', linewidth=1, label='Normal Dist.') + + # Add vertical line at 0 + ax.axvline(x=0, color='black', linestyle='-', alpha=0.3) + + # Add labels and title + ax.set_xlabel('Return', fontsize=12) + ax.set_ylabel('Frequency', fontsize=12) + ax.set_title('Return Distribution by Signal Type', fontsize=14) + + # Add summary statistics + long_stats = ( + f"Long Signals:\n" + f"Mean: {long_returns.mean():.2%}\n" + f"Std: {long_returns.std():.2%}\n" + f"Win: {(long_returns > 0).mean():.1%}" + ) + + short_stats = ( + f"Short Signals:\n" + f"Mean: {short_returns.mean():.2%}\n" + f"Std: {short_returns.std():.2%}\n" + f"Win: {(short_returns > 0).mean():.1%}" + ) + + ax.text(0.05, 0.95, long_stats, + transform=ax.transAxes, va='top', + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8), + fontsize=9) + + ax.text(0.95, 0.95, short_stats, + transform=ax.transAxes, va='top', ha='right', + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8), + fontsize=9) + + # Add legend + ax.legend() + +def _plot_signal_consistency(ax, results): + """Plot signal consistency over time""" + # Calculate rolling signal consistency + window = 10 # Number of signals to check + + # Get signal direction changes + signal_data = results[results['signal'] != 0].copy() + signal_data['prev_signal'] = signal_data['signal'].shift(1) + signal_data['direction_change'] = (signal_data['signal'] != signal_data['prev_signal']) & (signal_data['prev_signal'] != 0) + + # Calculate rolling consistency + signal_data['consistency'] = 1 - signal_data['direction_change'].rolling(window).mean() + + # Plot + ax.plot(signal_data.index, signal_data['consistency'] * 100, color='purple') + + # Add horizontal line at 50% + ax.axhline(y=50, color='gray', linestyle='--', alpha=0.5) + + # Format y-axis as percentage + ax.yaxis.set_major_formatter(mtick.PercentFormatter()) + + # Format x-axis for dates + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + ax.xaxis.set_major_locator(mdates.WeekdayLocator(interval=7)) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=45, ha='right') + + # Add labels and title + ax.set_xlabel('Date', fontsize=12) + ax.set_ylabel('Signal Consistency (%)', fontsize=12) + ax.set_title(f'Signal Consistency (Window={window})', fontsize=14) + + # Set y-limit + ax.set_ylim(0, 100) + + # Add average line + avg_consistency = signal_data['consistency'].mean() * 100 + ax.axhline(y=avg_consistency, color='red', linestyle='-', alpha=0.5) + ax.text(signal_data.index[10], avg_consistency + 5, + f'Avg: {avg_consistency:.1f}%', color='red') + +def _plot_model_agreement(ax, results, returns): + """Plot performance by model agreement level""" + # Create bins for model agreement + results['agreement_bin'] = pd.cut(results['model_agreement'], + bins=[0, 0.6, 0.7, 0.8, 0.9, 1.0], + labels=['0-60%', '60-70%', '70-80%', '80-90%', '90-100%']) + + # Combine with returns + performance = results.copy() + performance['return'] = returns + + # Keep only actual signals + performance = performance[performance['signal'] != 0] + + # Group by agreement bin + agreement_perf = performance.groupby('agreement_bin') + + # Calculate metrics + win_rates = agreement_perf['return'].apply(lambda x: (x > 0).mean() if len(x) > 0 else 0) + avg_returns = agreement_perf['return'].apply(lambda x: x.mean() if len(x) > 0 else 0) + counts = agreement_perf.size() + + # Create bar plot + bars = ax.bar(win_rates.index, win_rates, color='skyblue', alpha=0.7) + + # Add count annotations + for i, (bar, count) in enumerate(zip(bars, counts)): + if count > 0: + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width()/2., height + 0.02, + f'n={count}', ha='center', va='bottom', fontsize=9) + + # Set up secondary axis for average return + ax2 = ax.twinx() + ax2.plot(avg_returns.index, avg_returns, color='darkblue', marker='o') + ax2.set_ylabel('Average Return', fontsize=12, color='darkblue') + + # Add horizontal reference lines + ax.axhline(y=0.5, color='gray', linestyle='--', alpha=0.5) + ax2.axhline(y=0, color='darkblue', linestyle='--', alpha=0.5) + + # Add labels and title + ax.set_xlabel('Model Agreement Level', fontsize=12) + ax.set_ylabel('Win Rate', fontsize=12) + ax.set_title('Performance by Model Agreement', fontsize=14) + ax.set_ylim(0, 1) + + # Rotate x-labels for better readability + plt.setp(ax.get_xticklabels(), rotation=30, ha='right') + + # Add custom legend + from matplotlib.lines import Line2D + legend_elements = [ + Line2D([0], [0], color='skyblue', lw=0, marker='s', markersize=10, label='Win Rate'), + Line2D([0], [0], color='darkblue', marker='o', markersize=6, label='Avg Return') + ] + ax.legend(handles=legend_elements, loc='upper left') + +def _plot_signal_probability(ax, results, returns): + """Plot performance by signal probability""" + # Create bins for signal probability + results['proba_bin'] = pd.cut( + np.where(results['signal'] == 1, results['proba_up'], + np.where(results['signal'] == -1, results['proba_down'], 0)), + bins=[0, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.0], + labels=['0-65%', '65-70%', '70-75%', '75-80%', '80-85%', '85-90%', '90-95%', '95-100%'] + ) + + # Combine with returns + performance = results.copy() + performance['return'] = returns + + # Keep only actual signals + performance = performance[performance['signal'] != 0] + + # Group by probability bin + proba_perf = performance.groupby('proba_bin') + + # Calculate metrics + win_rates = proba_perf['return'].apply(lambda x: (x > 0).mean() if len(x) > 0 else 0) + avg_returns = proba_perf['return'].apply(lambda x: x.mean() if len(x) > 0 else 0) + counts = proba_perf.size() + + # Create bar plot + bars = ax.bar(win_rates.index, win_rates, color='lightcoral', alpha=0.7) + + # Add count annotations + for i, (bar, count) in enumerate(zip(bars, counts)): + if count > 0: + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width()/2., height + 0.02, + f'n={count}', ha='center', va='bottom', fontsize=9) + + # Set up secondary axis for average return + ax2 = ax.twinx() + ax2.plot(avg_returns.index, avg_returns, color='darkred', marker='o') + ax2.set_ylabel('Average Return', fontsize=12, color='darkred') + + # Add horizontal reference lines + ax.axhline(y=0.5, color='gray', linestyle='--', alpha=0.5) + ax2.axhline(y=0, color='darkred', linestyle='--', alpha=0.5) + + # Add labels and title + ax.set_xlabel('Signal Probability', fontsize=12) + ax.set_ylabel('Win Rate', fontsize=12) + ax.set_title('Performance by Signal Probability', fontsize=14) + ax.set_ylim(0, 1) + + # Rotate x-labels for better readability + plt.setp(ax.get_xticklabels(), rotation=45, ha='right') + + # Add custom legend + from matplotlib.lines import Line2D + legend_elements = [ + Line2D([0], [0], color='lightcoral', lw=0, marker='s', markersize=10, label='Win Rate'), + Line2D([0], [0], color='darkred', marker='o', markersize=6, label='Avg Return') + ] + ax.legend(handles=legend_elements, loc='upper left') + +def _plot_signal_strength_performance(ax, results, returns): + """Plot performance by signal strength""" + # Create bins for signal strength + results['strength_bin'] = pd.cut(results['strength'], + bins=[0, 20, 40, 60, 80, 100], + labels=['0-20', '20-40', '40-60', '60-80', '80-100']) + + # Combine with returns + performance = results.copy() + performance['return'] = returns + + # Separate long and short signals + long_data = performance[performance['signal'] == 1] + short_data = performance[performance['signal'] == -1] + + # Group by strength bin + long_perf = long_data.groupby('strength_bin') + short_perf = short_data.groupby('strength_bin') + + # Calculate win rates + long_wins = long_perf['return'].apply(lambda x: (x > 0).mean() if len(x) > 0 else 0) + short_wins = short_perf['return'].apply(lambda x: (x > 0).mean() if len(x) > 0 else 0) + long_counts = long_perf.size() + short_counts = short_perf.size() + + # Set width for bars + width = 0.35 + x = np.arange(len(long_wins)) + + # Create grouped bar chart + long_bars = ax.bar(x - width/2, long_wins, width, label='Long Signals', color='green', alpha=0.7) + short_bars = ax.bar(x + width/2, short_wins, width, label='Short Signals', color='red', alpha=0.7) + + # Add count annotations + for i, (bar, count) in enumerate(zip(long_bars, long_counts)): + if count > 0: + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width()/2., height + 0.02, + f'{count}', ha='center', va='bottom', fontsize=8, color='green') + + for i, (bar, count) in enumerate(zip(short_bars, short_counts)): + if count > 0: + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width()/2., height + 0.02, + f'{count}', ha='center', va='bottom', fontsize=8, color='red') + + # Add horizontal reference line + ax.axhline(y=0.5, color='gray', linestyle='--', alpha=0.5) + + # Set up x-ticks + ax.set_xticks(x) + ax.set_xticklabels(long_wins.index) + + # Add labels and title + ax.set_xlabel('Signal Strength (%)', fontsize=12) + ax.set_ylabel('Win Rate', fontsize=12) + ax.set_title('Win Rate by Signal Strength', fontsize=14) + ax.set_ylim(0, 1) + + # Add legend + ax.legend() + +def _plot_drawdown_analysis(ax, returns): + """Plot drawdown analysis""" + # Calculate cumulative returns and drawdowns + cumulative_returns = (1 + returns).cumprod() - 1 + drawdown = cumulative_returns - cumulative_returns.cummax() + + # Plot drawdown + ax.fill_between(drawdown.index, 0, drawdown * 100, color='red', alpha=0.3) + ax.plot(drawdown.index, drawdown * 100, color='red', linewidth=1) + + # Format y-axis as percentage + ax.yaxis.set_major_formatter(mtick.PercentFormatter()) + + # Format x-axis for dates + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + ax.xaxis.set_major_locator(mdates.WeekdayLocator(interval=7)) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=45, ha='right') + + # Find worst drawdowns + def find_drawdown_periods(drawdown_series, top_n=5): + periods = [] + current_dd = 0 + start_date = None + end_date = None + + for date, value in drawdown_series.items(): + if value < current_dd: + current_dd = value + end_date = date + elif value == 0 and current_dd < 0: + # Drawdown ended + periods.append((start_date, end_date, current_dd)) + current_dd = 0 + start_date = None + end_date = None + elif current_dd == 0 and value < 0: + # New drawdown started + start_date = date + current_dd = value + end_date = date + + # Add any ongoing drawdown + if current_dd < 0: + periods.append((start_date, end_date, current_dd)) + + # Sort by largest drawdown and return top_n + return sorted(periods, key=lambda x: x[2])[:top_n] + + # Get top drawdowns + top_drawdowns = find_drawdown_periods(drawdown, top_n=3) + + # Highlight top drawdowns + colors = ['darkred', 'firebrick', 'indianred'] + for i, (start, end, magnitude) in enumerate(top_drawdowns): + if start and end: + ax.axvspan(start, end, color=colors[i], alpha=0.2) + + # Add annotation + mid_point = start + (end - start) / 2 + ax.annotate(f"{magnitude*100:.1f}%", (mid_point, magnitude*100 - 0.5), + ha='center', fontsize=10, color=colors[i]) + + # Add labels and title + ax.set_ylabel('Drawdown (%)', fontsize=12) + ax.set_title('Drawdown Analysis', fontsize=14) + + # Calculate and display statistics + max_dd = drawdown.min() * 100 + avg_dd = drawdown[drawdown < 0].mean() * 100 + + stats_text = ( + f"Max Drawdown: {max_dd:.2f}%\n" + f"Avg Drawdown: {avg_dd:.2f}%\n" + f"# of DDs >1%: {(drawdown < -0.01).sum()}" + ) + + ax.text(0.02, 0.05, stats_text, + transform=ax.transAxes, + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8), + fontsize=10) + + +def analyze_trading_model(data_path, forecast_bars=24, confidence_threshold=0.65, save_path=None): + """ + Complete function to run model and create visualizations + + Parameters: + data_path: Path to CSV file with OHLCV data + forecast_bars: Number of 5-min bars to forecast + confidence_threshold: Threshold for signal generation + save_path: Optional path to save visualization images + + Returns: + predictor: Trained model + metrics: Performance metrics + results: Signal results + returns: Strategy returns + """ + # Import run_model function (assuming it's in your environment) + from tradingbot.models.tree_ensemble import run_model + + # Run the model + predictor, metrics, results, returns = run_model( + data_path, forecast_bars, confidence_threshold + ) + + # Load original data + data = pd.read_csv(data_path) + data['timestamp'] = pd.to_datetime(data['timestamp']) + data = data.drop_duplicates(subset=['timestamp']) + data.set_index('timestamp', inplace=True) + + # Create visualizations + print("\nGenerating visualizations...") + visualize_signals(data, results, returns, metrics, save_path) + + return predictor, metrics, results, returns + + +def plot_price_signals(data, signals, price_col="close", title="Price with Trading Signals"): + """Quick scatter of buy (1) / sell (-1) signals over the price series. + + ``signals`` is a Series (or array) aligned with ``data`` holding -1/0/1. + """ + df = data.copy() + df["signal"] = signals + + plt.figure(figsize=(12, 6)) + plt.plot(df.index, df[price_col], label="Price", color="blue", alpha=0.7) + + buy = df[df["signal"] == 1][price_col] + plt.scatter(buy.index, buy, label="Buy Signal", color="green", marker="^", s=100) + + sell = df[df["signal"] == -1][price_col] + plt.scatter(sell.index, sell, label="Sell Signal", color="red", marker="v", s=100) + + plt.title(title) + plt.xlabel("Date") + plt.ylabel("Price") + plt.legend() + plt.grid(True) + plt.show()