Add project scaffold with config and dependencies

.gitignore

@@ -0,0 +1,49 @@
+# Byte-compiled / optimized / cache
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info/
+.eggs/
+
+# Secrets / environment
+.env
+*.env
+
+# Trained model artifacts (root-level only; not tradingbot/models/ package)
+/models/
+*.pkl
+*.joblib
+*.pth
+*.cbm
+
+# Data files (don't commit large market data)
+*.csv
+data/*.csv
+*.parquet
+
+# Generated plots
+*.png
+*.jpg
+*.svg
+
+# Notebook checkpoints
+.ipynb_checkpoints/
+
+# Optuna / experiment DBs
+*.db
+
+# IDE / OS
+.idea/
+.vscode/
+.DS_Store

README.md

@@ -0,0 +1,173 @@
+# tradingbot
+
+A research framework for **5-minute gold / forex machine-learning trading**.
+
+It covers the full pipeline: downloading and cleaning candle data, engineering
+technical features, training several model architectures, turning predictions
+into risk-managed signals, and backtesting them with walk-forward optimization.
+
+---
+
+## Project structure
+
+```
+tradingbot/
+├── tradingbot/
+│   ├── config.py                 # central config (reads secrets from env)
+│   ├── data/
+│   │   ├── oanda_connector.py     # OANDA v20 historical candle download
+│   │   ├── loader.py              # CSV loading + time-gap analysis
+│   │   ├── processor.py           # GoldDataProcessor: cleaning, gap fill, outliers
+│   │   └── features.py            # GoldFeatureEngineer + GoldFeaturePipeline
+│   ├── models/
+│   │   ├── tree_ensemble.py       # TreeEnsemblePredictor (stacked trees + HMM)
+│   │   ├── neural_ensemble.py     # NeuralEnsemblePredictor (Transformer/LSTM + Bayesian NN)
+│   │   └── model_manager.py       # GoldModelManager (Optuna, Boruta, SHAP, 5-class)
+│   ├── signals/
+│   │   └── generator.py           # GoldSignalGenerator (signals + stops/targets)
+│   ├── backtest/
+│   │   ├── walk_forward.py        # WalkForwardOptimizer
+│   │   └── metrics.py             # compute_performance_metrics
+│   └── viz/
+│       └── visualize.py           # visualize_signals + plotting helpers
+├── notebooks/                     # exploratory research notebook
+├── requirements.txt
+└── .gitignore
+```
+
+## Model approaches
+
+The framework ships three model architectures that can be used independently:
+
+1. `**TreeEnsemblePredictor**` (`models/tree_ensemble.py`) — a binary up/down
+  classifier that stacks XGBoost, LightGBM, CatBoost, RandomForest and
+   ExtraTrees, with two meta-models and an HMM market-regime filter.
+2. `**NeuralEnsemblePredictor**` (`models/neural_ensemble.py`) — stacks
+  gradient-boosted trees with a Transformer + BiLSTM network and a Bayesian
+   network (Monte-Carlo dropout) for uncertainty-aware filtering.
+3. `**GoldModelManager` pipeline** (`models/model_manager.py` +
+  `signals/generator.py` + `backtest/walk_forward.py`) — the most complete
+   path: a 5-class classifier (strong/weak up, sideways, weak/strong down) with
+   Boruta/RFE/PCA feature selection, Optuna tuning, SHAP analysis, volatility
+   regime-aware ensembling, and walk-forward backtesting.
+
+---
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+```
+
+**TA-Lib** also requires the underlying C library:
+
+- macOS: `brew install ta-lib`
+- Ubuntu/Debian: install `ta-lib` (package or build from source)
+- Windows: install a prebuilt TA-Lib wheel / binaries
+
+For live data downloads, set your OANDA token (never commit it):
+
+```bash
+export OANDA_ACCESS_TOKEN="your-token-here"
+```
+
+---
+
+## Usage
+
+### 1. Download data (optional — or bring your own OHLCV CSV)
+
+```python
+from tradingbot.data.oanda_connector import download
+
+# Saves e.g. EURUSD_M5_<start>_to_<end>.csv in the current directory.
+download(instrument="EUR_USD", timeframe="M5", lookback_days=1825)
+```
+
+A CSV is expected with a `timestamp` column plus `open, high, low, close, volume`.
+
+### 2. Quick look at the data
+
+```python
+from tradingbot.data.loader import load_ohlcv_csv, summarize, analyze_time_gaps
+
+data = load_ohlcv_csv("EURUSD_M5_20200309_to_20250308.csv")
+summarize(data)
+analyze_time_gaps(data)
+```
+
+### 3a. Tree-ensemble approach
+
+```python
+from tradingbot.models.tree_ensemble import run_model
+
+predictor, metrics, results, returns = run_model(
+    data_path="XAUUSD_M5_20200222_to_20250220.csv",
+    forecast_bars=24,            # 2 hours of 5-minute bars
+    confidence_threshold=0.67,
+)
+```
+
+Visualize the result:
+
+```python
+from tradingbot.viz.visualize import visualize_signals
+
+visualize_signals(data, results, returns, metrics, save_path="signals.png")
+```
+
+### 3b. Neural-ensemble approach
+
+```python
+import pandas as pd
+from tradingbot.models.neural_ensemble import NeuralEnsemblePredictor
+
+data = pd.read_csv("XAUUSD_H1_...csv", parse_dates=["timestamp"], index_col="timestamp")
+train, test = data[:int(len(data) * 0.9)], data[int(len(data) * 0.9):]
+
+predictor = NeuralEnsemblePredictor(forecast_period=12, confidence_threshold=0.6)
+predictor.fit(train)
+signals = predictor.predict(test)
+```
+
+### 3c. Full pipeline + walk-forward backtest
+
+```python
+from tradingbot.data.features import GoldFeaturePipeline
+from tradingbot.backtest.walk_forward import WalkForwardOptimizer
+
+# Engineer features and the 5-class target.
+processed = GoldFeaturePipeline(forecast_horizon=6).process(
+    "XAUUSD_M5_20200222_to_20250220.csv"
+)
+
+# Walk-forward optimization (windows are in days).
+wf = WalkForwardOptimizer(
+    train_window=35,
+    step_size=5,
+    test_window=5,
+    feature_selection_interval=4,
+)
+results = wf.optimize(processed)
+```
+
+Generate signals from a trained model:
+
+```python
+from tradingbot.models.model_manager import GoldModelManager
+from tradingbot.signals.generator import GoldSignalGenerator
+
+manager = GoldModelManager(model_path="models/step_1", feature_selection_method="importance")
+manager.load_models()
+
+gen = GoldSignalGenerator(confidence_threshold=0.7, risk_reward_min=1.5, model_manager=manager)
+signals = gen.generate_signals(processed.iloc[-2880:])  # last ~10 days
+active = signals[signals["signal"] != 0]
+```
+
+---
+
+## Disclaimer
+
+This is research / educational code for exploring ML trading ideas on historical
+data. It is **not** financial advice and makes no guarantee of profitability. Markets carry real risk of loss, validate any strategy thoroughly before risking capital.

requirements.txt

@@ -0,0 +1,35 @@
+# Core
+numpy
+pandas
+scipy
+scikit-learn
+matplotlib
+seaborn
+joblib
+
+# Gradient boosting / tree models
+xgboost
+lightgbm
+catboost
+
+# Regime detection
+hmmlearn
+
+# Feature selection / hyperparameter tuning / explainability
+Boruta
+optuna
+scikit-optimize
+shap
+
+# Deep learning (neural ensemble)
+torch
+
+# Technical indicators
+# NOTE: TA-Lib also needs the underlying C library installed on your system.
+#   macOS:   brew install ta-lib
+#   Ubuntu:  apt-get install ta-lib   (or build from source)
+#   Windows: install a prebuilt wheel / the ta-lib binaries
+TA-Lib
+
+# Live data download (optional)
+oandapyV20

tradingbot/__init__.py

@@ -0,0 +1,11 @@
+"""tradingbot: a 5-minute gold/forex ML trading research framework.
+
+Sub-packages:
+    data      - data download, loading, cleaning and feature engineering
+    models    - the three model approaches (tree ensemble, neural ensemble, manager)
+    signals   - turning model predictions into risk-managed trade signals
+    backtest  - walk-forward optimization and performance metrics
+    viz       - signal / performance visualization
+"""
+
+__version__ = "0.1.0"

tradingbot/backtest/__init__.py

@@ -0,0 +1,4 @@
+from tradingbot.backtest.walk_forward import WalkForwardOptimizer
+from tradingbot.backtest.metrics import compute_performance_metrics
+
+__all__ = ["WalkForwardOptimizer", "compute_performance_metrics"]

tradingbot/backtest/metrics.py

@@ -0,0 +1,60 @@
+"""Simple strategy performance metrics.
+
+Scores a ``signal`` column against forward returns: total return, annualized
+Sharpe, win rate and max drawdown.
+"""
+import numpy as np
+
+
+def compute_performance_metrics(data, signal_col="signal", forecast_periods=12,
+                                annualization=24 * 365, verbose=True):
+    """Compute strategy performance from a signal column and forward returns.
+
+    Parameters
+    ----------
+    data : pd.DataFrame
+        Must contain a ``close`` column and ``signal_col`` (values in {-1, 0, 1}).
+    forecast_periods : int
+        Holding horizon (in bars) used to compute the forward return.
+    annualization : float
+        Factor applied under the square root when annualizing the Sharpe ratio.
+
+    Returns
+    -------
+    dict
+        total_return, sharpe_ratio, win_rate, max_drawdown and the augmented frame.
+    """
+    df = data.copy()
+
+    # Forward return over the forecast horizon.
+    df["future_return"] = df["close"].pct_change(periods=forecast_periods).shift(-forecast_periods)
+
+    # Strategy return: signal * future return (long: +return, short: -return).
+    df["strategy_return"] = df[signal_col] * df["future_return"]
+    df = df.dropna(subset=["strategy_return"])
+
+    # Cumulative return.
+    df["cumulative_return"] = (1 + df["strategy_return"]).cumprod() - 1
+
+    total_return = df["cumulative_return"].iloc[-1] if len(df) else float("nan")
+    sharpe_ratio = (
+        df["strategy_return"].mean() / df["strategy_return"].std() * np.sqrt(annualization)
+        if df["strategy_return"].std() else float("nan")
+    )
+    active = df[df["strategy_return"] != 0]
+    win_rate = len(df[df["strategy_return"] > 0]) / len(active) if len(active) else float("nan")
+    max_drawdown = (df["cumulative_return"].cummax() - df["cumulative_return"]).max()
+
+    if verbose:
+        print(f"Total Return: {total_return:.2%}")
+        print(f"Sharpe Ratio: {sharpe_ratio:.2f}")
+        print(f"Win Rate: {win_rate:.2%}")
+        print(f"Max Drawdown: {max_drawdown:.2%}")
+
+    return {
+        "total_return": total_return,
+        "sharpe_ratio": sharpe_ratio,
+        "win_rate": win_rate,
+        "max_drawdown": max_drawdown,
+        "data": df,
+    }

tradingbot/backtest/walk_forward.py

@@ -0,0 +1,280 @@
+"""Walk-forward optimization harness."""
+import numpy as np
+import pandas as pd
+from pathlib import Path
+
+from tradingbot.models.model_manager import GoldModelManager
+from tradingbot.signals.generator import GoldSignalGenerator
+
+
+class WalkForwardOptimizer:
+    """
+    Implements walk-forward optimization for model training and validation
+    """
+    def __init__(
+        self,
+        train_window=120,  # Initial training days (approx. 34560 5-min bars)
+        step_size=20,     # Days to move forward in each step (approx. 5760 bars)
+        feature_selection_interval=3,  # How often to perform feature selection
+        test_window=10,    # Days to test on after each training (approx. 2880 bars)
+        n_jobs=-1,
+        model_path='models',
+        random_state=42
+    ):
+        """
+        Initialize the walk-forward optimizer
+
+        Parameters:
+        -----------
+        train_window : int
+            Number of days for initial training window
+        step_size : int
+            Number of days to move forward in each step
+        feature_selection_interval : int
+            How many steps between feature selection (to save time)
+        test_window : int
+            Number of days to test on after each training
+        n_jobs : int
+            Number of parallel jobs for training
+        model_path : str
+            Directory to save models
+        """
+        self.train_window = train_window
+        self.step_size = step_size
+        self.feature_selection_interval = feature_selection_interval
+        self.test_window = test_window
+        self.n_jobs = n_jobs
+        self.model_path = Path(model_path)
+        self.model_path.mkdir(exist_ok=True)
+        self.random_state = random_state
+
+        # Initialize metrics storage
+        self.metrics = []
+        self.signals = []
+
+    def optimize(self, data, min_train_size=8000):
+        """
+        Perform walk-forward optimization with improved error handling
+
+        Parameters:
+        -----------
+        data : pd.DataFrame
+            DataFrame with features and target
+        min_train_size : int
+            Minimum number of samples required for training
+
+        Returns:
+        --------
+        results : dict
+            Walk-forward optimization results
+        """
+        # Convert time windows from days to number of 5-minute bars
+        # Assuming 288 5-minute bars per day (24 hours)
+        bars_per_day = 288
+        train_bars = self.train_window * bars_per_day
+        step_bars = self.step_size * bars_per_day
+        test_bars = self.test_window * bars_per_day
+
+        # Ensure data has a datetime index
+        if not isinstance(data.index, pd.DatetimeIndex):
+            raise ValueError("Data must have a DatetimeIndex")
+
+        # Calculate number of steps
+        total_bars = len(data)
+        n_steps = max(1, (total_bars - train_bars) // step_bars)
+
+        print(f"Starting walk-forward optimization with {n_steps} steps")
+
+        # Initialize results storage
+        all_signals = pd.DataFrame()
+        all_metrics = []
+
+        # Track whether to do feature selection in this step
+        do_feature_selection = True
+        selected_features = None
+
+        # For each step
+        for step in range(n_steps):
+            print(f"\n{'-'*50}")
+            print(f"Step {step+1}/{n_steps}")
+            print(f"{'-'*50}")
+
+            try:
+                # Calculate indices for this step
+                train_start = step * step_bars
+                train_end = train_start + train_bars
+                test_start = train_end
+                test_end = min(test_start + test_bars, total_bars)
+
+                # Get data for this step
+                train_data = data.iloc[train_start:train_end]
+                test_data = data.iloc[test_start:test_end]
+
+                print(f"Train period: {train_data.index[0]} to {train_data.index[-1]}")
+                print(f"Test period: {test_data.index[0]} to {test_data.index[-1]}")
+
+                # Skip if not enough training data
+                if len(train_data) < min_train_size:
+                    print(f"Skipping step {step+1} - not enough training data ({len(train_data)} < {min_train_size})")
+                    continue
+
+                # Create model manager for this step
+                model_dir = self.model_path / f'step_{step+1}'
+                model_dir.mkdir(exist_ok=True, parents=True)
+
+                model_manager = GoldModelManager(
+                    n_splits=3,
+                    feature_selection_method='boruta' if do_feature_selection else 'importance',
+                    max_features=25,
+                    ensemble_models=3,
+                    volatility_based_models=True,
+                    n_jobs=self.n_jobs,
+                    model_path=model_dir,
+                    random_state=self.random_state + step  # Vary random state by step
+                )
+
+                # Extract y_train for class distribution check before training
+                X_train, y_train = model_manager.prepare_data(train_data)
+                unique_classes = np.unique(y_train)
+                class_counts = {cls: np.sum(y_train == cls) for cls in unique_classes}
+
+                print("Class distribution in training data:")
+                expected_classes = set(range(-2, 3))  # -2, -1, 0, 1, 2
+                missing_classes = expected_classes - set(unique_classes)
+
+                for cls in sorted(expected_classes):
+                    count = class_counts.get(cls, 0)
+                    percentage = (count / len(y_train)) * 100 if len(y_train) > 0 else 0
+                    status = "PRESENT" if cls in unique_classes else "MISSING"
+                    print(f"  Class {cls}: {count} samples ({percentage:.2f}%) - {status}")
+
+                if missing_classes:
+                    print(f"Warning: Missing classes in training data: {missing_classes}")
+                    print("Continuing anyway with appropriate class weights...")
+
+                # Set selected features if not doing feature selection
+                if not do_feature_selection and selected_features is not None:
+                    model_manager.selected_features = selected_features
+
+                # Train models
+                model_manager.fit(train_data)
+
+                # Store selected features for future steps
+                if do_feature_selection or selected_features is None:
+                    selected_features = model_manager.selected_features
+
+                # Update feature selection flag for next step
+                do_feature_selection = ((step + 1) % self.feature_selection_interval == 0)
+
+                # Skip signal generation if model training failed
+                if model_manager.meta_model is None:
+                    print("Warning: Meta-model training failed, skipping signal generation")
+                    continue
+
+                # Create signal generator
+                signal_generator = GoldSignalGenerator(
+                    confidence_threshold=0.7,
+                    risk_reward_min=1.5,
+                    model_manager=model_manager
+                )
+
+                # Generate signals on test data
+                signals = signal_generator.generate_signals(test_data)
+
+                # Check if signals DataFrame is empty
+                if signals.empty or 'signal' not in signals.columns:
+                    print("Warning: No signals generated, skipping analysis")
+                    continue
+
+                # Check if any signals were generated
+                signal_count = (signals['signal'] != 0).sum() if 'signal' in signals.columns else 0
+                if signal_count == 0:
+                    print("Warning: No active signals found in test period")
+                else:
+                    print(f"Generated {signal_count} active signals")
+
+                # Analyze signals
+                metrics = signal_generator.analyze_signals(signals, test_data)
+
+                # Store results
+                signals['step'] = step + 1
+                all_signals = pd.concat([all_signals, signals])
+
+                metrics['step'] = step + 1
+                metrics['train_start'] = train_data.index[0]
+                metrics['train_end'] = train_data.index[-1]
+                metrics['test_start'] = test_data.index[0]
+                metrics['test_end'] = test_data.index[-1]
+                all_metrics.append(metrics)
+
+            except Exception as e:
+                import traceback
+                print(f"Error in step {step+1}: {str(e)}")
+                print("Detailed traceback:")
+                print(traceback.format_exc())
+                continue
+
+        # Compile final results
+        results = {
+            'signals': all_signals,
+            'metrics': all_metrics
+        }
+
+        # Print overall performance
+        self._print_overall_performance(all_metrics)
+
+        return results
+
+    def _print_overall_performance(self, metrics):
+        """Print overall performance statistics"""
+        if not metrics:
+            print("No metrics available for performance analysis")
+            return
+
+        print("\n=== Overall Walk-Forward Performance ===")
+
+        # Calculate average metrics
+        total_signals = sum(m['total_signals'] for m in metrics)
+        avg_win_rate = np.mean([m['overall_win_rate'] for m in metrics if not np.isnan(m['overall_win_rate'])])
+        avg_return = np.mean([m['overall_avg_return'] for m in metrics if not np.isnan(m['overall_avg_return'])])
+
+        print(f"Total Steps: {len(metrics)}")
+        print(f"Total Signals: {total_signals}")
+        print(f"Average Win Rate: {avg_win_rate:.2%}")
+        print(f"Average Return: {avg_return:.2%}")
+
+        # Calculate performance by regime if available
+        regime_performance = {}
+        for m in metrics:
+            if m['regime_stats'] is not None:
+                for regime, stats in m['regime_stats'].items():
+                    if regime not in regime_performance:
+                        regime_performance[regime] = {
+                            'count': 0,
+                            'win_rate': [],
+                            'avg_return': []
+                        }
+
+                    regime_performance[regime]['count'] += stats['count']
+
+                    # Long performance
+                    if not np.isnan(stats['long_win_rate']):
+                        regime_performance[regime]['win_rate'].append(stats['long_win_rate'])
+
+                    if not np.isnan(stats['long_avg_return']):
+                        regime_performance[regime]['avg_return'].append(stats['long_avg_return'])
+
+                    # Short performance
+                    if not np.isnan(stats['short_win_rate']):
+                        regime_performance[regime]['win_rate'].append(stats['short_win_rate'])
+
+                    if not np.isnan(stats['short_avg_return']):
+                        regime_performance[regime]['avg_return'].append(stats['short_avg_return'])
+
+        if regime_performance:
+            print("\nPerformance by Volatility Regime:")
+            for regime, stats in regime_performance.items():
+                avg_win_rate = np.mean(stats['win_rate']) if stats['win_rate'] else np.nan
+                avg_return = np.mean(stats['avg_return']) if stats['avg_return'] else np.nan
+
+                print(f"Regime {regime}: {stats['count']} signals, Win Rate: {avg_win_rate:.2%}, Avg Return: {avg_return:.2%}")

tradingbot/config.py

@@ -0,0 +1,18 @@
+"""Central configuration.
+
+Secrets are read from environment variables so they never live in source code.
+Set them in your shell or a local ``.env`` file (which is git-ignored):
+
+    export OANDA_ACCESS_TOKEN="your-token-here"
+"""
+import os
+
+# OANDA v20 REST access token (required only for live data downloads).
+OANDA_ACCESS_TOKEN = os.environ.get("OANDA_ACCESS_TOKEN", "")
+
+# Default instrument / granularity for downloads.
+DEFAULT_INSTRUMENT = os.environ.get("INSTRUMENT", "EUR_USD")
+DEFAULT_GRANULARITY = os.environ.get("GRANULARITY", "M5")
+
+# Where trained models are written / loaded from.
+MODEL_DIR = os.environ.get("MODEL_DIR", "models")

tradingbot/data/__init__.py

@@ -0,0 +1,12 @@
+from tradingbot.data.loader import load_ohlcv_csv, summarize, analyze_time_gaps
+from tradingbot.data.processor import GoldDataProcessor
+from tradingbot.data.features import GoldFeatureEngineer, GoldFeaturePipeline
+
+__all__ = [
+    "load_ohlcv_csv",
+    "summarize",
+    "analyze_time_gaps",
+    "GoldDataProcessor",
+    "GoldFeatureEngineer",
+    "GoldFeaturePipeline",
+]

tradingbot/data/features.py

@@ -0,0 +1,485 @@
+"""Feature engineering and the end-to-end feature pipeline."""
+import numpy as np
+import pandas as pd
+import talib
+from sklearn.base import BaseEstimator, TransformerMixin
+from hmmlearn import hmm
+from scipy import stats
+import warnings
+
+from tradingbot.data.processor import GoldDataProcessor
+
+warnings.filterwarnings("ignore")
+
+
+class GoldFeatureEngineer(BaseEstimator, TransformerMixin):
+    """
+    Feature engineering for 5-minute gold trading data.
+    Implements market microstructure, volatility, indicator adjustments,
+    and time-based features.
+    """
+    def __init__(self, forecast_horizon=6, volatility_regime_states=3, look_back_window=120):
+        """
+        Initialize the feature engineer
+
+        Parameters:
+        -----------
+        forecast_horizon : int
+            Number of bars to look ahead for target creation (default: 6 bars = 30 minutes)
+        volatility_regime_states : int
+            Number of states for HMM volatility regime detection
+        look_back_window : int
+            Window size for rolling computations (default: 120 bars = 10 hours)
+        """
+        self.forecast_horizon = forecast_horizon
+        self.volatility_regime_states = volatility_regime_states
+        self.look_back_window = look_back_window
+        self.regime_model = hmm.GaussianHMM(
+            n_components=volatility_regime_states,
+            random_state=42,
+            covariance_type="diag"
+        )
+
+    def fit(self, X, y=None):
+        """
+        Fit method to comply with sklearn transformer interface
+        Learns volatility regimes from the data
+        """
+        # Ensure X is a DataFrame
+        if not isinstance(X, pd.DataFrame):
+            X = pd.DataFrame(X)
+
+        # Detect volatility regimes for later transformation
+        self._fit_volatility_regimes(X)
+
+        return self
+
+    def _clean_features(self, data):
+        """
+        Clean feature data by handling infinities, NaNs, and extreme values
+        """
+        # Replace infinities with NaN
+        data = data.replace([np.inf, -np.inf], np.nan)
+
+        # For each feature column, handle NaNs and clip extreme values
+        for col in data.columns:
+            if col not in ['open', 'high', 'low', 'close', 'volume', 'target', 'is_weekend']:
+                # Fill NaNs with median (or forward fill if median is NaN)
+                median_val = data[col].median()
+                if pd.isna(median_val):
+                    data[col] = data[col].fillna(method='ffill')
+                    if data[col].isna().any():
+                        data[col] = data[col].fillna(method='bfill')
+                        if data[col].isna().any():
+                            data[col] = data[col].fillna(0)
+                else:
+                    data[col] = data[col].fillna(median_val)
+
+                # Clip extreme values to 5 standard deviations from mean
+                if not data[col].isna().all():
+                    mean_val = data[col].mean()
+                    std_val = data[col].std()
+                    if std_val > 0:  # Only clip if standard deviation is positive
+                        data[col] = data[col].clip(
+                            lower=mean_val - 5*std_val,
+                            upper=mean_val + 5*std_val
+                        )
+
+        return data
+
+    def transform(self, X):
+        """
+        Transform the data by adding engineered features
+        """
+        # Ensure X is a DataFrame and create a copy to avoid modifying original
+        if not isinstance(X, pd.DataFrame):
+            X = pd.DataFrame(X)
+
+        data = X.copy()
+
+        # Add all feature groups
+        data = self._add_market_microstructure_features(data)
+        data = self._add_volatility_features(data)
+        data = self._add_indicator_features(data)
+        data = self._add_time_based_features(data)
+
+        # Create target variable if close column exists
+        if 'close' in data.columns:
+            data = self._create_target_variable(data)
+
+        data = self._clean_features(data)
+
+        # Drop rows with NaN values resulting from indicators that need lookback
+        data = data.dropna()
+
+        return data
+
+    def _fit_volatility_regimes(self, data):
+        """
+        Fit HMM model to detect volatility regimes
+        """
+        # Calculate log returns
+        if 'close' in data.columns:
+            returns = np.log(data['close'] / data['close'].shift(1)).dropna()
+
+            # Calculate rolling volatility features for HMM
+            rolling_vol = returns.rolling(window=20).std().dropna()
+            rolling_range = ((data['high'] / data['low'] - 1)
+                             .rolling(window=20).mean().dropna())
+
+            # Combine features for regime detection
+            features = pd.DataFrame({
+                'returns_volatility': rolling_vol,
+                'price_range': rolling_range
+            }).dropna()
+
+            if len(features) > self.volatility_regime_states * 5:  # Ensure enough data to fit
+                # Normalize features for HMM
+                features = (features - features.mean()) / features.std()
+
+                # Fit the HMM model
+                self.regime_model.fit(features.values)
+
+    def _add_market_microstructure_features(self, data):
+        """
+        Add market microstructure features:
+        - Order flow imbalance
+        - Price velocity
+        - Relative volume analysis
+        """
+        # Order Flow Imbalance (approximated from OHLC)
+        data['bar_sentiment'] = np.where(
+            data['close'] > data['open'],
+            (data['close'] - data['open']) / (data['high'] - data['low'] + 1e-8),
+            -1 * (data['open'] - data['close']) / (data['high'] - data['low'] + 1e-8)
+        )
+
+        # Smooth bar sentiment
+        data['smooth_sentiment'] = data['bar_sentiment'].rolling(5).mean()
+
+        # Price Velocity - Rate of change over multiple timeframes
+        for window in [1, 3, 5, 15]:
+            data[f'price_velocity_{window}'] = data['close'].pct_change(window) / (window * 5)  # Normalize by minutes
+
+        # Acceleration (change in velocity)
+        data['price_acceleration'] = data['price_velocity_3'].diff()
+
+        # Relative Volume Analysis
+        # Calculate average volume by time of day first
+        data['hour'] = data.index.hour
+        data['minute'] = data.index.minute
+        data['time_of_day'] = data['hour'] * 60 + data['minute']
+
+        # Group by time of day and calculate average volume
+        avg_volume_by_time = data.groupby('time_of_day')['volume'].transform('mean')
+        data['relative_volume'] = data['volume'] / (avg_volume_by_time + 1e-8)  # Avoid division by zero
+
+        # Volume momentum
+        data['volume_momentum'] = data['volume'].pct_change(5)
+
+        # Drop temporary columns
+        data = data.drop(['hour', 'minute', 'time_of_day'], axis=1)
+
+        return data
+
+    def _add_volatility_features(self, data):
+        """
+        Add volatility-related features:
+        - Micro-volatility clusters
+        - Bollinger Band Width
+        - Volatility regime detection
+        """
+        # ATR with shorter periods for micro-volatility
+        for period in [5, 10]:
+            data[f'atr_{period}'] = talib.ATR(
+                data['high'].values,
+                data['low'].values,
+                data['close'].values,
+                timeperiod=period
+            )
+            # Normalize ATR by price level
+            data[f'atr_{period}_pct'] = data[f'atr_{period}'] / data['close']
+
+        # Bollinger Band Width (20 periods, 2 std)
+        upper, middle, lower = talib.BBANDS(
+            data['close'].values,
+            timeperiod=20,
+            nbdevup=2,
+            nbdevdn=2
+        )
+        data['bb_width'] = (upper - lower) / middle
+
+        # Rate of change of BB width
+        data['bb_width_change'] = data['bb_width'].pct_change(3)
+
+        # Volatility Regime Detection
+        if hasattr(self, 'regime_model') and hasattr(self.regime_model, 'transmat_'):
+            # Calculate the same features used during fitting
+            returns = np.log(data['close'] / data['close'].shift(1))
+            rolling_vol = returns.rolling(window=20).std()
+            rolling_range = (data['high'] / data['low'] - 1).rolling(window=20).mean()
+
+            # Combine and normalize features
+            features = pd.DataFrame({
+                'returns_volatility': rolling_vol,
+                'price_range': rolling_range
+            })
+
+            # Handle NaN values
+            features = features.fillna(method='bfill')
+
+            # Normalize using the same approach as in fitting
+            features = (features - features.mean()) / features.std()
+
+            # Predict regimes where we have data
+            valid_idx = ~features.isnull().any(axis=1)
+            if valid_idx.sum() > 0:
+                regimes = self.regime_model.predict(features[valid_idx].values)
+
+                # Create a series with index aligned to original data
+                regime_series = pd.Series(index=data.index, dtype='float64')
+                regime_series.loc[features[valid_idx].index] = regimes
+
+                # Forward fill regime values
+                data['volatility_regime'] = regime_series.fillna(method='ffill')
+            else:
+                # If we don't have enough data, use a default regime
+                data['volatility_regime'] = 1
+        else:
+            # If regime model isn't fitted, use a simpler approach
+            returns = np.log(data['close'] / data['close'].shift(1))
+            vol = returns.rolling(window=20).std() * np.sqrt(252 * 288)  # Annualized (288 5-min bars/day)
+            data['volatility_regime'] = pd.qcut(
+                vol,
+                q=self.volatility_regime_states,
+                labels=False,
+                duplicates='drop'
+            ).fillna(self.volatility_regime_states // 2)
+
+        return data
+
+    def _add_indicator_features(self, data):
+        """
+        Add adjusted technical indicators specific for 5-minute data:
+        - Fast & Slow EMAs
+        - RSI
+        - Stochastic Oscillator
+        - MACD
+        """
+        # EMAs with shorter periods
+        for period in [10, 20, 50]:
+            data[f'ema_{period}'] = talib.EMA(data['close'].values, timeperiod=period)
+
+            # Add relative position to EMA
+            data[f'close_to_ema_{period}'] = (data['close'] / data[f'ema_{period}'] - 1) * 100
+
+        # EMA Cross Features
+        data['ema_10_20_cross'] = data['ema_10'] - data['ema_20']
+        data['ema_10_50_cross'] = data['ema_10'] - data['ema_50']
+
+        # Shorter-period RSI
+        data['rsi_7'] = talib.RSI(data['close'].values, timeperiod=7)
+        data['rsi_14'] = talib.RSI(data['close'].values, timeperiod=14)
+
+        # RSI momentum and mean-reversion features
+        data['rsi_7_change'] = data['rsi_7'].diff(3)
+        data['rsi_divergence'] = data['rsi_14'] - data['rsi_7']
+
+        # Stochastic Oscillator - Fast and Slow
+        slowk, slowd = talib.STOCH(
+            data['high'].values,
+            data['low'].values,
+            data['close'].values,
+            fastk_period=5,
+            slowk_period=3,
+            slowk_matype=0,
+            slowd_period=3,
+            slowd_matype=0
+        )
+        data['stoch_k_fast'] = slowk
+        data['stoch_d_fast'] = slowd
+
+        slowk, slowd = talib.STOCH(
+            data['high'].values,
+            data['low'].values,
+            data['close'].values,
+            fastk_period=14,
+            slowk_period=3,
+            slowk_matype=0,
+            slowd_period=3,
+            slowd_matype=0
+        )
+        data['stoch_k_slow'] = slowk
+        data['stoch_d_slow'] = slowd
+
+        # Stochastic crossover signal
+        data['stoch_crossover'] = data['stoch_k_fast'] - data['stoch_d_fast']
+
+        # MACD with faster settings
+        macd, macdsignal, macdhist = talib.MACD(
+            data['close'].values,
+            fastperiod=8,
+            slowperiod=17,
+            signalperiod=9
+        )
+        data['macd'] = macd
+        data['macd_signal'] = macdsignal
+        data['macd_hist'] = macdhist
+
+        # Add momentum oscillator
+        data['mom_10'] = talib.MOM(data['close'].values, timeperiod=10)
+
+        # Add Average Directional Index for trend strength
+        data['adx_14'] = talib.ADX(
+            data['high'].values,
+            data['low'].values,
+            data['close'].values,
+            timeperiod=14
+        )
+
+        return data
+
+    def _add_time_based_features(self, data):
+        """
+        Add time-based features:
+        - Time of day (hour, minute)
+        - Day of week
+        - Session (Tokyo, London, New York)
+        """
+        # Extract basic time components
+        data['hour'] = data.index.hour
+        data['minute'] = data.index.minute
+        data['day_of_week'] = data.index.dayofweek
+
+        # Create cyclical time features (circular encoding to handle continuity)
+        # For hour of day (24 hours)
+        data['hour_sin'] = np.sin(2 * np.pi * data['hour'] / 24)
+        data['hour_cos'] = np.cos(2 * np.pi * data['hour'] / 24)
+
+        # For minute within hour
+        data['minute_sin'] = np.sin(2 * np.pi * data['minute'] / 60)
+        data['minute_cos'] = np.cos(2 * np.pi * data['minute'] / 60)
+
+        # For day of week (5-day trading week)
+        data['day_sin'] = np.sin(2 * np.pi * data['day_of_week'] / 7)
+        data['day_cos'] = np.cos(2 * np.pi * data['day_of_week'] / 7)
+
+        # Create Trading Session markers
+        # Define trading sessions (UTC times)
+        # Tokyo: 00:00-09:00
+        # London: 08:00-17:00
+        # New York: 13:00-22:00
+
+        # Convert hour to UTC assuming index is in UTC
+        # If index is not in UTC, this would need to be adjusted
+        utc_hour = data['hour']
+
+        # Trading session flags
+        data['tokyo_session'] = ((utc_hour >= 0) & (utc_hour < 9)).astype(int)
+        data['london_session'] = ((utc_hour >= 8) & (utc_hour < 17)).astype(int)
+        data['ny_session'] = ((utc_hour >= 13) & (utc_hour < 22)).astype(int)
+
+        # Session overlap flags
+        data['tokyo_london_overlap'] = ((utc_hour >= 8) & (utc_hour < 9)).astype(int)
+        data['london_ny_overlap'] = ((utc_hour >= 13) & (utc_hour < 17)).astype(int)
+
+        # Drop original time columns
+        data = data.drop(['hour', 'minute', 'day_of_week'], axis=1)
+
+        return data
+
+    def _create_target_variable(self, data):
+        """
+        Create target variable for prediction:
+        Multi-class classification based on future price movement
+        """
+        # Calculate future return over forecast_horizon
+        future_close = data['close'].shift(-self.forecast_horizon)
+        future_return = (future_close / data['close'] - 1) * 100  # Percentage return
+
+        # Create multi-class target
+        # Calculate dynamic thresholds based on recent volatility
+        volatility = data['close'].pct_change().rolling(window=20).std() * 100  # Convert to percentage
+
+        # Define thresholds as a function of volatility
+        # More volatile periods have wider thresholds
+        strong_threshold = volatility * 0.75  # 75% of recent volatility
+        weak_threshold = volatility * 0.25    # 25% of recent volatility
+
+        # Ensure minimum thresholds
+        strong_threshold = np.maximum(strong_threshold, 0.15)  # At least 0.15%
+        weak_threshold = np.maximum(weak_threshold, 0.05)     # At least 0.05%
+
+        # Create targets
+        conditions = [
+            future_return > strong_threshold,
+            (future_return > weak_threshold) & (future_return <= strong_threshold),
+            (future_return >= -weak_threshold) & (future_return <= weak_threshold),
+            (future_return < -weak_threshold) & (future_return >= -strong_threshold),
+            future_return < -strong_threshold
+        ]
+
+        choices = [2, 1, 0, -1, -2]  # Strong Up, Weak Up, Sideways, Weak Down, Strong Down
+
+        data['target'] = np.select(conditions, choices, default=np.nan)
+
+        # Store the thresholds used for reference
+        data['strong_threshold'] = strong_threshold
+        data['weak_threshold'] = weak_threshold
+
+        # Also store raw future return for potential regression tasks
+        data['future_return'] = future_return
+
+        return data
+
+class GoldFeaturePipeline:
+    """
+    Complete pipeline for gold 5-minute data preprocessing and feature engineering
+    """
+    def __init__(self, forecast_horizon=6, volatility_regime_states=3, remove_outliers=True):
+        """
+        Initialize the complete pipeline
+
+        Parameters:
+        -----------
+        forecast_horizon : int
+            Number of bars to look ahead for target creation
+        volatility_regime_states : int
+            Number of states for HMM volatility regime detection
+        remove_outliers : bool
+            Whether to remove price outliers
+        """
+        self.data_processor = GoldDataProcessor(
+            fill_gaps=True,
+            remove_outliers=remove_outliers,
+            handle_after_hours=True
+        )
+
+        self.feature_engineer = GoldFeatureEngineer(
+            forecast_horizon=forecast_horizon,
+            volatility_regime_states=volatility_regime_states
+        )
+
+    def process(self, file_path):
+        """
+        Process data from file through the complete pipeline
+
+        Parameters:
+        -----------
+        file_path : str
+            Path to the CSV file with 5-minute OHLCV data
+
+        Returns:
+        --------
+        processed_data : pandas.DataFrame
+            Processed data with all features and target
+        """
+        # Load and preprocess data
+        data = self.data_processor.load_data(file_path)
+
+        # Apply feature engineering
+        self.feature_engineer.fit(data)
+        processed_data = self.feature_engineer.transform(data)
+
+        # Return the processed data
+        return processed_data

tradingbot/data/loader.py

@@ -0,0 +1,55 @@
+"""Lightweight CSV loading and data-quality helpers.
+
+Loads raw 5-minute candles and inspects the gaps between consecutive timestamps.
+"""
+import pandas as pd
+
+
+def load_ohlcv_csv(file_path, timestamp_col="timestamp"):
+    """Load an OHLCV CSV with a datetime index.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to a CSV containing a timestamp column plus OHLCV columns.
+    timestamp_col : str
+        Name of the timestamp column to parse and use as the index.
+    """
+    data = pd.read_csv(file_path, index_col=timestamp_col, parse_dates=True)
+    data = data[~data.index.duplicated(keep="first")]
+    return data.sort_index()
+
+
+def summarize(data):
+    """Print basic information about a loaded OHLCV frame."""
+    print("Data Info:")
+    print(data.info())
+    print("\nFirst few rows:")
+    print(data.head())
+    print("\nLast few rows:")
+    print(data.tail())
+    print(f"\nTotal candles: {len(data)}")
+
+
+def analyze_time_gaps(data, expected_gap=pd.Timedelta(minutes=5),
+                      big_gap=pd.Timedelta(hours=1), verbose=True):
+    """Report gaps between consecutive timestamps.
+
+    Returns the Series of gaps larger than ``expected_gap``.
+    """
+    time_diffs = data.index.to_series().diff()
+    gaps = time_diffs[time_diffs > expected_gap]
+
+    if verbose:
+        print(f"Total gaps larger than {expected_gap}: {len(gaps)}")
+        print("\nGap statistics:")
+        print(gaps.describe())
+
+        big_gaps = gaps[gaps > big_gap]
+        print(f"\nNumber of gaps > {big_gap}: {len(big_gaps)}")
+        print(f"Big gaps (> {big_gap}):")
+        for gap_start, gap_size in big_gaps.items():
+            gap_end = gap_start + gap_size
+            print(f"Gap from {gap_start} to {gap_end} ({gap_size})")
+
+    return gaps

tradingbot/data/oanda_connector.py

@@ -0,0 +1,113 @@
+"""OANDA v20 historical candle downloader.
+
+The access token is read from configuration (the ``OANDA_ACCESS_TOKEN``
+environment variable) rather than being hard-coded.
+"""
+import time
+from datetime import datetime, timedelta
+
+import pandas as pd
+import oandapyV20
+import oandapyV20.endpoints.instruments as instruments
+from oandapyV20.exceptions import V20Error
+
+from tradingbot.config import OANDA_ACCESS_TOKEN
+
+
+class OandaDataConnector:
+    def __init__(self, access_token):
+        self.client = oandapyV20.API(access_token=access_token)
+        self.last_request_time = 0
+        self.request_limit_delay = 0.01  # 100 requests per second max
+
+    def _respect_rate_limit(self):
+        current_time = time.time()
+        time_passed = current_time - self.last_request_time
+        if time_passed < self.request_limit_delay:
+            time.sleep(self.request_limit_delay - time_passed)
+        self.last_request_time = time.time()
+
+    def get_historical_data_chunked(self, instrument, timeframe, start_date, end_date=None, chunk_days=5):
+        if end_date is None:
+            end_date = datetime.now()
+
+        all_candles = []
+        current_date = start_date
+
+        while current_date < end_date:
+            self._respect_rate_limit()
+
+            chunk_end = min(current_date + timedelta(days=chunk_days), end_date)
+
+            params = {
+                "from": current_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
+                "to": chunk_end.strftime('%Y-%m-%dT%H:%M:%SZ'),
+                "granularity": timeframe,
+                "price": "MBA"  # Mid, Bid, Ask prices
+            }
+
+            try:
+                r = instruments.InstrumentsCandles(instrument=instrument, params=params)
+                self.client.request(r)
+
+                for candle in r.response['candles']:
+                    if candle['complete']:
+                        all_candles.append({
+                            'timestamp': candle['time'],
+                            'open': float(candle['mid']['o']),
+                            'high': float(candle['mid']['h']),
+                            'low': float(candle['mid']['l']),
+                            'close': float(candle['mid']['c']),
+                            'volume': float(candle['volume'])
+                        })
+
+                print(f"Downloaded data from {current_date.date()} to {chunk_end.date()}")
+                current_date = chunk_end
+
+            except V20Error as e:
+                print(f"Error fetching data: {e}")
+                return None
+
+        df = pd.DataFrame(all_candles)
+        if not df.empty:
+            df['timestamp'] = pd.to_datetime(df['timestamp'])
+            df.set_index('timestamp', inplace=True)
+            df = df.sort_index()
+        return df
+
+
+def download(instrument="EUR_USD", timeframe="M5", lookback_days=1825,
+             access_token=None, out_dir="."):
+    """Download ``lookback_days`` of candles and save them to a CSV.
+
+    Returns the downloaded DataFrame (or ``None`` on failure).
+    """
+    token = access_token or OANDA_ACCESS_TOKEN
+    if not token:
+        raise ValueError(
+            "No OANDA access token. Set the OANDA_ACCESS_TOKEN environment "
+            "variable or pass access_token=..."
+        )
+
+    connector = OandaDataConnector(token)
+    start_date = datetime.now() - timedelta(days=lookback_days)
+    data = connector.get_historical_data_chunked(
+        instrument=instrument,
+        timeframe=timeframe,
+        start_date=start_date,
+    )
+
+    if data is not None:
+        name = instrument.replace("_", "")
+        filename = (
+            f"{out_dir}/{name}_{timeframe}_"
+            f"{start_date.strftime('%Y%m%d')}_to_{datetime.now().strftime('%Y%m%d')}.csv"
+        )
+        data.to_csv(filename)
+        print(f"Data saved to {filename}")
+        print(f"Total candles downloaded: {len(data)}")
+    return data
+
+
+if __name__ == "__main__":
+    download()

tradingbot/data/processor.py

@@ -0,0 +1,140 @@
+"""Raw 5-minute OHLCV loading and cleaning."""
+import numpy as np
+import pandas as pd
+
+
+class GoldDataProcessor:
+    """
+    Data processor for 5-minute gold price data.
+    Handles data loading, cleaning, and preprocessing.
+    """
+    def __init__(self, fill_gaps=True, remove_outliers=True, handle_after_hours=True):
+        self.fill_gaps = fill_gaps
+        self.remove_outliers = remove_outliers
+        self.handle_after_hours = handle_after_hours
+
+    def load_data(self, file_path):
+        """
+        Load data from CSV file and set appropriate index
+        """
+        # Load data from CSV
+        data = pd.read_csv(file_path)
+
+        # Ensure timestamp column exists
+        if 'timestamp' not in data.columns:
+            raise ValueError("CSV must contain a 'timestamp' column")
+
+        # Convert timestamp to datetime and set as index
+        data['timestamp'] = pd.to_datetime(data['timestamp'])
+        data = data.drop_duplicates(subset=['timestamp'])
+        data.set_index('timestamp', inplace=True)
+
+        # Ensure all necessary columns exist
+        required_columns = ['open', 'high', 'low', 'close', 'volume']
+        missing_columns = [col for col in required_columns if col not in data.columns]
+        if missing_columns:
+            raise ValueError(f"Missing required columns: {missing_columns}")
+
+        # Make column names lowercase if they aren't already
+        data.columns = [col.lower() for col in data.columns]
+
+        return self._preprocess_data(data)
+
+    def _preprocess_data(self, data):
+        """
+        Apply preprocessing steps to clean and prepare the data
+        """
+        # Sort data by timestamp to ensure chronological order
+        data = data.sort_index()
+
+        # Fill gaps in 5-minute data
+        if self.fill_gaps:
+            data = self._fill_time_gaps(data)
+
+        # Remove outliers
+        if self.remove_outliers:
+            data = self._remove_price_outliers(data)
+
+        # Handle after-hours and weekend data
+        if self.handle_after_hours:
+            data = self._handle_after_hours_data(data)
+
+        return data
+
+    def _fill_time_gaps(self, data):
+        """
+        Fill gaps in 5-minute data by reindexing with complete 5-minute intervals
+        during trading hours
+        """
+        # Create complete 5-minute intervals
+        start_date = data.index.min()
+        end_date = data.index.max()
+        full_range = pd.date_range(start=start_date, end=end_date, freq='5min')
+
+        # Reindex the data
+        reindexed_data = data.reindex(full_range)
+
+        # Forward fill price data (OHLC)
+        for col in ['open', 'high', 'low', 'close']:
+            reindexed_data[col] = reindexed_data[col].ffill()
+
+        # Fill volume with zeros
+        reindexed_data['volume'] = reindexed_data['volume'].fillna(0)
+
+        return reindexed_data
+
+    def _remove_price_outliers(self, data, z_threshold=10):
+        """
+        Remove extreme price outliers based on z-score of returns
+        """
+        # Calculate returns
+        returns = data['close'].pct_change()
+
+        # Calculate rolling z-score (20 periods ~ 100 minutes)
+        rolling_mean = returns.rolling(window=20).mean()
+        rolling_std = returns.rolling(window=20).std()
+        z_scores = (returns - rolling_mean) / rolling_std
+
+        # Identify outliers
+        outliers = abs(z_scores) > z_threshold
+
+        if outliers.sum() > 0:
+            print(f"Identified {outliers.sum()} outliers out of {len(data)} records")
+
+            # Replace outlier prices with interpolated values
+            clean_data = data.copy()
+            outlier_indices = outliers[outliers].index
+
+            for col in ['open', 'high', 'low', 'close']:
+                clean_data.loc[outlier_indices, col] = np.nan
+                clean_data[col] = clean_data[col].interpolate(method='linear')
+
+            return clean_data
+
+        return data
+
+    def _handle_after_hours_data(self, data):
+        """
+        Handle after-hours and weekend data in gold trading
+        """
+        # Create day of week and hour features
+        data['day_of_week'] = data.index.dayofweek
+        data['hour'] = data.index.hour
+
+        # Filter out weekends (Saturday and Sunday)
+        # For gold, market typically closes Friday ~5PM ET and opens Sunday ~6PM ET
+        weekend_mask = (data['day_of_week'] == 5) & (data['hour'] >= 17)  # After Friday 5PM
+        weekend_mask |= (data['day_of_week'] == 6)  # All of Saturday
+        weekend_mask |= (data['day_of_week'] == 0) & (data['hour'] < 18)  # Before Sunday 6PM
+
+        # Mark weekend data
+        data['is_weekend'] = weekend_mask
+
+        # Forward fill prices for weekend gaps
+        # We don't remove weekend data to maintain continuity
+        # but we mark it so we can filter it later if needed
+
+        # Remove temporary columns if needed
+        data = data.drop(['day_of_week', 'hour'], axis=1)
+
+        return data

tradingbot/models/__init__.py

@@ -0,0 +1,10 @@
+from tradingbot.models.tree_ensemble import TreeEnsemblePredictor, run_model
+from tradingbot.models.neural_ensemble import NeuralEnsemblePredictor
+from tradingbot.models.model_manager import GoldModelManager
+
+__all__ = [
+    "TreeEnsemblePredictor",
+    "run_model",
+    "NeuralEnsemblePredictor",
+    "GoldModelManager",
+]

tradingbot/models/neural_ensemble.py

@@ -0,0 +1,340 @@
+"""Neural + tree hybrid ensemble predictor.
+
+Stacks gradient-boosted trees with a Transformer/BiLSTM network and a Bayesian
+network (Monte-Carlo dropout) for uncertainty-aware signal filtering.
+"""
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import RobustScaler
+from sklearn.model_selection import TimeSeriesSplit
+import xgboost as xgb
+import lightgbm as lgb
+from catboost import CatBoostClassifier
+from hmmlearn import hmm
+import talib
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import Dataset, DataLoader
+
+
+class TimeSeriesDataset(Dataset):
+    def __init__(self, X, y):
+        self.X = torch.tensor(X, dtype=torch.float32)
+        self.y = torch.tensor(y, dtype=torch.long)
+
+    def __len__(self):
+        return len(self.y)
+
+    def __getitem__(self, idx):
+        return self.X[idx], self.y[idx]
+
+class TransformerBlock(nn.Module):
+    def __init__(self, d_model, nhead, dropout=0.3):
+        super().__init__()
+        self.attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout, batch_first=True)
+        self.norm1 = nn.LayerNorm(d_model)
+        self.ff = nn.Sequential(
+            nn.Linear(d_model, 64),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+            nn.Linear(64, d_model)
+        )
+        self.norm2 = nn.LayerNorm(d_model)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        attn_out, _ = self.attn(x, x, x)
+        x = self.norm1(x + self.dropout(attn_out))
+        ff_out = self.ff(x)
+        x = self.norm2(x + self.dropout(ff_out))
+        return x
+
+class NeuralEnsemblePredictor:
+    def __init__(self, forecast_period=12, confidence_threshold=0.6, device='cuda' if torch.cuda.is_available() else 'cpu'):
+        self.forecast_period = forecast_period
+        self.confidence_threshold = confidence_threshold
+        self.device = device
+        self.scaler = RobustScaler()
+        self.regime_model = hmm.GaussianHMM(n_components=3, random_state=42)
+
+        # Tree-based models with pre-tuned parameters
+        self.models = {
+            'xgboost': xgb.XGBClassifier(
+                colsample_bytree=0.9821911945239713,
+                learning_rate=0.07305593001592295,
+                max_depth=9,
+                min_child_weight=5,
+                subsample=0.7524259059189972,
+                random_state=42,
+                eval_metric='logloss'
+            ),
+            'lightgbm': lgb.LGBMClassifier(
+                feature_fraction=0.9249583953429453,
+                learning_rate=0.025468440525690465,
+                max_depth=7,
+                min_child_samples=41,
+                subsample=0.8092209312217534,
+                random_state=44
+            ),
+            'catboost': CatBoostClassifier(
+                colsample_bylevel=0.7762414086064304,
+                depth=4,
+                learning_rate=0.08067885793496102,
+                subsample=0.858134636983408,
+                random_state=45,
+                verbose=0
+            )
+        }
+
+        # PyTorch neural networks
+        self.nn_model = self._build_neural_network().to(self.device)
+        self.bnn_model = self._build_bayesian_network().to(self.device)
+
+        # Non-linear meta-model
+        self.meta_model = lgb.LGBMClassifier(
+            n_estimators=100, max_depth=3, learning_rate=0.05, random_state=46
+        )
+
+    def _build_neural_network(self):
+        """Transformer-based neural network"""
+        class Net(nn.Module):
+            def __init__(self):
+                super().__init__()
+                self.transformer = nn.ModuleList([
+                    TransformerBlock(d_model=6, nhead=2, dropout=0.3)
+                    for _ in range(2)
+                ])
+                self.lstm = nn.LSTM(6, 32, batch_first=True, bidirectional=True)
+                self.attn = nn.Linear(64, 1)  # Attention over 64 from bidirectional LSTM
+                self.fc = nn.Sequential(
+                    nn.Linear(64, 64),
+                    nn.ReLU(),
+                    nn.Dropout(0.4),
+                    nn.Linear(64, 2)
+                )
+
+            def forward(self, x):
+                for t in self.transformer:
+                    x = t(x)
+                x, _ = self.lstm(x)  # Shape: (batch, 20, 64)
+                attn_weights = torch.softmax(self.attn(x), dim=1)  # Shape: (batch, 20, 1)
+                x = (x * attn_weights).sum(dim=1)  # Shape: (batch, 64)
+                x = self.fc(x)
+                return x
+
+        return Net()
+
+    def _build_bayesian_network(self):
+        """Bayesian neural network with Monte Carlo dropout"""
+        class BNN(nn.Module):
+            def __init__(self):
+                super().__init__()
+                self.flatten = nn.Flatten()
+                self.fc1 = nn.Linear(20 * 6, 64)
+                self.fc2 = nn.Linear(64, 32)
+                self.fc3 = nn.Linear(32, 2)
+                self.dropout = nn.Dropout(0.3)
+                self.relu = nn.ReLU()
+
+            def forward(self, x, training=False):
+                x = self.flatten(x)
+                x = self.relu(self.fc1(x))
+                x = self.dropout(x) if training else x
+                x = self.relu(self.fc2(x))
+                x = self.dropout(x) if training else x
+                x = self.fc3(x)
+                return x
+
+        return BNN()
+
+    def detect_market_regime(self, data):
+        returns = np.log(data['close'] / data['close'].shift(1))
+        volatility = returns.rolling(window=20).std()
+        combined = pd.DataFrame({'returns': returns, 'volatility': volatility}).dropna()
+        self.regime_model.fit(combined.values)
+        regimes = self.regime_model.predict(combined.values)
+        regime_series = pd.Series(index=data.index, dtype='float64')
+        regime_series.iloc[len(data)-len(regimes):] = regimes
+        return regime_series
+
+    def create_advanced_features(self, df):
+        data = df.copy()
+        data['market_regime'] = self.detect_market_regime(data)
+        for period in [21, 55]:
+            data[f'ema_{period}'] = talib.EMA(data['close'], timeperiod=period)
+            data[f'trend_{period}'] = (data[f'ema_{period}'] - data[f'ema_{period}'].shift(period)) / data[f'ema_{period}'].shift(period)
+        data['atr_ratio'] = talib.ATR(data['high'], data['low'], data['close'], 14) / data['close']
+        data['rsi'] = talib.RSI(data['close'], 14)
+        data['volume_ma'] = talib.EMA(data['volume'], timeperiod=20)
+        data['volume_ratio'] = data['volume'] / data['volume_ma']
+
+        returns = data['close'].shift(-self.forecast_period) / data['close'] - 1
+        data['target'] = np.where(returns > 0.005, 1, np.where(returns < -0.005, 0, None))
+        return data.dropna()
+
+    def prepare_features(self, data, for_nn=False):
+        feature_columns = [
+            'market_regime', 'atr_ratio', 'rsi', 'volume_ratio',
+            'trend_21', 'trend_55'
+        ]
+        X = data[feature_columns]
+        y = data['target'].astype(int)
+
+        if for_nn:
+            X_3d = np.array([X.iloc[i-20:i].values for i in range(20, len(X))])
+            y_3d = y.iloc[20:].values
+            return X_3d, y_3d
+        return X, y
+
+    def train_nn(self, X_nn, y_nn, model, epochs=50, batch_size=32):
+        dataset = TimeSeriesDataset(X_nn, y_nn)
+        train_size = int(0.8 * len(dataset))
+        val_size = len(dataset) - train_size
+        train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])
+        train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
+        val_loader = DataLoader(val_dataset, batch_size=batch_size)
+
+        criterion = nn.CrossEntropyLoss()
+        optimizer = optim.Adam(model.parameters(), lr=0.0001)
+
+        best_val_loss = float('inf')
+        patience = 15
+        patience_counter = 0
+
+        for epoch in range(epochs):
+            model.train()
+            train_loss = 0
+            for X_batch, y_batch in train_loader:
+                X_batch, y_batch = X_batch.to(self.device), y_batch.to(self.device)
+                optimizer.zero_grad()
+                outputs = model(X_batch)
+                loss = criterion(outputs, y_batch)
+                loss.backward()
+                optimizer.step()
+                train_loss += loss.item()
+
+            model.eval()
+            val_loss = 0
+            with torch.no_grad():
+                for X_batch, y_batch in val_loader:
+                    X_batch, y_batch = X_batch.to(self.device), y_batch.to(self.device)
+                    outputs = model(X_batch)
+                    val_loss += criterion(outputs, y_batch).item()
+
+            train_loss /= len(train_loader)
+            val_loss /= len(val_loader)
+            print(f"Epoch {epoch+1}/{epochs}, Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}")
+
+            if val_loss < best_val_loss:
+                best_val_loss = val_loss
+                patience_counter = 0
+                torch.save(model.state_dict(), 'nn_best.pth')
+            else:
+                patience_counter += 1
+                if patience_counter >= patience:
+                    print("Early stopping")
+                    break
+
+        model.load_state_dict(torch.load('nn_best.pth'))
+        return model
+
+    def fit(self, train_data):
+        processed_data = self.create_advanced_features(train_data)
+        X, y = self.prepare_features(processed_data)
+        X_nn, y_nn = self.prepare_features(processed_data, for_nn=True)
+        X_scaled = self.scaler.fit_transform(X)
+        X_scaled = pd.DataFrame(X_scaled, columns=X.columns)
+
+        # Train tree-based models (no tuning)
+        tscv = TimeSeriesSplit(n_splits=5)
+        oof_preds = np.zeros((len(X_scaled), len(self.models) + 2))
+
+        for fold, (train_idx, val_idx) in enumerate(tscv.split(X_scaled)):
+            X_train, X_val = X_scaled.iloc[train_idx], X_scaled.iloc[val_idx]
+            y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]
+
+            for i, (name, model) in enumerate(self.models.items()):
+                if name == 'xgboost':
+                    model.fit(X_train, y_train, eval_set=[(X_val, y_val)],verbose=False)
+                elif name == 'lightgbm':
+                    model.fit(X_train, y_train, eval_set=[(X_val, y_val)])
+                elif name == 'catboost':
+                    model.fit(X_train, y_train, eval_set=(X_val, y_val))
+                oof_preds[val_idx, i] = model.predict_proba(X_val)[:, 1]
+
+        # Train neural network
+        nn_idx_shift = len(X_scaled) - len(X_nn)
+        self.nn_model = self.train_nn(X_nn, y_nn, self.nn_model)
+        self.nn_model.eval()
+        with torch.no_grad():
+            nn_preds = torch.softmax(self.nn_model(torch.tensor(X_nn, dtype=torch.float32).to(self.device)), dim=1)[:, 1].cpu().numpy()
+        oof_preds[nn_idx_shift:, len(self.models)] = nn_preds
+
+        # Train Bayesian NN with Monte Carlo dropout
+        self.bnn_model = self.train_nn(X_nn, y_nn, self.bnn_model)
+        self.bnn_model.train()  # Enable dropout for MC estimation
+        mc_preds = []
+        with torch.no_grad():
+            for _ in range(10):  # 10 Monte Carlo samples
+                preds = torch.softmax(self.bnn_model(torch.tensor(X_nn, dtype=torch.float32).to(self.device), training=True), dim=1)[:, 1].cpu().numpy()
+                mc_preds.append(preds)
+        bnn_mean = np.mean(mc_preds, axis=0)
+        bnn_std = np.std(mc_preds, axis=0)
+        oof_preds[nn_idx_shift:, len(self.models) + 1] = bnn_mean
+
+        # Train meta-model
+        self.meta_model.fit(oof_preds, y, sample_weight=np.exp(np.linspace(-1, 0, len(y))))
+
+        # Final training of tree models
+        for name, model in self.models.items():
+            if name in ['xgboost']:
+                model.fit(X_scaled, y, eval_set=[(X_scaled, y)], verbose=False)
+            elif name in ['lightgbm']:
+                model.fit(X_scaled, y, eval_set=[(X_scaled, y)])
+            else:
+                model.fit(X_scaled, y, eval_set=(X_scaled, y))
+
+        return processed_data
+
+    def predict(self, data):
+        processed_data = self.create_advanced_features(data)
+        X, _ = self.prepare_features(processed_data)
+        X_nn, _ = self.prepare_features(processed_data, for_nn=True)
+        X_scaled = self.scaler.transform(X)
+        X_scaled = pd.DataFrame(X_scaled, columns=X.columns)
+
+        base_preds = np.zeros((len(X_scaled), len(self.models) + 2))
+        for i, (name, model) in enumerate(self.models.items()):
+            base_preds[:, i] = model.predict_proba(X_scaled)[:, 1]
+
+        nn_idx_shift = len(X_scaled) - len(X_nn)
+        self.nn_model.eval()
+        with torch.no_grad():
+            nn_preds = torch.softmax(self.nn_model(torch.tensor(X_nn, dtype=torch.float32).to(self.device)), dim=1)[:, 1].cpu().numpy()
+        base_preds[nn_idx_shift:, len(self.models)] = nn_preds
+
+        # Bayesian NN predictions with uncertainty
+        self.bnn_model.train()  # Enable dropout
+        mc_preds = []
+        with torch.no_grad():
+            for _ in range(10):
+                preds = torch.softmax(self.bnn_model(torch.tensor(X_nn, dtype=torch.float32).to(self.device), training=True), dim=1)[:, 1].cpu().numpy()
+                mc_preds.append(preds)
+        bnn_mean = np.mean(mc_preds, axis=0)
+        bnn_std = np.std(mc_preds, axis=0)
+        base_preds[nn_idx_shift:, len(self.models) + 1] = bnn_mean
+
+        # Meta-model prediction
+        meta_proba = self.meta_model.predict_proba(base_preds)
+
+        signals = pd.Series(0, index=processed_data.index)
+        valid_indices = processed_data.index[20:]  # Skip first 20 due to lookback
+        meta_proba_valid = meta_proba[nn_idx_shift:]  # Align with NN predictions
+
+        long_mask = (meta_proba_valid[:, 1] > self.confidence_threshold) & (bnn_std < 0.2)
+        short_mask = (meta_proba_valid[:, 0] > self.confidence_threshold) & (bnn_std < 0.2)
+        signals.loc[valid_indices[long_mask]] = 1
+        signals.loc[valid_indices[short_mask]] = -1
+
+        return signals

tradingbot/models/tree_ensemble.py

@@ -0,0 +1,562 @@
+"""Stacked tree-ensemble predictor for the 5-minute timeframe.
+
+A binary (up/down) classifier that stacks five gradient-boosted / bagged tree
+models with two meta-models and an HMM market-regime filter.
+"""
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import RobustScaler
+from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import TimeSeriesSplit
+import xgboost as xgb
+import lightgbm as lgb
+from catboost import CatBoostClassifier
+from hmmlearn import hmm
+import talib
+
+
+class TreeEnsemblePredictor:
+    def __init__(self, forecast_bars=24, confidence_threshold=0.65):
+        """
+        Optimized gold price prediction model for 5-minute timeframe
+
+        Parameters:
+        forecast_bars (int): Number of future 5-min bars to predict (default=24, which is 2 hours)
+        confidence_threshold (float): Minimum probability threshold for signal generation
+        """
+        self.forecast_bars = forecast_bars
+        self.confidence_threshold = confidence_threshold
+        self.scaler = RobustScaler()
+        self.regime_model = hmm.GaussianHMM(n_components=3, random_state=42)
+
+        # Tree-based models with parameters optimized for 5-min timeframe
+        self.models = {
+            'xgboost': xgb.XGBClassifier(
+                colsample_bytree=0.85,
+                learning_rate=0.05,
+                max_depth=6,  # Reduced depth for faster market dynamics
+                min_child_weight=3,
+                subsample=0.8,
+                n_estimators=100,
+                random_state=42,
+                eval_metric='logloss',
+                use_label_encoder=False
+            ),
+            'lightgbm': lgb.LGBMClassifier(
+                feature_fraction=0.9,
+                learning_rate=0.02,
+                max_depth=5,
+                min_child_samples=20,  # Smaller sample for 5-min data
+                subsample=0.8,
+                n_estimators=100,
+                random_state=44,
+                boosting_type='dart'  # More robust to noise in high-frequency data
+            ),
+            'catboost': CatBoostClassifier(
+                depth=4,
+                learning_rate=0.07,
+                subsample=0.85,
+                n_estimators=100,
+                random_state=45,
+                verbose=0
+            ),
+            'randomforest': RandomForestClassifier(
+                n_estimators=100,
+                max_depth=5,
+                max_features='sqrt',
+                min_samples_leaf=5,  # Captures more granular patterns
+                random_state=46
+            ),
+            'extratrees': ExtraTreesClassifier(
+                n_estimators=100,
+                max_depth=5,
+                max_features='sqrt',
+                min_samples_leaf=5,
+                random_state=47
+            )
+        }
+
+        # Primary meta-model
+        self.meta_model = lgb.LGBMClassifier(
+            n_estimators=100,
+            max_depth=3,
+            learning_rate=0.03,
+            random_state=48,
+            boosting_type='gbdt'
+        )
+
+        # Secondary meta-model for consensus validation
+        self.meta_model_backup = LogisticRegression(
+            C=0.1,
+            solver='liblinear',
+            random_state=49
+        )
+
+        # Track feature importance
+        self.feature_importances = {}
+        self.feature_names = []
+
+    def detect_market_regime(self, data):
+        """
+        Detects market regimes using HMM on returns and volatility
+        Optimized for 5-minute data with shorter lookback windows
+
+        Returns regime classifications (0=low vol, 1=normal, 2=high vol)
+        """
+        # Calculate returns and volatility
+        returns = np.log(data['close'] / data['close'].shift(1))
+        # Shorter window for 5-minute data (60 periods = 5 hours)
+        volatility = returns.rolling(window=60).std()
+        combined = pd.DataFrame({'returns': returns, 'volatility': volatility}).dropna()
+
+        # Fit HMM model if we have enough data
+        if len(combined) > 100:
+            self.regime_model.fit(combined.values)
+            regimes = self.regime_model.predict(combined.values)
+            regime_series = pd.Series(index=data.index, dtype='float64')
+            regime_series.iloc[len(data)-len(regimes):] = regimes
+            return regime_series
+        else:
+            # Default to moderate regime if not enough data
+            return pd.Series(1, index=data.index)
+
+    def create_advanced_features(self, df):
+        """
+        Create features optimized for 5-minute gold price prediction
+
+        Features are organized in categories:
+        1. Market regime
+        2. Time-based features
+        3. Price action features
+        4. Volatility indicators
+        5. Momentum indicators
+        6. Volume indicators
+        7. Support/Resistance
+        8. Pattern recognition
+        """
+        data = df.copy()
+
+        # 1. Market regime detection
+        data['market_regime'] = self.detect_market_regime(data)
+
+        # 2. Time-based features for intraday seasonality
+        data['hour'] = data.index.hour
+        data['minute'] = data.index.minute
+        data['day_of_week'] = data.index.dayofweek
+        # Cyclical encoding of time (circular features)
+        data['hour_sin'] = np.sin(2 * np.pi * data['hour']/24)
+        data['hour_cos'] = np.cos(2 * np.pi * data['hour']/24)
+
+        # 3. Short-term price action features
+        # Moving averages adapted for 5-min timeframe
+        for period in [12, 24, 48, 96, 144]:  # 1h, 2h, 4h, 8h, 12h in 5-minute bars
+            # Exponential moving averages
+            data[f'ema_{period}'] = talib.EMA(data['close'], timeperiod=period)
+
+            # Price relative to moving average (normalized distance)
+            data[f'price_to_ema_{period}'] = data['close'] / data[f'ema_{period}'] - 1
+
+            # Trend strength
+            data[f'trend_{period}'] = (data[f'ema_{period}'] - data[f'ema_{period}'].shift(period//4)) / data[f'ema_{period}'].shift(period//4)
+
+        # 4. Volatility indicators
+        # ATR with periods suitable for 5-minute bars
+        for period in [12, 24, 48, 96]:  # 1h, 2h, a
+            data[f'atr_{period}'] = talib.ATR(data['high'], data['low'], data['close'], timeperiod=period)
+            data[f'atr_ratio_{period}'] = data[f'atr_{period}'] / data['close']
+
+        # Bollinger Bands - essential for mean-reversion detection
+        for period in [24, 48, 96]:
+            upper, middle, lower = talib.BBANDS(data['close'], timeperiod=period, nbdevup=2, nbdevdn=2)
+            data[f'bb_width_{period}'] = (upper - lower) / middle
+            data[f'bb_position_{period}'] = (data['close'] - lower) / (upper - lower)
+
+        # 5. Momentum indicators
+        # RSI with different lookback periods for 5-min data
+        for period in [12, 24, 48, 96]:
+            data[f'rsi_{period}'] = talib.RSI(data['close'], timeperiod=period)
+
+        # MACD for 5-minute data (faster parameters)
+        macd, macd_signal, macd_hist = talib.MACD(
+            data['close'],
+            fastperiod=6,    # Faster for 5-min data
+            slowperiod=19,   # Faster for 5-min data
+            signalperiod=5   # Faster for 5-min data
+        )
+        data['macd'] = macd
+        data['macd_signal'] = macd_signal
+        data['macd_hist'] = macd_hist
+
+        # 6. Volume indicators (crucial for 5-minute signals)
+        # Volume relative to moving average
+        for period in [12, 24, 48]:
+            data[f'volume_ma_{period}'] = talib.SMA(data['volume'], timeperiod=period)
+            data[f'volume_ratio_{period}'] = data['volume'] / data[f'volume_ma_{period}']
+
+        # On-balance volume - good for measuring buying/selling pressure
+        data['obv'] = talib.OBV(data['close'], data['volume'])
+        data['obv_ma'] = talib.SMA(data['obv'], timeperiod=24)
+        data['obv_ratio'] = data['obv'] / data['obv_ma']
+
+        # 7. Support/Resistance levels
+        # Pivot points for 5-min (using 96 periods = 8 hours)
+        data['pivot'] = (data['high'].rolling(96).max() + data['low'].rolling(96).min() + data['close'].rolling(96).mean()) / 3
+        data['dist_to_pivot'] = (data['close'] - data['pivot']) / data['close']
+
+        # 8. Candlestick pattern features
+        # Candle size metrics
+        data['candle_range'] = (data['high'] - data['low']) / data['close']
+        data['candle_body'] = abs(data['open'] - data['close']) / data['close']
+
+        # Rate of change - important for 5-min momentum
+        for period in [6, 12, 24]:
+            data[f'roc_{period}'] = talib.ROC(data['close'], timeperiod=period)
+
+        # Target definition for 5-minute timeframe
+        # Using appropriate thresholds for smaller price moves
+        future_return = data['close'].shift(-self.forecast_bars) / data['close'] - 1
+        # Lower threshold for 5-minute bars (approximately 0.1-0.15% move)
+        data['target'] = np.where(future_return > 0.0012, 1, np.where(future_return < -0.0012, 0, None))
+
+        # Drop rows with missing data
+        return data.dropna()
+
+    def prepare_features(self, data):
+        """
+        Prepare and select optimal features for the model
+        Features are grouped by category for easier selection
+        """
+        # Most important features for 5-minute gold prediction
+        feature_columns = [
+            # Market regime
+            'market_regime',
+
+            # Time features for intraday patterns
+            'hour_sin', 'hour_cos', 'day_of_week',
+
+            # Price action features
+            'price_to_ema_12', 'price_to_ema_24', 'price_to_ema_48',
+            'trend_24', 'trend_48', 'trend_96',
+
+            # Volatility indicators
+            'atr_ratio_12', 'atr_ratio_24',
+            'bb_width_24', 'bb_width_48',
+            'bb_position_24', 'bb_position_48',
+
+            # Momentum indicators
+            'rsi_12', 'rsi_24', 'rsi_48',
+            'macd', 'macd_hist',
+
+            # Volume indicators
+            'volume_ratio_12', 'volume_ratio_24',
+            'obv_ratio',
+
+            # Support/Resistance
+            'dist_to_pivot',
+
+            # Pattern recognition
+            'candle_range', 'candle_body',
+            'roc_6', 'roc_12'
+        ]
+
+        X = data[feature_columns]
+
+        if 'target' in data.columns:
+            y = data['target'].astype(int)
+            return X, y
+        else:
+            return X, None
+
+    def fit(self, train_data):
+        """Train the ensemble model on historical data"""
+        print("Creating features...")
+        processed_data = self.create_advanced_features(train_data)
+        X, y = self.prepare_features(processed_data)
+
+        # Store feature names for importance tracking
+        self.feature_names = X.columns.tolist()
+
+        # Scale features
+        X_scaled = self.scaler.fit_transform(X)
+        X_scaled = pd.DataFrame(X_scaled, columns=X.columns)
+
+        # Time series cross-validation
+        print("Performing time series cross-validation...")
+        tscv = TimeSeriesSplit(n_splits=5)
+        oof_preds = np.zeros((len(X_scaled), len(self.models)))
+
+        for fold, (train_idx, val_idx) in enumerate(tscv.split(X_scaled)):
+            X_train, X_val = X_scaled.iloc[train_idx], X_scaled.iloc[val_idx]
+            y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]
+
+            for i, (name, model) in enumerate(self.models.items()):
+                print(f"Training {name} for fold {fold+1}/5...")
+
+                if name == 'xgboost':
+                    model.fit(X_train, y_train, eval_set=[(X_val, y_val)], verbose=False)
+                elif name == 'lightgbm':
+                    model.fit(X_train, y_train, eval_set=[(X_val, y_val)])
+                elif name == 'catboost':
+                    model.fit(X_train, y_train, eval_set=(X_val, y_val))
+                else:
+                    model.fit(X_train, y_train)
+
+                # Store out-of-fold predictions
+                oof_preds[val_idx, i] = model.predict_proba(X_val)[:, 1]
+
+                # Track feature importance
+                if hasattr(model, 'feature_importances_'):
+                    if name not in self.feature_importances:
+                        self.feature_importances[name] = np.zeros(len(self.feature_names))
+                    self.feature_importances[name] += model.feature_importances_
+
+        # Evaluate base models
+        for i, name in enumerate(self.models.keys()):
+            auc = roc_auc_score(y, oof_preds[:, i])
+            print(f"{name} Out-of-fold AUC: {auc:.4f}")
+
+        # Train meta-models on out-of-fold predictions
+        print("Training meta-models...")
+        # Weight recent data more heavily for financial time series
+        sample_weights = np.exp(np.linspace(-1, 0, len(y)))
+
+        # Train primary meta-model
+        self.meta_model.fit(oof_preds, y, sample_weight=sample_weights)
+
+        # Train backup meta-model for consensus
+        self.meta_model_backup.fit(oof_preds, y, sample_weight=sample_weights)
+
+        # Display feature importance summary
+        self._print_feature_importance()
+
+        # Final training of all models on full dataset
+        print("Final training on complete dataset...")
+        for name, model in self.models.items():
+            if name in ['xgboost']:
+                model.fit(X_scaled, y, eval_set=[(X_scaled, y)], verbose=False)
+            elif name in ['lightgbm']:
+                model.fit(X_scaled, y, eval_set=[(X_scaled, y)])
+            elif name in ['catboost']:
+                model.fit(X_scaled, y, eval_set=(X_scaled, y))
+            else:
+                model.fit(X_scaled, y)
+
+        return processed_data
+
+    def _print_feature_importance(self):
+        """Display top features by importance for each model"""
+        print("\n=== Feature Importance Analysis ===")
+        for name, importances in self.feature_importances.items():
+            # Normalize importances to percentages
+            importances = importances / np.sum(importances) * 100
+            # Sort by importance
+            sorted_idx = np.argsort(importances)[::-1]
+            print(f"\n{name.upper()} Top 10 Features:")
+            for i in range(min(10, len(sorted_idx))):
+                idx = sorted_idx[i]
+                print(f"  {self.feature_names[idx]}: {importances[idx]:.2f}%")
+
+    def predict(self, data):
+        """
+        Generate trading signals for 5-minute gold price data
+
+        Returns:
+            DataFrame with columns:
+            - signal: Trading signal (-1=short, 0=neutral, 1=long)
+            - strength: Signal strength (0-100%)
+            - proba_up: Probability of price increase
+            - proba_down: Probability of price decrease
+            - model_agreement: Agreement level between meta-models
+            - market_regime: Detected market regime
+        """
+        processed_data = self.create_advanced_features(data)
+        X, _ = self.prepare_features(processed_data)
+        X_scaled = self.scaler.transform(X)
+        X_scaled = pd.DataFrame(X_scaled, columns=X.columns)
+
+        # Get predictions from base models
+        base_preds = np.zeros((len(X_scaled), len(self.models)))
+        for i, (name, model) in enumerate(self.models.items()):
+            base_preds[:, i] = model.predict_proba(X_scaled)[:, 1]
+
+        # Get predictions from both meta-models
+        meta_proba = self.meta_model.predict_proba(base_preds)
+        backup_proba = self.meta_model_backup.predict_proba(base_preds)
+
+        # Calculate consensus level between meta-models (0-1)
+        models_agreement = 1 - np.abs(meta_proba[:, 1] - backup_proba[:, 1])
+
+        # Initialize signals
+        signals = pd.Series(0, index=processed_data.index)
+
+        # Generate signals with sophisticated filtering
+        # Long signal: high probability of price increase + high model agreement
+        long_mask = (meta_proba[:, 1] > self.confidence_threshold) & (models_agreement > 0.8)
+
+        # Short signal: high probability of price decrease + high model agreement
+        short_mask = (meta_proba[:, 0] > self.confidence_threshold) & (models_agreement > 0.8)
+
+        # Add market regime filter - only take signals in appropriate regimes
+        market_regimes = processed_data['market_regime']
+
+        # Only generate long signals in trending or normal regimes (0 or 1)
+        long_regime_mask = (market_regimes == 0) | (market_regimes == 1)
+
+        # Only generate short signals in trending or high volatility regimes (0 or 2)
+        short_regime_mask = (market_regimes == 0) | (market_regimes == 2)
+
+        # Apply regime filters to signals
+        signals.loc[long_mask & long_regime_mask] = 1
+        signals.loc[short_mask & short_regime_mask] = -1
+
+        # Calculate signal strength (0-100%) based on prediction confidence
+        signal_strength = pd.Series(0.0, index=processed_data.index)
+        signal_strength.loc[long_mask] = (meta_proba[long_mask, 1] - self.confidence_threshold) * (1 / (1 - self.confidence_threshold)) * 100
+        signal_strength.loc[short_mask] = (meta_proba[short_mask, 0] - self.confidence_threshold) * (1 / (1 - self.confidence_threshold)) * 100
+
+        # Risk management: additional signal filters
+        # 1. Minimum signal duration (prevent rapid flipping)
+        min_bars = 3  # Minimum 15 minutes
+        for i in range(min_bars, len(signals)):
+            if signals.iloc[i] != 0 and signals.iloc[i] == -signals.iloc[i-1]:
+                # If signal flips too soon, maintain previous signal
+                if sum(signals.iloc[i-min_bars:i] == signals.iloc[i-1]) >= min_bars-1:
+                    signals.iloc[i] = signals.iloc[i-1]
+
+        # 2. Filter out signals during extreme volatility
+        high_vol_mask = processed_data['atr_ratio_24'] > processed_data['atr_ratio_24'].quantile(0.95)
+        signals.loc[high_vol_mask] = 0
+
+        # Combine results into a DataFrame
+        results = pd.DataFrame({
+            'signal': signals,
+            'strength': signal_strength,
+            'proba_up': meta_proba[:, 1],
+            'proba_down': meta_proba[:, 0],
+            'model_agreement': models_agreement,
+            'market_regime': processed_data['market_regime']
+        }, index=processed_data.index)
+
+        return results
+
+    def evaluate_performance(self, test_data):
+        """
+        Evaluate model performance with trading simulation - corrected version
+        """
+        results = self.predict(test_data)
+
+        # Calculate forward returns for evaluation period
+        close_prices = test_data['close']
+        forward_returns = close_prices.shift(-self.forecast_bars) / close_prices - 1
+
+        # Apply signals to returns (long = 1x return, short = -1x return)
+        strategy_returns = results['signal'] * forward_returns
+
+        # Properly handle NaN values that might appear from shifts
+        strategy_returns = strategy_returns.dropna()
+
+        # Calculate performance metrics
+        total_return = strategy_returns.sum()
+
+        # Correct annualization factor: 252 trading days, 12 hours per day, 12 bars per hour
+        annualization_factor = np.sqrt(252 * 12 * 12)
+        sharpe_ratio = strategy_returns.mean() / strategy_returns.std() * annualization_factor
+
+        # Correct win rate calculation - account for signal direction
+        wins = ((strategy_returns > 0) & (results['signal'] != 0)).sum()
+        total_trades = (results['signal'] != 0).sum()
+        win_rate = wins / total_trades if total_trades > 0 else 0
+
+        # Correct drawdown calculation
+        cumulative_returns = strategy_returns.cumsum()
+        drawdowns = cumulative_returns - cumulative_returns.cummax()
+        max_drawdown = drawdowns.min()
+
+        # Signal statistics
+        signal_count = (results['signal'] != 0).sum()
+        signal_changes = results['signal'].diff().abs()
+        signal_changes = signal_changes[signal_changes > 0].sum() / 2  # Each change counts twice in diff
+
+        # Correct trading days calculation (typically 5 days a week for forex)
+        # Assuming 12 hours of active trading per day and 12 5-min bars per hour
+        trading_days = len(results) / (12 * 12)
+        avg_signals_per_day = signal_count / trading_days
+
+        # Calculate profit factor
+        profitable_trades = strategy_returns[strategy_returns > 0].sum()
+        losing_trades = abs(strategy_returns[strategy_returns < 0].sum())
+        profit_factor = profitable_trades / losing_trades if losing_trades != 0 else float('inf')
+
+        # Calculate average profit per trade
+        avg_profit_per_trade = total_return / total_trades if total_trades > 0 else 0
+
+        # Compile metrics
+        metrics = {
+            'total_return': total_return,
+            'annualized_return': total_return * (252 / trading_days),
+            'sharpe_ratio': sharpe_ratio,
+            'win_rate': win_rate,
+            'max_drawdown': max_drawdown,
+            'profit_factor': profit_factor,
+            'signal_count': signal_count,
+            'signal_changes': signal_changes,
+            'avg_signals_per_day': avg_signals_per_day,
+            'avg_profit_per_trade': avg_profit_per_trade
+        }
+
+        print("\n=== Performance Evaluation ===")
+        print(f"Total Return: {total_return:.2%}")
+        print(f"Annualized Return: {metrics['annualized_return']:.2%}")
+        print(f"Annualized Sharpe Ratio: {sharpe_ratio:.2f}")
+        print(f"Win Rate: {win_rate:.2%}")
+        print(f"Maximum Drawdown: {max_drawdown:.2%}")
+        print(f"Profit Factor: {profit_factor:.2f}")
+        print(f"Total Signals: {signal_count}")
+        print(f"Signal Changes: {signal_changes}")
+        print(f"Average Signals Per Day: {avg_signals_per_day:.2f}")
+        print(f"Average Profit Per Trade: {avg_profit_per_trade:.4%}")
+
+        return metrics, results, strategy_returns
+
+# Example usage
+def run_model(data_path, forecast_bars=24, confidence_threshold=0.65):
+    """
+    Run the model on input data
+
+    Parameters:
+        data_path: Path to CSV file with OHLCV data
+        forecast_bars: Number of 5-min bars to forecast
+        confidence_threshold: Threshold for signal generation
+
+    Returns:
+        predictor: Trained model
+        metrics: Performance metrics
+        results: Signal results
+        returns: Strategy returns
+    """
+    # Load and prepare data
+    data = pd.read_csv(data_path)
+    data['timestamp'] = pd.to_datetime(data['timestamp'])
+    data = data.drop_duplicates(subset=['timestamp'])
+    data.set_index('timestamp', inplace=True)
+
+    # Split into train/test
+    #pick the last 40% of data before the last 80% from 40% to 80%
+    train_size = int(len(data) * 0.95)
+    train_data = data[int(len(data) * 0.80):train_size].copy()
+    test_data = data[train_size:].copy()
+
+    # Create and train model
+    predictor = TreeEnsemblePredictor(
+        forecast_bars=forecast_bars,
+        confidence_threshold=confidence_threshold
+    )
+    print(f"Training model with forecast_bars={forecast_bars}, confidence_threshold={confidence_threshold}")
+    predictor.fit(train_data)
+
+    # Evaluate model
+    metrics, results, returns = predictor.evaluate_performance(test_data)
+
+    return predictor, metrics, results, returns

tradingbot/signals/__init__.py

@@ -0,0 +1,3 @@
+from tradingbot.signals.generator import GoldSignalGenerator
+
+__all__ = ["GoldSignalGenerator"]

tradingbot/signals/generator.py

@@ -0,0 +1,383 @@
+"""Convert model predictions into trading signals with risk management."""
+import numpy as np
+import pandas as pd
+
+from tradingbot.models.model_manager import GoldModelManager
+
+
+class GoldSignalGenerator:
+    """
+    Generates trading signals from model predictions with risk management
+    """
+    def __init__(
+        self,
+        confidence_threshold=0.7,
+        risk_reward_min=1.5,
+        stop_atr_factor=1.5,
+        target_atr_factor=2.25,
+        model_manager=None,
+        model_path='models'
+    ):
+        """
+        Initialize the signal generator
+
+        Parameters:
+        -----------
+        confidence_threshold : float
+            Minimum probability threshold for generating signals
+        risk_reward_min : float
+            Minimum risk/reward ratio for valid trades
+        stop_atr_factor : float
+            Factor to multiply ATR for stop loss calculation
+        target_atr_factor : float
+            Factor to multiply ATR for take profit calculation
+        model_manager : GoldModelManager, optional
+            Model manager instance (if None, will load from model_path)
+        model_path : str
+            Directory to load models from (if model_manager is None)
+        """
+        self.confidence_threshold = confidence_threshold
+        self.risk_reward_min = risk_reward_min
+        self.stop_atr_factor = stop_atr_factor
+        self.target_atr_factor = target_atr_factor
+
+        # Use provided model manager or create a new one
+        if model_manager is not None:
+            self.model_manager = model_manager
+        else:
+            self.model_manager = GoldModelManager(model_path=model_path)
+            self.model_manager.load_models()
+
+    def generate_signals(self, data):
+        """
+        Generate trading signals from data with improved error handling
+
+        Parameters:
+        -----------
+        data : pd.DataFrame
+            Data with features
+
+        Returns:
+        --------
+        signals : pd.DataFrame
+            DataFrame with trading signals and risk management
+        """
+        try:
+            # Prepare features
+            X, _ = self.model_manager.prepare_data(data, remove_cols=None)
+
+            # Check if model manager has a trained meta model
+            if self.model_manager.meta_model is None:
+                print("Warning: No trained meta-model available for prediction")
+                return pd.DataFrame(index=data.index)
+
+            # Get model predictions
+            predictions, probabilities = self.model_manager.predict(X)
+
+            # Check if predictions or probabilities are empty
+            if predictions.empty or probabilities.empty:
+                print("Warning: Empty predictions or probabilities")
+                return pd.DataFrame(index=data.index)
+
+            # Initialize signals DataFrame
+            signals = pd.DataFrame(index=data.index)
+            signals['prediction'] = predictions
+
+            # Add class probabilities
+            for col in probabilities.columns:
+                signals[col] = probabilities[col]
+
+            # Calculate signal confidence with NaN handling
+            if probabilities.values.size > 0:
+                confidence_values = np.nanmax(probabilities.values, axis=1)
+                # Replace any NaN confidence values with 0
+                confidence_values = np.nan_to_num(confidence_values, nan=0)
+                signals['confidence'] = confidence_values
+            else:
+                signals['confidence'] = 0
+
+            # Generate directional signals
+            signals['signal'] = 0  # Default: no signal
+
+            # Long signals (Strong Up or Weak Up with high confidence)
+            long_mask = (
+                ((signals['prediction'] == 2) & (signals['confidence'] > self.confidence_threshold * 1.1)) |  # Higher threshold for strong up
+                ((signals['prediction'] == 1) & (signals['confidence'] > self.confidence_threshold))
+            )
+            if not long_mask.empty:
+                signals.loc[long_mask, 'signal'] = 1
+
+            # Short signals (Strong Down or Weak Down with high confidence)
+            short_mask = (
+                ((signals['prediction'] == -2) & (signals['confidence'] > self.confidence_threshold * 1.1)) |  # Higher threshold for strong down
+                ((signals['prediction'] == -1) & (signals['confidence'] > self.confidence_threshold))
+            )
+            if not short_mask.empty:
+                signals.loc[short_mask, 'signal'] = -1
+
+            # Add risk management
+            if 'atr_10' in data.columns:
+                # Use ATR for stop loss and take profit calculation
+                signals['atr'] = data['atr_10']
+
+                # Calculate stops and targets
+                signals['stop_distance'] = signals['atr'] * self.stop_atr_factor
+                signals['target_distance'] = signals['atr'] * self.target_atr_factor
+
+                # Set specific stop and target levels
+                signals['stop_price'] = np.where(
+                    signals['signal'] == 1,
+                    data['close'] - signals['stop_distance'],  # Long stop
+                    np.where(
+                        signals['signal'] == -1,
+                        data['close'] + signals['stop_distance'],  # Short stop
+                        np.nan
+                    )
+                )
+
+                signals['target_price'] = np.where(
+                    signals['signal'] == 1,
+                    data['close'] + signals['target_distance'],  # Long target
+                    np.where(
+                        signals['signal'] == -1,
+                        data['close'] - signals['target_distance'],  # Short target
+                        np.nan
+                    )
+                )
+
+                # Calculate risk-reward ratio
+                signals['risk_reward'] = np.where(
+                    signals['signal'] == 1,
+                    signals['target_distance'] / signals['stop_distance'],  # Long R:R
+                    np.where(
+                        signals['signal'] == -1,
+                        signals['target_distance'] / signals['stop_distance'],  # Short R:R
+                        np.nan
+                    )
+                )
+
+                # Filter signals by risk-reward ratio
+                poor_rr_mask = (signals['signal'] != 0) & (signals['risk_reward'] < self.risk_reward_min)
+                if not poor_rr_mask.empty:
+                    signals.loc[poor_rr_mask, 'signal'] = 0
+
+            # Add signal strength (1-3)
+            signals['signal_strength'] = 0
+
+            # Strength 3: Very high confidence predictions
+            strong_mask = (signals['signal'] != 0) & (signals['confidence'] > 0.85)
+            if not strong_mask.empty:
+                signals.loc[strong_mask, 'signal_strength'] = 3
+
+            # Strength 2: High confidence predictions
+            medium_mask = (signals['signal'] != 0) & (signals['confidence'] > 0.75) & (signals['confidence'] <= 0.85)
+            if not medium_mask.empty:
+                signals.loc[medium_mask, 'signal_strength'] = 2
+
+            # Strength 1: Moderate confidence predictions
+            weak_mask = (signals['signal'] != 0) & (signals['confidence'] <= 0.75)
+            if not weak_mask.empty:
+                signals.loc[weak_mask, 'signal_strength'] = 1
+
+            # Add market context
+            if 'volatility_regime' in data.columns:
+                signals['volatility_regime'] = data['volatility_regime']
+
+            # Add key price levels
+            signals['close'] = data['close']
+
+            # Add signal label for easier interpretation
+            signals['signal_label'] = 'NO_SIGNAL'
+            long_label_mask = signals['signal'] == 1
+            short_label_mask = signals['signal'] == -1
+
+            if not long_label_mask.empty:
+                signals.loc[long_label_mask, 'signal_label'] = 'LONG'
+
+            if not short_label_mask.empty:
+                signals.loc[short_label_mask, 'signal_label'] = 'SHORT'
+
+            # Count active signals
+            signal_count = (signals['signal'] != 0).sum()
+            print(f"Generated {signal_count} active signals out of {len(signals)} bars")
+
+            return signals
+
+        except Exception as e:
+            import traceback
+            print(f"Signal generation error: {str(e)}")
+            print(f"Traceback: {traceback.format_exc()}")
+
+            # Return empty DataFrame with same index as data
+            return pd.DataFrame(index=data.index)
+
+    def analyze_signals(self, signals, data):
+        """
+        Analyze generated signals performance with improved error handling
+
+        Parameters:
+        -----------
+        signals : pd.DataFrame
+            DataFrame with trading signals
+        data : pd.DataFrame
+            Original data with price information
+
+        Returns:
+        --------
+        analysis : dict
+            Dictionary with signal statistics
+        """
+        try:
+            # Ensure we have price data
+            if 'close' not in data.columns:
+                raise ValueError("Price data required for signal analysis")
+
+            # Check if signals DataFrame is empty or has no signal column
+            if signals.empty or 'signal' not in signals.columns:
+                print("Warning: Empty signals DataFrame or missing 'signal' column")
+                return {
+                    'total_signals': 0,
+                    'signal_frequency': 0,
+                    'long_count': 0,
+                    'short_count': 0,
+                    'overall_win_rate': np.nan,
+                    'overall_avg_return': np.nan
+                }
+
+            # Copy signals to avoid modifying the original
+            signals_copy = signals.copy()
+
+            # Calculate forward returns for performance assessment
+            for period in [1, 3, 6, 12]:  # Multiple forward periods
+                signals_copy[f'fwd_return_{period}'] = data['close'].pct_change(period).shift(-period)
+
+            # Count signals
+            total_signals = (signals_copy['signal'] != 0).sum()
+
+            # If no signals were generated, return empty stats
+            if total_signals == 0:
+                print("No active signals found for analysis")
+                return {
+                    'total_signals': 0,
+                    'signal_frequency': 0,
+                    'long_count': 0,
+                    'short_count': 0,
+                    'overall_win_rate': np.nan,
+                    'overall_avg_return': np.nan
+                }
+
+            # Separate long and short signals
+            long_signals = signals_copy[signals_copy['signal'] == 1]
+            short_signals = signals_copy[signals_copy['signal'] == -1]
+
+            # Calculate win rates with error handling
+            if len(long_signals) > 0 and 'fwd_return_6' in long_signals.columns:
+                long_win_rate = (long_signals['fwd_return_6'] > 0).mean()
+                long_avg_return = long_signals['fwd_return_6'].mean()
+            else:
+                long_win_rate = np.nan
+                long_avg_return = np.nan
+
+            if len(short_signals) > 0 and 'fwd_return_6' in short_signals.columns:
+                short_win_rate = (short_signals['fwd_return_6'] < 0).mean()
+                short_avg_return = -short_signals['fwd_return_6'].mean()
+            else:
+                short_win_rate = np.nan
+                short_avg_return = np.nan
+
+            # Calculate overall metrics
+            win_rates = [r for r in [long_win_rate, short_win_rate] if not np.isnan(r)]
+            returns = [r for r in [long_avg_return, short_avg_return] if not np.isnan(r)]
+
+            overall_win_rate = np.mean(win_rates) if win_rates else np.nan
+            overall_avg_return = np.mean(returns) if returns else np.nan
+
+            # Signal frequency
+            signal_frequency = total_signals / len(signals_copy)
+
+            # Analyze by volatility regime if available
+            regime_stats = None
+            if 'volatility_regime' in signals_copy.columns:
+                regime_stats = {}
+                for regime in signals_copy['volatility_regime'].unique():
+                    regime_signals = signals_copy[signals_copy['volatility_regime'] == regime]
+
+                    # Skip if too few signals
+                    if (regime_signals['signal'] != 0).sum() < 5:
+                        continue
+
+                    regime_long = regime_signals[regime_signals['signal'] == 1]
+                    regime_short = regime_signals[regime_signals['signal'] == -1]
+
+                    # Calculate regime metrics with error handling
+                    if len(regime_long) > 0 and 'fwd_return_6' in regime_long.columns:
+                        regime_long_win_rate = (regime_long['fwd_return_6'] > 0).mean()
+                        regime_long_avg_return = regime_long['fwd_return_6'].mean()
+                    else:
+                        regime_long_win_rate = np.nan
+                        regime_long_avg_return = np.nan
+
+                    if len(regime_short) > 0 and 'fwd_return_6' in regime_short.columns:
+                        regime_short_win_rate = (regime_short['fwd_return_6'] < 0).mean()
+                        regime_short_avg_return = -regime_short['fwd_return_6'].mean()
+                    else:
+                        regime_short_win_rate = np.nan
+                        regime_short_avg_return = np.nan
+
+                    regime_stats[int(regime)] = {
+                        'count': (regime_signals['signal'] != 0).sum(),
+                        'frequency': (regime_signals['signal'] != 0).sum() / len(regime_signals),
+                        'long_win_rate': regime_long_win_rate,
+                        'short_win_rate': regime_short_win_rate,
+                        'long_avg_return': regime_long_avg_return,
+                        'short_avg_return': regime_short_avg_return
+                    }
+
+            # Compile analysis results
+            analysis = {
+                'total_signals': total_signals,
+                'signal_frequency': signal_frequency,
+                'long_count': len(long_signals),
+                'short_count': len(short_signals),
+                'long_win_rate': long_win_rate,
+                'short_win_rate': short_win_rate,
+                'overall_win_rate': overall_win_rate,
+                'long_avg_return': long_avg_return,
+                'short_avg_return': short_avg_return,
+                'overall_avg_return': overall_avg_return,
+                'regime_stats': regime_stats
+            }
+
+            # Print summary
+            print("\nSignal Analysis:")
+            print(f"Total Signals: {total_signals} ({signal_frequency:.2%} of bars)")
+            print(f"Long Signals: {len(long_signals)}, Short Signals: {len(short_signals)}")
+
+            win_rate_str = f"{overall_win_rate:.2%}" if not np.isnan(overall_win_rate) else "N/A"
+            long_win_rate_str = f"{long_win_rate:.2%}" if not np.isnan(long_win_rate) else "N/A"
+            short_win_rate_str = f"{short_win_rate:.2%}" if not np.isnan(short_win_rate) else "N/A"
+
+            long_return_str = f"{long_avg_return:.2%}" if not np.isnan(long_avg_return) else "N/A"
+            short_return_str = f"{short_avg_return:.2%}" if not np.isnan(short_avg_return) else "N/A"
+            overall_return_str = f"{overall_avg_return:.2%}" if not np.isnan(overall_avg_return) else "N/A"
+
+            print(f"Win Rates - Long: {long_win_rate_str}, Short: {short_win_rate_str}, Overall: {win_rate_str}")
+            print(f"Avg Returns - Long: {long_return_str}, Short: {short_return_str}, Overall: {overall_return_str}")
+
+            return analysis
+
+        except Exception as e:
+            import traceback
+            print(f"Signal analysis error: {str(e)}")
+            print(f"Traceback: {traceback.format_exc()}")
+
+            # Return basic metrics
+            return {
+                'total_signals': 0,
+                'signal_frequency': 0,
+                'long_count': 0,
+                'short_count': 0,
+                'overall_win_rate': np.nan,
+                'overall_avg_return': np.nan,
+                'error': str(e)
+            }

tradingbot/viz/__init__.py

@@ -0,0 +1,3 @@
+from tradingbot.viz.visualize import visualize_signals, analyze_trading_model, plot_price_signals
+
+__all__ = ["visualize_signals", "analyze_trading_model", "plot_price_signals"]