quantumbotx/core/backtesting/enhanced_engine.py

# core/backtesting/enhanced_engine.py
# Enhanced Backtesting Engine with ATR-based Risk Management and Spread Modeling

import math
import logging
import os
from core.strategies.strategy_map import resolve_strategy_class

logger = logging.getLogger(__name__)
# Set appropriate logging level
backtest_log_level = os.getenv('BACKTEST_LOG_LEVEL', 'INFO')
if backtest_log_level == 'DEBUG':
    logger.setLevel(logging.DEBUG)
else:
    logger.disabled = True
    logger.propagate = False

class InstrumentConfig:
    """Configuration for different trading instruments"""
    
    FOREX_MAJOR = {
        'contract_size': 100000,
        'pip_size': 0.0001,
        'typical_spread_pips': 1.0,  # Reduced from 2.0 for more realistic backtesting
        'max_risk_percent': 2.0,
        'max_lot_size': 10.0,
        'slippage_pips': 0.2  # Reduced from 0.5 for backtesting
    }
    
    FOREX_JPY = {
        'contract_size': 100000,
        'pip_size': 0.01,
        'typical_spread_pips': 1.5,  # Reduced from 2.0
        'max_risk_percent': 2.0,
        'max_lot_size': 10.0,
        'slippage_pips': 0.3  # Reduced from 0.5
    }
    
    GOLD = {
        'contract_size': 100,
        'pip_size': 0.01,
        'typical_spread_pips': 8.0,  # Reduced from 15.0 but still higher than forex
        'max_risk_percent': 1.0,      # Conservative for gold
        'max_lot_size': 0.10,         # Much smaller max lot
        'slippage_pips': 1.0,         # Reduced from 2.0
        'atr_volatility_threshold_high': 20.0,
        'atr_volatility_threshold_extreme': 30.0,
        'emergency_brake_percent': 0.05  # 5% emergency brake
    }
    
    CRYPTO = {
        'contract_size': 1,
        'pip_size': 0.01,
        'typical_spread_pips': 2.0,  # Reduced from 5.0
        'max_risk_percent': 1.5,
        'max_lot_size': 1.0,
        'slippage_pips': 0.5  # Reduced from 1.0
    }
    
    INDICES = {
        'contract_size': 1,  # 1 point = $1 for index CFDs
        'pip_size': 0.01,  # 0.01 point = 1 pip
        'typical_spread_pips': 3.0,  # Index spreads are typically higher
        'max_risk_percent': 0.5,  # Very conservative for indices
        'max_lot_size': 0.1,  # Small lot sizes for indices
        'slippage_pips': 0.5,
        'atr_volatility_threshold_high': 50.0,  # Index-specific thresholds
        'atr_volatility_threshold_extreme': 100.0,
        'emergency_brake_percent': 0.1  # 10% emergency brake
    }
    
    @classmethod
    def get_config(cls, symbol_name):
        """Get configuration for a specific instrument"""
        symbol_upper = symbol_name.upper()
        
        # Index detection (US30, US100, US500, DE30, etc.)
        if any(index in symbol_upper for index in ['US30', 'US100', 'US500', 'DE30', 'UK100', 'JP225', 'NAS100', 'SPX500']):
            return cls.INDICES
        elif 'XAU' in symbol_upper or 'GOLD' in symbol_upper:
            return cls.GOLD
        elif any(jpy in symbol_upper for jpy in ['JPY', 'USDJPY', 'EURJPY', 'GBPJPY']):
            return cls.FOREX_JPY
        elif any(crypto in symbol_upper for crypto in ['BTC', 'ETH', 'CRYPTO']):
            return cls.CRYPTO
        else:
            return cls.FOREX_MAJOR

class EnhancedBacktestEngine:
    """Enhanced backtesting engine with realistic cost modeling"""
    
    def __init__(self, enable_spread_costs=True, enable_slippage=True, enable_realistic_execution=True):
        self.enable_spread_costs = enable_spread_costs
        self.enable_slippage = enable_slippage
        self.enable_realistic_execution = enable_realistic_execution
        
    def calculate_realistic_entry_price(self, signal, close_price, spread_pips, pip_size, slippage_pips=0):
        """Calculate realistic entry price with spread and slippage"""
        spread_cost = spread_pips * pip_size
        slippage_cost = slippage_pips * pip_size if self.enable_slippage else 0
        
        if signal == 'BUY':
            # Buy at ask price + slippage
            return close_price + (spread_cost / 2) + slippage_cost
        else:  # SELL
            # Sell at bid price - slippage
            return close_price - (spread_cost / 2) - slippage_cost
    
    def calculate_realistic_exit_price(self, position_type, target_price, spread_pips, pip_size, slippage_pips=0):
        """Calculate realistic exit price with spread and slippage"""
        spread_cost = spread_pips * pip_size
        slippage_cost = slippage_pips * pip_size if self.enable_slippage else 0
        
        if position_type == 'BUY':
            # Close BUY at bid price - slippage
            return target_price - (spread_cost / 2) - slippage_cost
        else:  # SELL
            # Close SELL at ask price + slippage
            return target_price + (spread_cost / 2) + slippage_cost
    
    def calculate_position_size(self, symbol_name, capital, risk_percent, sl_distance, atr_value, config):
        """Enhanced position sizing with instrument-specific rules"""
        
        # Apply instrument-specific risk limits
        risk_percent = min(risk_percent, config['max_risk_percent'])
        
        amount_to_risk = capital * (risk_percent / 100.0)
        
        # Special handling for high-risk instruments
        if config == InstrumentConfig.GOLD:
            return self._calculate_gold_position_size(risk_percent, atr_value, amount_to_risk, sl_distance, config)
        elif config == InstrumentConfig.INDICES:
            return self._calculate_index_position_size(risk_percent, atr_value, amount_to_risk, sl_distance, config)
        else:
            return self._calculate_standard_position_size(amount_to_risk, sl_distance, config)
    
    def _calculate_gold_position_size(self, risk_percent, atr_value, amount_to_risk, sl_distance, config):
        """Ultra-conservative position sizing for gold"""
        
        # Base lot size based on risk percentage (ultra-conservative)
        if risk_percent <= 0.25:
            base_lot_size = 0.01
        elif risk_percent <= 0.5:
            base_lot_size = 0.01
        elif risk_percent <= 0.75:
            base_lot_size = 0.02
        elif risk_percent <= 1.0:
            base_lot_size = 0.02
        else:
            base_lot_size = 0.03  # Maximum for any gold trade
        
        # ATR-based volatility adjustments
        atr_threshold_high = config.get('atr_volatility_threshold_high', 20.0)
        atr_threshold_extreme = config.get('atr_volatility_threshold_extreme', 30.0)
        
        if atr_value > atr_threshold_extreme:
            lot_size = 0.01  # Extreme volatility
            logger.warning(f"GOLD EXTREME VOLATILITY: ATR={atr_value:.1f}, lot=0.01")
        elif atr_value > atr_threshold_high:
            lot_size = max(0.01, base_lot_size * 0.5)  # High volatility
            logger.warning(f"GOLD HIGH VOLATILITY: ATR={atr_value:.1f}, lot={lot_size}")
        else:
            lot_size = base_lot_size  # Normal volatility
        
        # Final safety cap
        lot_size = min(lot_size, config['max_lot_size'])
        
        return round(lot_size, 2)
    
    def _calculate_index_position_size(self, risk_percent, atr_value, amount_to_risk, sl_distance, config):
        """Ultra-conservative position sizing for stock indices (US500, US30, etc.)"""
        
        # Base lot size for indices (extremely conservative)
        if risk_percent <= 0.25:
            base_lot_size = 0.01
        elif risk_percent <= 0.5:
            base_lot_size = 0.01
        elif risk_percent <= 0.75:
            base_lot_size = 0.02
        elif risk_percent <= 1.0:
            base_lot_size = 0.02
        else:
            base_lot_size = 0.03  # Maximum for any index trade
        
        # ATR-based volatility adjustments for indices
        atr_threshold_high = config.get('atr_volatility_threshold_high', 50.0)
        atr_threshold_extreme = config.get('atr_volatility_threshold_extreme', 100.0)
        
        if atr_value > atr_threshold_extreme:
            lot_size = 0.01  # Extreme volatility - minimum size
            logger.warning(f"INDEX EXTREME VOLATILITY: ATR={atr_value:.1f}, lot=0.01")
        elif atr_value > atr_threshold_high:
            lot_size = max(0.01, base_lot_size * 0.5)  # High volatility - reduce size
            logger.warning(f"INDEX HIGH VOLATILITY: ATR={atr_value:.1f}, lot={lot_size}")
        else:
            lot_size = base_lot_size  # Normal volatility
        
        # Final safety cap
        lot_size = min(lot_size, config['max_lot_size'])
        
        logger.debug(f"INDEX POSITION: Risk={risk_percent}%, ATR={atr_value:.1f}, Lot={lot_size}")
        
        return round(lot_size, 2)
    
    def _calculate_standard_position_size(self, amount_to_risk, sl_distance, config):
        """Standard position sizing for forex and other instruments"""
        
        risk_in_currency_per_lot = sl_distance * config['contract_size']
        
        if risk_in_currency_per_lot <= 0:
            return 0
        
        calculated_lot_size = amount_to_risk / risk_in_currency_per_lot
        
        # Apply limits
        if calculated_lot_size < 0.01:
            return 0.01
        elif calculated_lot_size > config['max_lot_size']:
            return config['max_lot_size']
        
        return round(calculated_lot_size, 2)
    
    def calculate_spread_cost(self, lot_size, spread_pips, config):
        """Calculate the cost of spread for a round-trip trade"""
        if not self.enable_spread_costs:
            return 0
        
        # Calculate pip value per lot based on instrument type
        if config == InstrumentConfig.GOLD:
            # For gold: $1 per 0.01 pip per 1 oz
            pip_value_per_lot = 1.0
        elif config == InstrumentConfig.INDICES:
            # For indices: $1 per point per lot (very conservative for backtesting)
            pip_value_per_lot = 0.1  # Much more conservative for indices
        elif config['contract_size'] == 100:  # Other instruments with 100 contract size
            pip_value_per_lot = 1.0
        else:  # Forex
            # For major pairs: Use conservative pip value for backtesting
            pip_value_per_lot = 1.0
        
        spread_cost = spread_pips * pip_value_per_lot * lot_size
        
        return spread_cost

def run_enhanced_backtest(strategy_id, params, historical_data_df, symbol_name=None, engine_config=None):
    """
    Run enhanced backtesting with realistic cost modeling
    
    Args:
        strategy_id: Strategy to test
        params: Strategy parameters
        historical_data_df: Historical OHLC data
        symbol_name: Symbol name for instrument detection
        engine_config: Engine configuration options
    """
    
    # Initialize engine
    engine_config = engine_config or {}
    engine = EnhancedBacktestEngine(
        enable_spread_costs=engine_config.get('enable_spread_costs', True),
        enable_slippage=engine_config.get('enable_slippage', True),
        enable_realistic_execution=engine_config.get('enable_realistic_execution', True)
    )
    
    # Get strategy
    strategy_class = resolve_strategy_class(strategy_id)
    if not strategy_class:
        return {"error": "Strategy not found"}
    
    # Detect instrument and get configuration
    if symbol_name:
        instrument_symbol = symbol_name
    elif historical_data_df.columns[0].count('_') > 0:
        instrument_symbol = historical_data_df.columns[0].split('_')[0]
    else:
        instrument_symbol = "UNKNOWN"
    
    config = InstrumentConfig.get_config(instrument_symbol)
    
    # Initialize strategy
    class MockBot:
        def __init__(self):
            self.market_for_mt5 = instrument_symbol
            self.timeframe = "H1"
            self.tf_map = {}
    
    strategy_instance = strategy_class(bot_instance=MockBot(), params=params)
    df = historical_data_df.copy()
    df_with_signals = strategy_instance.analyze_df(df)
    df_with_signals.ta.atr(length=14, append=True)
    df_with_signals.dropna(inplace=True)
    df_with_signals.reset_index(inplace=True)
    
    if df_with_signals.empty:
        return {"error": "Insufficient data for analysis"}
    
    # Initialize state
    trades = []
    in_position = False
    initial_capital = 10000.0
    capital = initial_capital
    equity_curve = [initial_capital]
    peak_equity = initial_capital
    max_drawdown = 0.0
    total_spread_costs = 0.0
    stop_out_triggered = False
    
    position_type = None
    entry_price = 0.0
    sl_price = 0.0
    tp_price = 0.0
    lot_size = 0.0
    entry_time = None
    
    # Enhanced parameter handling
    risk_percent = float(params.get('risk_percent', params.get('lot_size', 1.0)))
    sl_atr_multiplier = float(params.get('sl_atr_multiplier', params.get('sl_pips', 2.0)))
    tp_atr_multiplier = float(params.get('tp_atr_multiplier', params.get('tp_pips', 4.0)))
    
    # Apply instrument-specific parameter limits
    if config == InstrumentConfig.GOLD:
        risk_percent = min(risk_percent, 1.0)
        sl_atr_multiplier = min(sl_atr_multiplier, 1.0)
        tp_atr_multiplier = min(tp_atr_multiplier, 2.0)
        logger.debug(f"GOLD PROTECTION: Risk={risk_percent}%, SL={sl_atr_multiplier}x ATR, TP={tp_atr_multiplier}x ATR")
    
    # Main backtesting loop
    for i in range(1, len(df_with_signals)):
        current_bar = df_with_signals.iloc[i]
        
        if capital <= 0:
            break
        
        if in_position:
            # Check for exit conditions with realistic execution
            exit_price = None
            exit_reason = None
            
            if position_type == 'BUY':
                if current_bar['low'] <= sl_price:
                    exit_price = engine.calculate_realistic_exit_price(
                        'BUY', sl_price, config['typical_spread_pips'], 
                        config['pip_size'], config.get('slippage_pips', 0)
                    )
                    exit_reason = 'Stop Loss'
                elif current_bar['high'] >= tp_price:
                    exit_price = engine.calculate_realistic_exit_price(
                        'BUY', tp_price, config['typical_spread_pips'], 
                        config['pip_size'], config.get('slippage_pips', 0)
                    )
                    exit_reason = 'Take Profit'
            else:  # SELL
                if current_bar['high'] >= sl_price:
                    exit_price = engine.calculate_realistic_exit_price(
                        'SELL', sl_price, config['typical_spread_pips'], 
                        config['pip_size'], config.get('slippage_pips', 0)
                    )
                    exit_reason = 'Stop Loss'
                elif current_bar['low'] <= tp_price:
                    exit_price = engine.calculate_realistic_exit_price(
                        'SELL', tp_price, config['typical_spread_pips'], 
                        config['pip_size'], config.get('slippage_pips', 0)
                    )
                    exit_reason = 'Take Profit'
            
            if exit_price is not None:
                # Calculate profit with realistic execution
                profit_multiplier = lot_size * config['contract_size']
                
                if position_type == 'BUY':
                    profit = (exit_price - entry_price) * profit_multiplier
                else:
                    profit = (entry_price - exit_price) * profit_multiplier
                
                # Deduct spread costs
                spread_cost = engine.calculate_spread_cost(lot_size, config['typical_spread_pips'], config)
                profit -= spread_cost
                total_spread_costs += spread_cost
                
                if not math.isfinite(profit):
                    profit = 0.0
                
                capital += profit
                if capital <= 0:
                    # Simulate account stop-out in backtesting mode.
                    capital = 0.0
                    stop_out_triggered = True

                trades.append({
                    'entry_time': str(entry_time),
                    'exit_time': str(current_bar['time']),
                    'entry': entry_price,
                    'exit': exit_price,
                    'profit': profit,
                    'spread_cost': spread_cost,
                    'reason': exit_reason,
                    'position_type': position_type,
                    'lot_size': lot_size
                })
                
                equity_curve.append(capital)
                peak_equity = max(peak_equity, capital)
                drawdown = (peak_equity - capital) / peak_equity if peak_equity > 0 else 0
                max_drawdown = min(1.0, max(max_drawdown, drawdown))
                in_position = False
                
                logger.debug(f"Trade closed: {position_type} | Entry: {entry_price:.4f} | Exit: {exit_price:.4f} | Profit: ${profit:.2f} | Spread Cost: ${spread_cost:.2f}")

                if stop_out_triggered:
                    logger.warning("Backtest stop-out triggered: capital reached zero.")
                    break
        
        if not in_position:
            signal = current_bar.get("signal", "HOLD")
            if signal in ['BUY', 'SELL']:
                atr_value = current_bar['ATRr_14']
                if atr_value <= 0:
                    continue
                
                # Calculate SL/TP distances
                sl_distance = atr_value * sl_atr_multiplier
                tp_distance = atr_value * tp_atr_multiplier
                
                # Calculate position size
                lot_size = engine.calculate_position_size(
                    instrument_symbol, capital, risk_percent, sl_distance, atr_value, config
                )
                
                if lot_size <= 0:
                    continue
                
                # Emergency brake for high-risk trades (especially gold)
                if config == InstrumentConfig.GOLD:
                    estimated_risk = sl_distance * lot_size * config['contract_size']
                    max_risk_dollar = capital * config.get('emergency_brake_percent', 0.05)
                    if estimated_risk > max_risk_dollar:
                        logger.warning(f"EMERGENCY BRAKE: Risk ${estimated_risk:.0f} > ${max_risk_dollar:.0f}, trade SKIPPED")
                        continue
                
                # Calculate realistic entry price
                entry_price = engine.calculate_realistic_entry_price(
                    signal, current_bar['close'], config['typical_spread_pips'], 
                    config['pip_size'], config.get('slippage_pips', 0)
                )
                entry_time = current_bar['time']
                
                # Set SL/TP levels
                if signal == 'BUY':
                    sl_price = entry_price - sl_distance
                    tp_price = entry_price + tp_distance
                else:
                    sl_price = entry_price + sl_distance
                    tp_price = entry_price - tp_distance
                
                in_position = True
                position_type = signal
                
                logger.debug(f"New {signal} position: Entry={entry_price:.4f}, SL={sl_price:.4f}, TP={tp_price:.4f}, Lot={lot_size}")
    
    # Calculate final results
    total_profit = capital - initial_capital
    wins = len([t for t in trades if t['profit'] > 0])
    losses = len(trades) - wins
    win_rate = (wins / len(trades) * 100) if trades else 0
    
    # Clean up results
    capital = max(0.0, capital)
    total_profit = capital - initial_capital
    final_capital = round(capital, 2) if math.isfinite(capital) else initial_capital
    total_profit_clean = round(total_profit, 2) if math.isfinite(total_profit) else 0.0
    max_drawdown_clean = round(max_drawdown * 100, 2) if math.isfinite(max_drawdown) else 0.0
    win_rate_clean = round(win_rate, 2) if math.isfinite(win_rate) else 0.0
    
    gross_profit_clean = round(total_profit_clean + round(total_spread_costs, 2), 2)

    logger.info(
        f"Enhanced Backtest Complete: {len(trades)} trades, "
        f"gross ${gross_profit_clean:+.0f}, net ${total_profit_clean:+.0f}, "
        f"{win_rate_clean:.0f}% win rate, ${total_spread_costs:.0f} spread costs"
    )
    
    return {
        "strategy_name": strategy_class.name,
        "instrument": instrument_symbol,
        "total_trades": len(trades),
        "final_capital": final_capital,
        "gross_profit_usd": gross_profit_clean,
        "total_profit_usd": total_profit_clean,
        "total_spread_costs": round(total_spread_costs, 2),
        "net_profit_after_costs": round(total_profit_clean, 2),
        "win_rate_percent": win_rate_clean,
        "wins": wins,
        "losses": losses,
        "max_drawdown_percent": max_drawdown_clean,
        "equity_curve": equity_curve,
        "trades": trades[-20:],  # Last 20 trades
        "engine_config": {
            "spread_costs_enabled": engine.enable_spread_costs,
            "slippage_enabled": engine.enable_slippage,
            "realistic_execution": engine.enable_realistic_execution,
            "instrument_config": config
        }
    }

# Wrapper function for backward compatibility
def run_backtest(strategy_id, params, historical_data_df, symbol_name=None):
    """Backward compatible wrapper for enhanced backtesting"""
    return run_enhanced_backtest(strategy_id, params, historical_data_df, symbol_name)