diff --git a/config/system.yaml b/config/system.yaml index 5c4b191..dc9061a 100644 --- a/config/system.yaml +++ b/config/system.yaml @@ -102,3 +102,30 @@ supabase: url: https://.supabase.co key_env_name: SUPABASE_KEY table: fx_candles +phase2: + strategies: + - name: S7_Tight + pair: GBP_JPY + timeframe: H1 + enabled: true + - name: S9 + pair: GBP_USD + timeframe: H1 + enabled: true + - name: S9_Filtered + pair: GBP_AUD + timeframe: H1 + enabled: true + - name: S4F + pair: EUR_AUD + timeframe: M15 + htf_timeframe: H1 + enabled: true + - name: S3 + pair: GBP_JPY + timeframe: H1 + enabled: true + poll_interval_seconds: 60 + paper_mode: true + starting_equity: 100000 + max_daily_drawdown_pct: 5.0 diff --git a/results/phase2/correlation_analysis.json b/results/phase2/correlation_analysis.json new file mode 100644 index 0000000..db18cc4 --- /dev/null +++ b/results/phase2/correlation_analysis.json @@ -0,0 +1,23 @@ +{ + "S7_S3_overlap": { + "overlap_count": 21, + "same_dir": 21, + "opposite_dir": 0, + "ratio": 0.169 + }, + "S9_S9F_temporal": { + "s9_trade_days": 275, + "s9f_trade_days": 65, + "shared_trade_days": 33, + "temporal_overlap_ratio": 0.12 + }, + "portfolio": { + "total_trades": 694, + "win_rate_pct": 53.3, + "profit_factor": 0.99, + "total_pnl_pips": -175.1, + "total_pnl_dollars": -23036.66, + "max_drawdown_pct": -47.8, + "sharpe_ratio": -0.71 + } +} \ No newline at end of file diff --git a/results/phase2/kelly_sizing.json b/results/phase2/kelly_sizing.json new file mode 100644 index 0000000..ede8b81 --- /dev/null +++ b/results/phase2/kelly_sizing.json @@ -0,0 +1,82 @@ +{ + "S7_Tight_GBP_JPY": { + "trades": 98, + "win_rate": 59.2, + "avg_win_pips": 42.8, + "avg_loss_pips": 62.3, + "avg_win_loss_ratio": 0.69, + "kelly_full_pct": 0, + "kelly_half_pct": 0, + "monte_carlo_half_kelly": { + "p50_dd": 0, + "p75_dd": 0, + "p95_dd": 0, + "p99_dd": 0, + "ruin_pct": 0 + } + }, + "S9_GBP_USD": { + "trades": 285, + "win_rate": 53.7, + "avg_win_pips": 29.3, + "avg_loss_pips": 41.7, + "avg_win_loss_ratio": 0.7, + "kelly_full_pct": 0, + "kelly_half_pct": 0, + "monte_carlo_half_kelly": { + "p50_dd": 0, + "p75_dd": 0, + "p95_dd": 0, + "p99_dd": 0, + "ruin_pct": 0 + } + }, + "S9_Filtered_GBP_AUD": { + "trades": 66, + "win_rate": 60.6, + "avg_win_pips": 49.5, + "avg_loss_pips": 57.9, + "avg_win_loss_ratio": 0.85, + "kelly_full_pct": 14.53, + "kelly_half_pct": 7.26, + "monte_carlo_half_kelly": { + "p50_dd": 2.5, + "p75_dd": 3.3, + "p95_dd": 4.8, + "p99_dd": 6.3, + "ruin_pct": 0.0 + } + }, + "S4F_EUR_AUD": { + "trades": 95, + "win_rate": 45.3, + "avg_win_pips": 38.8, + "avg_loss_pips": 28.7, + "avg_win_loss_ratio": 1.35, + "kelly_full_pct": 4.7, + "kelly_half_pct": 2.35, + "monte_carlo_half_kelly": { + "p50_dd": 0.7, + "p75_dd": 1.0, + "p95_dd": 1.5, + "p99_dd": 1.9, + "ruin_pct": 0.0 + } + }, + "S3_GBP_JPY": { + "trades": 150, + "win_rate": 50.7, + "avg_win_pips": 43.5, + "avg_loss_pips": 41.9, + "avg_win_loss_ratio": 1.04, + "kelly_full_pct": 3.11, + "kelly_half_pct": 1.56, + "monte_carlo_half_kelly": { + "p50_dd": 0.9, + "p75_dd": 1.2, + "p95_dd": 1.9, + "p99_dd": 2.4, + "ruin_pct": 0.0 + } + } +} \ No newline at end of file diff --git a/results/phase2/regime_analysis.json b/results/phase2/regime_analysis.json new file mode 100644 index 0000000..ca0c6fb --- /dev/null +++ b/results/phase2/regime_analysis.json @@ -0,0 +1,192 @@ +{ + "S7_Tight_GBP_JPY": { + "pair": "GBP_JPY", + "timeframe": "H1", + "years": { + "2021": { + "trades": 34, + "wr": 47.1, + "pf": 0.47, + "pnl_pips": -465.5, + "max_dd_pips": -618.3, + "expectancy": -13.69 + }, + "2022": { + "trades": 29, + "wr": 58.6, + "pf": 0.89, + "pnl_pips": -102.1, + "max_dd_pips": -412.7, + "expectancy": -3.52 + }, + "2023": { + "trades": 35, + "wr": 71.4, + "pf": 1.8, + "pnl_pips": 558.9, + "max_dd_pips": -219.9, + "expectancy": 15.97 + } + }, + "full": { + "trades": 98, + "wr": 59.2, + "pf": 1.0, + "pnl_pips": -8.6, + "max_dd_pips": -723.9, + "expectancy": -0.09 + } + }, + "S9_GBP_USD": { + "pair": "GBP_USD", + "timeframe": "H1", + "years": { + "2021": { + "trades": 138, + "wr": 52.2, + "pf": 0.71, + "pnl_pips": -701.8, + "max_dd_pips": -856.6, + "expectancy": -5.09 + }, + "2022": { + "trades": 115, + "wr": 51.3, + "pf": 0.83, + "pnl_pips": -470.0, + "max_dd_pips": -1030.5, + "expectancy": -4.09 + }, + "2023": { + "trades": 32, + "wr": 68.8, + "pf": 1.38, + "pnl_pips": 154.8, + "max_dd_pips": -132.7, + "expectancy": 4.84 + } + }, + "full": { + "trades": 285, + "wr": 53.7, + "pf": 0.82, + "pnl_pips": -1017.0, + "max_dd_pips": -1775.8, + "expectancy": -3.57 + } + }, + "S9_Filtered_GBP_AUD": { + "pair": "GBP_AUD", + "timeframe": "H1", + "years": { + "2021": { + "trades": 31, + "wr": 45.2, + "pf": 0.58, + "pnl_pips": -412.3, + "max_dd_pips": -535.5, + "expectancy": -13.3 + }, + "2022": { + "trades": 23, + "wr": 73.9, + "pf": 2.57, + "pnl_pips": 544.1, + "max_dd_pips": -116.9, + "expectancy": 23.65 + }, + "2023": { + "trades": 12, + "wr": 75.0, + "pf": 2.98, + "pnl_pips": 342.6, + "max_dd_pips": -49.6, + "expectancy": 28.55 + } + }, + "full": { + "trades": 66, + "wr": 60.6, + "pf": 1.32, + "pnl_pips": 474.3, + "max_dd_pips": -561.2, + "expectancy": 7.19 + } + }, + "S4F_EUR_AUD": { + "pair": "EUR_AUD", + "timeframe": "M15", + "years": { + "2021": { + "trades": 42, + "wr": 42.9, + "pf": 1.11, + "pnl_pips": 57.8, + "max_dd_pips": -119.7, + "expectancy": 1.38 + }, + "2022": { + "trades": 33, + "wr": 60.6, + "pf": 1.79, + "pnl_pips": 370.0, + "max_dd_pips": -183.7, + "expectancy": 11.21 + }, + "2023": { + "trades": 20, + "wr": 25.0, + "pf": 0.48, + "pnl_pips": -254.7, + "max_dd_pips": -337.2, + "expectancy": -12.74 + } + }, + "full": { + "trades": 95, + "wr": 45.3, + "pf": 1.12, + "pnl_pips": 173.0, + "max_dd_pips": -337.2, + "expectancy": 1.82 + } + }, + "S3_GBP_JPY": { + "pair": "GBP_JPY", + "timeframe": "H1", + "years": { + "2021": { + "trades": 65, + "wr": 46.2, + "pf": 0.9, + "pnl_pips": -112.6, + "max_dd_pips": -345.7, + "expectancy": -1.73 + }, + "2022": { + "trades": 47, + "wr": 53.2, + "pf": 1.21, + "pnl_pips": 262.9, + "max_dd_pips": -307.3, + "expectancy": 5.59 + }, + "2023": { + "trades": 38, + "wr": 55.3, + "pf": 1.07, + "pnl_pips": 56.9, + "max_dd_pips": -182.9, + "expectancy": 1.5 + } + }, + "full": { + "trades": 150, + "wr": 50.7, + "pf": 1.07, + "pnl_pips": 203.1, + "max_dd_pips": -533.2, + "expectancy": 1.35 + } + } +} \ No newline at end of file diff --git a/src/backtester/engine.py b/src/backtester/engine.py index e6f5f77..9c4c793 100644 --- a/src/backtester/engine.py +++ b/src/backtester/engine.py @@ -13,30 +13,12 @@ Supports: """ import numpy as np import pandas as pd -from dataclasses import dataclass, field from typing import Optional - -# --------------------------------------------------------------------------- -# Spread Configuration (in pips) -# --------------------------------------------------------------------------- -SPREAD_PIPS = { - "EUR_USD": 1.5, "GBP_USD": 1.5, - "GBP_AUD": 2.0, "EUR_AUD": 2.0, - "EUR_GBP": 2.0, "GBP_JPY": 2.5, - "USD_JPY": 1.5, "GBP_CAD": 2.5, - "EUR_CAD": 2.5, "EUR_NZD": 2.5, - "GBP_NZD": 2.5, -} - -# Pip value per pair -PIP_SIZE = { - "EUR_USD": 0.0001, "GBP_USD": 0.0001, "EUR_AUD": 0.0001, - "GBP_AUD": 0.0001, "EUR_GBP": 0.0001, "GBP_CAD": 0.0001, - "EUR_CAD": 0.0001, "EUR_NZD": 0.0001, - "GBP_NZD": 0.0001, - "USD_JPY": 0.01, "GBP_JPY": 0.01, -} +from src.position_manager import ( + Position, TradeRecord, PositionManager, + SPREAD_PIPS, PIP_SIZE, +) # Major news events (simplified: first Friday of month = NFP, plus key dates) # In production, use a calendar API. For backtesting 2021-2024, we hardcode @@ -46,72 +28,6 @@ MAJOR_NEWS_DAY_OF_WEEK = 4 # Friday MAJOR_NEWS_WEEK = 1 # First full week of month -@dataclass -class Position: - """Represents an open position.""" - entry_time: pd.Timestamp - direction: str # 'LONG' or 'SHORT' - entry_price: float - sl_price: float - tp1_price: float - tp2_price: float - tp3_price: float - initial_size: float - current_size: float - tp_splits: tuple # e.g. (0.40, 0.40, 0.20) - trail_atr_mult: float - max_bars: int - bars_held: int = 0 - tp1_hit: bool = False - tp2_hit: bool = False - tp3_hit: bool = False - trailing_sl: Optional[float] = None - strategy_id: int = 0 - confluence_score: int = 0 - signal_features: dict = field(default_factory=dict) - realized_pnl: float = 0.0 # Tracks PnL from partial closes - no_breakeven: bool = False # When True, don't move SL to breakeven after TP1 - - -@dataclass -class TradeRecord: - """Rich trade log entry for Phase 3 feature engineering.""" - timestamp: pd.Timestamp = None - strategy_id: int = 0 - pair: str = "" - signal_direction: str = "" - entry_price: float = 0.0 - sl_price: float = 0.0 - tp1_price: float = 0.0 - tp2_price: float = 0.0 - tp3_price: float = 0.0 - lot_size: float = 0.0 - confluence_score: int = 0 - session: str = "" - spread_at_entry: float = 0.0 - atr_at_entry: float = 0.0 - adx_at_entry: float = 0.0 - rsi_at_entry: float = 0.0 - ema_50_value: float = 0.0 - ema_200_value: float = 0.0 - vwap_deviation: float = 0.0 - news_within_60min: bool = False - macd_hist_at_entry: float = 0.0 - stoch_k_at_entry: float = 0.0 - distance_from_ema50_pips: float = 0.0 - candle_body_ratio: float = 0.0 - hour_of_day: int = 0 - day_of_week: int = 0 - entry_pattern: str = "" - exit_price: float = 0.0 - exit_reason: str = "" - exit_time: pd.Timestamp = None - pnl_pips: float = 0.0 - pnl_dollars: float = 0.0 - hold_time_minutes: int = 0 - win: bool = False - - class Backtester: """Event-driven backtesting engine.""" @@ -129,18 +45,50 @@ class Backtester: self.data = data self.strategy = strategy self.pair = pair - self.equity = starting_equity self.starting_equity = starting_equity - self.peak_equity = starting_equity self.htf_data = htf_data - self.pip = PIP_SIZE.get(pair, 0.0001) - self.spread = SPREAD_PIPS.get(pair, 2.0) * self.pip + # Delegate position management to PositionManager + self._pm = PositionManager(pair, starting_equity) - self.open_positions: list[Position] = [] - self.closed_trades: list[TradeRecord] = [] self.equity_curve = [] - self.daily_pnl = {} + + # Proxy properties so existing code that reads these still works + @property + def equity(self): + return self._pm.equity + + @equity.setter + def equity(self, value): + self._pm.equity = value + + @property + def peak_equity(self): + return self._pm.peak_equity + + @peak_equity.setter + def peak_equity(self, value): + self._pm.peak_equity = value + + @property + def open_positions(self): + return self._pm.open_positions + + @property + def closed_trades(self): + return self._pm.closed_trades + + @property + def daily_pnl(self): + return self._pm.daily_pnl + + @property + def pip(self): + return self._pm.pip + + @property + def spread(self): + return self._pm.spread def _get_session(self, hour: int) -> str: if 0 <= hour < 8: @@ -152,63 +100,16 @@ class Backtester: else: return "NY" - def _get_slippage(self, hour: int) -> float: - """Slippage in price units. High-volume sessions get less slippage.""" - if 8 <= hour < 17: # London + NY overlap - return 0.4 * self.pip - else: - return 1.0 * self.pip - def _is_near_news(self, timestamp: pd.Timestamp) -> bool: """Simple news filter: first Friday of each month (NFP proxy) +/- 30 min.""" - # Check if current day is first Friday of month - if timestamp.weekday() != 4: # Not Friday + if timestamp.weekday() != 4: return False - if timestamp.day > 7: # Not first week + if timestamp.day > 7: return False - # NFP typically at 13:30 UTC if 13 <= timestamp.hour <= 14: return True return False - def _calculate_position_size(self, sl_distance: float, - confluence_score: int, - risk_pct: float = 0.01) -> float: - """Risk-based position sizing (default 1%, strategies can override).""" - if sl_distance <= 0: - return 0.0 - - risk_amount = self.equity * risk_pct - # Position size = risk_amount / SL distance in price - position_size = risk_amount / sl_distance - return round(position_size, 2) - - def _get_htf_row(self, timestamp: pd.Timestamp) -> Optional[pd.Series]: - """Get the most recent FULLY CLOSED higher-timeframe candle.""" - if self.htf_data is None: - return None - # Only use HTF candles that closed BEFORE current timestamp - valid = self.htf_data[self.htf_data.index < timestamp] - if len(valid) == 0: - return None - return valid.iloc[-1] - - def _apply_spread_to_entry(self, price: float, direction: str) -> float: - """Apply spread to entry price.""" - if direction == "LONG": - return price + self.spread # Buy at ask - else: - return price - self.spread # Sell at bid (lower) - - def _apply_slippage_to_entry(self, price: float, direction: str, - hour: int) -> float: - """Apply slippage to entry price.""" - slip = self._get_slippage(hour) - if direction == "LONG": - return price + slip # Slippage works against us - else: - return price - slip - def _check_daily_drawdown(self, timestamp: pd.Timestamp) -> bool: """Check if 5% daily drawdown has been breached.""" date_key = timestamp.date() @@ -216,204 +117,14 @@ class Backtester: self.daily_pnl[date_key] = 0.0 return self.daily_pnl[date_key] <= -0.05 * self.starting_equity - def _update_positions(self, candle: pd.Series, i: int): - """Check SL/TP/trailing/time exits for all open positions.""" - to_close = [] - - for pos in self.open_positions: - pos.bars_held += 1 - high = candle["high"] - low = candle["low"] - close_price = candle["close"] - current_atr = candle.get("atr_14", 0) - - # Determine effective SL - effective_sl = pos.trailing_sl if pos.trailing_sl is not None else pos.sl_price - - if pos.direction == "LONG": - # Check SL - if low <= effective_sl: - self._close_position(pos, effective_sl, "SL", candle) - to_close.append(pos) - continue - - # Check TP1 - if not pos.tp1_hit and high >= pos.tp1_price: - close_size = pos.initial_size * pos.tp_splits[0] - self._partial_close(pos, pos.tp1_price, close_size, "TP1", candle) - pos.tp1_hit = True - if not pos.no_breakeven: - # Move SL to breakeven after TP1 - pos.trailing_sl = pos.entry_price - - # Check TP2 - if not pos.tp2_hit and pos.tp1_hit and high >= pos.tp2_price: - close_size = pos.initial_size * pos.tp_splits[1] - if close_size > 0: - self._partial_close(pos, pos.tp2_price, close_size, "TP2", candle) - pos.tp2_hit = True - # If no breakeven was set, start trailing from original SL - if pos.trailing_sl is None: - pos.trailing_sl = pos.sl_price - - # Check TP3 - if not pos.tp3_hit and pos.tp2_hit and high >= pos.tp3_price: - self._close_position(pos, pos.tp3_price, "TP3", candle) - to_close.append(pos) - continue - - # Trailing stop for runner (after TP2) - if pos.tp2_hit and current_atr > 0: - new_trail = high - pos.trail_atr_mult * current_atr - if pos.trailing_sl is None or new_trail > pos.trailing_sl: - pos.trailing_sl = new_trail - - else: # SHORT - # Check SL - if high >= effective_sl: - self._close_position(pos, effective_sl, "SL", candle) - to_close.append(pos) - continue - - # Check TP1 - if not pos.tp1_hit and low <= pos.tp1_price: - close_size = pos.initial_size * pos.tp_splits[0] - self._partial_close(pos, pos.tp1_price, close_size, "TP1", candle) - pos.tp1_hit = True - if not pos.no_breakeven: - pos.trailing_sl = pos.entry_price - - # Check TP2 - if not pos.tp2_hit and pos.tp1_hit and low <= pos.tp2_price: - close_size = pos.initial_size * pos.tp_splits[1] - if close_size > 0: - self._partial_close(pos, pos.tp2_price, close_size, "TP2", candle) - pos.tp2_hit = True - if pos.trailing_sl is None: - pos.trailing_sl = pos.sl_price - - # Check TP3 - if not pos.tp3_hit and pos.tp2_hit and low <= pos.tp3_price: - self._close_position(pos, pos.tp3_price, "TP3", candle) - to_close.append(pos) - continue - - # Trailing stop for runner - if pos.tp2_hit and current_atr > 0: - new_trail = low + pos.trail_atr_mult * current_atr - if pos.trailing_sl is None or new_trail < pos.trailing_sl: - pos.trailing_sl = new_trail - - # Time-based exit - if pos.max_bars > 0 and pos.bars_held >= pos.max_bars: - self._close_position(pos, close_price, "TIME", candle) - to_close.append(pos) - continue - - for pos in to_close: - if pos in self.open_positions: - self.open_positions.remove(pos) - - def _partial_close(self, pos: Position, exit_price: float, - close_size: float, reason: str, candle: pd.Series): - """Close a partial portion of a position.""" - if pos.direction == "LONG": - pnl_per_unit = exit_price - pos.entry_price - else: - pnl_per_unit = pos.entry_price - exit_price - - pnl = pnl_per_unit * close_size - self.equity += pnl - pos.current_size -= close_size - pos.realized_pnl += pnl - - date_key = candle.name.date() if hasattr(candle.name, 'date') else None - if date_key: - self.daily_pnl[date_key] = self.daily_pnl.get(date_key, 0.0) + pnl - - if self.equity > self.peak_equity: - self.peak_equity = self.equity - - def _close_position(self, pos: Position, exit_price: float, - reason: str, candle: pd.Series): - """Fully close remaining position and log the trade.""" - remaining = pos.current_size - if remaining <= 0: - remaining = 0.01 # avoid zero - - if pos.direction == "LONG": - pnl_per_unit = exit_price - pos.entry_price - else: - pnl_per_unit = pos.entry_price - exit_price - - final_pnl = pnl_per_unit * remaining - self.equity += final_pnl - - # Total PnL = partial closes + final close - total_pnl = pos.realized_pnl + final_pnl - total_pnl_pips = total_pnl / (pos.initial_size * self.pip) if pos.initial_size > 0 else 0 - - date_key = candle.name.date() if hasattr(candle.name, 'date') else None - if date_key: - self.daily_pnl[date_key] = self.daily_pnl.get(date_key, 0.0) + final_pnl - - if self.equity > self.peak_equity: - self.peak_equity = self.equity - - exit_time = candle.name - hold_minutes = 0 - if hasattr(exit_time, 'timestamp') and hasattr(pos.entry_time, 'timestamp'): - hold_minutes = int((exit_time - pos.entry_time).total_seconds() / 60) - - # Build detailed exit reason (e.g. "TP1+TP2+SL" instead of just "SL") - exit_detail = reason - if reason != "TP3": - parts = [] - if pos.tp1_hit: - parts.append("TP1") - if pos.tp2_hit: - parts.append("TP2") - parts.append(reason) - exit_detail = "+".join(parts) - - features = pos.signal_features - record = TradeRecord( - timestamp=pos.entry_time, - strategy_id=pos.strategy_id, - pair=self.pair, - signal_direction=pos.direction, - entry_price=pos.entry_price, - sl_price=pos.sl_price, - tp1_price=pos.tp1_price, - tp2_price=pos.tp2_price, - tp3_price=pos.tp3_price, - lot_size=pos.initial_size, - confluence_score=pos.confluence_score, - session=features.get("session", ""), - spread_at_entry=features.get("spread_at_entry", 0), - atr_at_entry=features.get("atr_at_entry", 0), - adx_at_entry=features.get("adx_at_entry", 0), - rsi_at_entry=features.get("rsi_at_entry", 0), - ema_50_value=features.get("ema_50_value", 0), - ema_200_value=features.get("ema_200_value", 0), - vwap_deviation=features.get("vwap_deviation", 0), - news_within_60min=features.get("news_within_60min", False), - macd_hist_at_entry=features.get("macd_hist_at_entry", 0), - stoch_k_at_entry=features.get("stoch_k_at_entry", 0), - distance_from_ema50_pips=features.get("distance_from_ema50_pips", 0), - candle_body_ratio=features.get("candle_body_ratio", 0), - hour_of_day=features.get("hour_of_day", 0), - day_of_week=features.get("day_of_week", 0), - entry_pattern=features.get("entry_pattern", ""), - exit_price=exit_price, - exit_reason=exit_detail, - exit_time=exit_time, - pnl_pips=total_pnl_pips, - pnl_dollars=total_pnl, - hold_time_minutes=hold_minutes, - win=total_pnl > 0, - ) - self.closed_trades.append(record) + def _get_htf_row(self, timestamp: pd.Timestamp) -> Optional[pd.Series]: + """Get the most recent FULLY CLOSED higher-timeframe candle.""" + if self.htf_data is None: + return None + valid = self.htf_data[self.htf_data.index < timestamp] + if len(valid) == 0: + return None + return valid.iloc[-1] def _build_signal_features(self, candle: pd.Series, i: int) -> dict: """Extract features from current candle for trade logging.""" @@ -447,18 +158,15 @@ class Backtester: def run(self) -> dict: """Run the backtest. Returns performance report dict.""" - # Give the strategy access to full HTF data (strategy filters by timestamp) self.strategy.htf_data = self.htf_data - - # Need at least 200 bars for indicators to warm up warmup = 200 for i in range(warmup, len(self.data)): candle = self.data.iloc[i] timestamp = self.data.index[i] - # Update open positions first (SL/TP/trail/time checks) - self._update_positions(candle, i) + # Update open positions (SL/TP/trail/time checks) + self._pm.update_positions(candle) # Record equity self.equity_curve.append({ @@ -475,83 +183,38 @@ class Backtester: continue # Skip if already have an open position (1 at a time per strategy) - if self.open_positions: + if self._pm.open_positions: continue - # Get HTF context (only fully closed candles) + # Get HTF context htf_row = self._get_htf_row(timestamp) - # Pass full data + current index. Strategy must only access [:i+1]. + # Check for signal signal = self.strategy.check_signal(self.data, i, candle, htf_row) - if signal is None: continue - direction = signal["direction"] - sl = signal["sl"] - tp1 = signal["tp1"] - tp2 = signal["tp2"] - tp3 = signal["tp3"] - confluence = signal.get("confluence", 0) - tp_splits = signal.get("tp_splits", (0.40, 0.40, 0.20)) - trail_mult = signal.get("trail_atr_mult", 1.5) - max_bars = signal.get("max_bars", 200) - no_breakeven = signal.get("no_breakeven", False) - risk_pct = signal.get("risk_pct", 0.01) - - # Minimum RR check (TP1 vs SL distance) - entry = candle["close"] - sl_dist = abs(entry - sl) - tp1_dist = abs(tp1 - entry) - if sl_dist == 0 or tp1_dist / sl_dist < 0.5: - continue - - # Apply spread and slippage to entry - hour = timestamp.hour if hasattr(timestamp, 'hour') else 0 - adj_entry = self._apply_spread_to_entry(entry, direction) - adj_entry = self._apply_slippage_to_entry(adj_entry, direction, hour) - - # Recalculate SL distance after adjustment - sl_dist_adj = abs(adj_entry - sl) - if sl_dist_adj <= 0: - continue - - # Position sizing - size = self._calculate_position_size(sl_dist_adj, confluence, risk_pct) - if size <= 0: - continue - # Build features for logging features = self._build_signal_features(candle, i) features["entry_pattern"] = signal.get("entry_pattern", "") - # Open position - pos = Position( - entry_time=timestamp, - direction=direction, - entry_price=adj_entry, - sl_price=sl, - tp1_price=tp1, - tp2_price=tp2, - tp3_price=tp3, - initial_size=size, - current_size=size, - tp_splits=tp_splits, - trail_atr_mult=trail_mult, - max_bars=max_bars, + risk_pct = signal.get("risk_pct", 0.01) + + # Open position via PositionManager + self._pm.open_position( + signal=signal, + entry_price=candle["close"], + timestamp=timestamp, + atr=candle.get("atr_14", 0), strategy_id=self.strategy.strategy_id, - confluence_score=confluence, signal_features=features, - no_breakeven=no_breakeven, + risk_pct=risk_pct, ) - self.open_positions.append(pos) # Force-close any remaining positions at last candle - if self.open_positions: + if self._pm.open_positions: last_candle = self.data.iloc[-1] - for pos in list(self.open_positions): - self._close_position(pos, last_candle["close"], "END", last_candle) - self.open_positions.clear() + self._pm.force_close_all(last_candle) return self.generate_report() @@ -596,7 +259,7 @@ class Backtester: if len(eq) > 0: peak = eq.cummax() dd = (eq - peak) / peak - max_dd = dd.min() * 100 # negative percentage + max_dd = dd.min() * 100 else: max_dd = 0 @@ -615,10 +278,8 @@ class Backtester: if not r and current_streak > max_consec_losses: max_consec_losses = current_streak - # Average hold time avg_hold = np.mean([t.hold_time_minutes for t in trades]) - # Best/worst trade best_trade_pips = max(pnl_pips) worst_trade_pips = min(pnl_pips) best_trade_dollars = max(pnls) diff --git a/src/live/__init__.py b/src/live/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/live/data_feed.py b/src/live/data_feed.py new file mode 100644 index 0000000..2fbdc24 --- /dev/null +++ b/src/live/data_feed.py @@ -0,0 +1,140 @@ +""" +OANDA candle polling + indicator computation for the live engine. + +Fetches recent candles from OANDA REST API, computes technical indicators, +and exposes them for strategy signal checks. +""" +import os +import time +from datetime import datetime, timezone, timedelta +from pathlib import Path + +import requests +import pandas as pd + +from src.indicators.technical import compute_all_indicators + +# Load OANDA credentials from config/.env +ROOT = Path(__file__).resolve().parent.parent.parent +_env_path = ROOT / "config" / ".env" +if _env_path.exists(): + for line in _env_path.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + os.environ.setdefault(k.strip(), v.strip()) + +API_KEY = os.getenv("OANDA_API_KEY") +OANDA_ENV = os.getenv("OANDA_ENV", "practice") +BASE_URL = ( + "https://api-fxpractice.oanda.com" + if OANDA_ENV == "practice" + else "https://api-fxtrade.oanda.com" +) +HEADERS = {"Authorization": f"Bearer {API_KEY}"} + + +def _fetch_candles(instrument: str, granularity: str, count: int) -> list[dict]: + """Fetch the most recent `count` candles from OANDA.""" + url = f"{BASE_URL}/v3/instruments/{instrument}/candles" + params = { + "granularity": granularity, + "count": count, + "price": "M", # mid prices + } + r = requests.get(url, headers=HEADERS, params=params, timeout=30) + r.raise_for_status() + return r.json().get("candles", []) + + +def _candles_to_df(candles: list[dict]) -> pd.DataFrame: + """Convert OANDA candle JSON to a DataFrame matching our data format.""" + rows = [] + for c in candles: + if not c.get("complete", False): + continue # skip incomplete candles for indicator calculation + mid = c["mid"] + rows.append({ + "timestamp": c["time"], + "open": float(mid["o"]), + "high": float(mid["h"]), + "low": float(mid["l"]), + "close": float(mid["c"]), + "volume": c.get("volume", 0), + }) + if not rows: + return pd.DataFrame() + df = pd.DataFrame(rows) + df["timestamp"] = pd.to_datetime(df["timestamp"]) + df = df.drop_duplicates(subset=["timestamp"], keep="last") + df = df.set_index("timestamp").sort_index() + df.index.name = "timestamp" + return df + + +class OandaDataFeed: + """Polls OANDA for latest candles and computes indicators. + + Args: + pair: OANDA instrument name (e.g. "GBP_JPY"). + granularity: Candle granularity (e.g. "H1", "M15"). + candle_count: How many candles to fetch (enough for indicator warmup). + """ + + def __init__(self, pair: str, granularity: str = "H1", + candle_count: int = 250): + self.pair = pair + self.granularity = granularity + self.candle_count = candle_count + self._cached_df: pd.DataFrame = pd.DataFrame() + self._last_fetch_time: float = 0 + + def fetch_latest(self) -> pd.DataFrame: + """Fetch latest candles and compute all indicators. + + Returns DataFrame with OHLCV + all indicator columns. + Only includes fully closed candles. + """ + candles = _fetch_candles(self.pair, self.granularity, self.candle_count) + df = _candles_to_df(candles) + if df.empty: + return self._cached_df + + df = compute_all_indicators(df) + self._cached_df = df + self._last_fetch_time = time.time() + return df + + def get_current_candle(self) -> pd.Series: + """Return the most recent fully closed candle with indicators.""" + if self._cached_df.empty: + self.fetch_latest() + if self._cached_df.empty: + return pd.Series() + return self._cached_df.iloc[-1] + + def get_htf_row(self, timestamp=None) -> pd.Series: + """Get the most recent fully closed candle as HTF context. + + For H1 strategies where HTF=H1 (self-reference), this is just + the latest closed candle. For M15 strategies needing H1 context, + a separate OandaDataFeed for H1 should be used. + """ + if self._cached_df.empty: + return pd.Series() + if timestamp is not None: + valid = self._cached_df[self._cached_df.index < timestamp] + if valid.empty: + return pd.Series() + return valid.iloc[-1] + return self._cached_df.iloc[-1] + + @property + def is_stale(self) -> bool: + """True if data hasn't been fetched recently.""" + if self._last_fetch_time == 0: + return True + # H1 = stale after 65 min, M15 = stale after 20 min + gran_minutes = {"H1": 65, "M15": 20, "M5": 8} + stale_seconds = gran_minutes.get(self.granularity, 65) * 60 + return (time.time() - self._last_fetch_time) > stale_seconds diff --git a/src/live/engine.py b/src/live/engine.py new file mode 100644 index 0000000..3109fb7 --- /dev/null +++ b/src/live/engine.py @@ -0,0 +1,281 @@ +""" +Live trading engine orchestrator. + +Ties together data feeds, strategies, position managers, and the executor +into a single run loop that polls OANDA and manages positions. +""" +import json +import time +import os +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import pandas as pd + +from src.position_manager import PositionManager, PIP_SIZE, SPREAD_PIPS +from src.live.data_feed import OandaDataFeed +from src.live.executor import OandaExecutor + +ROOT = Path(__file__).resolve().parent.parent.parent +LOGS_DIR = ROOT / "logs" +LOGS_DIR.mkdir(exist_ok=True) + + +@dataclass +class StrategySlot: + """One strategy-pair combination running in the live engine.""" + name: str + strategy: object # BaseStrategy instance + pair: str + timeframe: str + data_feed: OandaDataFeed + position_manager: PositionManager + htf_feed: Optional[OandaDataFeed] = None # for multi-TF strategies + enabled: bool = True + + +class LiveEngine: + """Main orchestrator for live paper/live trading. + + Each cycle: + 1. Check kill switch + 2. Fetch latest candles for each slot + 3. Compute indicators + 4. Check for new signals (only if no open position) + 5. Update open positions (SL/TP/trail/time) + 6. Log state + 7. Print status summary + """ + + def __init__(self, slots: list[StrategySlot], executor: OandaExecutor, + interval_seconds: int = 60, max_daily_drawdown_pct: float = 5.0): + self.slots = slots + self.executor = executor + self.interval_seconds = interval_seconds + self.max_daily_drawdown_pct = max_daily_drawdown_pct + self._state_path = LOGS_DIR / "live_state.json" + self._trades_path = LOGS_DIR / "live_trades.csv" + self._cycle_count = 0 + + def run_once(self): + """Single iteration: fetch, check signals, manage positions.""" + self._cycle_count += 1 + now = datetime.now(timezone.utc) + print(f"\n{'='*70}") + print(f"Cycle #{self._cycle_count} | {now.strftime('%Y-%m-%d %H:%M:%S UTC')}") + print(f"{'='*70}") + + # 1. Kill switch + if self.executor.check_kill_switch(): + print("KILL SWITCH ACTIVE — skipping all trading.") + return + + for slot in self.slots: + if not slot.enabled: + continue + + slot_label = f"[{slot.name}/{slot.pair}]" + + try: + # 2. Fetch latest candles + df = slot.data_feed.fetch_latest() + if df.empty: + print(f" {slot_label} No data fetched, skipping.") + continue + + htf_df = None + if slot.htf_feed is not None: + htf_df = slot.htf_feed.fetch_latest() + + # 3. Current candle + HTF row + current = df.iloc[-1] + idx = len(df) - 1 + timestamp = df.index[-1] + + htf_row = None + if htf_df is not None and not htf_df.empty: + valid = htf_df[htf_df.index < timestamp] + if not valid.empty: + htf_row = valid.iloc[-1] + elif slot.timeframe == "H1": + # H1 self-reference for HTF + valid = df[df.index < timestamp] + if not valid.empty: + htf_row = valid.iloc[-1] + + # 4. Update existing positions + pm = slot.position_manager + closed = pm.update_positions(current) + for trade in closed: + self._log_closed_trade(trade, slot.name) + print(f" {slot_label} CLOSED: {trade.signal_direction} " + f"exit={trade.exit_reason} pnl={trade.pnl_pips:+.1f}p") + + # 5. Check for new signal (only if no open position) + if not pm.get_open_positions(): + # Daily drawdown check + today = now.date() + daily_pnl = pm.daily_pnl.get(today, 0.0) + dd_limit = pm.starting_equity * self.max_daily_drawdown_pct / 100 + if daily_pnl <= -dd_limit: + print(f" {slot_label} Daily DD limit hit ({daily_pnl:.0f}), skipping signals.") + continue + + # Give strategy access to HTF data + if htf_df is not None: + slot.strategy.htf_data = htf_df + + signal = slot.strategy.check_signal(df, idx, current, htf_row) + + if signal is not None: + atr = current.get("atr_14", 0) + pos = pm.open_position( + signal=signal, + entry_price=current["close"], + timestamp=timestamp, + atr=atr, + strategy_id=slot.strategy.strategy_id, + signal_features={"entry_pattern": signal.get("entry_pattern", "")}, + ) + if pos is not None: + print(f" {slot_label} NEW {pos.direction} @ {pos.entry_price:.5f} " + f"SL={pos.sl_price:.5f} TP1={pos.tp1_price:.5f}") + + # Log to executor + self.executor.place_market_order( + pair=slot.pair, + units=int(pos.initial_size), + direction=pos.direction, + sl=pos.sl_price, + tp=pos.tp1_price, + ) + else: + print(f" {slot_label} No signal.") + else: + open_pos = pm.get_open_positions()[0] + print(f" {slot_label} Open: {open_pos.direction} " + f"bars={open_pos.bars_held} " + f"tp1={'Y' if open_pos.tp1_hit else 'N'} " + f"tp2={'Y' if open_pos.tp2_hit else 'N'}") + + except Exception as e: + print(f" {slot_label} ERROR: {e}") + + # 6. Save state + self.save_state() + + # 7. Summary + self._print_summary() + + def run_loop(self): + """Continuous loop with interval sleep.""" + print(f"Starting live engine loop (interval={self.interval_seconds}s)") + print(f"Slots: {len(self.slots)}") + for s in self.slots: + print(f" - {s.name} / {s.pair} / {s.timeframe}") + print() + + while True: + try: + self.run_once() + print(f"\nSleeping {self.interval_seconds}s...") + time.sleep(self.interval_seconds) + except KeyboardInterrupt: + print("\nShutting down gracefully.") + break + + def save_state(self): + """Persist current state to JSON for crash recovery.""" + state = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "cycle": self._cycle_count, + "slots": [], + } + for slot in self.slots: + pm = slot.position_manager + slot_state = { + "name": slot.name, + "pair": slot.pair, + "equity": pm.get_equity(), + "open_positions": len(pm.get_open_positions()), + "closed_trades": len(pm.closed_trades), + "enabled": slot.enabled, + } + if pm.get_open_positions(): + pos = pm.get_open_positions()[0] + slot_state["position"] = { + "direction": pos.direction, + "entry_price": pos.entry_price, + "sl": pos.sl_price, + "bars_held": pos.bars_held, + "tp1_hit": pos.tp1_hit, + "tp2_hit": pos.tp2_hit, + } + state["slots"].append(slot_state) + + with open(self._state_path, "w") as f: + json.dump(state, f, indent=2) + + def load_state(self): + """Restore state after restart (basic — restores equity only).""" + if not self._state_path.exists(): + return + try: + with open(self._state_path) as f: + state = json.load(f) + for slot_state in state.get("slots", []): + for slot in self.slots: + if slot.name == slot_state["name"] and slot.pair == slot_state["pair"]: + slot.position_manager.equity = slot_state.get("equity", + slot.position_manager.equity) + print(f"Loaded state from {self._state_path}") + except Exception as e: + print(f"Warning: Could not load state: {e}") + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _log_closed_trade(self, trade, slot_name: str): + """Append a closed trade record to live_trades.csv.""" + columns = [ + "slot", "timestamp", "pair", "direction", "entry_price", + "exit_price", "exit_reason", "pnl_pips", "pnl_dollars", + "hold_minutes", + ] + row = { + "slot": slot_name, + "timestamp": str(trade.timestamp), + "pair": trade.pair, + "direction": trade.signal_direction, + "entry_price": trade.entry_price, + "exit_price": trade.exit_price, + "exit_reason": trade.exit_reason, + "pnl_pips": round(trade.pnl_pips, 2), + "pnl_dollars": round(trade.pnl_dollars, 2), + "hold_minutes": trade.hold_time_minutes, + } + import csv + file_exists = self._trades_path.exists() + with open(self._trades_path, "a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns) + if not file_exists: + writer.writeheader() + writer.writerow(row) + + def _print_summary(self): + """Print a compact status summary.""" + print(f"\n{'─'*50}") + print(f"{'Slot':<18} {'Equity':>10} {'Open':>5} {'Trades':>7}") + print(f"{'─'*50}") + for slot in self.slots: + if not slot.enabled: + continue + pm = slot.position_manager + label = f"{slot.name}/{slot.pair}" + print(f"{label:<18} {pm.get_equity():>10,.0f} " + f"{len(pm.get_open_positions()):>5} " + f"{len(pm.closed_trades):>7}") + print(f"{'─'*50}") diff --git a/src/live/executor.py b/src/live/executor.py new file mode 100644 index 0000000..30bd949 --- /dev/null +++ b/src/live/executor.py @@ -0,0 +1,204 @@ +""" +Paper/live order execution for the live engine. + +Thin wrapper around OANDA v20 order API. Reuses auth patterns from +src/order_executor.py but stripped of Supabase/old strategy coupling. +""" +import os +import csv +import time +from datetime import datetime, timezone +from pathlib import Path + +import requests + +# Project root +ROOT = Path(__file__).resolve().parent.parent.parent + +# Load .env +_env_path = ROOT / "config" / ".env" +if _env_path.exists(): + for line in _env_path.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + os.environ.setdefault(k.strip(), v.strip()) + + +def _oanda_base_url() -> str: + env = os.getenv("OANDA_ENV", "practice").strip() + if env == "practice": + return "https://api-fxpractice.oanda.com" + return "https://api-fxtrade.oanda.com" + + +def _oanda_headers() -> dict: + api_key = os.getenv("OANDA_API_KEY") + return { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + +class OandaExecutor: + """Executes orders in paper or live mode. + + Paper mode: logs trades to CSV, tracks positions locally. + Live mode: POSTs to OANDA v20 REST API with SL/TP attached. + """ + + def __init__(self, paper_mode: bool = True): + self.paper_mode = paper_mode + self._logs_dir = ROOT / "logs" + self._logs_dir.mkdir(exist_ok=True) + self._csv_path = self._logs_dir / "paper_trades.csv" + self._account_id = os.getenv("OANDA_ACCOUNT_ID", "") + + def check_kill_switch(self) -> bool: + """Check for STOP_ALL_TRADING file in project root.""" + return (ROOT / "STOP_ALL_TRADING").exists() + + def place_market_order(self, pair: str, units: int, direction: str, + sl: float = None, tp: float = None) -> dict: + """Place a market order with optional SL/TP. + + Args: + pair: Instrument (e.g. "GBP_JPY"). + units: Position size in units. + direction: "LONG" or "SHORT". + sl: Stop loss price. + tp: Take profit price. + + Returns dict with status and details. + """ + if self.check_kill_switch(): + self._log_trade(pair, direction, units, 0, "BLOCKED_KILL_SWITCH", sl, tp) + return {"status": "BLOCKED_KILL_SWITCH"} + + signed_units = units if direction == "LONG" else -units + + if self.paper_mode: + now = datetime.now(timezone.utc).isoformat() + self._log_trade(pair, direction, units, 0, "FILLED_PAPER", sl, tp) + return { + "status": "FILLED", + "mode": "paper", + "units": signed_units, + "sl": sl, + "tp": tp, + } + + # Live mode + return self._place_live_order(pair, signed_units, sl, tp) + + def modify_trade(self, trade_id: str, sl: float = None, + tp: float = None) -> dict: + """Modify SL/TP on an existing OANDA trade.""" + if self.paper_mode: + return {"status": "MODIFIED_PAPER", "trade_id": trade_id} + + base = _oanda_base_url() + url = f"{base}/v3/accounts/{self._account_id}/trades/{trade_id}/orders" + body = {} + if sl is not None: + body["stopLoss"] = {"price": f"{sl:.5f}", "timeInForce": "GTC"} + if tp is not None: + body["takeProfit"] = {"price": f"{tp:.5f}", "timeInForce": "GTC"} + + r = requests.put(url, headers=_oanda_headers(), json=body, timeout=10) + return {"status": "OK" if r.status_code == 200 else f"ERROR_{r.status_code}", + "response": r.json()} + + def close_trade(self, trade_id: str) -> dict: + """Close a specific OANDA trade.""" + if self.paper_mode: + return {"status": "CLOSED_PAPER", "trade_id": trade_id} + + base = _oanda_base_url() + url = f"{base}/v3/accounts/{self._account_id}/trades/{trade_id}/close" + r = requests.put(url, headers=_oanda_headers(), timeout=10) + return {"status": "OK" if r.status_code == 200 else f"ERROR_{r.status_code}", + "response": r.json()} + + def get_open_trades(self) -> list: + """Get all open trades from OANDA.""" + if self.paper_mode: + return [] # paper trades tracked by PositionManager + + base = _oanda_base_url() + url = f"{base}/v3/accounts/{self._account_id}/openTrades" + r = requests.get(url, headers=_oanda_headers(), timeout=10) + r.raise_for_status() + return r.json().get("trades", []) + + def get_account_summary(self) -> dict: + """Get account balance, NAV, open position count.""" + base = _oanda_base_url() + url = f"{base}/v3/accounts/{self._account_id}/summary" + r = requests.get(url, headers=_oanda_headers(), timeout=10) + r.raise_for_status() + acct = r.json()["account"] + return { + "balance": float(acct["balance"]), + "NAV": float(acct["NAV"]), + "open_trade_count": int(acct["openTradeCount"]), + } + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _place_live_order(self, pair: str, signed_units: int, + sl: float, tp: float) -> dict: + base = _oanda_base_url() + url = f"{base}/v3/accounts/{self._account_id}/orders" + + order = { + "type": "MARKET", + "instrument": pair, + "units": str(signed_units), + "timeInForce": "FOK", + "positionFill": "DEFAULT", + } + if sl is not None: + order["stopLossOnFill"] = {"price": f"{sl:.5f}", "timeInForce": "GTC"} + if tp is not None: + order["takeProfitOnFill"] = {"price": f"{tp:.5f}", "timeInForce": "GTC"} + + r = requests.post(url, headers=_oanda_headers(), + json={"order": order}, timeout=10) + resp = r.json() + status = "FILLED" if r.status_code == 201 else f"ERROR_{r.status_code}" + direction = "LONG" if signed_units > 0 else "SHORT" + + fill_price = resp.get("orderFillTransaction", {}).get("price", 0) + self._log_trade(pair, direction, abs(signed_units), fill_price, status, sl, tp) + + return {"status": status, "mode": "live", "response": resp} + + def _log_trade(self, pair: str, direction: str, units: int, + price: float, status: str, sl: float, tp: float): + """Append trade to paper_trades.csv.""" + columns = [ + "timestamp", "pair", "direction", "units", "price", + "sl", "tp", "status", "mode", + ] + now = datetime.now(timezone.utc).isoformat() + row = { + "timestamp": now, + "pair": pair, + "direction": direction, + "units": units, + "price": price, + "sl": sl or "", + "tp": tp or "", + "status": status, + "mode": "paper" if self.paper_mode else "live", + } + + file_exists = self._csv_path.exists() + with open(self._csv_path, "a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns) + if not file_exists: + writer.writeheader() + writer.writerow(row) diff --git a/src/live/run.py b/src/live/run.py new file mode 100644 index 0000000..58e2589 --- /dev/null +++ b/src/live/run.py @@ -0,0 +1,150 @@ +""" +Entry point for the Phase 2 live paper trading engine. + +Usage: + python src/live/run.py # continuous loop + python src/live/run.py --once # single iteration (for testing) +""" +import os +import sys +import yaml + +# Ensure project root on path +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, ROOT) + +from src.position_manager import PositionManager +from src.live.data_feed import OandaDataFeed +from src.live.executor import OandaExecutor +from src.live.engine import LiveEngine, StrategySlot + +# Strategy imports +from src.strategies_pkg.s7_liquidity_sweep import S7_Liquidity_Sweep +from src.strategies_pkg.s9_london_session import S9_London_Session +from src.strategies_pkg.s4f_ema_ribbon import S4F_EMA_Ribbon +from src.strategies_pkg.s3_key_level_breakout import S3_KeyLevel_Breakout + + +def load_config() -> dict: + """Load Phase 2 config from system.yaml.""" + cfg_path = os.path.join(ROOT, "config", "system.yaml") + with open(cfg_path) as f: + cfg = yaml.safe_load(f) + return cfg.get("phase2", {}) + + +def build_slots(config: dict) -> list[StrategySlot]: + """Build StrategySlot instances from config.""" + starting_equity = config.get("starting_equity", 100_000) + slots = [] + + # Strategy factory + strategy_map = config.get("strategies", []) + for entry in strategy_map: + if not entry.get("enabled", True): + continue + + name = entry["name"] + pair = entry["pair"] + tf = entry.get("timeframe", "H1") + htf_tf = entry.get("htf_timeframe", None) + + # Create strategy instance + if name == "S7_Tight": + strategy = S7_Liquidity_Sweep() + elif name == "S9": + strategy = S9_London_Session() + elif name == "S9_Filtered": + strategy = S9_London_Session(pair=pair, filtered=True) + elif name == "S4F": + strategy = S4F_EMA_Ribbon() + elif name == "S3": + strategy = S3_KeyLevel_Breakout() + else: + print(f"Unknown strategy: {name}, skipping.") + continue + + # Data feeds + data_feed = OandaDataFeed(pair=pair, granularity=tf, candle_count=250) + htf_feed = None + if htf_tf and htf_tf != tf: + htf_feed = OandaDataFeed(pair=pair, granularity=htf_tf, candle_count=250) + + # Position manager (each slot gets its own) + pm = PositionManager(pair=pair, starting_equity=starting_equity) + + slot = StrategySlot( + name=name, + strategy=strategy, + pair=pair, + timeframe=tf, + data_feed=data_feed, + position_manager=pm, + htf_feed=htf_feed, + enabled=True, + ) + slots.append(slot) + + return slots + + +def main(): + config = load_config() + if not config: + print("ERROR: No phase2 config found in config/system.yaml") + sys.exit(1) + + paper_mode = config.get("paper_mode", True) + interval = config.get("poll_interval_seconds", 60) + max_dd = config.get("max_daily_drawdown_pct", 5.0) + + # Check OANDA credentials + env_path = os.path.join(ROOT, "config", ".env") + if os.path.exists(env_path): + for line in open(env_path).read().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + os.environ.setdefault(k.strip(), v.strip()) + + account_id = os.getenv("OANDA_ACCOUNT_ID", "") + if not account_id and not paper_mode: + print("ERROR: OANDA_ACCOUNT_ID not set in config/.env") + print("Paper trading can run without it, but live mode requires it.") + sys.exit(1) + + if not account_id: + print("WARNING: OANDA_ACCOUNT_ID empty — paper mode only, " + "data feeds will still work for signal testing.") + + # Build components + executor = OandaExecutor(paper_mode=paper_mode) + slots = build_slots(config) + + if not slots: + print("ERROR: No strategy slots configured.") + sys.exit(1) + + print(f"Phase 2 Live Engine") + print(f" Mode: {'PAPER' if paper_mode else 'LIVE'}") + print(f" Slots: {len(slots)}") + print(f" Poll interval: {interval}s") + print(f" Max daily DD: {max_dd}%") + print() + + engine = LiveEngine( + slots=slots, + executor=executor, + interval_seconds=interval, + max_daily_drawdown_pct=max_dd, + ) + engine.load_state() + + if "--once" in sys.argv: + engine.run_once() + else: + engine.run_loop() + + +if __name__ == "__main__": + main() diff --git a/src/position_manager.py b/src/position_manager.py new file mode 100644 index 0000000..08a8854 --- /dev/null +++ b/src/position_manager.py @@ -0,0 +1,403 @@ +""" +Shared position management for backtester and live engine. + +Extracted from backtester/engine.py to allow both the backtest engine +and the live trading engine to share the same position update logic. +""" +import numpy as np +import pandas as pd +from dataclasses import dataclass, field +from typing import Optional + + +# --------------------------------------------------------------------------- +# Spread Configuration (in pips) +# --------------------------------------------------------------------------- +SPREAD_PIPS = { + "EUR_USD": 1.5, "GBP_USD": 1.5, + "GBP_AUD": 2.0, "EUR_AUD": 2.0, + "EUR_GBP": 2.0, "GBP_JPY": 2.5, + "USD_JPY": 1.5, "GBP_CAD": 2.5, + "EUR_CAD": 2.5, "EUR_NZD": 2.5, + "GBP_NZD": 2.5, +} + +# Pip value per pair +PIP_SIZE = { + "EUR_USD": 0.0001, "GBP_USD": 0.0001, "EUR_AUD": 0.0001, + "GBP_AUD": 0.0001, "EUR_GBP": 0.0001, "GBP_CAD": 0.0001, + "EUR_CAD": 0.0001, "EUR_NZD": 0.0001, + "GBP_NZD": 0.0001, + "USD_JPY": 0.01, "GBP_JPY": 0.01, +} + + +@dataclass +class Position: + """Represents an open position.""" + entry_time: pd.Timestamp + direction: str # 'LONG' or 'SHORT' + entry_price: float + sl_price: float + tp1_price: float + tp2_price: float + tp3_price: float + initial_size: float + current_size: float + tp_splits: tuple # e.g. (0.40, 0.40, 0.20) + trail_atr_mult: float + max_bars: int + bars_held: int = 0 + tp1_hit: bool = False + tp2_hit: bool = False + tp3_hit: bool = False + trailing_sl: Optional[float] = None + strategy_id: int = 0 + confluence_score: int = 0 + signal_features: dict = field(default_factory=dict) + realized_pnl: float = 0.0 + no_breakeven: bool = False + + +@dataclass +class TradeRecord: + """Rich trade log entry for Phase 3 feature engineering.""" + timestamp: pd.Timestamp = None + strategy_id: int = 0 + pair: str = "" + signal_direction: str = "" + entry_price: float = 0.0 + sl_price: float = 0.0 + tp1_price: float = 0.0 + tp2_price: float = 0.0 + tp3_price: float = 0.0 + lot_size: float = 0.0 + confluence_score: int = 0 + session: str = "" + spread_at_entry: float = 0.0 + atr_at_entry: float = 0.0 + adx_at_entry: float = 0.0 + rsi_at_entry: float = 0.0 + ema_50_value: float = 0.0 + ema_200_value: float = 0.0 + vwap_deviation: float = 0.0 + news_within_60min: bool = False + macd_hist_at_entry: float = 0.0 + stoch_k_at_entry: float = 0.0 + distance_from_ema50_pips: float = 0.0 + candle_body_ratio: float = 0.0 + hour_of_day: int = 0 + day_of_week: int = 0 + entry_pattern: str = "" + exit_price: float = 0.0 + exit_reason: str = "" + exit_time: pd.Timestamp = None + pnl_pips: float = 0.0 + pnl_dollars: float = 0.0 + hold_time_minutes: int = 0 + win: bool = False + + +class PositionManager: + """Manages open positions and produces TradeRecords on close. + + Used by both the Backtester and the LiveEngine so that position + update logic (SL/TP/trail/time exits, partial closes) is shared. + """ + + def __init__(self, pair: str, starting_equity: float = 100_000.0, + pip_size: float = None, spread_pips: float = None): + self.pair = pair + self.equity = starting_equity + self.starting_equity = starting_equity + self.peak_equity = starting_equity + + self.pip = pip_size if pip_size is not None else PIP_SIZE.get(pair, 0.0001) + spread_p = spread_pips if spread_pips is not None else SPREAD_PIPS.get(pair, 2.0) + self.spread = spread_p * self.pip + + self.open_positions: list[Position] = [] + self.closed_trades: list[TradeRecord] = [] + self.daily_pnl: dict = {} + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def open_position(self, signal: dict, entry_price: float, + timestamp: pd.Timestamp, atr: float, + strategy_id: int = 0, signal_features: dict = None, + risk_pct: float = 0.01) -> Optional[Position]: + """Create and register a new Position from a strategy signal. + + Returns the Position or None if sizing fails. + """ + direction = signal["direction"] + sl = signal["sl"] + + # Apply spread + if direction == "LONG": + adj_entry = entry_price + self.spread + else: + adj_entry = entry_price - self.spread + + # Apply slippage + hour = timestamp.hour if hasattr(timestamp, 'hour') else 0 + slip = 0.4 * self.pip if 8 <= hour < 17 else 1.0 * self.pip + if direction == "LONG": + adj_entry += slip + else: + adj_entry -= slip + + sl_dist = abs(adj_entry - sl) + if sl_dist <= 0: + return None + + # Minimum RR check + tp1_dist = abs(signal["tp1"] - adj_entry) + if tp1_dist / sl_dist < 0.5: + return None + + # Position sizing + size = self._calculate_position_size(sl_dist, risk_pct) + if size <= 0: + return None + + tp_splits = signal.get("tp_splits", (0.40, 0.40, 0.20)) + trail_mult = signal.get("trail_atr_mult", 1.5) + max_bars = signal.get("max_bars", 200) + no_breakeven = signal.get("no_breakeven", False) + confluence = signal.get("confluence", 0) + + pos = Position( + entry_time=timestamp, + direction=direction, + entry_price=adj_entry, + sl_price=sl, + tp1_price=signal["tp1"], + tp2_price=signal["tp2"], + tp3_price=signal["tp3"], + initial_size=size, + current_size=size, + tp_splits=tp_splits, + trail_atr_mult=trail_mult, + max_bars=max_bars, + strategy_id=strategy_id, + confluence_score=confluence, + signal_features=signal_features or {}, + no_breakeven=no_breakeven, + ) + self.open_positions.append(pos) + return pos + + def update_positions(self, candle: pd.Series) -> list[TradeRecord]: + """Check SL/TP/trailing/time exits for all open positions. + + Returns list of TradeRecords for positions that were fully closed. + """ + to_close = [] + new_records_start = len(self.closed_trades) + + for pos in self.open_positions: + pos.bars_held += 1 + high = candle["high"] + low = candle["low"] + close_price = candle["close"] + current_atr = candle.get("atr_14", 0) + + effective_sl = pos.trailing_sl if pos.trailing_sl is not None else pos.sl_price + + if pos.direction == "LONG": + if low <= effective_sl: + self._close_position(pos, effective_sl, "SL", candle) + to_close.append(pos) + continue + + if not pos.tp1_hit and high >= pos.tp1_price: + close_size = pos.initial_size * pos.tp_splits[0] + self._partial_close(pos, pos.tp1_price, close_size, "TP1", candle) + pos.tp1_hit = True + if not pos.no_breakeven: + pos.trailing_sl = pos.entry_price + + if not pos.tp2_hit and pos.tp1_hit and high >= pos.tp2_price: + close_size = pos.initial_size * pos.tp_splits[1] + if close_size > 0: + self._partial_close(pos, pos.tp2_price, close_size, "TP2", candle) + pos.tp2_hit = True + if pos.trailing_sl is None: + pos.trailing_sl = pos.sl_price + + if not pos.tp3_hit and pos.tp2_hit and high >= pos.tp3_price: + self._close_position(pos, pos.tp3_price, "TP3", candle) + to_close.append(pos) + continue + + if pos.tp2_hit and current_atr > 0: + new_trail = high - pos.trail_atr_mult * current_atr + if pos.trailing_sl is None or new_trail > pos.trailing_sl: + pos.trailing_sl = new_trail + + else: # SHORT + if high >= effective_sl: + self._close_position(pos, effective_sl, "SL", candle) + to_close.append(pos) + continue + + if not pos.tp1_hit and low <= pos.tp1_price: + close_size = pos.initial_size * pos.tp_splits[0] + self._partial_close(pos, pos.tp1_price, close_size, "TP1", candle) + pos.tp1_hit = True + if not pos.no_breakeven: + pos.trailing_sl = pos.entry_price + + if not pos.tp2_hit and pos.tp1_hit and low <= pos.tp2_price: + close_size = pos.initial_size * pos.tp_splits[1] + if close_size > 0: + self._partial_close(pos, pos.tp2_price, close_size, "TP2", candle) + pos.tp2_hit = True + if pos.trailing_sl is None: + pos.trailing_sl = pos.sl_price + + if not pos.tp3_hit and pos.tp2_hit and low <= pos.tp3_price: + self._close_position(pos, pos.tp3_price, "TP3", candle) + to_close.append(pos) + continue + + if pos.tp2_hit and current_atr > 0: + new_trail = low + pos.trail_atr_mult * current_atr + if pos.trailing_sl is None or new_trail < pos.trailing_sl: + pos.trailing_sl = new_trail + + if pos.max_bars > 0 and pos.bars_held >= pos.max_bars: + self._close_position(pos, close_price, "TIME", candle) + to_close.append(pos) + continue + + for pos in to_close: + if pos in self.open_positions: + self.open_positions.remove(pos) + + return self.closed_trades[new_records_start:] + + def get_open_positions(self) -> list[Position]: + return list(self.open_positions) + + def get_equity(self) -> float: + return self.equity + + def force_close_all(self, candle: pd.Series) -> list[TradeRecord]: + """Force-close all open positions at candle close price.""" + new_records_start = len(self.closed_trades) + for pos in list(self.open_positions): + self._close_position(pos, candle["close"], "END", candle) + self.open_positions.clear() + return self.closed_trades[new_records_start:] + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _calculate_position_size(self, sl_distance: float, + risk_pct: float = 0.01) -> float: + if sl_distance <= 0: + return 0.0 + risk_amount = self.equity * risk_pct + return round(risk_amount / sl_distance, 2) + + def _partial_close(self, pos: Position, exit_price: float, + close_size: float, reason: str, candle: pd.Series): + if pos.direction == "LONG": + pnl_per_unit = exit_price - pos.entry_price + else: + pnl_per_unit = pos.entry_price - exit_price + + pnl = pnl_per_unit * close_size + self.equity += pnl + pos.current_size -= close_size + pos.realized_pnl += pnl + + date_key = candle.name.date() if hasattr(candle.name, 'date') else None + if date_key: + self.daily_pnl[date_key] = self.daily_pnl.get(date_key, 0.0) + pnl + + if self.equity > self.peak_equity: + self.peak_equity = self.equity + + def _close_position(self, pos: Position, exit_price: float, + reason: str, candle: pd.Series): + remaining = pos.current_size + if remaining <= 0: + remaining = 0.01 + + if pos.direction == "LONG": + pnl_per_unit = exit_price - pos.entry_price + else: + pnl_per_unit = pos.entry_price - exit_price + + final_pnl = pnl_per_unit * remaining + self.equity += final_pnl + + total_pnl = pos.realized_pnl + final_pnl + total_pnl_pips = total_pnl / (pos.initial_size * self.pip) if pos.initial_size > 0 else 0 + + date_key = candle.name.date() if hasattr(candle.name, 'date') else None + if date_key: + self.daily_pnl[date_key] = self.daily_pnl.get(date_key, 0.0) + final_pnl + + if self.equity > self.peak_equity: + self.peak_equity = self.equity + + exit_time = candle.name + hold_minutes = 0 + if hasattr(exit_time, 'timestamp') and hasattr(pos.entry_time, 'timestamp'): + hold_minutes = int((exit_time - pos.entry_time).total_seconds() / 60) + + exit_detail = reason + if reason != "TP3": + parts = [] + if pos.tp1_hit: + parts.append("TP1") + if pos.tp2_hit: + parts.append("TP2") + parts.append(reason) + exit_detail = "+".join(parts) + + features = pos.signal_features + record = TradeRecord( + timestamp=pos.entry_time, + strategy_id=pos.strategy_id, + pair=self.pair, + signal_direction=pos.direction, + entry_price=pos.entry_price, + sl_price=pos.sl_price, + tp1_price=pos.tp1_price, + tp2_price=pos.tp2_price, + tp3_price=pos.tp3_price, + lot_size=pos.initial_size, + confluence_score=pos.confluence_score, + session=features.get("session", ""), + spread_at_entry=features.get("spread_at_entry", 0), + atr_at_entry=features.get("atr_at_entry", 0), + adx_at_entry=features.get("adx_at_entry", 0), + rsi_at_entry=features.get("rsi_at_entry", 0), + ema_50_value=features.get("ema_50_value", 0), + ema_200_value=features.get("ema_200_value", 0), + vwap_deviation=features.get("vwap_deviation", 0), + news_within_60min=features.get("news_within_60min", False), + macd_hist_at_entry=features.get("macd_hist_at_entry", 0), + stoch_k_at_entry=features.get("stoch_k_at_entry", 0), + distance_from_ema50_pips=features.get("distance_from_ema50_pips", 0), + candle_body_ratio=features.get("candle_body_ratio", 0), + hour_of_day=features.get("hour_of_day", 0), + day_of_week=features.get("day_of_week", 0), + entry_pattern=features.get("entry_pattern", ""), + exit_price=exit_price, + exit_reason=exit_detail, + exit_time=exit_time, + pnl_pips=total_pnl_pips, + pnl_dollars=total_pnl, + hold_time_minutes=hold_minutes, + win=total_pnl > 0, + ) + self.closed_trades.append(record) diff --git a/src/run_correlation_analysis.py b/src/run_correlation_analysis.py new file mode 100644 index 0000000..ac18e4f --- /dev/null +++ b/src/run_correlation_analysis.py @@ -0,0 +1,313 @@ +""" +Phase 2 — Correlation Analysis (Step 8). + +For strategies sharing a pair (S7_Tight + S3 on GBP_JPY): +- Compute signal overlap and simultaneous position frequency. +- Combined equity curve analysis. + +Also compute portfolio-level metrics: combined PF, combined max DD, +Sharpe of combined equity curve. + +Output: results/phase2/correlation_analysis.json +""" +import os, sys, io, json, time +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +import pandas as pd +import numpy as np +from src.indicators.technical import compute_all_indicators +from src.backtester.engine import Backtester + +# Strategy imports +from src.strategies_pkg.s7_liquidity_sweep import S7_Liquidity_Sweep +from src.strategies_pkg.s9_london_session import S9_London_Session +from src.strategies_pkg.s4f_ema_ribbon import S4F_EMA_Ribbon +from src.strategies_pkg.s3_key_level_breakout import S3_KeyLevel_Breakout + +PROCESSED_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "processed") +RESULTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "results", "phase2") +os.makedirs(RESULTS_DIR, exist_ok=True) + +# All Phase 2 strategies +CONFIGS = [ + {"name": "S7_Tight", "pair": "GBP_JPY", "tf": "H1", + "factory": lambda: S7_Liquidity_Sweep()}, + {"name": "S9", "pair": "GBP_USD", "tf": "H1", + "factory": lambda: S9_London_Session()}, + {"name": "S9_Filtered", "pair": "GBP_AUD", "tf": "H1", + "factory": lambda: S9_London_Session(pair="GBP_AUD", filtered=True)}, + {"name": "S4F", "pair": "EUR_AUD", "tf": "M15", + "factory": lambda: S4F_EMA_Ribbon()}, + {"name": "S3", "pair": "GBP_JPY", "tf": "H1", + "factory": lambda: S3_KeyLevel_Breakout()}, +] + + +def load_data(pair, tf): + fp = os.path.join(PROCESSED_DIR, f"{pair}_{tf}.csv") + if not os.path.exists(fp): + return None + df = pd.read_csv(fp, index_col=0, parse_dates=True) + df.index.name = "timestamp" + return compute_all_indicators(df) + + +def run_backtest(cfg): + """Run backtest for a config, return trade log and equity curve.""" + pair = cfg["pair"] + tf = cfg["tf"] + data = load_data(pair, tf) + if data is None: + return None, None, None + + htf_data = data.copy() if tf == "H1" else load_data(pair, "H1") + + strategy = cfg["factory"]() + bt = Backtester(data=data, strategy=strategy, pair=pair, + starting_equity=100_000.0, htf_data=htf_data) + report = bt.run() + trade_log = bt.get_trade_log_df() + eq_curve = pd.DataFrame(bt.equity_curve) + return report, trade_log, eq_curve + + +def compute_signal_overlap(log_a, log_b, pair): + """Compute signal overlap between two strategies on the same pair. + + Returns: + - overlap_count: trades that are open at the same time + - same_direction_count: overlapping trades in same direction + - opposite_direction_count: overlapping trades in opposite direction + - overlap_ratio: fraction of trades that overlap + """ + if log_a.empty or log_b.empty: + return {"overlap_count": 0, "same_dir": 0, "opposite_dir": 0, "ratio": 0} + + overlap = 0 + same_dir = 0 + opp_dir = 0 + + for _, trade_a in log_a.iterrows(): + a_start = pd.Timestamp(trade_a["timestamp"]) + a_end = pd.Timestamp(trade_a["exit_time"]) if pd.notna(trade_a.get("exit_time")) else a_start + + for _, trade_b in log_b.iterrows(): + b_start = pd.Timestamp(trade_b["timestamp"]) + b_end = pd.Timestamp(trade_b["exit_time"]) if pd.notna(trade_b.get("exit_time")) else b_start + + # Check if time ranges overlap + if a_start <= b_end and b_start <= a_end: + overlap += 1 + if trade_a["signal_direction"] == trade_b["signal_direction"]: + same_dir += 1 + else: + opp_dir += 1 + + total = len(log_a) + len(log_b) + ratio = (overlap * 2) / total if total > 0 else 0 + + return { + "overlap_count": overlap, + "same_dir": same_dir, + "opposite_dir": opp_dir, + "ratio": round(ratio, 3), + } + + +def compute_combined_equity(eq_curves: list[pd.DataFrame]) -> pd.DataFrame: + """Combine equity curves from multiple strategies into portfolio equity.""" + combined = None + for eq in eq_curves: + if eq is None or eq.empty: + continue + eq = eq.set_index("timestamp")["equity"] + # Convert to returns relative to starting equity + returns = eq - 100_000.0 + if combined is None: + combined = returns + else: + combined = combined.add(returns, fill_value=0) + + if combined is None: + return pd.DataFrame() + + # Add back starting equity (100k per slot, or just use combined returns) + combined = combined + 100_000.0 + return combined.reset_index() + + +def compute_portfolio_metrics(all_trade_logs: list[pd.DataFrame]) -> dict: + """Compute portfolio-level metrics from combined trade logs.""" + combined = pd.concat([log for log in all_trade_logs if not log.empty], + ignore_index=True) + if combined.empty: + return {} + + n = len(combined) + wins = combined[combined["win"] == True] + losses = combined[combined["win"] == False] + + wr = len(wins) / n * 100 if n > 0 else 0 + gross_profit = wins["pnl_pips"].sum() if len(wins) > 0 else 0 + gross_loss = abs(losses["pnl_pips"].sum()) if len(losses) > 0 else 0 + pf = gross_profit / gross_loss if gross_loss > 0 else float("inf") + total_pnl = combined["pnl_pips"].sum() + + # Combined max drawdown + cum_pnl = combined.sort_values("timestamp")["pnl_dollars"].cumsum() + peak = cum_pnl.cummax() + dd = cum_pnl - peak + max_dd = dd.min() + max_dd_pct = max_dd / 100_000 * 100 if max_dd < 0 else 0 + + # Sharpe ratio + daily_pnl = combined.copy() + daily_pnl["date"] = pd.to_datetime(daily_pnl["timestamp"]).dt.date + daily = daily_pnl.groupby("date")["pnl_dollars"].sum() + if len(daily) > 1 and daily.std() > 0: + sharpe = (daily.mean() / daily.std()) * np.sqrt(252) + else: + sharpe = 0 + + return { + "total_trades": n, + "win_rate_pct": round(wr, 1), + "profit_factor": round(pf, 2), + "total_pnl_pips": round(total_pnl, 1), + "total_pnl_dollars": round(combined["pnl_dollars"].sum(), 2), + "max_drawdown_pct": round(max_dd_pct, 2), + "sharpe_ratio": round(sharpe, 2), + } + + +def main(): + print(f"{'='*80}") + print("PHASE 2 — CORRELATION ANALYSIS") + print(f"{'='*80}") + + results = {} + trade_logs = {} + eq_curves = {} + + # Run all backtests + for cfg in CONFIGS: + name = cfg["name"] + pair = cfg["pair"] + print(f"\nRunning {name} / {pair}...", end=" ", flush=True) + t0 = time.time() + report, log, eq = run_backtest(cfg) + elapsed = time.time() - t0 + n_trades = len(log) if log is not None and not log.empty else 0 + print(f"{n_trades} trades ({elapsed:.0f}s)") + trade_logs[name] = log + eq_curves[name] = eq + + # --- Signal Overlap: S7_Tight vs S3 on GBP_JPY --- + print(f"\n{'#'*60}") + print("# Signal Overlap: S7_Tight vs S3 on GBP_JPY") + print(f"{'#'*60}") + + log_s7 = trade_logs.get("S7_Tight", pd.DataFrame()) + log_s3 = trade_logs.get("S3", pd.DataFrame()) + + if not log_s7.empty and not log_s3.empty: + overlap = compute_signal_overlap(log_s7, log_s3, "GBP_JPY") + results["S7_S3_overlap"] = overlap + + print(f" S7 trades: {len(log_s7)}") + print(f" S3 trades: {len(log_s3)}") + print(f" Overlapping periods: {overlap['overlap_count']}") + print(f" Same direction: {overlap['same_dir']}") + print(f" Opposite direction: {overlap['opposite_dir']}") + print(f" Overlap ratio: {overlap['ratio']:.1%}") + + if overlap['ratio'] < 0.15: + print(" => LOW overlap: Good diversification!") + elif overlap['ratio'] < 0.30: + print(" => MODERATE overlap: Some clustering.") + else: + print(" => HIGH overlap: Significant clustering risk.") + else: + print(" Insufficient data for overlap analysis.") + + # --- S9 vs S9_Filtered (different pairs, should be independent) --- + print(f"\n{'#'*60}") + print("# Independence Check: S9 (GBP_USD) vs S9_Filtered (GBP_AUD)") + print(f"{'#'*60}") + + log_s9 = trade_logs.get("S9", pd.DataFrame()) + log_s9f = trade_logs.get("S9_Filtered", pd.DataFrame()) + + if not log_s9.empty and not log_s9f.empty: + # Check temporal clustering (same-day entries) + s9_dates = set(pd.to_datetime(log_s9["timestamp"]).dt.date) + s9f_dates = set(pd.to_datetime(log_s9f["timestamp"]).dt.date) + shared_dates = s9_dates & s9f_dates + temporal_overlap = len(shared_dates) / max(len(s9_dates), 1) + + results["S9_S9F_temporal"] = { + "s9_trade_days": len(s9_dates), + "s9f_trade_days": len(s9f_dates), + "shared_trade_days": len(shared_dates), + "temporal_overlap_ratio": round(temporal_overlap, 3), + } + + print(f" S9 trade days: {len(s9_dates)}") + print(f" S9_Filtered trade days: {len(s9f_dates)}") + print(f" Shared trade days: {len(shared_dates)}") + print(f" Temporal overlap: {temporal_overlap:.1%}") + else: + print(" Insufficient data.") + + # --- Portfolio Metrics --- + print(f"\n{'#'*60}") + print("# Portfolio-Level Metrics (All 5 Strategies Combined)") + print(f"{'#'*60}") + + all_logs = [log for log in trade_logs.values() + if log is not None and not log.empty] + if all_logs: + portfolio = compute_portfolio_metrics(all_logs) + results["portfolio"] = portfolio + + print(f" Total trades: {portfolio['total_trades']}") + print(f" Win rate: {portfolio['win_rate_pct']}%") + print(f" Profit factor: {portfolio['profit_factor']}") + print(f" Total PnL (pips): {portfolio['total_pnl_pips']:+.1f}") + print(f" Total PnL ($): {portfolio['total_pnl_dollars']:+,.2f}") + print(f" Max drawdown: {portfolio['max_drawdown_pct']:.2f}%") + print(f" Sharpe ratio: {portfolio['sharpe_ratio']:.2f}") + + # --- Per-Strategy Summary --- + print(f"\n{'='*80}") + print("STRATEGY SUMMARY") + print(f"{'='*80}") + print(f"{'Strategy':<16} {'Pair':<10} {'Trades':>6} {'WR%':>6} {'PF':>6} {'PnL(p)':>9}") + print(f"{'-'*60}") + + for cfg in CONFIGS: + name = cfg["name"] + pair = cfg["pair"] + log = trade_logs.get(name, pd.DataFrame()) + if log.empty: + print(f"{name:<16} {pair:<10} {'N/A':>6}") + continue + n = len(log) + wins = log[log["win"] == True] + wr = len(wins) / n * 100 if n > 0 else 0 + gp = wins["pnl_pips"].sum() if len(wins) > 0 else 0 + gl = abs(log[log["win"] == False]["pnl_pips"].sum()) + pf = gp / gl if gl > 0 else 0 + pnl = log["pnl_pips"].sum() + print(f"{name:<16} {pair:<10} {n:>6} {wr:>5.1f}% {pf:>5.2f} {pnl:>+8.1f}") + + # Save + out_path = os.path.join(RESULTS_DIR, "correlation_analysis.json") + with open(out_path, "w") as f: + json.dump(results, f, indent=2, default=str) + print(f"\nResults saved: {out_path}") + + +if __name__ == "__main__": + main() diff --git a/src/run_kelly_sizing.py b/src/run_kelly_sizing.py new file mode 100644 index 0000000..4cd2e4d --- /dev/null +++ b/src/run_kelly_sizing.py @@ -0,0 +1,252 @@ +""" +Phase 2 — Kelly Criterion Sizing (Step 9). + +For each Phase 2 strategy, compute: + - Kelly optimal fraction: f* = (W * R - L) / R + - Half-Kelly as practical recommendation + - Monte Carlo simulation (1000 paths) to estimate drawdown distribution + +Output: results/phase2/kelly_sizing.json +""" +import os, sys, io, json, time +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +import pandas as pd +import numpy as np +from src.indicators.technical import compute_all_indicators +from src.backtester.engine import Backtester + +# Strategy imports +from src.strategies_pkg.s7_liquidity_sweep import S7_Liquidity_Sweep +from src.strategies_pkg.s9_london_session import S9_London_Session +from src.strategies_pkg.s4f_ema_ribbon import S4F_EMA_Ribbon +from src.strategies_pkg.s3_key_level_breakout import S3_KeyLevel_Breakout + +PROCESSED_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "processed") +RESULTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "results", "phase2") +os.makedirs(RESULTS_DIR, exist_ok=True) + +CONFIGS = [ + {"name": "S7_Tight", "pair": "GBP_JPY", "tf": "H1", + "factory": lambda: S7_Liquidity_Sweep()}, + {"name": "S9", "pair": "GBP_USD", "tf": "H1", + "factory": lambda: S9_London_Session()}, + {"name": "S9_Filtered", "pair": "GBP_AUD", "tf": "H1", + "factory": lambda: S9_London_Session(pair="GBP_AUD", filtered=True)}, + {"name": "S4F", "pair": "EUR_AUD", "tf": "M15", + "factory": lambda: S4F_EMA_Ribbon()}, + {"name": "S3", "pair": "GBP_JPY", "tf": "H1", + "factory": lambda: S3_KeyLevel_Breakout()}, +] + +N_SIMULATIONS = 1000 +STARTING_EQUITY = 100_000.0 + + +def load_data(pair, tf): + fp = os.path.join(PROCESSED_DIR, f"{pair}_{tf}.csv") + if not os.path.exists(fp): + return None + df = pd.read_csv(fp, index_col=0, parse_dates=True) + df.index.name = "timestamp" + return compute_all_indicators(df) + + +def run_backtest(cfg): + pair = cfg["pair"] + tf = cfg["tf"] + data = load_data(pair, tf) + if data is None: + return None + + htf_data = data.copy() if tf == "H1" else load_data(pair, "H1") + strategy = cfg["factory"]() + bt = Backtester(data=data, strategy=strategy, pair=pair, + starting_equity=STARTING_EQUITY, htf_data=htf_data) + bt.run() + return bt.get_trade_log_df() + + +def compute_kelly(trade_log): + """Compute Kelly criterion from trade log. + + f* = (W * R - L) / R + where: + W = win rate (probability of winning) + L = loss rate (1 - W) + R = avg_win / avg_loss (win/loss ratio) + + Returns dict with kelly_full, kelly_half, and component values. + """ + if trade_log.empty or len(trade_log) < 10: + return { + "trades": len(trade_log) if trade_log is not None else 0, + "kelly_full_pct": 0, "kelly_half_pct": 0, + "win_rate": 0, "avg_win_loss_ratio": 0, + } + + wins = trade_log[trade_log["win"] == True] + losses = trade_log[trade_log["win"] == False] + + n = len(trade_log) + w = len(wins) / n # win probability + l = 1 - w # loss probability + + avg_win = wins["pnl_pips"].mean() if len(wins) > 0 else 0 + avg_loss = abs(losses["pnl_pips"].mean()) if len(losses) > 0 else 1 + + r = avg_win / avg_loss if avg_loss > 0 else 0 + + # Kelly formula: f* = (W * R - L) / R + if r > 0: + kelly = (w * r - l) / r + else: + kelly = 0 + + kelly = max(kelly, 0) # clamp at 0 (never bet negative) + + return { + "trades": n, + "win_rate": round(w * 100, 1), + "avg_win_pips": round(avg_win, 1), + "avg_loss_pips": round(avg_loss, 1), + "avg_win_loss_ratio": round(r, 2), + "kelly_full_pct": round(kelly * 100, 2), + "kelly_half_pct": round(kelly * 50, 2), + } + + +def monte_carlo_drawdown(trade_log, fraction: float, + n_sims: int = N_SIMULATIONS) -> dict: + """Monte Carlo simulation to estimate drawdown distribution. + + Randomly resamples trade outcomes (with replacement) to build + equity paths and measure max drawdown at the given Kelly fraction. + + Returns percentile drawdown estimates. + """ + if trade_log.empty or len(trade_log) < 10 or fraction <= 0: + return {"p50_dd": 0, "p75_dd": 0, "p95_dd": 0, "p99_dd": 0, "ruin_pct": 0} + + # Use pnl_pips as return per trade (normalized) + returns = trade_log["pnl_pips"].values + n_trades = len(returns) + + max_drawdowns = [] + ruin_count = 0 + + rng = np.random.default_rng(42) + + for _ in range(n_sims): + # Resample trades with replacement + sampled = rng.choice(returns, size=n_trades, replace=True) + + # Scale by fraction (Kelly fraction applied to risk) + equity = STARTING_EQUITY + peak = equity + max_dd = 0 + + for ret in sampled: + # PnL scaled by fraction (relative to 1% base risk) + pnl = ret * fraction / 0.01 * equity / STARTING_EQUITY + equity += pnl + + if equity > peak: + peak = equity + dd = (peak - equity) / peak * 100 + if dd > max_dd: + max_dd = dd + + if equity <= 0: + ruin_count += 1 + max_dd = 100 + break + + max_drawdowns.append(max_dd) + + dd_array = np.array(max_drawdowns) + return { + "p50_dd": round(np.percentile(dd_array, 50), 1), + "p75_dd": round(np.percentile(dd_array, 75), 1), + "p95_dd": round(np.percentile(dd_array, 95), 1), + "p99_dd": round(np.percentile(dd_array, 99), 1), + "ruin_pct": round(ruin_count / n_sims * 100, 2), + } + + +def main(): + print(f"{'='*90}") + print("PHASE 2 — KELLY CRITERION SIZING") + print(f"{'='*90}") + + all_results = {} + + for cfg in CONFIGS: + name = cfg["name"] + pair = cfg["pair"] + + print(f"\n{'#'*60}") + print(f"# {name} / {pair}") + print(f"{'#'*60}") + + t0 = time.time() + trade_log = run_backtest(cfg) + elapsed = time.time() - t0 + + if trade_log is None or trade_log.empty: + print(f" No trades. Skipping.") + continue + + print(f" Backtest: {len(trade_log)} trades ({elapsed:.0f}s)") + + # Kelly computation + kelly = compute_kelly(trade_log) + print(f" Win rate: {kelly['win_rate']}%") + print(f" Avg win/loss ratio: {kelly['avg_win_loss_ratio']}") + print(f" Kelly full: {kelly['kelly_full_pct']:.1f}%") + print(f" Kelly half: {kelly['kelly_half_pct']:.1f}% (recommended)") + + # Monte Carlo at half-Kelly + half_kelly_frac = kelly["kelly_half_pct"] / 100 + if half_kelly_frac > 0: + print(f"\n Monte Carlo ({N_SIMULATIONS} paths at half-Kelly={kelly['kelly_half_pct']:.1f}%):") + mc = monte_carlo_drawdown(trade_log, half_kelly_frac) + print(f" Median max DD: {mc['p50_dd']:.1f}%") + print(f" 75th pctl DD: {mc['p75_dd']:.1f}%") + print(f" 95th pctl DD: {mc['p95_dd']:.1f}%") + print(f" 99th pctl DD: {mc['p99_dd']:.1f}%") + print(f" Ruin probability: {mc['ruin_pct']:.1f}%") + else: + mc = {"p50_dd": 0, "p75_dd": 0, "p95_dd": 0, "p99_dd": 0, "ruin_pct": 0} + print(f" Kelly <= 0, no MC simulation.") + + all_results[f"{name}_{pair}"] = { + **kelly, + "monte_carlo_half_kelly": mc, + } + + # Summary table + print(f"\n{'='*90}") + print("KELLY SIZING SUMMARY") + print(f"{'='*90}") + print(f"{'Strategy':<20} {'Trades':>6} {'WR%':>6} {'W/L':>5} " + f"{'Kelly%':>7} {'HalfK%':>7} {'MC-p95DD':>9}") + print(f"{'-'*70}") + + for key, result in all_results.items(): + mc = result.get("monte_carlo_half_kelly", {}) + print(f"{key:<20} {result['trades']:>6} {result['win_rate']:>5.1f}% " + f"{result['avg_win_loss_ratio']:>4.2f} " + f"{result['kelly_full_pct']:>6.1f}% {result['kelly_half_pct']:>6.1f}% " + f"{mc.get('p95_dd', 0):>8.1f}%") + + # Save + out_path = os.path.join(RESULTS_DIR, "kelly_sizing.json") + with open(out_path, "w") as f: + json.dump(all_results, f, indent=2, default=str) + print(f"\nResults saved: {out_path}") + + +if __name__ == "__main__": + main() diff --git a/src/run_regime_analysis.py b/src/run_regime_analysis.py new file mode 100644 index 0000000..84d0d2c --- /dev/null +++ b/src/run_regime_analysis.py @@ -0,0 +1,224 @@ +""" +Phase 2 — Regime Analysis (Step 7). + +Runs each Phase 2 strategy on the full dataset, breaks down results by year +(2021, 2022, 2023). Answers: "Is the edge strengthening or was OOS lucky?" + +Output: results/phase2/regime_analysis.json + console table. +""" +import os, sys, io, json, time +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +import pandas as pd +import numpy as np +from src.indicators.technical import compute_all_indicators +from src.backtester.engine import Backtester + +# Strategy imports +from src.strategies_pkg.s7_liquidity_sweep import S7_Liquidity_Sweep +from src.strategies_pkg.s9_london_session import S9_London_Session +from src.strategies_pkg.s4f_ema_ribbon import S4F_EMA_Ribbon +from src.strategies_pkg.s3_key_level_breakout import S3_KeyLevel_Breakout + +PROCESSED_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "processed") +RESULTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "results", "phase2") +os.makedirs(RESULTS_DIR, exist_ok=True) + +# Phase 2 strategy-pair configurations +CONFIGS = [ + {"name": "S7_Tight", "pair": "GBP_JPY", "tf": "H1", "htf_tf": "H1", + "factory": lambda: S7_Liquidity_Sweep()}, + {"name": "S9", "pair": "GBP_USD", "tf": "H1", "htf_tf": "H1", + "factory": lambda: S9_London_Session()}, + {"name": "S9_Filtered", "pair": "GBP_AUD", "tf": "H1", "htf_tf": "H1", + "factory": lambda: S9_London_Session(pair="GBP_AUD", filtered=True)}, + {"name": "S4F", "pair": "EUR_AUD", "tf": "M15", "htf_tf": "H1", + "factory": lambda: S4F_EMA_Ribbon()}, + {"name": "S3", "pair": "GBP_JPY", "tf": "H1", "htf_tf": "H1", + "factory": lambda: S3_KeyLevel_Breakout()}, +] + + +def load_data(pair, tf): + """Load price data with indicators.""" + fp = os.path.join(PROCESSED_DIR, f"{pair}_{tf}.csv") + if not os.path.exists(fp): + print(f" WARNING: {fp} not found") + return None + df = pd.read_csv(fp, index_col=0, parse_dates=True) + df.index.name = "timestamp" + return compute_all_indicators(df) + + +def slice_year(df, year): + """Slice dataframe to a specific year, with 60 days warmup prepended.""" + # Match timezone awareness of the dataframe index + if df.index.tz is not None: + start = pd.Timestamp(f"{year}-01-01", tz=df.index.tz) + end = pd.Timestamp(f"{year}-12-31 23:59:59", tz=df.index.tz) + else: + start = pd.Timestamp(f"{year}-01-01") + end = pd.Timestamp(f"{year}-12-31 23:59:59") + warmup_start = start - pd.DateOffset(days=60) + sliced = df[(df.index >= warmup_start) & (df.index <= end)].copy() + return sliced, start + + +def run_backtest(strategy, data, htf_data, pair, start_after=None): + """Run backtest, optionally filtering trades after a start date.""" + bt = Backtester(data=data, strategy=strategy, pair=pair, + starting_equity=100_000.0, htf_data=htf_data) + report = bt.run() + trade_log = bt.get_trade_log_df() + + # Filter trades to only those after start_after (exclude warmup trades) + if start_after is not None and not trade_log.empty: + ts = pd.to_datetime(trade_log["timestamp"]) + start_ts = pd.Timestamp(start_after) + # Align timezone awareness + if ts.dt.tz is not None and start_ts.tz is None: + start_ts = start_ts.tz_localize(ts.dt.tz) + elif ts.dt.tz is None and start_ts.tz is not None: + start_ts = start_ts.tz_localize(None) + trade_log = trade_log[ts >= start_ts] + + return report, trade_log + + +def compute_metrics(trade_log): + """Compute metrics from a trade log DataFrame.""" + if trade_log.empty or len(trade_log) == 0: + return { + "trades": 0, "wr": 0, "pf": 0, "pnl_pips": 0, + "max_dd_pips": 0, "expectancy": 0, + } + + n = len(trade_log) + wins = trade_log[trade_log["win"] == True] + losses = trade_log[trade_log["win"] == False] + + wr = len(wins) / n * 100 if n > 0 else 0 + gross_profit = wins["pnl_pips"].sum() if len(wins) > 0 else 0 + gross_loss = abs(losses["pnl_pips"].sum()) if len(losses) > 0 else 0 + pf = gross_profit / gross_loss if gross_loss > 0 else float("inf") + total_pnl = trade_log["pnl_pips"].sum() + expectancy = total_pnl / n if n > 0 else 0 + + # Max drawdown in pips (cumulative PnL drawdown) + cum_pnl = trade_log["pnl_pips"].cumsum() + peak = cum_pnl.cummax() + dd = cum_pnl - peak + max_dd = dd.min() if len(dd) > 0 else 0 + + return { + "trades": n, + "wr": round(wr, 1), + "pf": round(pf, 2), + "pnl_pips": round(total_pnl, 1), + "max_dd_pips": round(max_dd, 1), + "expectancy": round(expectancy, 2), + } + + +def main(): + years = [2021, 2022, 2023] + all_results = {} + + print(f"{'='*90}") + print("PHASE 2 — REGIME ANALYSIS (Per-Year Breakdown)") + print(f"{'='*90}") + + for cfg in CONFIGS: + name = cfg["name"] + pair = cfg["pair"] + tf = cfg["tf"] + htf_tf = cfg["htf_tf"] + + print(f"\n{'#'*70}") + print(f"# {name} / {pair} ({tf})") + print(f"{'#'*70}") + + # Load data + data = load_data(pair, tf) + if data is None: + continue + + htf_data = data.copy() if htf_tf == tf else load_data(pair, htf_tf) + if htf_data is None: + continue + + strategy_results = {"pair": pair, "timeframe": tf, "years": {}} + + # Full dataset first + strategy = cfg["factory"]() + _, full_log = run_backtest(strategy, data, htf_data, pair) + full_metrics = compute_metrics(full_log) + strategy_results["full"] = full_metrics + print(f" {'FULL':>6}: {full_metrics['trades']:>4} trades | " + f"WR {full_metrics['wr']:>5.1f}% | PF {full_metrics['pf']:>5.2f} | " + f"PnL {full_metrics['pnl_pips']:>+8.1f}p | " + f"DD {full_metrics['max_dd_pips']:>+8.1f}p | " + f"Exp {full_metrics['expectancy']:>+6.2f}p") + + # Per-year breakdown + for year in years: + sliced, start = slice_year(data, year) + if len(sliced) < 250: + print(f" {year:>6}: insufficient data ({len(sliced)} bars)") + strategy_results["years"][str(year)] = {"trades": 0} + continue + + htf_sliced = sliced.copy() if htf_tf == tf else slice_year(htf_data, year)[0] + strategy = cfg["factory"]() + _, year_log = run_backtest(strategy, sliced, htf_sliced, pair, start_after=start) + metrics = compute_metrics(year_log) + strategy_results["years"][str(year)] = metrics + + print(f" {year:>6}: {metrics['trades']:>4} trades | " + f"WR {metrics['wr']:>5.1f}% | PF {metrics['pf']:>5.2f} | " + f"PnL {metrics['pnl_pips']:>+8.1f}p | " + f"DD {metrics['max_dd_pips']:>+8.1f}p | " + f"Exp {metrics['expectancy']:>+6.2f}p") + + all_results[f"{name}_{pair}"] = strategy_results + + # Save results + out_path = os.path.join(RESULTS_DIR, "regime_analysis.json") + with open(out_path, "w") as f: + json.dump(all_results, f, indent=2, default=str) + print(f"\nResults saved: {out_path}") + + # Summary table + print(f"\n{'='*90}") + print("REGIME SUMMARY — Per-Year Profit Factor") + print(f"{'='*90}") + print(f"{'Strategy':<20} {'Pair':<10} {'Full':>6} {'2021':>6} {'2022':>6} {'2023':>6} {'Trend':>8}") + print(f"{'-'*90}") + for key, result in all_results.items(): + name_pair = key.split("_", 1) + name = result.get("pair", key) + pf_full = result.get("full", {}).get("pf", 0) + pf_years = [] + for y in ["2021", "2022", "2023"]: + pf = result.get("years", {}).get(y, {}).get("pf", 0) + pf_years.append(pf) + + # Trend: compare first year to last year + if pf_years[0] > 0 and pf_years[-1] > 0: + if pf_years[-1] > pf_years[0] * 1.1: + trend = "UP" + elif pf_years[-1] < pf_years[0] * 0.9: + trend = "DOWN" + else: + trend = "STABLE" + else: + trend = "N/A" + + print(f"{key:<20} {result.get('pair',''):<10} " + f"{pf_full:>5.2f} {pf_years[0]:>5.2f} {pf_years[1]:>5.2f} " + f"{pf_years[2]:>5.2f} {trend:>8}") + + +if __name__ == "__main__": + main()