Refactor trading logic and enhance risk management features

- Removed AdaptiveRiskManager trade limits from optimization and training outputs. - Introduced DailyEquityGate to monitor daily equity and prevent excessive losses. - Updated backtester to utilize new risk management features and removed session limits. - Enhanced metrics computation to include recovery factor for better performance evaluation. - Implemented Gaussian Mixture Model (GMM) for volatility clustering in processor and validator. - Adjusted optimizer scoring to reward lower drawdown and incorporate recovery factor. - Cleaned up unused model metadata files and updated database for consistency. - Improved logging and error handling across various modules for better traceability.
2026-11-02 06:26:15 +00:00 · 2026-04-15 08:29:13 +03:00
parent 00d550dbc1
commit ab0f2d6052
13 changed files with 401 additions and 123 deletions
@@ -311,13 +311,10 @@ def cmd_optimize(args):
    for tf in tfs:
        logger.info("Optimizing [%s] broker=%s balance=$%.0f trials=%d", tf, broker, balance, args.trials)
        study = run_optimization(n_trials=args.trials, balance=balance, broker=broker, tf=tf, n_jobs=args.n_jobs)
-        arm = AdaptiveRiskManager(balance, broker=broker)
-        limits = arm.get_trade_limits()
        print(f"\n=== Best Result [{tf}] ===")
        print(f"Score:         {study.best_value:.3f}")
        print(f"Broker:        {broker}")
-        print(f"Balance:       ${balance:.0f} USD  ({arm}")
-        print(f"Session Limit: {limits['max_daily_trades']} trade(s)/day | {limits['pos_per_trade']} pos/trade")
+        print(f"Balance:       ${balance:.0f} USD")
        print("Best Params:")
        for k, v in study.best_params.items():
            print(f"  {k}: {v}")
@@ -371,10 +368,8 @@ def cmd_train(args):
    save_xgb_ensemble(models_ensemble, thresholds, metrics, get_ensemble_path(tf, broker))

    arm = AdaptiveRiskManager(balance, broker=broker)
-    limits = arm.get_trade_limits()
    print(f"\n=== Training Results [{tf}] ===")
    print(f"Broker: {broker} | Balance: ${balance:.0f} | Tier: {'small' if arm.is_small_account else 'growth'}")
-    print(f"Session: {limits['max_daily_trades']}/day | Positions: {limits['pos_per_trade']}/trade")
    print(f"Sharpe: {result['sharpe_ratio']:.3f} | MaxDD: {result['max_drawdown']*100:.1f}% "
          f"| WR: {result['win_rate']*100:.1f}% | Trades: {result['n_trades']}")
    if "oos_sharpe_ratio" in result:
@@ -650,7 +645,6 @@ def cmd_report(args):
        "broker": broker,
        "balance_usd": balance,
        "tier": "small" if arm.is_small_account else "growth",
-        "session_limit": limits["max_daily_trades"],
        "pos_per_trade": limits["pos_per_trade"],
        "tf": tf,
    })
@@ -1 +0,0 @@
-{"timestamp": 1775749655.5421815, "tf": "M5", "best_score": 5.150384458557411}
@@ -1 +0,0 @@
-{"timestamp": 1776092348.3391533, "tf": "M5", "best_score": 2.4810574957119442}
@@ -1 +0,0 @@
-{"timestamp": 1775754958.6410613, "tf": "M5", "best_score": 4.882846936049799}
@@ -144,12 +144,25 @@ def _compute_metrics(strategy_returns, signals, tf: str = "H1"):
                      if len(active_returns) > 0 else 0.0)
    total_return   = float(np.exp(np.sum(strategy_returns)) - 1)

+    # Recovery Factor = Net Profit / Max Drawdown.
+    # Rewards strategies that achieve high return relative to worst trough —
+    # safer than raw Sharpe for small accounts where a single DD event is fatal.
+    # Capped at 20 to prevent extreme low-DD outliers from distorting Optuna's
+    # surrogate model.
+    if max_dd > 0:
+        recovery_factor = float(min(total_return / max_dd, 20.0))
+    elif total_return > 0:
+        recovery_factor = 20.0   # no drawdown, positive return → best possible
+    else:
+        recovery_factor = 0.0
+
    return {
-        "sharpe_ratio": float(sharpe),
-        "max_drawdown": max_dd,
-        "win_rate":     win_rate,
-        "total_return": total_return,
-        "n_trades":     n_trades,
+        "sharpe_ratio":    float(sharpe),
+        "max_drawdown":    max_dd,
+        "win_rate":        win_rate,
+        "total_return":    total_return,
+        "n_trades":        n_trades,
+        "recovery_factor": recovery_factor,
    }


@@ -189,7 +202,7 @@ def vectorized_backtest(
        threshold=buy_th,
        short_threshold=short_threshold,   # None → symmetric default inside
    )
-    signals = apply_session_limits(raw_signals, df.index, account_size, hmm_states, tf=tf, broker=broker)
+    signals = raw_signals  # daily trade-count cap removed; DailyEquityGate governs live risk

    # Determine pos_per_trade from the adaptive risk manager
    arm           = AdaptiveRiskManager(account_size, broker=broker)
@@ -206,11 +219,10 @@ def vectorized_backtest(

    result = _compute_metrics(strategy_returns, signals, tf=tf)
    result.update({
-        "account_size":      account_size,
-        "broker":            broker,
-        "tf":                tf,
-        "session_max_trades": arm.get_trade_limits(tf=tf)["max_daily_trades"],
-        "pos_per_trade":     pos_per_trade,
+        "account_size":  account_size,
+        "broker":        broker,
+        "tf":            tf,
+        "pos_per_trade": pos_per_trade,
    })

    if split_idx is not None and 0 < split_idx < len(strategy_returns):
@@ -237,11 +249,10 @@ def vectorized_backtest(
        )

    logger.info(
-        "%s: account=$%.2f | tier=%s | session_limit=%d/day | pos_per_trade=%d | costs=%.4f/trade",
+        "%s: account=$%.2f | tier=%s | pos_per_trade=%d | costs=%.4f/trade",
        broker,
        account_size,
        "small" if arm.is_small_account else "growth",
-        arm.get_trade_limits(tf=tf)["max_daily_trades"],
        pos_per_trade,
        BROKER_CONFIGS.get(broker, {}).get("spread_frac", 0)
        + BROKER_CONFIGS.get(broker, {}).get("commission_frac", 0),
@@ -9,6 +9,14 @@ logger = setup_logger(__name__)

 MODEL_PATH = Path("models/hmm_model.pkl")

+# TF-specific diagonal boost applied to the HMM transition matrix after fitting.
+# Adds this value to each self-transition probability then renormalises rows.
+# Higher boost = stickier regime (fewer flips per bar).
+# H1: 0.25 — hourly bars have genuine multi-hour trends; bias hard toward persistence.
+# M15: 0.15 — moderate; 15-min noise warrants some smoothing but less than H1.
+# M5: 0.10 — flexible; scalping benefits from faster regime detection.
+TF_TRANS_BOOST = {"H1": 0.25, "M15": 0.15, "M5": 0.10}
+

 def get_model_path(tf: str, broker: str = "headway_cent") -> Path:
    """Return the TF+broker-specific HMM model path.
@@ -86,6 +94,7 @@ def fit_hmm(
    n_states: int = 3,
    n_iter: int = 200,
    random_state: int = 42,
+    tf: str = "H1",
 ):
    X = df[["kalman_return", "volatility"]].values
    model = GaussianHMM(
@@ -96,6 +105,16 @@ def fit_hmm(
        verbose=False,
    )
    model.fit(X)
+
+    # ── HMM Persistence Boost ─────────────────────────────────────────────────
+    # Add a TF-specific value to each diagonal of the transition matrix, then
+    # renormalise rows to sum to 1.  This biases the model toward staying in the
+    # current regime, reducing bar-to-bar jitter ("regime flip every 2 bars").
+    boost = TF_TRANS_BOOST.get(tf.upper(), 0.10)
+    mat = model.transmat_.copy()
+    np.fill_diagonal(mat, mat.diagonal() + boost)
+    model.transmat_ = mat / mat.sum(axis=1, keepdims=True)
+
    raw_states = model.predict(X)

    states, state_names = _sort_states(model, raw_states, n_states)
@@ -22,9 +22,12 @@ def get_ensemble_path(tf: str, broker: str = "headway_cent") -> Path:
    """
    return Path(f"models/xgb_ensemble_{tf.upper()}_{broker}.pkl")

-# Base features always present; usdchf_log_return is added when the USDCHF
-# master CSV is available (run  python main.py --mode consolidate  first).
-FEATURE_COLS    = ["hmm_state", "rsi_slope", "atr_normalized", "prev_log_return"]
+# Base features always present; gmm_vol_cluster and usdchf_log_return are
+# added conditionally when the columns are present in the processed DataFrame.
+# Feature order (canonical): hmm_state | gmm_vol_cluster | rsi_slope |
+#                             atr_normalized | prev_log_return | usdchf_log_return
+FEATURE_COLS    = ["hmm_state", "gmm_vol_cluster", "rsi_slope", "atr_normalized", "prev_log_return"]
+GMM_FEATURE     = "gmm_vol_cluster"
 USDCHF_FEATURE  = "usdchf_log_return"
 DXY_FEATURE     = USDCHF_FEATURE   # legacy alias — kept so old imports don't crash

@@ -35,10 +38,17 @@ VOL_BUCKETS = ["low", "med", "high"]
 def get_feature_cols(df: pd.DataFrame) -> list[str]:
    """Return the feature column list for this DataFrame.

-    Includes ``usdchf_log_return`` only when the column is present and has
-    enough non-null values to be useful (>50% coverage).
+    Starts from FEATURE_COLS (which includes gmm_vol_cluster), then:
+    - Drops gmm_vol_cluster if not present (old parquet without GMM).
+    - Appends usdchf_log_return when present and >50% non-null.
    """
-    cols = list(FEATURE_COLS)
+    cols = []
+    for c in FEATURE_COLS:
+        if c == GMM_FEATURE:
+            if c in df.columns and df[c].notna().mean() > 0.5:
+                cols.append(c)
+        else:
+            cols.append(c)
    if USDCHF_FEATURE in df.columns and df[USDCHF_FEATURE].notna().mean() > 0.5:
        cols.append(USDCHF_FEATURE)
    return cols
@@ -36,7 +36,7 @@ from src.engine_xgb import (
    load_xgb_ensemble, get_predictions_ensemble, assign_vol_bucket, FEATURE_COLS,
    get_ensemble_path, ENSEMBLE_PKL_PATH as XGB_GENERIC_PATH,
 )
-from src.risk_manager import AdaptiveRiskManager, CENT_MULTIPLIER
+from src.risk_manager import AdaptiveRiskManager, CENT_MULTIPLIER, DailyEquityGate

 logger = setup_logger(__name__)

@@ -58,17 +58,19 @@ MIN_SPREAD_RATIO = {"headway_cent": 1.5, "standard": 3.0}  # TP1 vs spread floor
 # SL = ATR × multiplier (per TF — M5 tighter to avoid noise-outs on scalps)
 TF_ATR_MULTIPLIER  = {"M5": 1.5, "M15": 2.0, "H1": 2.0}

-# Quick-profit target for M5 scalping: close early when floating P&L reaches
-# this USD amount rather than waiting for the full TP.  Re-entry is allowed on
-# the next bar if signal conditions are still met.  Set to None to disable.
-QUICK_PROFIT_TARGET_M5 = 4.0
+# ── ATR-linked Hybrid Trailing Stop ──────────────────────────────────────────
+# PROFIT_ACTIVATION_USD: floating P&L threshold that triggers Phase 1 (move SL
+# to breakeven + 2× spread) and optional partial close.  Applies to all TFs.
+PROFIT_ACTIVATION_USD = 2.50

-# Trailing P&L stop ("Chop Buffer"): once a position's peak floating P&L has
-# reached TRAILING_ACTIVATION_USD, close it if P&L drops back below
-# peak × TRAILING_DRAWDOWN_PCT.  Catches stalling/reversing scalps that never
-# reach the fixed target, bypassing the 1-bar HMM state detection delay.
-TRAILING_ACTIVATION_USD  = 2.0   # start protecting once P&L first exceeds this
-TRAILING_DRAWDOWN_PCT    = 0.50  # close if P&L falls below 50 % of observed peak
+# ATR trail multiplier per TF.  Phase 2 trails SL at current_price ∓ (mult × ATR).
+# Larger multiplier for H1 gives hourly swings room before the SL is touched.
+ATR_TRAIL_MULTIPLIER = {"M5": 1.5, "M15": 1.5, "H1": 2.5}
+
+# Minimum lot for partial close.  MT5 rejects close volumes below 0.01.
+# When a position is already at 0.01 lots the partial close is skipped and the
+# ATR trail runs on the full position instead.
+MIN_LOT_GUARD = 0.01

 # ── Per-timeframe signal thresholds and order parameters ──────────────────────
 TF_PROB_THRESHOLD  = {"M5": 0.52, "M15": 0.54, "H1": 0.54}   # fallback — Optuna value used when available
@@ -337,6 +339,89 @@ def _apply_profit_guard(signal_tracker: dict, mt5) -> None:
        )


+def _set_trailing_sl(
+    ticket: int, new_sl: float, current_sl: float, direction: str, mt5
+) -> bool:
+    """Update an open position's SL only if it improves on the current SL.
+
+    For BUY positions: new_sl must be > current_sl (ratchet upward).
+    For SELL positions: new_sl must be < current_sl (ratchet downward).
+    Returns True if the SL was actually updated.
+    """
+    if direction == "BUY"  and new_sl <= current_sl:
+        return False
+    if direction == "SELL" and new_sl >= current_sl:
+        return False
+    positions = mt5.positions_get(ticket=ticket)
+    if not positions:
+        return False
+    pos = positions[0]
+    request = {
+        "action":   mt5.TRADE_ACTION_SLTP,
+        "position": ticket,
+        "sl":       round(new_sl, 2),
+        "tp":       pos.tp,
+    }
+    res = mt5.order_send(request)
+    if res and res.retcode == mt5.TRADE_RETCODE_DONE:
+        logger.info(
+            "ATR trail SL updated: ticket=%d  sl=%.2f  (was %.2f)",
+            ticket, new_sl, current_sl,
+        )
+        return True
+    logger.warning(
+        "ATR trail SL failed: ticket=%d  retcode=%s",
+        ticket, res.retcode if res else "None",
+    )
+    return False
+
+
+def _execute_partial_close(ticket: int, symbol: str, mt5) -> bool:
+    """Close 50 % of a position's volume to bank partial profit.
+
+    Skips and returns False if volume <= MIN_LOT_GUARD (0.01) since MT5
+    rejects close volumes below the broker minimum lot.
+    """
+    positions = mt5.positions_get(ticket=ticket)
+    if not positions:
+        return False
+    pos = positions[0]
+    close_vol = round(pos.volume / 2.0, 2)
+    if close_vol < MIN_LOT_GUARD:
+        logger.info(
+            "Partial close skipped: ticket=%d  vol=%.2f  half=%.2f < MIN_LOT_GUARD %.2f",
+            ticket, pos.volume, close_vol, MIN_LOT_GUARD,
+        )
+        return False
+    close_type = mt5.ORDER_TYPE_SELL if pos.type == mt5.ORDER_TYPE_BUY else mt5.ORDER_TYPE_BUY
+    tick  = mt5.symbol_info_tick(symbol)
+    price = tick.bid if pos.type == mt5.ORDER_TYPE_BUY else tick.ask
+    request = {
+        "action":       mt5.TRADE_ACTION_DEAL,
+        "position":     ticket,
+        "symbol":       symbol,
+        "volume":       close_vol,
+        "type":         close_type,
+        "price":        price,
+        "magic":        MAGIC_NUMBER,
+        "comment":      "GRX_Partial_Profit",
+        "type_filling": mt5.ORDER_FILLING_IOC,
+    }
+    res = mt5.order_send(request)
+    if res and res.retcode == mt5.TRADE_RETCODE_DONE:
+        logger.info(
+            "Partial close: ticket=%d  vol_closed=%.2f  price=%.2f",
+            ticket, close_vol, price,
+        )
+        return True
+    logger.warning(
+        "Partial close failed: ticket=%d  retcode=%s  vol=%.2f",
+        ticket, res.retcode if res else "None", close_vol,
+    )
+    return False
+
+
+
 def _close_position(ticket: int, mt5, comment: str = "GRX_close_chop") -> None:
    """Close a specific open position at market price."""
    positions = mt5.positions_get(ticket=ticket)
@@ -640,7 +725,7 @@ def run_live_loop(
        account_size:  USD balance used for lot-sizing.  If None, reads from MT5
                       and normalises for cent accounts automatically.
        profit_target: Close early when floating P&L reaches this USD amount.
-                       Defaults to QUICK_PROFIT_TARGET_M5 for M5, disabled for
+                       Defaults to PROFIT_ACTIVATION_USD for all TFs (legacy param,
                       other TFs.  Pass 0 to disable on M5 explicitly.
    """
    import MetaTrader5 as mt5
@@ -734,9 +819,15 @@ def _run_loop_inner(tf: str, broker: str, account_size: float, mt5,
    # so the break-even SL logic can fire when TP1 closes position 1.
    signal_tracker = {"tickets": [], "entry_price": 0.0,
                      "direction": None, "tp1_hit": False,
-                      "tp1_level": None, "guard_hit": False}
-    # Per-ticket high-water-mark for trailing P&L stop  {ticket: peak_usd}
+                      "tp1_level": None, "guard_hit": False,
+                      "atr_price": 0.0}   # cached price-denom ATR for between-bar trail
+    # Per-ticket ATR trail state: {ticket: {"activated": bool, "partial_done": bool, "current_sl": float}}
+    atr_state_tracker: dict = {}
+    # Deprecated peak-pnl tracker kept for cleanup compat (atr_state_tracker supersedes it)
    peak_pnl_tracker: dict = {}
+    # Daily floating-equity safety switch: suspends trading if account drops 5%
+    equity_gate = DailyEquityGate()
+    equity_gate.reset_day(account_size)

    arm = AdaptiveRiskManager(account_size, tf=tf, broker=broker)
    logger.info(
@@ -750,17 +841,15 @@ def _run_loop_inner(tf: str, broker: str, account_size: float, mt5,
        else ("optuna" if prob_threshold_opt is not None else "hardcoded"),
    )

-    # Resolve quick-profit target: CLI override > M5 default > disabled
-    if profit_target is None:
-        profit_target = QUICK_PROFIT_TARGET_M5 if tf.upper() == "M5" else None
-    elif profit_target <= 0:
-        profit_target = None   # explicit CLI disable (--profit_target 0)
-    if profit_target is not None:
-        logger.info(
-            "Hybrid Scalp Protection — Fixed target: $%.2f | "
-            "Trailing guard: activation $%.2f / drawdown %.0f%%.",
-            profit_target, TRAILING_ACTIVATION_USD, TRAILING_DRAWDOWN_PCT * 100,
-        )
+    # ATR-linked trailing stop applies to all TFs unconditionally.
+    # The legacy profit_target CLI param is kept for backward compat but is no
+    # longer used — the ATR trail supersedes the old fixed-exit logic.
+    _atr_mult_log = ATR_TRAIL_MULTIPLIER.get(tf.upper(), 1.5)
+    logger.info(
+        "ATR Trailing Stop — activation $%.2f | "
+        "multiplier %.1fx ATR [%s] | partial close (lot>%.2f).",
+        PROFIT_ACTIVATION_USD, _atr_mult_log, tf.upper(), MIN_LOT_GUARD,
+    )

    # Warn if the MQL5 EA (same MAGIC_NUMBER) already has open positions.
    # Running both simultaneously causes Python to see EA positions as its own
@@ -783,7 +872,8 @@ def _run_loop_inner(tf: str, broker: str, account_size: float, mt5,
            if today != current_day:
                daily_trades = 0
                current_day  = today
-                logger.info("New UTC day %s — session counter reset.", today)
+                equity_gate.reset_day(account_size)
+                logger.info("New UTC day %s — session counter and equity gate reset.", today)

            # ── 2. Bar-change detection ───────────────────────────────────
            bars = mt5.copy_rates_from_pos(DEFAULT_SYMBOL, tf_mt5, 1, 1)
@@ -804,53 +894,98 @@ def _run_loop_inner(tf: str, broker: str, account_size: float, mt5,
                        _log_closed_pnl(closed, mt5)
                        for _t in closed:
                            peak_pnl_tracker.pop(_t, None)
+                            atr_state_tracker.pop(_t, None)
                        signal_tracker["tickets"] = [
                            t for t in signal_tracker["tickets"] if t in open_set
                        ]
                        if not signal_tracker["tickets"]:
                            signal_tracker["tp1_hit"] = False

-                # ── Hybrid Scalp Protection (per-position) ────────────────
-                # Runs every poll cycle (5 s) — bypasses the 5-min bar delay.
-                # Condition A — Fixed target  : close when P&L >= profit_target
-                # Condition B — Trailing guard: once P&L peaked >= $2, close if
-                #               it pulls back to ≤ 50 % of that peak (chop buffer)
-                if profit_target is not None and signal_tracker["tickets"]:
-                    _pnl_divisor = 100 if broker == "headway_cent" else 1
-                    _open_pos = [
+                # ── Daily equity protection gate ──────────────────────────
+                _pnl_divisor = 100 if broker == "headway_cent" else 1
+                _open_pnl = sum(
+                    p.profit for p in (mt5.positions_get(symbol=DEFAULT_SYMBOL) or [])
+                    if p.magic == MAGIC_NUMBER
+                ) / _pnl_divisor
+                if equity_gate.check(account_size + _open_pnl):
+                    if equity_gate.needs_notification:
+                        _loss_usd = -_open_pnl
+                        logger.warning(
+                            "Daily loss limit hit: equity=%.2f  loss=%.2f  "
+                            "(%.0f%% of $%.2f) — closing all & locking until UTC midnight.",
+                            account_size + _open_pnl, _loss_usd,
+                            equity_gate.loss_pct * 100, account_size,
+                        )
+                        for _gp in (mt5.positions_get(symbol=DEFAULT_SYMBOL) or []):
+                            if _gp.magic == MAGIC_NUMBER:
+                                _close_position(_gp.ticket, mt5,
+                                                comment="GRX_Daily_Loss_Limit")
+                        try:
+                            send_telegram_msg(
+                                f"⚠️ <b>Daily Loss Limit Reached</b>\n"
+                                f"Loss: <b>${_loss_usd:.2f}</b> "
+                                f"({equity_gate.loss_pct*100:.0f}% of ${account_size:.2f})\n"
+                                "🔒 Trading suspended until next UTC day."
+                            )
+                        except Exception:
+                            pass
+                    time.sleep(POLL_INTERVAL_SEC)
+                    continue
+
+                # ── ATR-linked Hybrid Trailing Stop ───────────────────────
+                # Phase 1 (one-time per ticket): when P&L >= PROFIT_ACTIVATION_USD,
+                #   move SL to breakeven+2×spread; optionally close 50% volume.
+                # Phase 2 (every poll): trail SL at price ∓ (ATR_mult × cached ATR).
+                if signal_tracker["tickets"]:
+                    _atr_mult  = ATR_TRAIL_MULTIPLIER.get(tf.upper(), 1.5)
+                    _atr_cache = signal_tracker.get("atr_price", 0.0)
+                    _open_pos  = [
                        p for p in (mt5.positions_get(symbol=DEFAULT_SYMBOL) or [])
                        if p.magic == MAGIC_NUMBER
                        and p.ticket in set(signal_tracker["tickets"])
                    ]
                    for _pos in _open_pos:
-                        _cur  = _pos.profit / _pnl_divisor
-                        _peak = peak_pnl_tracker.get(_pos.ticket, _cur)
+                        _cur    = _pos.profit / _pnl_divisor
+                        _ticket = _pos.ticket
+                        _atr_st = atr_state_tracker.setdefault(
+                            _ticket,
+                            {"activated": False, "partial_done": False,
+                             "current_sl": _pos.sl},
+                        )

-                        # Update high-water mark
-                        if _cur > _peak:
-                            _peak = _cur
-                            peak_pnl_tracker[_pos.ticket] = _peak
-
-                        # Condition A: fixed scalp target (per position)
-                        if _cur >= profit_target:
+                        if not _atr_st["activated"] and _cur >= PROFIT_ACTIVATION_USD:
+                            # Phase 1: lock in break-even + 2× spread (one time)
+                            _tick   = mt5.symbol_info_tick(DEFAULT_SYMBOL)
+                            _spread = _tick.ask - _tick.bid
+                            _dir    = signal_tracker["direction"]
+                            if _dir == "BUY":
+                                _be_sl = round(signal_tracker["entry_price"] + _spread * 2, 2)
+                            else:
+                                _be_sl = round(signal_tracker["entry_price"] - _spread * 2, 2)
+                            _move_sl_to_breakeven(_ticket, _be_sl, mt5)
+                            _atr_st["activated"]  = True
+                            _atr_st["current_sl"] = _be_sl
                            logger.info(
-                                "Fixed scalp target #%d: P&L=+$%.2f >= $%.2f — closing.",
-                                _pos.ticket, _cur, profit_target,
+                                "ATR trail activated: ticket=%d  P&L=+$%.2f  BE_SL=%.2f",
+                                _ticket, _cur, _be_sl,
                            )
-                            _close_position(_pos.ticket, mt5,
-                                            comment="GRX_Fixed_Scalp_Target")
-                            continue
+                            # Partial close — skipped automatically if lot <= MIN_LOT_GUARD
+                            if not _atr_st["partial_done"]:
+                                if _execute_partial_close(_ticket, DEFAULT_SYMBOL, mt5):
+                                    _atr_st["partial_done"] = True
+
+                        elif _atr_st["activated"] and _atr_cache > 0:
+                            # Phase 2: ratchet SL toward price at ATR distance
+                            _tick = mt5.symbol_info_tick(DEFAULT_SYMBOL)
+                            _dir  = signal_tracker["direction"]
+                            if _dir == "BUY":
+                                _trail_sl = round(_tick.bid - _atr_mult * _atr_cache, 2)
+                            else:
+                                _trail_sl = round(_tick.ask + _atr_mult * _atr_cache, 2)
+                            if _set_trailing_sl(_ticket, _trail_sl,
+                                                _atr_st["current_sl"], _dir, mt5):
+                                _atr_st["current_sl"] = _trail_sl

-                        # Condition B: trailing guard — activation $2, drawdown 50 %
-                        if (_peak >= TRAILING_ACTIVATION_USD
-                                and _cur <= _peak * TRAILING_DRAWDOWN_PCT):
-                            logger.info(
-                                "Trailing guard #%d: P&L=$%.2f <= 50%% of peak $%.2f "
-                                "(activation $%.2f) — closing.",
-                                _pos.ticket, _cur, _peak, TRAILING_ACTIVATION_USD,
-                            )
-                            _close_position(_pos.ticket, mt5,
-                                            comment="GRX_Trailing_Profit_Guard")
                time.sleep(POLL_INTERVAL_SEC)
                continue
            last_bar_time = bar_time
@@ -871,6 +1006,7 @@ def _run_loop_inner(tf: str, broker: str, account_size: float, mt5,
                DEFAULT_SYMBOL, tf_mt5, model_hmm, obs_cov, trans_cov,
                feature_cols=feature_cols, mt5=mt5,
            )
+            signal_tracker["atr_price"] = atr_price   # cache for between-bar ATR trail
            _, _probs = get_predictions_ensemble(models_xgb, thresholds_xgb, features_df)
            prob = float(_probs[0])

@@ -71,25 +71,18 @@ def _get_tier(balance: float) -> str:


 def _score_result(result: dict, tier: str, broker: str, tf: str = "H1") -> float:
-    cfg  = TIER_CONFIGS[tier]
-    dd   = result["max_drawdown"]
-    # H1 on a small account is more vulnerable to drawdown: a single "normal"
-    # H1 swing can consume 10-15 % of a $15 balance.  Double the penalty to
-    # force the optimizer towards lower-drawdown, safer entries over raw Sharpe.
-    if tf.upper() == "H1" and tier == "small":
-        dd_penalty = cfg["dd_penalty"] * 2.0
-    else:
-        dd_penalty = cfg["dd_penalty"]
-    base = (result["sharpe_ratio"] - dd * dd_penalty
-            if broker == "headway_cent"
-            else result["sharpe_ratio"])
+    cfg = TIER_CONFIGS[tier]
+    dd  = result["max_drawdown"]
+    # Recovery Factor = Net Profit / Max Drawdown.
+    # Rewards the optimizer for finding the safest path to profit on a $15
+    # account rather than just the highest Sharpe.  Capped at 20 to prevent
+    # extreme low-DD outliers from distorting Optuna's TPE surrogate.
+    rf = float(min(result.get("recovery_factor", 0.0), 20.0))
    if dd > cfg["dd_limit"]:
        # Sliding penalty: -2.0 per 1% of DD over the limit.
-        # Replaces the hard -10.0 so trials with, e.g., 16% DD on a 15% limit
-        # can still score positively instead of being discarded outright.
-        overshoot = (dd - cfg["dd_limit"]) * 200   # 0.01 excess → -2.0
-        return base - overshoot
-    return base
+        overshoot = (dd - cfg["dd_limit"]) * 200
+        return rf - overshoot
+    return rf


 def make_objective(balance: float = 15.0, broker: str = "standard", tf: str = "H1"):
@@ -103,16 +96,10 @@ def make_objective(balance: float = 15.0, broker: str = "standard", tf: str = "H
        # M5: floor=1.0 (5-min noise needs high obs_cov to separate states).
        # H1/M15: floor=0.5 (sub-hourly bars still degenerate below this threshold).
        # Other TFs: 0.1 (full range).
-        if tf.upper() == "M5":
-            obs_cov = trial.suggest_float("obs_cov", 1.0, 5.0, log=True)
-        elif tf.upper() in ("H1", "M15"):
-            # H1/M15: floor raised to 1.0 — values below 1.0 with n_states=4
-            # still produce ~70-75% positive-definite failures even with [4]-only
-            # search space.  M5 uses 1.0 floor for the same reason; M15 sub-hourly
-            # bars have similar noise characteristics.
-            obs_cov = trial.suggest_float("obs_cov", 1.0, 5.0, log=True)
-        else:
-            obs_cov = trial.suggest_float("obs_cov", 0.1, 5.0, log=True)
+        # obs_cov range: 0.5–5.0 for all TFs.  Values < 0.5 cause non-positive-
+        # definite covariance errors in ~70-75 % of trials; the persistence and
+        # vol-ratio guards catch the remaining bad trials in the 0.5–1.0 region.
+        obs_cov = trial.suggest_float("obs_cov", 0.5, 5.0, log=True)
        # M5 Kalman is calibrated for 5-min noise; H1/M15 have smoother signals
        # so trans_cov above 0.03 tends to produce chaotic Bull/Chop oscillation
        # (49K+ transitions, identical state means) that the persistence guard
@@ -200,7 +187,7 @@ def make_objective(balance: float = 15.0, broker: str = "standard", tf: str = "H
            df = process_pipeline(
                obs_cov=obs_cov, trans_cov=trans_cov, save=False, tf=tf
            )
-            _hmm, states, _ = fit_hmm(df, n_states=n_states)
+            _hmm, states, _ = fit_hmm(df, n_states=n_states, tf=tf)

            # ── HMM quality gate ─────────────────────────────────────────────
            # Reject trials where any state has low self-transition probability.
@@ -263,8 +250,8 @@ def make_objective(balance: float = 15.0, broker: str = "standard", tf: str = "H
                if result.get("oos_n_trades", 0) < min_oos_trades:
                    return -10.0
                oos_result = {
-                    "sharpe_ratio": result["oos_sharpe_ratio"],
-                    "max_drawdown": result["oos_max_drawdown"],
+                    "recovery_factor": result.get("oos_recovery_factor", 0.0),
+                    "max_drawdown":    result["oos_max_drawdown"],
                }
                score = _score_result(oos_result, tier, broker, tf)
            else:
@@ -272,11 +259,11 @@ def make_objective(balance: float = 15.0, broker: str = "standard", tf: str = "H

            logger.info(
                "Trial %d [%s/%s tier=%s $%.0f]: score=%.3f  "
-                "IS=%.3f  OOS=%.3f  OOS_DD=%.1f%%  trades=%d",
+                "IS_RF=%.3f  OOS_RF=%.3f  OOS_DD=%.1f%%  trades=%d",
                trial.number, tf, broker, tier, balance,
                score,
-                result.get("is_sharpe_ratio",  result["sharpe_ratio"]),
-                result.get("oos_sharpe_ratio", result["sharpe_ratio"]),
+                result.get("is_recovery_factor",  result.get("recovery_factor", 0)),
+                result.get("oos_recovery_factor", result.get("recovery_factor", 0)),
                result.get("oos_max_drawdown", result["max_drawdown"]) * 100,
                result.get("oos_n_trades",     result["n_trades"]),
            )
@@ -346,7 +333,7 @@ def _make_callbacks(total_target: int, study_name: str, already_done: int = 0) -
            )
            print(
                f"  [{total_done:>4}/{total_target}]  "
-                f"Best Sharpe: {best:+.3f}  |  "
+                f"Best RF: {best:+.3f}  |  "
                f"ETA: {eta_str}{ram_str}"
            )

@@ -1,6 +1,7 @@
 import numpy as np
 import pandas as pd
 from pathlib import Path
+from sklearn.mixture import GaussianMixture
 from src.logger import setup_logger

 logger = setup_logger(__name__)
@@ -198,6 +199,54 @@ def compute_atr(df: pd.DataFrame, period: int = 14) -> pd.Series:
    return (atr / df["Close"]).rename("atr_normalized")


+def compute_gmm_vol_cluster(volatility: np.ndarray, n_components: int = 3) -> np.ndarray:
+    """Cluster bars into Volatility Buckets (Low=0, Med=1, High=2) via GMM.
+
+    Fits a Gaussian Mixture Model on the rolling-volatility series and returns
+    integer bucket labels sorted by component mean.  This provides XGBoost a
+    second 'opinion' on market state — e.g. a breakout bar (High Vol, label 2)
+    versus genuine Chop (Low Vol, label 0) can look identical to the HMM but
+    the GMM cluster separates them reliably.
+
+    Falls back to quantile-based labelling if GMM collapses to fewer than
+    ``n_components`` unique labels (degenerate convergence on uniform data).
+
+    Args:
+        volatility: 1-D array of rolling-volatility values (from compute_volatility).
+        n_components: Number of GMM components (default 3 = Low/Med/High).
+
+    Returns:
+        Integer array of shape (N,) with values in {0, 1, …, n_components-1}.
+    """
+    valid_mask = ~np.isnan(volatility)
+    labels_out = np.zeros(len(volatility), dtype=np.int8)  # NaN rows → 0; dropped later by dropna
+
+    vol_valid = volatility[valid_mask]
+    X = vol_valid.reshape(-1, 1)
+    gmm = GaussianMixture(n_components=n_components, random_state=42, n_init=5)
+    raw_labels = gmm.fit_predict(X)
+
+    # Remap: 0 = lowest-volatility component, n_components-1 = highest
+    order = np.argsort(gmm.means_.ravel())
+    remap = np.empty(n_components, dtype=int)
+    for rank, original_idx in enumerate(order):
+        remap[original_idx] = rank
+    labels_valid = remap[raw_labels].astype(np.int8)
+
+    # Fallback: if GMM collapsed to fewer unique labels, use quantile buckets
+    if len(np.unique(labels_valid)) < n_components:
+        logger.warning(
+            "GMM collapsed to %d clusters (expected %d) — using quantile fallback.",
+            len(np.unique(labels_valid)), n_components,
+        )
+        boundaries = np.percentile(vol_valid, [100 / n_components * i
+                                               for i in range(1, n_components)])
+        labels_valid = np.digitize(vol_valid, boundaries).astype(np.int8)
+
+    labels_out[valid_mask] = labels_valid
+    return labels_out
+
+
 def process_pipeline(
    obs_cov: float = None,
    trans_cov: float = None,
@@ -230,6 +279,7 @@ def process_pipeline(
    df["rsi"] = compute_rsi(df["Close"])
    df["rsi_slope"] = df["rsi"].diff()
    df["atr_normalized"] = compute_atr(df)
+    df["gmm_vol_cluster"] = compute_gmm_vol_cluster(df["volatility"].values)

    # ── Optional USDCHF cross-asset feature ─────────────────────────────────
    # Each TF uses its own USDCHF master so bar frequencies stay in sync:
@@ -188,3 +188,75 @@ class AdaptiveRiskManager:
            f"AdaptiveRiskManager(balance=${self.balance:.0f}, tier={tier}, "
            f"broker={self.broker}, daily_trades={self.daily_trades})"
        )
+
+
+class DailyEquityGate:
+    """Floating-equity safety switch for live trading.
+
+    Monitors the sum of realised balance and open floating P&L.  If that
+    running equity drops >= ``loss_pct`` below the start-of-day baseline, the
+    gate locks and remains locked until ``reset_day`` is called (UTC midnight).
+
+    Usage in the live loop::
+
+        gate = DailyEquityGate()
+        gate.reset_day(account_size)   # call once at start of day
+
+        # each poll cycle:
+        current_eq = account_size + open_pnl
+        if gate.check(current_eq):
+            if gate.needs_notification:
+                <close all positions + send Telegram alert>
+            continue   # skip new signals until tomorrow
+    """
+
+    DAILY_LOSS_PCT = 0.05   # 5 % default loss limit
+
+    def __init__(self, loss_pct: float = DAILY_LOSS_PCT):
+        self.loss_pct      = loss_pct
+        self._start_equity: float | None = None
+        self._locked       = False
+        self._notified     = False
+
+    def reset_day(self, equity: float) -> None:
+        """Initialise or reset for a new UTC trading day."""
+        self._start_equity = float(equity)
+        self._locked       = False
+        self._notified     = False
+        logger.debug("DailyEquityGate reset: start_equity=%.2f  limit=%.0f%%",
+                     equity, self.loss_pct * 100)
+
+    def check(self, current_equity: float) -> bool:
+        """Return True if the daily loss limit has been breached.
+
+        Calling this repeatedly after lock is safe — it stays True and does
+        NOT re-trigger ``needs_notification``.
+        """
+        if self._start_equity is None or self._start_equity <= 0:
+            return False
+        dd_pct = (self._start_equity - current_equity) / self._start_equity
+        if dd_pct >= self.loss_pct:
+            self._locked = True
+        return self._locked
+
+    @property
+    def needs_notification(self) -> bool:
+        """True exactly once — the first poll cycle after the lock engages.
+
+        Designed so the caller sends a single Telegram alert without repeated
+        messages on subsequent polls.
+        """
+        if self._locked and not self._notified:
+            self._notified = True
+            return True
+        return False
+
+    @property
+    def locked(self) -> bool:
+        return self._locked
+
+    def __repr__(self) -> str:
+        return (
+            f"DailyEquityGate(loss_pct={self.loss_pct*100:.0f}%, "
+            f"locked={self._locked}, start=${self._start_equity or 0:.2f})"
+        )
@@ -32,6 +32,7 @@ from src.processor import (
    compute_volatility,
    compute_rsi,
    compute_atr,
+    compute_gmm_vol_cluster,
 )
 from src.engine_hmm import load_model as load_hmm, predict_states, get_model_path as hmm_model_path, MODEL_PATH as HMM_GENERIC_PATH
 from src.engine_xgb import load_xgb_ensemble, prepare_features, get_predictions_ensemble, get_ensemble_path, ENSEMBLE_PKL_PATH as XGB_GENERIC_PATH
@@ -72,8 +73,9 @@ def _apply_features(
    df["kalman_return"]  = kalman_smooth(df["log_return"].values, obs_cov, trans_cov)
    df["volatility"]     = compute_volatility(df["log_return"])
    df["rsi"]            = compute_rsi(df["Close"])
-    df["rsi_slope"]      = df["rsi"].diff()
-    df["atr_normalized"] = compute_atr(df)
+    df["rsi_slope"]       = df["rsi"].diff()
+    df["atr_normalized"]  = compute_atr(df)
+    df["gmm_vol_cluster"] = compute_gmm_vol_cluster(df["volatility"].values)

    # Mirror process_pipeline: add USDCHF if the matching TF master file exists
    usdchf_path = _USDCHF_PATH_BY_TF.get(tf.upper(), USDCHF_MASTER_PATH)
				`@@ -1 +0,0 @@`
				`{"timestamp": 1775749655.5421815, "tf": "M5", "best_score": 5.150384458557411}`
				`@@ -1 +0,0 @@`
				`{"timestamp": 1776092348.3391533, "tf": "M5", "best_score": 2.4810574957119442}`
				`@@ -1 +0,0 @@`
				`{"timestamp": 1775754958.6410613, "tf": "M5", "best_score": 4.882846936049799}`