commit 1ecd7baa53d2b2c970ee86b7a47674d7fdbc4d01 Author: Luke Mosoti Date: Mon Mar 30 23:42:32 2026 +0300 Initial commit: Gold Regime X hybrid HMM+XGBoost trading system - Full pipeline: process → optimize → train → export → live - Optuna optimizer with OOS-only Sharpe scoring and prob_threshold in search space - MT5 live bridge with adaptive risk management and staged TPs - M5/M15/H1 timeframe support with per-TF Kalman, HMM, and XGBoost configs - Telegram remote control, guardian health monitor, and daily audit reports - MQL5 EA (v2.00) with ONNX inference and cent account support Co-Authored-By: Claude Sonnet 4.6 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f970940 --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +# Gold Regime X — Environment Variables +# Copy this file to .env and fill in your credentials. +# NEVER commit .env to git (it's in .gitignore). + +# ── Telegram Bot ────────────────────────────────────────────────────────────── +# 1. Open Telegram and search for @BotFather +# 2. Send /newbot, follow the prompts, copy the token below +TELEGRAM_BOT_TOKEN=8566397719:AAH7o9Lw7MQ3P6kEZDEYLjwwtTMOAvs0jNg + +# 3. Send /start to your new bot, then visit: +# https://api.telegram.org/bot/getUpdates +# Copy the "id" value from "chat" in the first result +TELEGRAM_CHAT_ID=557964386 + +# ── Remote Control Security ─────────────────────────────────────────────────── +# Your personal Telegram numeric user ID (not username). +# Only messages from this ID will be accepted by --mode listen. +# Get it from: https://t.me/userinfobot or the getUpdates response above +ALLOWED_USER_ID=557964386 + +# ── Live Trading Defaults for Remote Control ────────────────────────────────── +# These values are used when you press "START TRADING" from Telegram. +LIVE_TF=H1 +LIVE_BROKER=headway_cent +LIVE_BALANCE=15 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1003d3b --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* !text !filter !merge !diff diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b7c7545 --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# Secrets +.env + +# Python +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +.eggs/ +dist/ +build/ +.venv/ +venv/ + +# Data and models (large files) +data/raw/ +data/processed/ +models/*.pkl +models/*.joblib +models/*.onnx +models/study.db + +# Reports +reports/ + +# Logs +logs/ + +# Compiled MQL5 binary (source .mq5 is tracked) +mql5/*.ex5 + +# IDE +.vscode/settings.json +.idea/ diff --git a/README_LIVE.md b/README_LIVE.md new file mode 100644 index 0000000..85f2d47 --- /dev/null +++ b/README_LIVE.md @@ -0,0 +1,223 @@ +# Gold Regime X — Live Trading Guide + +This guide explains how to operate the Python MT5 Live Bridge for Gold Regime X. +The bridge connects directly to your running MetaTrader5 terminal, validates the +model against recent market data, and executes orders automatically on XAUUSD. + +--- + +## Prerequisites + +| Requirement | Notes | +|---|---| +| Windows OS | MetaTrader5 Python package is Windows-only | +| MT5 terminal running | Must be open and logged into your Headway account | +| XAUUSD in Market Watch | Right-click Market Watch → Show All, or search XAUUSD | +| Algorithmic trading enabled | MT5 → Tools → Options → Expert Advisors → Allow Algorithmic Trading | +| Models trained | `models/hmm_model.pkl` and `models/xgb_model.pkl` must exist | +| Python package | `pip install MetaTrader5>=5.0.45` | +| **EA removed from chart** | The GoldRegimeX.mq5 EA and the Python bridge share `MAGIC_NUMBER = 123456`. Running both simultaneously will double-count daily trades. Remove the EA from the XAUUSD chart before starting the Python bridge. | + +--- + +## MT5 Terminal Setup + +1. Open MetaTrader5. +2. Go to **Tools → Options → Expert Advisors**. +3. Check **Allow Algorithmic Trading**. +4. Check **Allow WebRequest for listed URL** and add `http://localhost` if using a local bridge. +5. Log into your Headway account (Cent or Live). +6. Add XAUUSD to the Market Watch if it is not already visible. + +--- + +## Quick-Start Workflow + +### Step 1 — Train the models (if not done already) + +``` +python main.py --mode process --tf H1 +python main.py --mode optimize --trials 250 --broker headway_cent --balance 15 --tf H1 +python main.py --mode train --broker headway_cent --balance 15 --tf H1 +``` + +### Step 2 — Sync recent data and validate the model + +Run this daily before going live to ensure the model is still healthy. + +``` +python main.py --mode sync_validate --period 3m --broker headway_cent --balance 15 --tf H1 +``` + +| Validation Status | Meaning | Action | +|---|---|---| +| **PASS** (Sharpe ≥ 0.8) | Model is stable | Proceed to Step 3 | +| **WARN** (Sharpe 0.5–0.8) | Borderline performance | Proceed with caution or re-optimise | +| **FAIL** (Sharpe < 0.5) | Market regime drift | Run optimize + train before going live | + +A FAIL status will exit with code 1 and block the script from continuing. + +### Step 3 — Paper trade on demo (recommended first) + +``` +python main.py --mode live --account demo --broker headway_cent --balance 15 --tf H1 +``` + +Demo mode logs every signal and simulates the full session-limit logic, but +sends no real orders to MT5. Use this to verify signals appear as expected. + +### Step 4 — Switch to live account + +``` +python main.py --mode live --account live --broker headway_cent --balance 15 --tf H1 +``` + +You will be prompted to type `YES` to confirm before the loop starts. + +--- + +## Command Reference + +``` +python main.py --mode sync_validate [OPTIONS] + --period 3m Lookback window (e.g. 3m, 6m, 12m) + --tf H1 Timeframe (H1 or M15) + --broker headway_cent + --balance 15 + +python main.py --mode live [OPTIONS] + --account demo|live demo = dry-run (no orders) live = real orders + --tf H1 Must match the timeframe the models were trained on + --broker headway_cent + --balance 15 USD balance used for lot-sizing + --period (not used in live mode) +``` + +--- + +## How to Monitor Live Trading + +### Terminal View (MT5) + +Open the **Trade** tab at the bottom of the MT5 terminal to see all open +positions. Each position placed by the Python bridge will show: + +- **Symbol**: XAUUSD +- **Comment**: `GRX_BUY_p1of1_s0_p0.73` (direction / position index / HMM state / probability) +- **Magic**: 123456 + +### Expert Logs + +Go to the **Journal** or **Experts** tab in MT5. Errors from API calls +(e.g. "Invalid S/L", "Not enough money") appear in red here. + +### Python Log File + +All signals, orders, and errors are written to `logs/goldregimex.log`. + +``` +# Key log lines to watch for: +SIGNAL BUY — signal fired, orders being placed +Order filled — order confirmed by broker +Session limit — daily cap enforced (no more trades today) +Order failed — execution error (check MT5 Journal tab) +Insufficient margin — reduce lot size or top up account +High-vol deviation — elevated deviation used due to regime instability +``` + +To follow the log in real time: + +``` +# PowerShell +Get-Content logs/goldregimex.log -Wait -Tail 30 +``` + +--- + +## Risk Management Reference + +| Account Balance | Max Trades/Day | Positions per Signal | Total Positions | +|---|---|---|---| +| ≤ $50 USD | 2 | 1 | 2 maximum | +| > $50 USD (Bull/Bear) | 3 | 2 | 6 maximum | +| > $50 USD (Chop) | 2 | 2 | 4 maximum | + +**All trades use the 1% risk rule:** + +``` +lot_size = (1% of USD balance) / (ATR(14) × 2.0) +``` + +The stop-loss is placed at `entry ± ATR(14) × 2.0`. No take-profit is set — +positions are managed by the next opposing signal or manual intervention. + +**Cent Account Note**: On Headway Cent, the terminal displays $15 USD as +`1500.00` in the balance field. The Python bridge detects `--broker headway_cent` +and automatically divides by 100 for all internal calculations. If you see a +balance of `1500.00` in MT5 and pass `--balance 15`, both will agree. + +--- + +## Order Specifications + +| Parameter | Value | Notes | +|---|---|---| +| Filling type | IOC (Immediate or Cancel) | Standard for Headway ECN | +| Deviation (normal) | 20 points | $0.20 on XAUUSD | +| Deviation (high-vol) | 50 points | When HMM self-transition prob < 0.70 | +| Magic number | 123456 | Matches GoldRegimeX.mq5 | +| SL | ATR × 2.0 below/above entry | Hard stop on every trade | +| TP | None | No fixed take-profit | + +IOC means: if the broker cannot fill the order within the deviation window, +the order is cancelled entirely (never partially filled). This prevents +unexpected partial exposure during gold news spikes. + +--- + +## Troubleshooting + +| Problem | Likely Cause | Fix | +|---|---|---| +| `ConnectionError: Could not connect to MT5` | Terminal not running | Open MT5 and log in first | +| `FileNotFoundError: models/hmm_model.pkl` | Models not trained | Run `--mode train` | +| Validation status FAIL every day | Model too old for current market | Run `--mode optimize` then `--mode train` | +| `Order failed: retcode=10006` | No connection to broker | Check MT5 connection indicator (bottom-right) | +| `Order failed: retcode=10015` | Price changed faster than deviation allows | Will retry on next bar; consider longer TF | +| `Insufficient margin` repeated | Account below minimum for lot size | Top up account or reduce `--balance` value | +| Signals appear but no trades fired | `dry_run=True` (demo mode) | Switch to `--account live` after confirming signals look correct | +| Double positions appearing | EA still on chart | Remove GoldRegimeX.mq5 EA from XAUUSD chart | + +--- + +## Emergency Stop + +Press **Ctrl+C** in the terminal window. The loop handles `KeyboardInterrupt` +gracefully: it logs the shutdown message and calls `mt5.shutdown()` to cleanly +disconnect the Python package. + +All open positions remain open in MT5 after the script exits. You must close +them manually from the Trade tab or by placing an opposing order. + +--- + +## Complete Workflow Example (Headway Cent, $15 balance, H1) + +```bash +# 1. First time setup +python main.py --mode process --tf H1 +python main.py --mode optimize --trials 250 --broker headway_cent --balance 15 --tf H1 +python main.py --mode train --broker headway_cent --balance 15 --tf H1 + +# 2. Daily routine (before market open) +python main.py --mode sync_validate --period 3m --broker headway_cent --balance 15 --tf H1 +# → PASS: continue +# → WARN: continue with reduced size or check market conditions +# → FAIL: run optimize + train again before going live + +# 3. Paper trade for several days to verify signals +python main.py --mode live --account demo --broker headway_cent --balance 15 --tf H1 + +# 4. Go live when satisfied +python main.py --mode live --account live --broker headway_cent --balance 15 --tf H1 +``` diff --git a/README_REMOTE.md b/README_REMOTE.md new file mode 100644 index 0000000..c26cbe8 --- /dev/null +++ b/README_REMOTE.md @@ -0,0 +1,134 @@ +# Gold Regime X — Remote Control & Telegram Setup + +## Prerequisites + +Install the required packages: + +```bash +pip install requests python-dotenv psutil schedule +``` + +--- + +## Step 1 — Create a Telegram Bot + +1. Open Telegram → search **@BotFather** → send `/newbot` +2. Follow the prompts and copy the **Bot Token** (looks like `123456789:AAF-abc...`) +3. Send **`/start`** to your new bot in Telegram +4. Find your **Chat ID** by opening this URL in a browser: + ``` + https://api.telegram.org/bot/getUpdates + ``` + Look for `"chat": {"id": 123456789}` — that number is your Chat ID. +5. Find your **User ID** (same as Chat ID for private chats, or use [@userinfobot](https://t.me/userinfobot)) + +--- + +## Step 2 — Configure .env + +```bash +copy .env.example .env +``` + +Fill in the three required values: + +``` +TELEGRAM_BOT_TOKEN=123456789:AAF-abc... +TELEGRAM_CHAT_ID=123456789 +ALLOWED_USER_ID=123456789 +``` + +Optionally set the live trading defaults (used by the START TRADING button): + +``` +LIVE_TF=H1 +LIVE_BROKER=headway_cent +LIVE_BALANCE=15 +``` + +--- + +## Step 3 — Start the Listener + +On your laptop/PC (while MT5 is running): + +```bash +python main.py --mode listen --broker headway_cent --balance 15 +``` + +You will see a confirmation message in your Telegram chat and a keyboard appears. + +--- + +## Telegram Keyboard Buttons + +| Button | Action | +|--------|--------| +| 🚀 START TRADING | Launches the live execution loop (uses `.env` TF/broker/balance) | +| 🛑 STOP TRADING | Terminates the running live loop process | +| 📉 START OPTIMIZE (M5) | Starts M5 Optuna study (resumes from study.db if interrupted) | +| 📊 BOT STATUS | Sends last 24h trade history with P&L, win rate, and floating positions | + +--- + +## Additional Modes + +### Guardian — Continuous health monitor +```bash +python main.py --mode guardian --tf M5,M15,H1 --period 3m --interval 3600 --broker headway_cent --balance 15 +``` +Checks rolling Sharpe for each TF every hour. Sends Telegram alert if Sharpe drops below 0.6. + +### Audit — On-demand performance report +```bash +python main.py --mode audit --broker headway_cent --balance 15 +``` +Prints and sends the last 24h deal summary to Telegram. + +--- + +## Nightly Auto-Report + +When `--mode listen` is running, a background thread automatically sends the +daily performance summary to Telegram at **23:55 UTC** every night. + +No extra configuration needed — just keep the listener running. + +--- + +## Security Notes + +- The `ALLOWED_USER_ID` env var is the single security gate — only your Telegram + ID can issue commands. Keep your `.env` file private. +- Never commit `.env` to git. It is listed in `.gitignore`. +- The listener uses Telegram's long-polling API (no public webhook needed). +- All commands are launched as separate subprocesses so the bot stays responsive. + +--- + +## Optimization Resume (Failsafe) + +The optimizer saves every completed trial to `models/study.db`. If interrupted: + +1. Press **Ctrl+C** in your terminal at any time — progress is saved +2. Re-run the same optimize command: + ```bash + python main.py --mode optimize --tf M5 --trials 500 + ``` +3. You will see: `Failsafe: XX trials found in study.db. Resuming from trial #XX...` + +To start a completely fresh study, delete `models/study.db` manually. + +### Parallel Optimization (Advanced) + +For true multi-process parallelism, open **multiple terminals** and run the same +command in each — they all share `study.db` safely via SQLite locking: + +``` +Terminal 1: python main.py --mode optimize --tf M5 --trials 500 +Terminal 2: python main.py --mode optimize --tf M5 --trials 500 +Terminal 3: python main.py --mode optimize --tf M5 --trials 500 +``` + +Each terminal runs an independent worker. Optuna coordinates via SQLite. +This is more reliable than `--n_jobs` for CPU-bound objectives like HMM+XGBoost. diff --git a/main.py b/main.py new file mode 100644 index 0000000..dd8a8a9 --- /dev/null +++ b/main.py @@ -0,0 +1,534 @@ +import argparse +import json +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +# Load .env for Telegram credentials (silently ignored if python-dotenv not installed) +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + pass + +from src.logger import setup_logger +from src.processor import process_pipeline, TF_CONFIG, PROCESSED_PATH +from src.engine_hmm import fit_hmm, save_model as save_hmm, load_model as load_hmm +from src.engine_xgb import ( + prepare_features, train_xgb, get_predictions, + export_onnx, save_xgb, load_xgb, ONNX_PATH, FEATURE_COLS, +) +from src.optimizer import run_optimization, get_best_params +from src.backtester import vectorized_backtest +from src.visualizer import generate_full_report +from src.risk_manager import AdaptiveRiskManager + +logger = setup_logger("main") + + +def _resolve_balance(args) -> float: + """--balance takes precedence over --min_cap.""" + return args.balance if args.balance is not None else args.min_cap + + +_M5_META_PATH = Path("models/m5_meta.json") +_M5_EXPIRY_HOURS = 120 # 5 days + + +def _check_m5_readiness(tf: str) -> bool: + """Return True if the M5 model is fresh enough for live trading. + + The M5 timeframe is sensitive to microstructure changes, so the model + must have been optimised within the last 5 days. A meta.json timestamp + is written by cmd_optimize after each successful M5 study. + """ + if tf.upper() != "M5": + return True + if not _M5_META_PATH.exists(): + print( + "\nERROR: M5 model meta-data not found.\n" + "Run python main.py --mode optimize --tf M5 before live trading." + ) + return False + meta = json.loads(_M5_META_PATH.read_text()) + age_h = (time.time() - meta.get("timestamp", 0)) / 3600 + if age_h > _M5_EXPIRY_HOURS: + print( + f"\nWARNING: M5 model is {age_h:.0f} hours old (limit: {_M5_EXPIRY_HOURS}h / 5 days).\n" + "REQUIRED ACTION: Run python main.py --mode optimize --tf M5 to refresh before live.\n" + "\n Pre-flight checklist:\n" + " 1. python main.py --mode sync_validate --period 3m --tf M5\n" + " 2. python main.py --mode optimize --tf M5 --trials 500 --balance 15\n" + " 3. python main.py --mode train --tf M5\n" + ) + return False + return True + + +def _train_for_tf(tf: str, balance: float, broker: str, params: dict): + """Shared train logic for a single timeframe. Returns (result, model_hmm, states, X, metrics).""" + df = process_pipeline( + obs_cov=params.get("obs_cov"), + trans_cov=params.get("trans_cov"), + save=True, + tf=tf, + ) + model_hmm, states, state_map = fit_hmm( + df, n_states=params.get("n_states", TF_CONFIG[tf].get("n_states_default", 3)) + ) + X, y, df_aligned = prepare_features(df, states) + model_xgb, metrics = train_xgb( + X, y, + max_depth=params.get("max_depth", 4), + learning_rate=params.get("learning_rate", 0.1), + n_estimators=params.get("n_estimators", 200), + subsample=params.get("subsample", 0.8), + colsample_bytree=params.get("colsample_bytree", 0.8), + min_child_weight=params.get("min_child_weight", 5), + gamma=params.get("gamma", 1.0), + reg_alpha=params.get("reg_alpha", 0.1), + ) + _, probabilities = get_predictions(model_xgb, X) + states_aligned = states[df.index.isin(df_aligned.index)] + split_idx = metrics.get("split_idx") + result = vectorized_backtest( + df_aligned, probabilities, states_aligned, + split_idx=split_idx, + account_size=balance, + broker=broker, + tf=tf, + ) + return result, model_hmm, state_map, model_xgb, metrics, df_aligned, states_aligned, X, probabilities + + +def cmd_process(args): + tfs = [t.strip().upper() for t in args.tf.split(",")] + for tf in tfs: + if tf not in TF_CONFIG: + logger.error("Unknown timeframe '%s'. Valid: %s", tf, list(TF_CONFIG)) + continue + try: + df = process_pipeline(save=True, tf=tf) + logger.info( + "[%s] Done: %d bars | Range: %s to %s", + tf, len(df), df.index.min(), df.index.max(), + ) + except FileNotFoundError as e: + logger.error(str(e)) + + +def cmd_optimize(args): + balance = _resolve_balance(args) + broker = args.broker + tfs = [t.strip().upper() for t in args.tf.split(",")] + for tf in tfs: + logger.info("Optimizing [%s] broker=%s balance=$%.0f trials=%d", tf, broker, balance, args.trials) + study = run_optimization(n_trials=args.trials, balance=balance, broker=broker, tf=tf, n_jobs=args.n_jobs) + arm = AdaptiveRiskManager(balance) + limits = arm.get_trade_limits() + print(f"\n=== Best Result [{tf}] ===") + print(f"Score: {study.best_value:.3f}") + print(f"Broker: {broker}") + print(f"Balance: ${balance:.0f} USD ({arm}") + print(f"Session Limit: {limits['max_daily_trades']} trade(s)/day | {limits['pos_per_trade']} pos/trade") + print("Best Params:") + for k, v in study.best_params.items(): + print(f" {k}: {v}") + if tf == "M5": + _M5_META_PATH.parent.mkdir(parents=True, exist_ok=True) + _M5_META_PATH.write_text(json.dumps({ + "timestamp": time.time(), + "tf": "M5", + "best_score": study.best_value, + })) + print( + "\nOptimization Complete. HMM States and XGBoost Weights have been updated. " + "You are now cleared for M5 Live Trading for the next 5 days." + ) + + +def cmd_train(args): + balance = _resolve_balance(args) + broker = args.broker + tf = args.tf.upper() + + try: + params = get_best_params(balance=balance, broker=broker, tf=tf) + logger.info("Using Optuna best params [%s/%s]: %s", tf, broker, params) + except Exception: + logger.warning("No Optuna study found for tf=%s broker=%s — using defaults", tf, broker) + params = {} + + result, model_hmm, state_map, model_xgb, metrics, *_ = _train_for_tf(tf, balance, broker, params) + save_hmm(model_hmm) + save_xgb(model_xgb, metrics) + + arm = AdaptiveRiskManager(balance) + limits = arm.get_trade_limits() + print(f"\n=== Training Results [{tf}] ===") + print(f"Broker: {broker} | Balance: ${balance:.0f} | Tier: {'small' if arm.is_small_account else 'growth'}") + print(f"Session: {limits['max_daily_trades']}/day | Positions: {limits['pos_per_trade']}/trade") + print(f"Sharpe: {result['sharpe_ratio']:.3f} | MaxDD: {result['max_drawdown']*100:.1f}% " + f"| WR: {result['win_rate']*100:.1f}% | Trades: {result['n_trades']}") + if "oos_sharpe_ratio" in result: + print(f"\n--- In-Sample ---") + print(f" Sharpe: {result['is_sharpe_ratio']:.3f} | DD: {result['is_max_drawdown']*100:.1f}% " + f"| WR: {result['is_win_rate']*100:.1f}% | Trades: {result['is_n_trades']}") + print(f"--- Out-of-Sample ---") + print(f" Sharpe: {result['oos_sharpe_ratio']:.3f} | DD: {result['oos_max_drawdown']*100:.1f}% " + f"| WR: {result['oos_win_rate']*100:.1f}% | Trades: {result['oos_n_trades']}") + print(f"\nModels saved. Run --mode export to generate ONNX.") + + +def cmd_compare(args): + """Train and backtest the specified timeframes, then show a side-by-side comparison.""" + balance = _resolve_balance(args) + broker = args.broker + tfs = [t.strip().upper() for t in args.tf.split(",")] + results = {} + + for tf in tfs: + try: + params = get_best_params(balance=balance, broker=broker, tf=tf) + except Exception: + params = {} + + try: + result, *_ = _train_for_tf(tf, balance, broker, params) + results[tf] = result + except FileNotFoundError as e: + logger.error("Skipping %s: %s", tf, e) + continue + + if len(results) < 2: + avail = ", ".join(tfs) + print(f"\nOnly {len(results)} timeframe(s) available. Run --mode process --tf {avail} first.") + if results: + tf, r = next(iter(results.items())) + print(f"\n[{tf}] Sharpe={r['sharpe_ratio']:.3f} | OOS={r.get('oos_sharpe_ratio',0):.3f} " + f"| DD={r['max_drawdown']*100:.1f}% | Trades={r['n_trades']}") + return + + def _oos_sharpe(r): + return r.get("oos_sharpe_ratio", r.get("sharpe_ratio", 0.0)) + + winner = max(results, key=lambda k: _oos_sharpe(results[k])) + col_w = 12 + tf_list = list(results.keys()) + header = f"{'Metric':<20}" + "".join(f" {tf:>{col_w}}" for tf in tf_list) + + print(f"\n{'='*50}") + print(f" TIMEFRAME COMPARISON | Balance: ${balance:.0f} | Broker: {broker}") + print(f"{'='*50}") + print(header) + print("-" * len(header)) + metrics_to_show = [ + ("OOS Sharpe", "oos_sharpe_ratio", ".3f"), + ("IS Sharpe", "is_sharpe_ratio", ".3f"), + ("OOS Max DD", "oos_max_drawdown", ".1%"), + ("OOS Win Rate", "oos_win_rate", ".1%"), + ("OOS Trades", "oos_n_trades", "d"), + ("Full Sharpe", "sharpe_ratio", ".3f"), + ] + for label, key, fmt in metrics_to_show: + row = f"{label:<20}" + for tf in tf_list: + r = results[tf] + val = r.get(key, r.get(key.replace("oos_", "").replace("is_", ""), 0)) + s = f"{val:{fmt}}" if fmt != "d" else str(int(val)) + row += f" {s:>{col_w}}" + print(row) + + for tf, r in results.items(): + logger.info( + " %s Sharpe=%.3f (OOS=%.3f) | DD=%.1f%% | WR=%.1f%% | Trades=%d", + tf, r["sharpe_ratio"], r.get("oos_sharpe_ratio", r["sharpe_ratio"]), + r["max_drawdown"] * 100, r["win_rate"] * 100, r["n_trades"], + ) + + print(f"\n-> Recommended timeframe: {winner} (higher OOS Sharpe)") + print(f" Run: python main.py --mode train --tf {winner} --broker {broker} --balance {balance:.0f}") + + +def cmd_export(args): + try: + model_xgb, _ = load_xgb() + except FileNotFoundError: + logger.error("No trained XGB model found. Run --mode train first.") + sys.exit(1) + export_onnx(model_xgb) + print(f"\nONNX model exported to {ONNX_PATH}") + print("Copy this file to your MT5 MQL5/Files/ directory.") + + +def cmd_sync_validate(args): + """Download recent MT5 bars then run the model validation gatekeeper.""" + from src.mt5_sync import sync_mt5_data + from src.validator import run_validation + + balance = _resolve_balance(args) + tf = args.tf.upper() + + logger.info("Syncing MT5 data [%s] period=%s ...", tf, args.period) + try: + df = sync_mt5_data(tf=tf, period=args.period) + logger.info( + "Sync complete: %d bars %s -> %s", + len(df), df.index.min(), df.index.max(), + ) + except Exception as exc: + logger.error("MT5 sync failed: %s", exc) + sys.exit(1) + + try: + result = run_validation(tf=tf, broker=args.broker, account_size=balance) + except Exception as exc: + logger.error("Validation error: %s", exc) + sys.exit(1) + + print(f"\n=== Validation Result [{tf}] ===") + print(f" Recent Sharpe: {result['sharpe']:.3f}") + print(f" Trades: {result['n_trades']}") + print(f" Win Rate: {result['win_rate']*100:.1f}%") + print(f" Max Drawdown: {result['max_dd']*100:.1f}%") + print(f" Status: {result['status'].upper()}") + print(f" {result['message']}") + + if result["status"] == "fail": + print( + "\nABORTING: Validation failed. " + "Retune with --mode optimize then --mode train before going live." + ) + sys.exit(1) + if result["status"] == "warn": + print("\nWARNING: Borderline performance — consider re-optimising before live trading.") + + +def cmd_live(args): + """Connect to MT5 and start the live signal execution loop.""" + from src.mt5_trader import run_live_loop + + balance = _resolve_balance(args) + tf = args.tf.upper() + dry_run = (args.account == "demo") + + if not _check_m5_readiness(tf): + sys.exit(1) + + if args.account == "live": + print("\n" + "=" * 60) + print(" WARNING: LIVE ACCOUNT — real money is at risk.") + print(" Ensure --mode sync_validate passed before continuing.") + print(" Ensure the GoldRegimeX EA is removed from the XAUUSD chart.") + confirm = input(" Type YES to confirm live trading: ").strip() + if confirm != "YES": + print("Aborted.") + sys.exit(0) + print("=" * 60 + "\n") + + logger.info( + "Starting live loop — TF=%s broker=%s balance=$%.0f dry_run=%s", + tf, args.broker, balance, dry_run, + ) + run_live_loop(tf=tf, broker=args.broker, account_size=balance, dry_run=dry_run) + + +def cmd_report(args): + balance = _resolve_balance(args) + broker = args.broker + tf = args.tf.upper() + + try: + params = get_best_params(balance=balance, broker=broker, tf=tf) + except Exception: + params = {} + + df = process_pipeline(obs_cov=params.get("obs_cov"), trans_cov=params.get("trans_cov"), + save=False, tf=tf) + + try: + model_hmm = load_hmm() + from src.engine_hmm import predict_states, STATE_NAMES_3, STATE_NAMES_2, STATE_NAMES_4 + states = predict_states(model_hmm, df) + n = model_hmm.n_components + state_names = {2: STATE_NAMES_2, 3: STATE_NAMES_3, 4: STATE_NAMES_4}.get(n, STATE_NAMES_3) + except Exception: + model_hmm, states, state_names = fit_hmm(df, n_states=params.get("n_states", 3)) + + X, y, df_aligned = prepare_features(df, states) + + try: + model_xgb, metrics = load_xgb() + except FileNotFoundError: + model_xgb, metrics = train_xgb( + X, y, + max_depth=params.get("max_depth", 4), + learning_rate=params.get("learning_rate", 0.1), + n_estimators=params.get("n_estimators", 200), + subsample=params.get("subsample", 0.8), + colsample_bytree=params.get("colsample_bytree", 0.8), + min_child_weight=params.get("min_child_weight", 5), + gamma=params.get("gamma", 1.0), + reg_alpha=params.get("reg_alpha", 0.1), + ) + + if "feature_importance" not in metrics: + metrics["feature_importance"] = dict(zip(FEATURE_COLS, model_xgb.feature_importances_)) + + _, probabilities = get_predictions(model_xgb, X) + states_aligned = states[df.index.isin(df_aligned.index)] + + # split_idx in saved metrics may come from a different TF's training run + # (e.g. M15 split_idx ~186k used in an H1 report with only ~58k bars). + # Recompute from current data when the stored value doesn't fit. + split_idx = metrics.get("split_idx") + if split_idx is None or not (0 < split_idx < len(X)): + split_idx = int(len(X) * 0.8) + + result = vectorized_backtest( + df_aligned, probabilities, states_aligned, + split_idx=split_idx, account_size=balance, broker=broker, tf=tf, + ) + + arm = AdaptiveRiskManager(balance) + limits = arm.get_trade_limits() + display_params = dict(params) + display_params.update({ + "broker": broker, + "balance_usd": balance, + "tier": "small" if arm.is_small_account else "growth", + "session_limit": limits["max_daily_trades"], + "pos_per_trade": limits["pos_per_trade"], + "tf": tf, + }) + + paths = generate_full_report( + df_aligned, states_aligned, state_names, model_hmm, + X, probabilities, metrics, result, display_params, + split_idx=split_idx, tf=tf, + ) + + print(f"\n=== Report Generated [{tf}] ===") + for p in paths: + print(f" {p}") + print(f"\n5 charts saved to reports/{tf}/") + + +def cmd_audit(args): + """Print (and optionally send) the daily MT5 performance report.""" + from src.auditor import get_daily_report + from src.notifier import send_telegram_msg + + balance = _resolve_balance(args) + report = get_daily_report(broker=args.broker) + print(report) + sent = send_telegram_msg(report) + if sent: + print("\nReport also sent to Telegram.") + else: + print("\n(Telegram not configured — see .env.example)") + + +def cmd_guardian(args): + """Start the multi-TF health monitor loop.""" + from src.guardian import run_guardian + + balance = _resolve_balance(args) + tfs = [t.strip().upper() for t in args.tf.split(",")] + interval = getattr(args, "interval", 3600) + + run_guardian( + tfs=tfs, + broker=args.broker, + account_size=balance, + period=args.period, + interval_sec=interval, + ) + + +def cmd_listen(args): + """Start the Telegram remote control listener + nightly report scheduler.""" + import threading + from src.remote_control import run_listener + from src.notifier import send_telegram_msg + from src.auditor import get_daily_report + + balance = _resolve_balance(args) + + # ── Nightly audit scheduler (runs in background thread) ────────────────── + def _run_scheduler(): + try: + import schedule + except ImportError: + logger.warning( + "schedule package not installed — nightly report disabled. " + "Install with: pip install schedule" + ) + return + + def _send_nightly(): + report = get_daily_report(broker=args.broker) + send_telegram_msg(f"Nightly Report\n{report}") + logger.info("Nightly report sent via Telegram.") + + schedule.every().day.at("23:55").do(_send_nightly) + logger.info("Nightly report scheduled at 23:55 UTC.") + while True: + schedule.run_pending() + time.sleep(60) + + sched_thread = threading.Thread(target=_run_scheduler, daemon=True) + sched_thread.start() + + # ── Blocking Telegram listener ──────────────────────────────────────────── + run_listener() + + +def main(): + parser = argparse.ArgumentParser(description="Gold Regime X — Hybrid ML Trading System") + parser.add_argument( + "--mode", + choices=["process", "optimize", "train", "compare", "export", "report", + "sync_validate", "live", "audit", "guardian", "listen"], + required=True, + ) + parser.add_argument("--trials", type=int, default=250) + parser.add_argument("--n_jobs", type=int, default=1, + help="Parallel Optuna trial workers (default 1). " + "See optimizer.py for caveats with n_jobs>1.") + parser.add_argument("--interval", type=int, default=3600, + help="Guardian check interval in seconds (default 3600 = 1h).") + parser.add_argument("--min_cap", type=float, default=15.0, + help="Account capital in USD (legacy, use --balance)") + parser.add_argument("--balance", type=float, default=None, + help="Account balance in USD — overrides --min_cap") + parser.add_argument("--broker", type=str, default="standard", + choices=["standard", "headway_cent"]) + parser.add_argument("--tf", type=str, default="H1", + help="Timeframe: H1 | M15 | M15,H1 (process/compare accept comma list)") + parser.add_argument("--period", type=str, default="3m", + help="Lookback window for MT5 sync, e.g. '3m' '6m' '12m'.") + parser.add_argument("--account", type=str, default="demo", + choices=["live", "demo"], + help="Account type for --mode live. 'live' requires confirmation.") + + args = parser.parse_args() + { + "process": cmd_process, + "optimize": cmd_optimize, + "train": cmd_train, + "compare": cmd_compare, + "export": cmd_export, + "report": cmd_report, + "sync_validate": cmd_sync_validate, + "live": cmd_live, + "audit": cmd_audit, + "guardian": cmd_guardian, + "listen": cmd_listen, + }[args.mode](args) + + +if __name__ == "__main__": + main() diff --git a/models/m5_meta.json b/models/m5_meta.json new file mode 100644 index 0000000..f3ddc10 --- /dev/null +++ b/models/m5_meta.json @@ -0,0 +1 @@ +{"timestamp": 1774801417.6893382, "tf": "M5", "best_score": 4.953615714300805} \ No newline at end of file diff --git a/mql5/GoldRegimeX.mq5 b/mql5/GoldRegimeX.mq5 new file mode 100644 index 0000000..5acd13d --- /dev/null +++ b/mql5/GoldRegimeX.mq5 @@ -0,0 +1,299 @@ +//+------------------------------------------------------------------+ +//| GoldRegimeX.mq5 | +//| Gold Regime X — Hybrid ML EA | +//+------------------------------------------------------------------+ +#property copyright "Gold Regime X" +#property version "2.00" +#property strict + +#resource "\\Files\\xgb_model.onnx" as uchar OnnxModel[] + +//--- Trading inputs +input double RiskPercent = 1.0; // Risk per trade (%) +input double ProbThreshold = 0.65; // XGB probability threshold for Long +input double ShortThreshold = 0.35; // XGB probability threshold for Short +input int ATRPeriod = 14; // ATR period +input double ATRMultiplier = 2.0; // ATR stop loss multiplier +input int RSIPeriod = 14; // RSI period +input int ChopState = 2; // HMM Chop state index (no trade) +input int MagicNumber = 123456; +input int NStates = 3; // Classes in trained model — must match n_states from Python (default 3) + +//--- Timeframe input (M15 or H1) +input ENUM_TIMEFRAMES SignalTimeframe = PERIOD_H1; // Timeframe for ONNX signals + +//--- Account / session inputs +// These are OVERRIDDEN at runtime by AdaptiveScaling logic below. +// Set only as fallback defaults. +input bool IsCentAccount = true; // Headway Cent micro-lot floor + +//--- HMM proxy thresholds (update from Python training output) +input double BullReturnThreshold = 0.0001; +input double BearReturnThreshold = -0.0001; +input double VolThreshold = 0.005; + +//--- Handles +long onnx_handle = INVALID_HANDLE; +int atr_handle = INVALID_HANDLE; +int rsi_handle = INVALID_HANDLE; + +//--- Session state +int session_trade_count = 0; +datetime last_session_day = 0; + +//+------------------------------------------------------------------+ +//| Adaptive scaling from Python spec | +//| ≤ $50 USD : max_trades=2, pos_per_trade=1 | +//| > $50 USD : max_trades=2|3 (Chop vs Bull/Bear), pos_per_trade=2 | +//+------------------------------------------------------------------+ +int GetMaxTrades(double balance, int hmm_state) +{ + if(balance <= 50.0) return 2; + return (hmm_state == ChopState) ? 2 : 3; +} + +int GetPosPerTrade(double balance) +{ + return (balance <= 50.0) ? 1 : 2; +} + +//+------------------------------------------------------------------+ +void CheckSessionReset() +{ + MqlDateTime dt; + TimeToStruct(TimeCurrent(), dt); + datetime today = StringToTime(StringFormat("%04d.%02d.%02d 00:00", + dt.year, dt.mon, dt.day)); + if(today != last_session_day) + { + if(last_session_day != 0) + PrintFormat("New session [%s]: reset count (was %d)", + TimeToString(today, TIME_DATE), session_trade_count); + session_trade_count = 0; + last_session_day = today; + } +} + +//+------------------------------------------------------------------+ +int OnInit() +{ + onnx_handle = OnnxCreateFromBuffer(OnnxModel, ONNX_DEFAULT); + if(onnx_handle == INVALID_HANDLE) + { + Print("ONNX load failed: ", GetLastError()); + return INIT_FAILED; + } + long input_shape[] = {1, 4}; + long label_shape[] = {1}; + long prob_shape[] = {1, NStates}; + if(!OnnxSetInputShape(onnx_handle, 0, input_shape)) + { + PrintFormat("ONNX input shape error (code %d)", GetLastError()); + return INIT_FAILED; + } + if(!OnnxSetOutputShape(onnx_handle, 0, label_shape)) + { + PrintFormat("ONNX label output[0] shape error (code %d)", GetLastError()); + return INIT_FAILED; + } + if(!OnnxSetOutputShape(onnx_handle, 1, prob_shape)) + { + PrintFormat("ONNX prob output[1] shape error — NStates=%d may not match trained model (code %d)", + NStates, GetLastError()); + return INIT_FAILED; + } + + atr_handle = iATR(_Symbol, SignalTimeframe, ATRPeriod); + rsi_handle = iRSI(_Symbol, SignalTimeframe, RSIPeriod, PRICE_CLOSE); + if(atr_handle == INVALID_HANDLE || rsi_handle == INVALID_HANDLE) + { + Print("Indicator handle error"); + return INIT_FAILED; + } + + double bal = AccountInfoDouble(ACCOUNT_BALANCE); + PrintFormat("GoldRegimeX v2.00 | TF=%s | Balance=%.2f | MaxTrades=%d | PosPerTrade=%d | Cent=%s", + EnumToString(SignalTimeframe), bal, + GetMaxTrades(bal, 0), GetPosPerTrade(bal), + IsCentAccount ? "YES" : "NO"); + return INIT_SUCCEEDED; +} + +//+------------------------------------------------------------------+ +void OnDeinit(const int reason) +{ + if(onnx_handle != INVALID_HANDLE) OnnxRelease(onnx_handle); + if(atr_handle != INVALID_HANDLE) IndicatorRelease(atr_handle); + if(rsi_handle != INVALID_HANDLE) IndicatorRelease(rsi_handle); +} + +//+------------------------------------------------------------------+ +double StdDev(double &arr[], int size) +{ + double sum = 0, sq = 0; + for(int i = 0; i < size; i++) { sum += arr[i]; sq += arr[i]*arr[i]; } + double mean = sum / size; + return MathSqrt(sq/size - mean*mean); +} + +int GetHMMState(double kalman_ret, double vol) +{ + if(kalman_ret > BullReturnThreshold && vol < VolThreshold) return 0; + if(kalman_ret < BearReturnThreshold) return 1; + return 2; +} + +bool ComputeFeatures(float &features[], int &hmm_state_out) +{ + ArrayResize(features, 4); + double cl[]; + if(CopyClose(_Symbol, SignalTimeframe, 0, 22, cl) < 22) return false; + + double rets[]; + ArrayResize(rets, 20); + for(int i = 0; i < 20; i++) rets[i] = MathLog(cl[i+1] / cl[i]); + + double vol = StdDev(rets, 20); + double lr = MathLog(cl[21] / cl[20]); + hmm_state_out = GetHMMState(lr, vol); + features[0] = (float)hmm_state_out; + + double rsi_buf[]; + if(CopyBuffer(rsi_handle, 0, 0, 2, rsi_buf) < 2) return false; + features[1] = (float)(rsi_buf[1] - rsi_buf[0]); + + double atr_buf[]; + if(CopyBuffer(atr_handle, 0, 0, 1, atr_buf) < 1) return false; + features[2] = (float)(atr_buf[0] / cl[21]); + features[3] = (float)MathLog(cl[20] / cl[19]); + return true; +} + +//+------------------------------------------------------------------+ +//| Lot sizing with micro-lot floor for Headway Cent | +//+------------------------------------------------------------------+ +double CalculateLotSize(double stop_pts, int pos_per_trade) +{ + double balance = AccountInfoDouble(ACCOUNT_BALANCE); + double risk_amt = balance * RiskPercent / 100.0 / pos_per_trade; // split across positions + double tv = SymbolInfoDouble(_Symbol, SYMBOL_TRADE_TICK_VALUE); + double ts = SymbolInfoDouble(_Symbol, SYMBOL_TRADE_TICK_SIZE); + double min_lot = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_MIN); + double max_lot = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_MAX); + double lot_step = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_STEP); + + if(IsCentAccount) min_lot = MathMax(min_lot, 0.01); + if(tv == 0 || ts == 0 || stop_pts == 0) return min_lot; + + double lot = risk_amt / (stop_pts / ts * tv); + lot = MathFloor(lot / lot_step) * lot_step; + return MathMax(min_lot, MathMin(max_lot, lot)); +} + +bool HasOpenPosition() +{ + for(int i = 0; i < PositionsTotal(); i++) + if(PositionGetSymbol(i) == _Symbol && + PositionGetInteger(POSITION_MAGIC) == MagicNumber) return true; + return false; +} + +bool SendOrder(ENUM_ORDER_TYPE type, double price, double sl, double lot, string comment) +{ + MqlTradeRequest req = {}; + MqlTradeResult res = {}; + req.action = TRADE_ACTION_DEAL; + req.symbol = _Symbol; + req.volume = lot; + req.type = type; + req.price = price; + req.sl = sl; + req.magic = MagicNumber; + req.comment = comment; + return OrderSend(req, res); +} + +//+------------------------------------------------------------------+ +void OnTick() +{ + CheckSessionReset(); + + double balance = AccountInfoDouble(ACCOUNT_BALANCE); + float features[]; + int hmm_state = 0; + + // Compute features first so we know the HMM state for adaptive limits + if(!ComputeFeatures(features, hmm_state)) return; + + int max_trades = GetMaxTrades(balance, hmm_state); + int pos_per_trade = GetPosPerTrade(balance); + + // Enforce daily session limit + if(session_trade_count >= max_trades) return; + + // Only trade on new bar for the configured timeframe + static datetime last_bar_time = 0; + datetime cur_bar = iTime(_Symbol, SignalTimeframe, 0); + if(cur_bar == last_bar_time) return; + last_bar_time = cur_bar; + + if(HasOpenPosition()) return; + + // ONNX inference + long labels[]; + float probs[]; + if(!OnnxRun(onnx_handle, ONNX_DEFAULT, features, labels, probs)) + { + Print("ONNX run failed: ", GetLastError()); + return; + } + + // probs[0] = P(Bull/class-0), probs[1] = P(Bear/class-1) + float bull_prob = probs[0]; + float bear_prob = probs[1]; + + double atr_buf[]; + CopyBuffer(atr_handle, 0, 0, 1, atr_buf); + double stop_dist = atr_buf[0] * ATRMultiplier; + double lot = CalculateLotSize(stop_dist, pos_per_trade); + + double ask = SymbolInfoDouble(_Symbol, SYMBOL_ASK); + double bid = SymbolInfoDouble(_Symbol, SYMBOL_BID); + + // Long signal: high Bull probability + if(bull_prob > ProbThreshold && hmm_state != ChopState) + { + double sl = NormalizeDouble(ask - stop_dist, _Digits); + bool ok = false; + for(int p = 0; p < pos_per_trade && session_trade_count < max_trades; p++) + { + if(SendOrder(ORDER_TYPE_BUY, ask, sl, lot, + StringFormat("GRX Long p%d", p+1))) + { + session_trade_count++; + ok = true; + } + } + if(ok) + PrintFormat("LONG ×%d | count=%d/%d | lot=%.2f | sl=%.2f | bull_prob=%.3f | state=%d | tf=%s", + pos_per_trade, session_trade_count, max_trades, + lot, sl, bull_prob, hmm_state, EnumToString(SignalTimeframe)); + } + + // Short signal: high Bear probability (bear_prob > 1 - ShortThreshold, default 0.65) + if(bear_prob > (1.0f - (float)ShortThreshold) && hmm_state != ChopState && + session_trade_count < max_trades) + { + double sl = NormalizeDouble(bid + stop_dist, _Digits); + for(int p = 0; p < pos_per_trade && session_trade_count < max_trades; p++) + { + if(SendOrder(ORDER_TYPE_SELL, bid, sl, lot, + StringFormat("GRX Short p%d", p+1))) + session_trade_count++; + } + PrintFormat("SHORT ×%d | count=%d/%d | lot=%.2f | bear_prob=%.3f | state=%d | tf=%s", + pos_per_trade, session_trade_count, max_trades, + lot, bear_prob, hmm_state, EnumToString(SignalTimeframe)); + } +} +//+------------------------------------------------------------------+ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6877222 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +numpy>=1.24 +pandas>=2.0 +hmmlearn>=0.3 +xgboost>=2.0 +optuna>=3.5 +scikit-learn>=1.3 +joblib>=1.3 +onnx>=1.15 +onnxmltools>=1.12 +skl2onnx>=1.16 +pyarrow>=14.0 +matplotlib>=3.8 +onnxconverter-common>=1.14 +metatrader5 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/auditor.py b/src/auditor.py new file mode 100644 index 0000000..d90b35d --- /dev/null +++ b/src/auditor.py @@ -0,0 +1,94 @@ +"""Daily performance auditor for Gold Regime X. + +Fetches closed deal history from MT5 for the last 24 hours and formats +a Telegram-ready HTML summary. Floating P&L from any currently open positions +is included so you see the real-time risk alongside historical results. + +Usage (via main.py): + python main.py --mode audit --broker headway_cent + +The report is also triggered automatically by: + - The Telegram remote control "📊 BOT STATUS" button + - The nightly scheduler inside --mode listen (fires at 23:55 UTC) +""" + +from datetime import datetime, timedelta, timezone + +from src.logger import setup_logger +from src.risk_manager import CENT_MULTIPLIER + +logger = setup_logger(__name__) + + +def get_daily_report(broker: str = "headway_cent", days: int = 1) -> str: + """Fetch and format a performance summary for the last *days* days. + + Returns an HTML-formatted string suitable for Telegram (parse_mode='HTML'). + Never raises — returns an error string if MT5 is unavailable. + """ + try: + import MetaTrader5 as mt5 + except ImportError: + return "MetaTrader5 package not installed — cannot fetch report." + + to_date = datetime.now() + from_date = to_date - timedelta(days=days) + + history = mt5.history_deals_get(from_date, to_date) + if history is None or len(history) == 0: + period = "24 hours" if days == 1 else f"{days} days" + return f"No trades closed in the last {period}." + + try: + import pandas as pd + except ImportError: + return "pandas not installed — cannot format report." + + df = pd.DataFrame(list(history), columns=history[0]._asdict().keys()) + + # DEAL_ENTRY_OUT = closed exit deals (excludes balance deposits, etc.) + df = df[df["entry"] == mt5.DEAL_ENTRY_OUT] + if df.empty: + return "No exit deals recorded in the last 24 hours." + + # Normalise P&L: cent accounts report in cents, convert to USD + divisor = float(CENT_MULTIPLIER) if broker == "headway_cent" else 1.0 + df["profit_usd"] = df["profit"] / divisor + + total_trades = len(df) + wins = int((df["profit"] > 0).sum()) + losses = total_trades - wins + win_rate = (wins / total_trades * 100) if total_trades else 0.0 + total_pnl = float(df["profit_usd"].sum()) + best_trade = float(df["profit_usd"].max()) + worst_trade = float(df["profit_usd"].min()) + + # Floating P&L from any currently open positions + positions = mt5.positions_get() + floating_line = "" + if positions: + floating = sum(p.profit / divisor for p in positions) + floating_line = f"Floating P&L: {floating:+.2f} USD\n" + + icon_pnl = "+" if total_pnl >= 0 else "" + period = "24h" if days == 1 else f"{days}d" + stamp = datetime.now(timezone.utc).strftime("%H:%M UTC") + + report = ( + f"Gold Regime X — Daily Report ({period})\n" + f"{'—'*28}\n" + f"Total PnL: {icon_pnl}{total_pnl:.2f} USD\n" + f"{floating_line}" + f"Total Trades: {total_trades} " + f"(Wins: {wins} | Losses: {losses})\n" + f"Win Rate: {win_rate:.1f}%\n" + f"Best Trade: +{best_trade:.2f} USD\n" + f"Worst Trade: {worst_trade:.2f} USD\n" + f"{'—'*28}\n" + f"Generated at {stamp}" + ) + logger.info( + "Daily report: pnl=%.2f trades=%d win_rate=%.1f%%", + total_pnl, total_trades, win_rate, + ) + return report diff --git a/src/backtester.py b/src/backtester.py new file mode 100644 index 0000000..7973290 --- /dev/null +++ b/src/backtester.py @@ -0,0 +1,239 @@ +import numpy as np +import pandas as pd +from src.logger import setup_logger +from src.risk_manager import BROKER_CONFIGS, AdaptiveRiskManager + +logger = setup_logger(__name__) + +ANNUALIZATION_FACTORS = { + "H1": np.sqrt(252 * 24), + "M15": np.sqrt(252 * 96), # 96 M15 bars per day + "M5": np.sqrt(252 * 288), # 288 M5 bars per day +} +ANNUALIZATION_FACTOR = ANNUALIZATION_FACTORS["H1"] # default + +RISK_PER_TRADE = 0.01 +PROB_THRESHOLD = 0.65 +CHOP_STATE = 2 + + +def compute_signals(probabilities, hmm_states, threshold=PROB_THRESHOLD): + return ((probabilities > threshold) & (hmm_states != CHOP_STATE)).astype(int) + + +def compute_position_sizes( + signals, + atr_normalized, + atr_multiplier=2.0, + risk_per_trade=RISK_PER_TRADE, + pos_per_trade: int = 1, +): + """Size positions using 1% risk / (2×ATR) with optional dual-position multiplier. + + Args: + pos_per_trade: 1 for small accounts (single), 2 for growth accounts (dual/hedging). + """ + stop_distance = atr_normalized * atr_multiplier + stop_distance = np.where(stop_distance > 0, stop_distance, 1e-8) + sizes = (risk_per_trade / stop_distance) * signals * pos_per_trade + return sizes + + +def apply_session_limits( + signals: np.ndarray, + dates, + account_size: float = 15.0, + hmm_states: np.ndarray = None, +) -> np.ndarray: + """Cap signals to the adaptive daily limit based on account size and HMM state. + + For growth accounts (> $50), the limit is market-state-dependent: + - Bull / Bear: 3 signals/day + - Chop: 2 signals/day + + Uses the majority HMM state within each calendar day to decide the cap. + """ + arm = AdaptiveRiskManager(account_size) + result = signals.copy() + day_labels = pd.DatetimeIndex(dates).normalize().values + + unique_days, day_starts = np.unique(day_labels, return_index=True) + day_ends = np.append(day_starts[1:], len(signals)) + + for start, end in zip(day_starts, day_ends): + # Market-state-aware limit for growth accounts + if hmm_states is not None and not arm.is_small_account: + day_states = hmm_states[start:end].astype(int) + day_mode = int(np.bincount(day_states).argmax()) + limits = arm.get_trade_limits(day_mode) + else: + limits = arm.get_trade_limits() + + max_daily = limits["max_daily_trades"] + seg = result[start:end] + trade_indices = np.where(seg == 1)[0] + if len(trade_indices) > max_daily: + seg[trade_indices[max_daily:]] = 0 + result[start:end] = seg + + original = int(np.sum(signals)) + limited = int(np.sum(result)) + if original != limited: + logger.debug( + "Session limit (acct=$%.0f): %d -> %d trades", + account_size, original, limited, + ) + return result + + +def compute_trade_costs(signals: np.ndarray, broker: str = "standard") -> np.ndarray: + config = BROKER_CONFIGS.get(broker, BROKER_CONFIGS["standard"]) + cost_per_trade = config["spread_frac"] + config["commission_frac"] + return signals * cost_per_trade + + +def _compute_metrics(strategy_returns, signals, tf: str = "H1"): + ann_factor = ANNUALIZATION_FACTORS.get(tf.upper(), ANNUALIZATION_FACTOR) + mean_ret = np.mean(strategy_returns) + std_ret = np.std(strategy_returns) + sharpe = (mean_ret / std_ret) * ann_factor if std_ret > 0 else 0.0 + + cumulative = np.cumsum(strategy_returns) + running_max = np.maximum.accumulate(cumulative) + drawdowns = running_max - cumulative + max_dd = float(np.max(drawdowns)) if len(drawdowns) > 0 else 0.0 + + active = signals == 1 + active_returns = strategy_returns[active[:len(strategy_returns)]] + n_trades = int(np.sum(active)) + win_rate = float(np.sum(active_returns > 0) / len(active_returns)) if len(active_returns) > 0 else 0.0 + total_return = float(np.exp(np.sum(strategy_returns)) - 1) + + return { + "sharpe_ratio": float(sharpe), + "max_drawdown": max_dd, + "win_rate": win_rate, + "total_return": total_return, + "n_trades": n_trades, + } + + +def vectorized_backtest( + df, + probabilities, + hmm_states, + split_idx=None, + account_size: float = 15.0, + broker: str = "standard", + tf: str = "H1", + prob_threshold: float = None, +): + """Vectorized backtest with adaptive session limits and broker costs. + + Args: + df: Processed DataFrame with ``log_return`` and ``atr_normalized``. + probabilities: XGB probability array aligned with df. + hmm_states: HMM state array aligned with df. + split_idx: Bar index for IS/OOS split (None = full-period only). + account_size: Account balance in USD; drives adaptive session limits. + broker: Broker profile key from ``BROKER_CONFIGS``. + tf: Timeframe string used for correct annualization ("H1" or "M15"). + prob_threshold: XGBoost probability threshold for signal generation. + Defaults to PROB_THRESHOLD (0.65) if not supplied. + """ + log_returns = df["log_return"].values + atr_norm = df["atr_normalized"].values + + raw_signals = compute_signals( + probabilities, hmm_states, + threshold=prob_threshold if prob_threshold is not None else PROB_THRESHOLD, + ) + signals = apply_session_limits(raw_signals, df.index, account_size, hmm_states) + + # Determine pos_per_trade from the adaptive risk manager + arm = AdaptiveRiskManager(account_size) + base_limits = arm.get_trade_limits() + pos_per_trade = base_limits["pos_per_trade"] + + sizes = compute_position_sizes(signals, atr_norm, pos_per_trade=pos_per_trade) + + next_returns = np.roll(log_returns, -1) + next_returns[-1] = 0.0 + gross_returns = sizes * next_returns + costs = compute_trade_costs(signals, broker=broker) + strategy_returns = gross_returns - costs + + result = _compute_metrics(strategy_returns, signals, tf=tf) + result.update({ + "account_size": account_size, + "broker": broker, + "tf": tf, + "session_max_trades": arm.get_trade_limits()["max_daily_trades"], + "pos_per_trade": pos_per_trade, + }) + + if split_idx is not None and 0 < split_idx < len(strategy_returns): + is_m = _compute_metrics(strategy_returns[:split_idx], signals[:split_idx], tf) + oos_m = _compute_metrics(strategy_returns[split_idx:], signals[split_idx:], tf) + result["split_idx"] = split_idx + for k, v in is_m.items(): + result[f"is_{k}"] = v + for k, v in oos_m.items(): + result[f"oos_{k}"] = v + logger.info( + "Backtest IS [%s]: Sharpe=%.3f | MaxDD=%.3f | WinRate=%.3f | Trades=%d", + tf, is_m["sharpe_ratio"], is_m["max_drawdown"], is_m["win_rate"], is_m["n_trades"], + ) + logger.info( + "Backtest OOS [%s]: Sharpe=%.3f | MaxDD=%.3f | WinRate=%.3f | Trades=%d", + tf, oos_m["sharpe_ratio"], oos_m["max_drawdown"], oos_m["win_rate"], oos_m["n_trades"], + ) + else: + logger.info( + "Backtest [%s]: Sharpe=%.3f | MaxDD=%.3f | WR=%.3f | Return=%.1f%% | Trades=%d", + tf, result["sharpe_ratio"], result["max_drawdown"], result["win_rate"], + result["total_return"] * 100, result["n_trades"], + ) + + logger.info( + "%s: account=$%.2f | tier=%s | session_limit=%d/day | pos_per_trade=%d | costs=%.4f/trade", + broker, + account_size, + "small" if arm.is_small_account else "growth", + arm.get_trade_limits()["max_daily_trades"], + pos_per_trade, + BROKER_CONFIGS.get(broker, {}).get("spread_frac", 0) + BROKER_CONFIGS.get(broker, {}).get("commission_frac", 0), + ) + return result + + +def compare_timeframes(m15_result: dict, h1_result: dict) -> tuple[dict, str]: + """Compare M15 vs H1 backtest results and recommend the better timeframe. + + Comparison is based on the OOS Sharpe ratio when available, otherwise + full-period Sharpe. + + Returns: + results: Dict mapping tf label to its result dict. + winner: ``"M15"`` or ``"H1"``, whichever has the higher Sharpe. + """ + results = {"M15": m15_result, "H1": h1_result} + + def _oos_sharpe(r): + return r.get("oos_sharpe_ratio", r.get("sharpe_ratio", 0.0)) + + winner = max(results, key=lambda k: _oos_sharpe(results[k])) + + logger.info("Timeframe comparison:") + for tf, r in results.items(): + logger.info( + " %s Sharpe=%.3f (OOS=%.3f) | DD=%.1f%% | WR=%.1f%% | Trades=%d", + tf, + r["sharpe_ratio"], + r.get("oos_sharpe_ratio", r["sharpe_ratio"]), + r["max_drawdown"] * 100, + r["win_rate"] * 100, + r["n_trades"], + ) + logger.info("Winner: %s", winner) + return results, winner diff --git a/src/engine_hmm.py b/src/engine_hmm.py new file mode 100644 index 0000000..66e5f48 --- /dev/null +++ b/src/engine_hmm.py @@ -0,0 +1,131 @@ +import numpy as np +import pandas as pd +import joblib +from pathlib import Path +from hmmlearn.hmm import GaussianHMM +from src.logger import setup_logger, log_regime_transition + +logger = setup_logger(__name__) + +MODEL_PATH = Path("models/hmm_model.pkl") +STATE_NAMES_3 = {0: "Bull", 1: "Bear", 2: "Chop"} +STATE_NAMES_2 = {0: "Bull", 1: "Bear"} +STATE_NAMES_4 = {0: "Bull", 1: "Bear", 2: "Chop_Low", 3: "Chop_High"} + + +def _sort_states(model: GaussianHMM, raw_states: np.ndarray, n_states: int): + mean_returns = model.means_[:, 0] + sorted_idx = np.argsort(mean_returns) + + if n_states == 2: + remap = {int(sorted_idx[0]): 1, int(sorted_idx[1]): 0} + state_names = STATE_NAMES_2 + elif n_states == 3: + remap = { + int(sorted_idx[0]): 1, # lowest mean -> Bear + int(sorted_idx[1]): 2, # middle -> Chop + int(sorted_idx[2]): 0, # highest -> Bull + } + state_names = STATE_NAMES_3 + else: + remap = {int(sorted_idx[i]): i for i in range(n_states)} + remap[int(sorted_idx[0])] = 1 # Bear + remap[int(sorted_idx[-1])] = 0 # Bull + for j in range(1, n_states - 1): + remap[int(sorted_idx[j])] = j + 1 + state_names = STATE_NAMES_4 + + remapped = np.array([remap[s] for s in raw_states]) + + perm = [0] * n_states + for old, new in remap.items(): + perm[new] = old + model.means_ = model.means_[perm] + model.covars_ = model.covars_[perm] + model.transmat_ = model.transmat_[np.ix_(perm, perm)] + model.startprob_ = model.startprob_[perm] + + return remapped, state_names + + +def _log_transition_matrix(model: GaussianHMM, state_names: dict): + n = model.n_components + logger.info("Transition Matrix:") + header = " " + " ".join(f"{state_names.get(j, str(j)):>8}" for j in range(n)) + logger.info(header) + for i in range(n): + row = f"{state_names.get(i, str(i)):>8} " + " ".join( + f"{model.transmat_[i, j]:8.4f}" for j in range(n) + ) + logger.info(row) + if model.transmat_[i, i] < 0.85: + logger.warning( + "State %s persistence %.3f < 0.85 -- consider higher Kalman smoothing", + state_names.get(i, str(i)), model.transmat_[i, i], + ) + + logger.info("State means (kalman_return, volatility):") + for i in range(n): + logger.info( + " %s: return=%.6f, vol=%.6f", + state_names.get(i, str(i)), model.means_[i, 0], model.means_[i, 1], + ) + + +def fit_hmm( + df: pd.DataFrame, + n_states: int = 3, + n_iter: int = 200, + random_state: int = 42, +): + X = df[["kalman_return", "volatility"]].values + model = GaussianHMM( + n_components=n_states, + covariance_type="full", + n_iter=n_iter, + random_state=random_state, + verbose=False, + ) + model.fit(X) + raw_states = model.predict(X) + + states, state_names = _sort_states(model, raw_states, n_states) + _log_transition_matrix(model, state_names) + + transitions = np.where(np.diff(states) != 0)[0] + 1 + logger.info("Total regime transitions: %d", len(transitions)) + for idx in transitions[:10]: + log_regime_transition( + logger, df.index[idx], states[idx - 1], states[idx], state_names + ) + + return model, states, state_names + + +def predict_states(model: GaussianHMM, df: pd.DataFrame) -> np.ndarray: + X = df[["kalman_return", "volatility"]].values + raw = model.predict(X) + mean_returns = model.means_[:, 0] + sorted_idx = np.argsort(mean_returns) + n = model.n_components + if n == 2: + remap = {int(sorted_idx[0]): 1, int(sorted_idx[1]): 0} + elif n == 3: + remap = {int(sorted_idx[0]): 1, int(sorted_idx[1]): 2, int(sorted_idx[2]): 0} + else: + remap = {int(sorted_idx[0]): 1, int(sorted_idx[-1]): 0} + for j in range(1, n - 1): + remap[int(sorted_idx[j])] = j + 1 + return np.array([remap.get(s, s) for s in raw]) + + +def save_model(model: GaussianHMM, path: Path = MODEL_PATH): + path.parent.mkdir(parents=True, exist_ok=True) + joblib.dump(model, path) + logger.info("HMM model saved to %s", path) + + +def load_model(path: Path = MODEL_PATH) -> GaussianHMM: + model = joblib.load(path) + logger.info("HMM model loaded from %s", path) + return model diff --git a/src/engine_xgb.py b/src/engine_xgb.py new file mode 100644 index 0000000..a27ac48 --- /dev/null +++ b/src/engine_xgb.py @@ -0,0 +1,190 @@ +import numpy as np +import pandas as pd +import xgboost as xgb +import joblib +import onnx +from pathlib import Path +from sklearn.metrics import accuracy_score +from src.logger import setup_logger + +logger = setup_logger(__name__) + +ONNX_PATH = Path("models/xgb_model.onnx") +XGB_PKL_PATH = Path("models/xgb_model.pkl") + +FEATURE_COLS = ["hmm_state", "rsi_slope", "atr_normalized", "prev_log_return"] + + +def prepare_features(df: pd.DataFrame, hmm_states: np.ndarray): + df = df.copy() + df["hmm_state"] = hmm_states + df["prev_log_return"] = df["log_return"].shift(1) + y = (df["log_return"].shift(-1) > 0).astype(int).rename("target") + + X = df[FEATURE_COLS] + valid = X.notna().all(axis=1) & y.notna() + X = X[valid] + y = y[valid] + df_aligned = df.loc[X.index] + + logger.info("Features prepared: %d samples, %d features", len(X), len(FEATURE_COLS)) + return X, y, df_aligned + + +def train_xgb( + X: pd.DataFrame, + y: pd.Series, + max_depth: int = 4, + learning_rate: float = 0.1, + n_estimators: int = 200, + subsample: float = 0.8, + min_child_weight: int = 5, + gamma: float = 1.0, + reg_alpha: float = 0.1, + colsample_bytree: float = 0.8, + train_ratio: float = 0.8, +): + split_idx = int(len(X) * train_ratio) + X_train, X_test = X.iloc[:split_idx], X.iloc[split_idx:] + y_train, y_test = y.iloc[:split_idx], y.iloc[split_idx:] + + model = xgb.XGBClassifier( + max_depth=max_depth, + learning_rate=learning_rate, + n_estimators=n_estimators, + subsample=subsample, + min_child_weight=min_child_weight, + gamma=gamma, + reg_alpha=reg_alpha, + colsample_bytree=colsample_bytree, + objective="binary:logistic", + eval_metric="logloss", + random_state=42, + n_jobs=-1, + verbosity=0, + ) + model.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False) + + train_acc = accuracy_score(y_train, model.predict(X_train)) + test_acc = accuracy_score(y_test, model.predict(X_test)) + importance = dict(zip(FEATURE_COLS, model.feature_importances_)) + + logger.info("XGB Train Acc: %.4f | Test Acc: %.4f", train_acc, test_acc) + logger.info("Feature importance: %s", importance) + + metrics = { + "train_accuracy": train_acc, + "test_accuracy": test_acc, + "feature_importance": importance, + "split_idx": split_idx, + } + return model, metrics + + +def get_predictions(model: xgb.XGBClassifier, X: pd.DataFrame): + predictions = model.predict(X) + probabilities = model.predict_proba(X)[:, 1] + return predictions, probabilities + + +def save_xgb(model: xgb.XGBClassifier, metrics: dict = None, path: Path = XGB_PKL_PATH): + path.parent.mkdir(parents=True, exist_ok=True) + joblib.dump({"model": model, "metrics": metrics or {}}, path) + logger.info("XGB model saved to %s", path) + + +def load_xgb(path: Path = XGB_PKL_PATH): + data = joblib.load(path) + if isinstance(data, dict) and "model" in data: + model = data["model"] + metrics = data.get("metrics", {}) + else: + # Backwards compat: old saves stored just the model + model = data + metrics = { + "feature_importance": dict(zip(FEATURE_COLS, model.feature_importances_)), + } + return model, metrics + + +def _strip_zipmap(onnx_model): + """Remove ZipMap node from ONNX graph, exposing the raw float probability tensor. + + onnxmltools converts XGBoost probabilities as ZipMap (sequence of maps). + MT5's OnnxRun expects a plain float32 tensor. This surgery replaces the + ZipMap output with the raw tensor it is wrapping, making the model fully + compatible with MT5's OnnxSetOutputShape / OnnxRun API. + """ + import onnx as _onnx + + graph = onnx_model.graph + + float_tensor_name = None + zipmap_output_name = None + nodes_to_keep = [] + + for node in graph.node: + if node.op_type == "ZipMap": + float_tensor_name = node.input[0] + zipmap_output_name = node.output[0] + else: + nodes_to_keep.append(node) + + if float_tensor_name is None: + return onnx_model # no ZipMap present — nothing to do + + new_outputs = [] + for output in graph.output: + if output.name == zipmap_output_name: + new_outputs.append( + _onnx.helper.make_tensor_value_info( + float_tensor_name, + _onnx.TensorProto.FLOAT, + None, # shape inferred at runtime + ) + ) + else: + new_outputs.append(output) + + new_graph = _onnx.helper.make_graph( + nodes_to_keep, + graph.name, + list(graph.input), + new_outputs, + list(graph.initializer), + ) + new_model = _onnx.helper.make_model( + new_graph, opset_imports=onnx_model.opset_import + ) + new_model.ir_version = onnx_model.ir_version + return new_model + + +def export_onnx(model: xgb.XGBClassifier, n_features: int = 4, path: Path = ONNX_PATH): + path.parent.mkdir(parents=True, exist_ok=True) + + # onnxmltools requires feature names in 'f%d' pattern + # Clone the model's booster with generic feature names + import copy + model_copy = copy.deepcopy(model) + model_copy.get_booster().feature_names = [f"f{i}" for i in range(n_features)] + + from onnxmltools import convert_xgboost + from onnxmltools.convert.common.data_types import FloatTensorType + + initial_type = [("input", FloatTensorType([None, n_features]))] + onnx_model = convert_xgboost(model_copy, initial_types=initial_type) + onnx_model = _strip_zipmap(onnx_model) # expose float tensor output for MT5 + + onnx.save_model(onnx_model, str(path)) + onnx.checker.check_model(onnx_model) + + inputs = [i.name for i in onnx_model.graph.input] + outputs = [o.name for o in onnx_model.graph.output] + n_classes = model.n_classes_ + logger.info( + "ONNX exported to %s | inputs: %s | outputs: %s | n_classes=%d", + path, inputs, outputs, n_classes, + ) + print(f"\n ONNX export OK — n_classes={n_classes}. " + f"Set NStates={n_classes} in the MT5 EA inputs.\n") diff --git a/src/guardian.py b/src/guardian.py new file mode 100644 index 0000000..0d86009 --- /dev/null +++ b/src/guardian.py @@ -0,0 +1,111 @@ +"""Multi-timeframe health monitor for Gold Regime X. + +Periodically re-validates HMM+XGBoost signal quality on the most recently +synced MT5 data. If rolling Sharpe for any timeframe drops below the alert +threshold a Telegram notification is fired immediately. + +Usage (via main.py): + python main.py --mode guardian --tf M5,M15,H1 --period 3m --interval 3600 + (defaults: all three TFs, last 3 months of data, check every hour) + +The check re-uses the same validator.run_validation() that gates live trading, +so the health score is directly comparable to sync_validate output. +""" + +import time + +from src.logger import setup_logger +from src.notifier import send_telegram_msg + +logger = setup_logger(__name__) + +SHARPE_ALERT_THRESHOLD = 0.6 # Telegram alert fires below this +SHARPE_CRITICAL = 0.4 # escalated "CRITICAL" label + + +def run_guardian( + tfs: list, + broker: str, + account_size: float, + period: str = "3m", + interval_sec: int = 3600, +) -> None: + """Monitor timeframe health continuously; alert via Telegram on degradation. + + Args: + tfs: List of timeframe strings, e.g. ["M5", "M15", "H1"]. + broker: Broker key used for backtest cost simulation. + account_size: USD balance for lot-sizing in the backtest. + period: MT5 lookback period for synced data ('3m', '6m', etc.). + interval_sec: Seconds between health checks. Default 3600 (1 hour). + """ + from src.validator import run_validation + + logger.info( + "Guardian started — TFs=%s interval=%ds alert_threshold=%.1f", + tfs, interval_sec, SHARPE_ALERT_THRESHOLD, + ) + send_telegram_msg( + f"Guardian online\n" + f"Monitoring: {', '.join(tfs)}\n" + f"Check interval: every {interval_sec // 60} min" + ) + + cycle = 0 + while True: + cycle += 1 + lines = [f"Guardian Health Check #{cycle}"] + alerts = [] + + for tf in tfs: + try: + result = run_validation( + tf=tf, + broker=broker, + account_size=account_size, + period=period, + ) + sharpe = result["sharpe"] + status = result["status"] + trades = result["n_trades"] + + if sharpe < SHARPE_CRITICAL: + icon = "CRITICAL" + elif sharpe < SHARPE_ALERT_THRESHOLD: + icon = "WARN" + elif status == "pass": + icon = "OK" + else: + icon = "WARN" + + lines.append( + f" [{tf}] Sharpe={sharpe:.3f} " + f"Trades={trades} [{icon}]" + ) + logger.info( + "Guardian [%s]: sharpe=%.3f status=%s trades=%d", + tf, sharpe, status, trades, + ) + + if sharpe < SHARPE_ALERT_THRESHOLD: + level = "CRITICAL" if sharpe < SHARPE_CRITICAL else "WARNING" + alerts.append( + f"GUARDIAN {level} — [{tf}]\n" + f"Rolling Sharpe: {sharpe:.3f} " + f"(threshold: {SHARPE_ALERT_THRESHOLD})\n" + f"Action: python main.py --mode optimize --tf {tf}" + ) + except Exception as exc: + msg = f" [{tf}] Check error: {exc}" + lines.append(msg) + logger.error("Guardian [%s] check failed: %s", tf, exc) + + # Send periodic digest and any alerts + send_telegram_msg("\n".join(lines)) + for alert in alerts: + send_telegram_msg(alert) + + logger.info( + "Guardian cycle #%d complete — next in %ds.", cycle, interval_sec + ) + time.sleep(interval_sec) diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 0000000..237070b --- /dev/null +++ b/src/logger.py @@ -0,0 +1,38 @@ +import logging +from pathlib import Path + +LOG_DIR = Path("logs") + + +def setup_logger(name: str, level: int = logging.INFO) -> logging.Logger: + LOG_DIR.mkdir(exist_ok=True) + logger = logging.getLogger(name) + if logger.handlers: + return logger + logger.setLevel(level) + fmt = logging.Formatter("[%(asctime)s %(levelname)s] %(name)s: %(message)s") + sh = logging.StreamHandler() + sh.setLevel(level) + sh.setFormatter(fmt) + logger.addHandler(sh) + fh = logging.FileHandler(LOG_DIR / "goldregimex.log", mode="a") + fh.setLevel(level) + fh.setFormatter(fmt) + logger.addHandler(fh) + return logger + + +def log_regime_transition(logger, timestamp, old_state, new_state, state_names): + logger.info( + "REGIME CHANGE at %s: %s -> %s", + timestamp, + state_names.get(old_state, str(old_state)), + state_names.get(new_state, str(new_state)), + ) + + +def log_trade_signal(logger, timestamp, direction, probability, hmm_state): + logger.info( + "SIGNAL at %s: %s (prob=%.3f, hmm_state=%d)", + timestamp, direction, probability, hmm_state, + ) diff --git a/src/mt5_sync.py b/src/mt5_sync.py new file mode 100644 index 0000000..7ca1231 --- /dev/null +++ b/src/mt5_sync.py @@ -0,0 +1,163 @@ +"""MT5 Data Sync — downloads recent OHLCV bars from the MetaTrader5 terminal. + +The MT5 terminal must already be running and logged into the desired account +before calling any function here. The MetaTrader5 Python package is imported +lazily so this module can be imported on machines that do not have MT5 installed +(e.g. a CI environment that only runs the backtest pipeline). +""" + +import pandas as pd +from datetime import datetime +from pathlib import Path +from src.logger import setup_logger + +logger = setup_logger(__name__) + +SYNC_OUTPUT_PATH = Path("data/processed/mt5_sync_data.csv") +DEFAULT_SYMBOL = "XAUUSD" + +# Lazy MT5 timeframe map — populated on first call to _get_tf_map() +_MT5_TF_MAP: dict | None = None + + +def _get_tf_map() -> dict: + global _MT5_TF_MAP + if _MT5_TF_MAP is None: + import MetaTrader5 as mt5 + _MT5_TF_MAP = { + "M5": mt5.TIMEFRAME_M5, + "M15": mt5.TIMEFRAME_M15, + "H1": mt5.TIMEFRAME_H1, + } + return _MT5_TF_MAP + + +# ───────────────────────────────────────────────────────────────────────────── +# Public helpers +# ───────────────────────────────────────────────────────────────────────────── + +def parse_period(period_str: str) -> int: + """Convert a period string such as ``'3m'`` to a month count integer.""" + period_str = period_str.strip().lower() + if period_str.endswith("m") and period_str[:-1].isdigit(): + return int(period_str[:-1]) + raise ValueError( + f"Unrecognised period format: '{period_str}'. " + "Expected a digit followed by 'm', e.g. '3m', '6m', '12m'." + ) + + +def connect_mt5(login: int = None, password: str = None, server: str = None) -> bool: + """Initialise the MT5 package and optionally log in programmatically. + + If *login* is None the function relies on the account that is already + active in the terminal. Returns ``True`` on success. + """ + import MetaTrader5 as mt5 + if not mt5.initialize(): + logger.error("MT5 initialize() failed: %s", mt5.last_error()) + return False + if login is not None: + if not mt5.login(login, password=password, server=server): + logger.error("MT5 login(%d) failed: %s", login, mt5.last_error()) + mt5.shutdown() + return False + info = mt5.account_info() + if info: + logger.info( + "MT5 connected — login=%d server=%s balance=%.2f %s", + info.login, info.server, info.balance, info.currency, + ) + return True + + +def disconnect_mt5() -> None: + """Shut down the MT5 Python connection (safe to call when not connected).""" + try: + import MetaTrader5 as mt5 + mt5.shutdown() + logger.debug("MT5 disconnected.") + except Exception: + pass + + +def fetch_bars(symbol: str, tf: str, months: int) -> pd.DataFrame: + """Download completed OHLCV bars for *symbol* on *tf* going back *months*. + + The currently open (incomplete) bar is always excluded. + + Returns a DataFrame with columns ``Open, High, Low, Close, Volume`` and a + UTC DatetimeIndex named ``Date`` — matching the convention expected by the + standalone feature-engineering functions in ``processor.py``. + """ + import MetaTrader5 as mt5 + from dateutil.relativedelta import relativedelta + + tf_map = _get_tf_map() + tf_key = tf.upper() + if tf_key not in tf_map: + raise ValueError(f"Unknown timeframe '{tf}'. Supported: {list(tf_map)}") + + date_from = datetime.utcnow() - relativedelta(months=months) + date_to = datetime.utcnow() + + rates = mt5.copy_rates_range(symbol, tf_map[tf_key], date_from, date_to) + if rates is None or len(rates) == 0: + raise RuntimeError( + f"MT5 returned no data for {symbol} {tf}: {mt5.last_error()}\n" + "Ensure the symbol is in Market Watch and the terminal is connected." + ) + + df = pd.DataFrame(rates) + df["time"] = pd.to_datetime(df["time"], unit="s", utc=True) + df = ( + df.rename(columns={ + "time": "Date", + "open": "Open", + "high": "High", + "low": "Low", + "close": "Close", + "tick_volume": "Volume", + }) + [["Date", "Open", "High", "Low", "Close", "Volume"]] + ) + df.set_index("Date", inplace=True) + df.sort_index(inplace=True) + df = df.iloc[:-1] # drop the currently open bar + + logger.info( + "Fetched %d %s bars for %s: %s -> %s", + len(df), tf_key, symbol, df.index.min(), df.index.max(), + ) + return df + + +# ───────────────────────────────────────────────────────────────────────────── +# Primary entry point +# ───────────────────────────────────────────────────────────────────────────── + +def sync_mt5_data( + symbol: str = DEFAULT_SYMBOL, + tf: str = "H1", + period: str = "3m", + output_path: Path = SYNC_OUTPUT_PATH, +) -> pd.DataFrame: + """Connect to MT5, fetch recent bars, save a CSV, then disconnect. + + Raises ``ConnectionError`` when the MT5 terminal cannot be reached. + """ + if not connect_mt5(): + raise ConnectionError( + "Could not connect to MetaTrader5 terminal. " + "Ensure MT5 is running and logged into your account." + ) + try: + months = parse_period(period) + df = fetch_bars(symbol, tf.upper(), months) + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(output_path) + logger.info("Saved %d bars -> %s", len(df), output_path) + return df + finally: + disconnect_mt5() diff --git a/src/mt5_trader.py b/src/mt5_trader.py new file mode 100644 index 0000000..36a40fb --- /dev/null +++ b/src/mt5_trader.py @@ -0,0 +1,721 @@ +"""MT5 Live Execution Engine. + +Provides account telemetry, feature engineering parity with the training +pipeline, margin validation, and live order placement through the +MetaTrader5 Python package. + +Usage (via main.py): + python main.py --mode live --account demo --broker headway_cent --balance 15 --tf H1 + +IMPORTANT: Remove the GoldRegimeX.mq5 EA from the XAUUSD chart before running +this script. Both the EA and the Python bridge use MAGIC_NUMBER = 123456 and +their session counters are independent — running both simultaneously will +double-count trades. +""" + +import time +import numpy as np +import pandas as pd +from datetime import datetime, timezone +from src.notifier import send_telegram_msg + +from src.logger import setup_logger +from src.processor import ( + TF_CONFIG, + compute_log_returns, + kalman_smooth, + compute_volatility, + compute_rsi, + compute_atr, +) +from src.engine_hmm import load_model as load_hmm, predict_states +from src.engine_xgb import load_xgb +from src.risk_manager import AdaptiveRiskManager, CENT_MULTIPLIER + +logger = setup_logger(__name__) + +# ───────────────────────────────────────────────────────────────────────────── +# Constants +# ───────────────────────────────────────────────────────────────────────────── + +DEFAULT_SYMBOL = "XAUUSD" +MAGIC_NUMBER = 123456 # must match GoldRegimeX.mq5 MagicNumber +CHOP_STATE = 2 # HMM Chop state index +ATR_MULTIPLIER = 2.0 # SL = entry ± ATR × 2.0 (all TFs) +DEFAULT_DEVIATION = 20 # fallback deviation for check_margin / send_market_order +N_BARS_WARMUP = 200 # bars fetched for Kalman/HMM warm-up +POLL_INTERVAL_SEC = 5 # seconds between bar-change checks +HIGH_VOL_SELF_TRANS_THRESHOLD = 0.70 # self-transition prob below this -> elevated deviation +MIN_SPREAD_RATIO = 1.5 # TP1 must be at least 1.5× spread to be viable + +# ── Per-timeframe signal thresholds and order parameters ────────────────────── +TF_PROB_THRESHOLD = {"M5": 0.70, "M15": 0.65, "H1": 0.65} +TF_SHORT_THRESHOLD = {"M5": 0.30, "M15": 0.35, "H1": 0.35} +TF_DEFAULT_DEV = {"M5": 30, "M15": 20, "H1": 20} +TF_HIGH_VOL_DEV = {"M5": 50, "M15": 50, "H1": 50} + +# ── State-aware multi-stage TP multipliers (relative to SL distance) ────────── +# Bull / Bear: TP1 quick partial, TP2 runner. +# Chop: tighter single TP — position 2 (runner) is skipped. +# Format: {regime: [tp1_mult, tp2_mult]}; single-element list = one TP, close all. +TF_TP_CONFIG = { + "M5": {"trending": [1.0, 3.0], "chop": [0.8]}, + "M15": {"trending": [1.5, 3.0], "chop": [1.5]}, + "H1": {"trending": [1.5, 3.0], "chop": [1.5]}, +} + +# Lazy MT5 timeframe map +_MT5_TF_MAP: dict | None = None + + +def _get_tf_map() -> dict: + global _MT5_TF_MAP + if _MT5_TF_MAP is None: + import MetaTrader5 as mt5 + _MT5_TF_MAP = { + "M5": mt5.TIMEFRAME_M5, + "M15": mt5.TIMEFRAME_M15, + "H1": mt5.TIMEFRAME_H1, + } + return _MT5_TF_MAP + + +def _tp_multipliers(tf: str, hmm_state: int) -> list[float]: + """Return the TP multiplier list for this TF and HMM state.""" + cfg = TF_TP_CONFIG.get(tf.upper(), TF_TP_CONFIG["H1"]) + return cfg["chop"] if hmm_state == CHOP_STATE else cfg["trending"] + + +def _normalise_balance(raw_balance: float, broker: str) -> float: + """Convert MT5 raw balance to USD for AdaptiveRiskManager. + + On Headway Cent accounts the terminal displays balance in cents + (e.g. $15 USD shows as 1500.00). The --broker headway_cent flag triggers + the divide-by-100 normalisation. + """ + if broker == "headway_cent": + return raw_balance / CENT_MULTIPLIER + return raw_balance + + +# ───────────────────────────────────────────────────────────────────────────── +# Account queries +# ───────────────────────────────────────────────────────────────────────────── + +def get_account_telemetry() -> dict: + """Return a snapshot of the current MT5 account state. + + Raises ``RuntimeError`` if not connected to MT5. + """ + import MetaTrader5 as mt5 + info = mt5.account_info() + if info is None: + raise RuntimeError( + f"mt5.account_info() returned None: {mt5.last_error()}. " + "Is the terminal connected?" + ) + telemetry = { + "balance": info.balance, + "equity": info.equity, + "margin": info.margin, + "free_margin": info.margin_free, + "margin_level": info.margin_level, + "currency": info.currency, + "login": info.login, + "server": info.server, + "company": info.company, + "trade_mode": info.trade_mode, + } + logger.debug( + "Telemetry: balance=%.2f free_margin=%.2f equity=%.2f", + telemetry["balance"], telemetry["free_margin"], telemetry["equity"], + ) + return telemetry + + +def display_account_info(trading_balance: float = None) -> None: + """Print a formatted account dashboard to stdout.""" + import MetaTrader5 as mt5 + t = get_account_telemetry() + is_demo = t["trade_mode"] == mt5.ACCOUNT_TRADE_MODE_DEMO + print("=" * 50) + print(f" GOLD REGIME X — LIVE BRIDGE") + print("=" * 50) + print(f" Broker: {t['company']}") + print(f" Login: {t['login']}") + print(f" Server: {t['server']}") + print(f" Mode: {'DEMO' if is_demo else 'LIVE'}") + print(f" MT5 Balance: {t['balance']:.2f} {t['currency']}") + if trading_balance is not None: + print(f" Risk Balance:{trading_balance:.2f} USD (used for lot sizing)") + print(f" Equity: {t['equity']:.2f} {t['currency']}") + print(f" Free Margin: {t['free_margin']:.2f} {t['currency']}") + print("=" * 50) + + +def has_open_position(symbol: str = DEFAULT_SYMBOL, magic: int = MAGIC_NUMBER) -> bool: + """Return True if there is at least one open position for *symbol* with *magic*.""" + import MetaTrader5 as mt5 + positions = mt5.positions_get(symbol=symbol) + if positions is None: + logger.debug("positions_get returned None: %s", mt5.last_error()) + return False + return any(p.magic == magic for p in positions) + + +def check_margin(symbol: str, lot: float, order_type: int, price: float) -> bool: + """Pre-flight margin check via mt5.order_check(). + + Returns True if the order passes margin validation without submitting it. + """ + import MetaTrader5 as mt5 + request = { + "action": mt5.TRADE_ACTION_DEAL, + "symbol": symbol, + "volume": float(lot), + "type": order_type, + "price": float(price), + "deviation": DEFAULT_DEVIATION, + "magic": MAGIC_NUMBER, + "type_filling": mt5.ORDER_FILLING_IOC, + } + check = mt5.order_check(request) + if check is None: + logger.warning("order_check returned None: %s", mt5.last_error()) + return False + ok = check.retcode == 0 + logger.debug("Margin check: retcode=%d margin=%.2f free_margin=%.2f", + check.retcode, check.margin, check.margin_free) + return ok + + +# ───────────────────────────────────────────────────────────────────────────── +# Signal derivation +# ───────────────────────────────────────────────────────────────────────────── + +def compute_deviation(model_hmm, current_state: int, tf: str = "H1") -> int: + """Select order deviation based on regime stability and timeframe. + + A low self-transition probability on the current HMM state means the regime + is likely to flip next bar — a proxy for elevated market volatility. + Deviation is widened to avoid requotes in those conditions. + M5 uses a higher base deviation (30) than M15/H1 (20) for scalping fills. + """ + self_prob = model_hmm.transmat_[current_state, current_state] + base_dev = TF_DEFAULT_DEV.get(tf.upper(), 20) + high_vol_dev = TF_HIGH_VOL_DEV.get(tf.upper(), 50) + if self_prob < HIGH_VOL_SELF_TRANS_THRESHOLD: + logger.debug( + "High-vol deviation: state=%d self_trans=%.3f < %.2f — using %d pts", + current_state, self_prob, HIGH_VOL_SELF_TRANS_THRESHOLD, high_vol_dev, + ) + return high_vol_dev + return base_dev + + +def _move_sl_to_breakeven(ticket: int, entry_price: float, mt5) -> None: + """Modify an open position's SL to the entry price (break-even).""" + positions = mt5.positions_get(ticket=ticket) + if not positions: + return + pos = positions[0] + request = { + "action": mt5.TRADE_ACTION_SLTP, + "position": ticket, + "sl": round(entry_price, 2), + "tp": pos.tp, + } + res = mt5.order_send(request) + if res and res.retcode == mt5.TRADE_RETCODE_DONE: + logger.info("Break-even SL set: ticket=%d sl=%.2f", ticket, entry_price) + else: + logger.warning("Break-even SL failed: ticket=%d retcode=%s", + ticket, res.retcode if res else "None") + + +def _close_position(ticket: int, mt5) -> None: + """Close a specific open position at market price.""" + positions = mt5.positions_get(ticket=ticket) + if not positions: + return + pos = positions[0] + close_type = mt5.ORDER_TYPE_SELL if pos.type == mt5.ORDER_TYPE_BUY else mt5.ORDER_TYPE_BUY + tick = mt5.symbol_info_tick(pos.symbol) + price = tick.bid if close_type == mt5.ORDER_TYPE_SELL else tick.ask + request = { + "action": mt5.TRADE_ACTION_DEAL, + "position": ticket, + "symbol": pos.symbol, + "volume": pos.volume, + "type": close_type, + "price": price, + "deviation": 20, + "magic": MAGIC_NUMBER, + "comment": "GRX_close_chop", + "type_filling": mt5.ORDER_FILLING_IOC, + } + res = mt5.order_send(request) + if res and res.retcode == mt5.TRADE_RETCODE_DONE: + logger.info("Position closed (Chop exit): ticket=%d", ticket) + else: + logger.warning("Position close failed: ticket=%d retcode=%s", + ticket, res.retcode if res else "None") + + +def _build_live_df( + symbol: str, + tf_mt5: int, + n_bars: int, + obs_cov: float, + trans_cov: float, +) -> pd.DataFrame: + """Fetch the last *n_bars* completed bars and return a featurised DataFrame. + + The currently open (incomplete) bar is excluded via offset=1. + Feature sequence mirrors processor.process_pipeline() exactly. + """ + import MetaTrader5 as mt5 + rates = mt5.copy_rates_from_pos(symbol, tf_mt5, 1, n_bars) + if rates is None or len(rates) == 0: + raise RuntimeError( + f"copy_rates_from_pos returned no data: {mt5.last_error()}" + ) + df = pd.DataFrame(rates) + df["time"] = pd.to_datetime(df["time"], unit="s", utc=True) + df = ( + df.rename(columns={ + "time": "Date", + "open": "Open", + "high": "High", + "low": "Low", + "close": "Close", + "tick_volume": "Volume", + }) + [["Date", "Open", "High", "Low", "Close", "Volume"]] + ) + df.set_index("Date", inplace=True) + df.sort_index(inplace=True) + + df["log_return"] = compute_log_returns(df["Close"]) + df["kalman_return"] = kalman_smooth(df["log_return"].values, obs_cov, trans_cov) + df["volatility"] = compute_volatility(df["log_return"]) + df["rsi"] = compute_rsi(df["Close"]) + df["rsi_slope"] = df["rsi"].diff() + df["atr_normalized"] = compute_atr(df) + df.dropna(inplace=True) + return df + + +def compute_live_features( + symbol: str, + tf_mt5: int, + model_hmm, + obs_cov: float, + trans_cov: float, +): + """Build the current-bar feature vector for XGBoost inference. + + Returns: + features (np.ndarray shape (1,4)): [hmm_state, rsi_slope, atr_normalized, prev_log_return] + hmm_state (int): current HMM regime index + atr_price (float): ATR in price terms for SL calculation + + NOTE: We build the 4-element feature vector manually (bypassing + prepare_features()) to avoid the last-row NaN drop that strips the signal + bar from training feature matrices. Column order must match FEATURE_COLS. + """ + df = _build_live_df(symbol, tf_mt5, N_BARS_WARMUP, obs_cov, trans_cov) + states = predict_states(model_hmm, df) + + current_state = int(states[-1]) + rsi_slope = float(df["rsi_slope"].iloc[-1]) + atr_normalized = float(df["atr_normalized"].iloc[-1]) + prev_log_return = float(df["log_return"].iloc[-2]) # bar N-1 + + # FEATURE_COLS = ["hmm_state", "rsi_slope", "atr_normalized", "prev_log_return"] + features = np.array( + [[float(current_state), rsi_slope, atr_normalized, prev_log_return]], + dtype=np.float32, + ) + + atr_price = atr_normalized * float(df["Close"].iloc[-1]) + return features, current_state, atr_price + + +# ───────────────────────────────────────────────────────────────────────────── +# Order execution +# ───────────────────────────────────────────────────────────────────────────── + +def send_market_order( + symbol: str = DEFAULT_SYMBOL, + order_type: int = None, + lot: float = 0.01, + sl: float = 0.0, + tp: float = 0.0, + deviation: int = DEFAULT_DEVIATION, + magic: int = MAGIC_NUMBER, + comment: str = "GRX_Python", +) -> dict: + """Send an IOC market order to the MT5 terminal. + + Uses ORDER_FILLING_IOC (Immediate or Cancel) — standard for ECN/Cent + brokers. If the fill price is outside the deviation window the order is + cancelled rather than partially filled, preventing unintended exposure. + + Returns a dict with keys: retcode, order, comment, success. + Does NOT raise on failure — inspect result['success']. + """ + import MetaTrader5 as mt5 + tick = mt5.symbol_info_tick(symbol) + if tick is None: + logger.error("Cannot get tick for %s: %s", symbol, mt5.last_error()) + return {"retcode": -1, "order": 0, "comment": "No tick data", "success": False} + + price = tick.ask if order_type == mt5.ORDER_TYPE_BUY else tick.bid + + request = { + "action": mt5.TRADE_ACTION_DEAL, + "symbol": symbol, + "volume": float(lot), + "type": order_type, + "price": float(price), + "sl": float(sl), + "tp": float(tp), + "deviation": int(deviation), + "magic": int(magic), + "comment": comment, + "type_time": mt5.ORDER_TIME_GTC, + "type_filling": mt5.ORDER_FILLING_IOC, + } + + result = mt5.order_send(request) + if result is None: + logger.error("order_send returned None: %s", mt5.last_error()) + return {"retcode": -1, "order": 0, "comment": str(mt5.last_error()), "success": False} + + success = result.retcode == mt5.TRADE_RETCODE_DONE + if success: + logger.info( + "Order filled: #%d %s lot=%.2f price=%.2f sl=%.2f tp=%.2f dev=%d", + result.order, + "BUY" if order_type == mt5.ORDER_TYPE_BUY else "SELL", + lot, price, sl, tp, deviation, + ) + else: + logger.error( + "Order rejected: retcode=%d comment=%s", + result.retcode, result.comment, + ) + return { + "retcode": result.retcode, + "order": result.order, + "comment": result.comment, + "success": success, + } + + +# ───────────────────────────────────────────────────────────────────────────── +# Live execution loop +# ───────────────────────────────────────────────────────────────────────────── + +def run_live_loop( + tf: str = "H1", + broker: str = "headway_cent", + account_size: float = None, + dry_run: bool = False, +) -> None: + """Connect to MT5 and run the signal → order loop until interrupted. + + Fires on each newly completed bar. Position sizing, session limits, and + margin validation are enforced before every order. + + Args: + tf: Timeframe to trade — "H1" or "M15". + broker: Broker config key from risk_manager.BROKER_CONFIGS. + account_size: USD balance used for lot-sizing. If None, reads from MT5 + and normalises for cent accounts automatically. + dry_run: When True, signals are logged but no orders are placed. + Use with --account demo for paper-trading simulation. + """ + import MetaTrader5 as mt5 + from src.mt5_sync import connect_mt5, disconnect_mt5 + + if not connect_mt5(): + raise ConnectionError("Could not connect to MT5 terminal.") + + try: + _run_loop_inner(tf, broker, account_size, dry_run, mt5) + finally: + disconnect_mt5() + logger.info("Live loop terminated. MT5 disconnected.") + + +def _run_loop_inner(tf: str, broker: str, account_size: float, dry_run: bool, mt5) -> None: + """Inner loop extracted to allow clean finally / disconnect in run_live_loop.""" + tf_mt5 = _get_tf_map()[tf.upper()] + + # Resolve live balance + telemetry = get_account_telemetry() + if account_size is None: + account_size = _normalise_balance(telemetry["balance"], broker) + logger.info( + "Balance auto-detected: MT5 raw=%.2f USD normalised=%.2f", + telemetry["balance"], account_size, + ) + + display_account_info(trading_balance=account_size) + + # Load models + try: + model_hmm = load_hmm() + except FileNotFoundError: + raise FileNotFoundError("HMM model not found. Run --mode train first.") + try: + model_xgb, _ = load_xgb() + except FileNotFoundError: + raise FileNotFoundError("XGB model not found. Run --mode train first.") + + # Resolve Kalman params from Optuna study or TF defaults + try: + from src.optimizer import get_best_params + params = get_best_params(balance=account_size, broker=broker, tf=tf) + obs_cov = params.get("obs_cov") + trans_cov = params.get("trans_cov") + prob_threshold_opt = params.get("prob_threshold") + except Exception: + params = {} + obs_cov = None + trans_cov = None + prob_threshold_opt = None + + cfg_tf = TF_CONFIG[tf.upper()] + obs_cov = obs_cov if obs_cov is not None else cfg_tf["obs_cov_default"] + trans_cov = trans_cov if trans_cov is not None else cfg_tf["trans_cov_default"] + + # Use optimized prob threshold if available; otherwise fall back to TF hardcoded values. + # Short threshold is symmetric: 1 - prob_threshold. + if prob_threshold_opt is not None: + prob_threshold = prob_threshold_opt + short_threshold = 1.0 - prob_threshold_opt + else: + prob_threshold = TF_PROB_THRESHOLD.get(tf.upper(), 0.65) + short_threshold = TF_SHORT_THRESHOLD.get(tf.upper(), 0.35) + + # Session state (persists across bars within a day) + daily_trades = 0 + last_bar_time = None + current_day = None + # Tracks the tickets and entry context of the most recently placed signal + # so the break-even SL logic can fire when TP1 closes position 1. + signal_tracker = {"tickets": [], "entry_price": 0.0, + "direction": None, "tp1_hit": False} + + arm = AdaptiveRiskManager(account_size) + logger.info( + "Live loop started — TF=%s broker=%s balance=$%.2f dry_run=%s %s", + tf, broker, account_size, dry_run, arm, + ) + logger.info( + "Signal thresholds — BUY>%.3f SELL<%.3f (source: %s)", + prob_threshold, short_threshold, + "optuna" if prob_threshold_opt is not None else "hardcoded", + ) + + while True: + try: + # ── 1. Daily reset at UTC midnight ──────────────────────────── + today = datetime.now(timezone.utc).date() + if today != current_day: + daily_trades = 0 + current_day = today + logger.info("New UTC day %s — session counter reset.", today) + + # ── 2. Bar-change detection ─────────────────────────────────── + bars = mt5.copy_rates_from_pos(DEFAULT_SYMBOL, tf_mt5, 1, 1) + if bars is None or len(bars) == 0: + time.sleep(POLL_INTERVAL_SEC) + continue + bar_time = bars[0]["time"] + if bar_time == last_bar_time: + time.sleep(POLL_INTERVAL_SEC) + continue + last_bar_time = bar_time + + # ── 3. Refresh telemetry for margin/display (risk sizing always + # uses the caller-supplied account_size, not the live MT5 + # balance, because demo accounts carry arbitrary balances). + try: + telemetry = get_account_telemetry() + except Exception: + telemetry = {} + + arm = AdaptiveRiskManager(account_size) + arm.daily_trades = daily_trades # restore session count + + # ── 4. Compute live features ────────────────────────────────── + features, hmm_state, atr_price = compute_live_features( + DEFAULT_SYMBOL, tf_mt5, model_hmm, obs_cov, trans_cov, + ) + + # ── 5. Session limit check ──────────────────────────────────── + if not arm.can_trade(hmm_state): + logger.info( + "Session limit reached (state=%d daily=%d/%d). Skipping bar.", + hmm_state, arm.daily_trades, + arm.get_trade_limits(hmm_state)["max_daily_trades"], + ) + time.sleep(POLL_INTERVAL_SEC) + continue + + # ── 6. Skip if position already open ───────────────────────── + if has_open_position(DEFAULT_SYMBOL, MAGIC_NUMBER): + logger.debug("Open position exists — skipping bar.") + time.sleep(POLL_INTERVAL_SEC) + continue + + bar_str = datetime.fromtimestamp(bar_time, timezone.utc).strftime("%Y-%m-%d %H:%M") + state_lbl = {0: "Bull", 1: "Bear", 2: "Chop"}.get(hmm_state, str(hmm_state)) + max_trades_today = arm.get_trade_limits(hmm_state)["max_daily_trades"] + prob = float(model_xgb.predict_proba(features)[:, 1][0]) + logger.info( + "Bar %s | state=%s | prob=%.3f | trades=%d/%d", + bar_str, hmm_state, prob, + arm.daily_trades, max_trades_today, + ) + send_telegram_msg( + f"📊 Bar {bar_str} [{tf}]\n" + f"Regime: {state_lbl} ({hmm_state}) | " + f"Prob: {prob:.3f}\n" + f"Trades today: {arm.daily_trades}/{max_trades_today}" + + (" 🔴 DRY RUN" if dry_run else "") + ) + + # ── 7b. Break-even SL and Chop-exit position management ─────── + if not dry_run and signal_tracker["tickets"]: + open_tickets = { + p.ticket + for p in (mt5.positions_get(symbol=DEFAULT_SYMBOL) or []) + if p.magic == MAGIC_NUMBER + } + active = [t for t in signal_tracker["tickets"] if t in open_tickets] + # TP1 hit: first position closed → move runner's SL to break-even + if len(active) < len(signal_tracker["tickets"]) and not signal_tracker["tp1_hit"]: + signal_tracker["tp1_hit"] = True + for ticket in active: + _move_sl_to_breakeven( + ticket, signal_tracker["entry_price"], mt5, + ) + # Regime shifted to Chop while runner is active → close immediately + if hmm_state == CHOP_STATE and active: + for ticket in active: + _close_position(ticket, mt5) + logger.info("Runner(s) closed: Chop state shift while position open.") + active = [] + signal_tracker["tickets"] = active + + # ── 8. Signal routing (TF-specific thresholds) ─────────────── + if prob > prob_threshold and hmm_state != CHOP_STATE: + direction = "BUY" + order_type = mt5.ORDER_TYPE_BUY + elif prob < short_threshold and hmm_state != CHOP_STATE: + direction = "SELL" + order_type = mt5.ORDER_TYPE_SELL + else: + logger.debug("No signal (prob=%.3f state=%d). Holding.", prob, hmm_state) + time.sleep(POLL_INTERVAL_SEC) + continue + + # ── 9. Position sizing ──────────────────────────────────────── + tp_mults = _tp_multipliers(tf, hmm_state) + limits = arm.get_trade_limits(hmm_state) + pos_per_trade = min(limits["pos_per_trade"], len(tp_mults)) + sl_distance = max(atr_price * ATR_MULTIPLIER, 0.01) + lot_total = arm.calculate_lot_size(stop_loss_pips=sl_distance) + lot_per_pos = max(0.01, round(lot_total / pos_per_trade, 2)) + + # ── 10. Deviation (TF-specific + regime stability) ──────────── + deviation = compute_deviation(model_hmm, hmm_state, tf) + + # ── 11. SL / TP prices ──────────────────────────────────────── + tick = mt5.symbol_info_tick(DEFAULT_SYMBOL) + if direction == "BUY": + entry_price = tick.ask + sl_price = round(entry_price - sl_distance, 2) + tp_levels = [round(entry_price + sl_distance * m, 2) for m in tp_mults] + else: + entry_price = tick.bid + sl_price = round(entry_price + sl_distance, 2) + tp_levels = [round(entry_price - sl_distance * m, 2) for m in tp_mults] + + # ── 11b. Spread viability guard (M5 scalping only) ─────────── + if tf.upper() == "M5": + sym_info = mt5.symbol_info(DEFAULT_SYMBOL) + spread_price = (sym_info.spread * sym_info.point) if sym_info else 0.0 + tp1_distance = sl_distance * tp_mults[0] + if spread_price > 0 and tp1_distance < spread_price * MIN_SPREAD_RATIO: + logger.warning( + "Unviable M5 trade: TP1=%.4f < %.1fx spread=%.4f. Skipping.", + tp1_distance, MIN_SPREAD_RATIO, spread_price, + ) + time.sleep(POLL_INTERVAL_SEC) + continue + + # ── 12. Margin check ────────────────────────────────────────── + if not check_margin(DEFAULT_SYMBOL, lot_per_pos, order_type, entry_price): + logger.warning( + "Insufficient margin for %s lot=%.2f. Skipping. Free margin: %.2f", + direction, lot_per_pos, telemetry.get("free_margin", 0), + ) + time.sleep(POLL_INTERVAL_SEC) + continue + + # ── 13. Send order(s) with state-aware staged TPs ───────────── + signal_tracker = {"tickets": [], "entry_price": entry_price, + "direction": direction, "tp1_hit": False} + logger.info( + "SIGNAL %s | state=%d | prob=%.3f | lot×%d=%.2f | sl=%.2f | tp=%s | dev=%d", + direction, hmm_state, prob, pos_per_trade, lot_per_pos, + sl_price, "/".join(f"{t:.2f}" for t in tp_levels), deviation, + ) + for p in range(pos_per_trade): + tp_price = tp_levels[p] + comment = f"GRX_{direction}_p{p+1}of{pos_per_trade}_s{hmm_state}_tp{p+1}" + if dry_run: + logger.info( + "[DRY RUN] Would send: %s lot=%.2f sl=%.2f tp=%.2f dev=%d comment=%s", + direction, lot_per_pos, sl_price, tp_price, deviation, comment, + ) + else: + result = send_market_order( + symbol=DEFAULT_SYMBOL, + order_type=order_type, + lot=lot_per_pos, + sl=sl_price, + tp=tp_price, + deviation=deviation, + magic=MAGIC_NUMBER, + comment=comment, + ) + if result["success"]: + signal_tracker["tickets"].append(result["order"]) + daily_trades += 1 + arm.log_trade() + else: + logger.error( + "Order %d/%d failed — retcode=%d %s", + p + 1, pos_per_trade, + result["retcode"], result["comment"], + ) + + if dry_run: + daily_trades += 1 # count dry-run signals for session limit testing + + except KeyboardInterrupt: + logger.info("KeyboardInterrupt — stopping live loop.") + break + except Exception as exc: + logger.error( + "Unhandled loop error: %s — sleeping 30 s before retry.", + exc, exc_info=True, + ) + time.sleep(30) diff --git a/src/notifier.py b/src/notifier.py new file mode 100644 index 0000000..88e8ff8 --- /dev/null +++ b/src/notifier.py @@ -0,0 +1,66 @@ +"""Telegram notification helper for Gold Regime X. + +Reads credentials from environment variables (or .env file if python-dotenv +is installed). All public functions degrade gracefully when credentials are +missing — the rest of the system continues to operate without Telegram. + +Environment variables required in .env: + TELEGRAM_BOT_TOKEN — BotFather token (e.g. 123456789:AAF-xxx…) + TELEGRAM_CHAT_ID — Your personal chat ID (see README_REMOTE.md) + +Install: + pip install requests python-dotenv +""" + +import os +import requests + +from src.logger import setup_logger + +logger = setup_logger(__name__) + +# Load .env on import if python-dotenv is available (silently ignored otherwise) +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + pass + +_TELEGRAM_API = "https://api.telegram.org/bot{token}/sendMessage" + + +def get_credentials() -> tuple: + """Return (bot_token, chat_id) from environment variables.""" + return os.getenv("TELEGRAM_BOT_TOKEN"), os.getenv("TELEGRAM_CHAT_ID") + + +def send_telegram_msg(message: str, parse_mode: str = "HTML") -> bool: + """Send a message to your configured Telegram chat. + + Returns True on success, False on any failure (never raises). + The ``parse_mode`` parameter accepts 'HTML' or 'Markdown'. + Use HTML tags like bold, monospace, italic. + """ + token, chat_id = get_credentials() + if not token or not chat_id: + logger.debug("Telegram credentials not set — skipping notification.") + return False + + payload = { + "chat_id": chat_id, + "text": message, + "parse_mode": parse_mode, + } + try: + r = requests.post( + _TELEGRAM_API.format(token=token), + data=payload, + timeout=10, + ) + if not r.ok: + logger.warning("Telegram API error %d: %s", r.status_code, r.text[:200]) + return False + return True + except Exception as exc: + logger.warning("Telegram send failed: %s", exc) + return False diff --git a/src/optimizer.py b/src/optimizer.py new file mode 100644 index 0000000..1a3d8e8 --- /dev/null +++ b/src/optimizer.py @@ -0,0 +1,356 @@ +"""Optuna hyperparameter optimiser for Gold Regime X. + +Key design decisions: +- SQLite study.db with load_if_exists=True: crash-safe resume at any point +- Scores on OOS Sharpe ONLY to prevent IS data leakage / overfitting +- Per-trial gc.collect() to prevent RAM accumulation over long M5 runs +- Optional n_jobs>1: Optuna uses threads; note that XGBoost independently + uses multiple cores, so n_jobs>2 typically yields diminishing returns + unless you have 16+ cores. Default n_jobs=1 is safest. +- psutil RAM guard: if RAM usage exceeds 90%, new trials pause 30s to let + the OS breathe before dispatching more work. +- Progress dashboard every 5 trials with ETA. +- Telegram heartbeat every 10% of requested trials (if credentials set). +""" + +import gc +import os +import time + +import optuna + +from src.processor import process_pipeline +from src.engine_hmm import fit_hmm +from src.engine_xgb import prepare_features, train_xgb, get_predictions +from src.backtester import vectorized_backtest +from src.risk_manager import SMALL_ACCOUNT_THRESHOLD +from src.logger import setup_logger + +logger = setup_logger(__name__) + +# Optional psutil for RAM guard — degrades gracefully if not installed +try: + import psutil + _PSUTIL_OK = True +except ImportError: + _PSUTIL_OK = False + logger.debug("psutil not installed — RAM guard disabled.") + +STUDY_DB = "sqlite:///models/study.db" +DD_HARD_LIMIT = 0.15 +MIN_OOS_TRADES = 50 +RAM_HIGH_PCT = 90 # pause new trials when used RAM exceeds this % +RAM_PAUSE_SEC = 30 # seconds to sleep when RAM is low + +# Study tier objectives -------------------------------------------------------- +# Small accounts ($15–$50): extreme drawdown protection — Sharpe - DD×5.0 +# Growth accounts (>$50): balanced with higher frequency — Sharpe - DD×3.0 +TIER_CONFIGS = { + "small": {"dd_penalty": 5.0, "dd_limit": 0.10}, + "growth": {"dd_penalty": 3.0, "dd_limit": 0.15}, +} + + +def _get_tier(balance: float) -> str: + return "small" if balance <= SMALL_ACCOUNT_THRESHOLD else "growth" + + +def _score_result(result: dict, tier: str, broker: str) -> float: + cfg = TIER_CONFIGS[tier] + if result["max_drawdown"] > cfg["dd_limit"]: + return -10.0 + if broker == "headway_cent": + return result["sharpe_ratio"] - result["max_drawdown"] * cfg["dd_penalty"] + return result["sharpe_ratio"] + + +def make_objective(balance: float = 15.0, broker: str = "standard", tf: str = "H1"): + """Return an Optuna objective function for the given account / TF context.""" + tier = _get_tier(balance) + + def objective(trial: optuna.Trial) -> float: + obs_cov = trial.suggest_float("obs_cov", 0.1, 5.0, log=True) + trans_cov = trial.suggest_float("trans_cov", 0.001, 0.1, log=True) + n_states = trial.suggest_int("n_states", 2, 4) + prob_threshold = trial.suggest_float("prob_threshold", 0.52, 0.68) + + max_depth = trial.suggest_int("max_depth", 3, 5) + learning_rate = trial.suggest_float("learning_rate", 0.01, 0.3, log=True) + n_estimators = trial.suggest_int("n_estimators", 100, 500, step=50) + subsample = trial.suggest_float("subsample", 0.6, 1.0) + colsample_bytree = trial.suggest_float("colsample_bytree", 0.5, 1.0) + min_child_weight = trial.suggest_int("min_child_weight", 1, 15) + gamma = trial.suggest_float("gamma", 0.01, 5.0, log=True) + reg_alpha = trial.suggest_float("reg_alpha", 0.01, 10.0, log=True) + + try: + df = process_pipeline( + obs_cov=obs_cov, trans_cov=trans_cov, save=False, tf=tf + ) + _hmm, states, _ = fit_hmm(df, n_states=n_states) + X, y, df_aligned = prepare_features(df, states) + model_xgb, metrics = train_xgb( + X, y, + max_depth=max_depth, + learning_rate=learning_rate, + n_estimators=n_estimators, + subsample=subsample, + colsample_bytree=colsample_bytree, + min_child_weight=min_child_weight, + gamma=gamma, + reg_alpha=reg_alpha, + ) + _, probabilities = get_predictions(model_xgb, X) + + states_aligned = states[df.index.isin(df_aligned.index)] + split_idx = metrics.get("split_idx") + result = vectorized_backtest( + df_aligned, probabilities, states_aligned, + split_idx=split_idx, + account_size=balance, + broker=broker, + tf=tf, + prob_threshold=prob_threshold, + ) + + # Score on OOS only to prevent IS data leakage + if split_idx and "oos_sharpe_ratio" in result: + if result.get("oos_n_trades", 0) < MIN_OOS_TRADES: + return -10.0 + oos_result = { + "sharpe_ratio": result["oos_sharpe_ratio"], + "max_drawdown": result["oos_max_drawdown"], + } + score = _score_result(oos_result, tier, broker) + else: + score = _score_result(result, tier, broker) + + logger.info( + "Trial %d [%s/%s tier=%s $%.0f]: score=%.3f " + "IS=%.3f OOS=%.3f OOS_DD=%.1f%% trades=%d", + trial.number, tf, broker, tier, balance, + score, + result.get("is_sharpe_ratio", result["sharpe_ratio"]), + result.get("oos_sharpe_ratio", result["sharpe_ratio"]), + result.get("oos_max_drawdown", result["max_drawdown"]) * 100, + result.get("oos_n_trades", result["n_trades"]), + ) + return score + + except Exception as e: + logger.warning("Trial %d failed: %s", trial.number, e) + return -10.0 + + finally: + # Release HMM/XGB objects and aligned arrays to prevent RAM creep + # over long M5 runs (~750K bars × 500 trials) + gc.collect() + + return objective + + +def _make_callbacks(n_trials: int, study_name: str, already_done: int = 0) -> list: + """Build Optuna callbacks for progress reporting, RAM guard, and Telegram. + + Args: + n_trials: Number of trials requested for THIS session. + study_name: Optuna study name (for Telegram messages). + already_done: Completed trials in the DB before this session started. + All progress percentages are relative to session trials only, + so a resumed run shows [1/500] not [373/500]. + """ + from src.notifier import send_telegram_msg + + start_time = [time.time()] # mutable list for closure mutation + heartbeat_pct = set() # tracks which 10% milestones were pinged + + def _callback(study: optuna.Study, _trial: optuna.trial.FrozenTrial) -> None: + total_done = len([ + t for t in study.trials + if t.state == optuna.trial.TrialState.COMPLETE + ]) + # session_done = trials completed in THIS session only + session_done = max(0, total_done - already_done) + + if session_done <= 0: + return + + # ── RAM guard: pause before dispatching the next trial ─────────────── + if _PSUTIL_OK: + mem = psutil.virtual_memory() + if mem.percent >= RAM_HIGH_PCT: + logger.warning( + "RAM at %.0f%% (>= %d%%) — pausing %ds.", + mem.percent, RAM_HIGH_PCT, RAM_PAUSE_SEC, + ) + time.sleep(RAM_PAUSE_SEC) + + # ── Progress dashboard every 5 session trials ───────────────────────── + if session_done % 5 == 0: + elapsed = time.time() - start_time[0] + rate = elapsed / session_done + remaining = max(0, n_trials - session_done) + eta_sec = remaining * rate + if eta_sec >= 3600: + eta_str = f"{int(eta_sec // 3600)}h {int((eta_sec % 3600) // 60)}m" + else: + eta_str = f"{int(eta_sec // 60)}m {int(eta_sec % 60)}s" + best = study.best_value if study.best_trial else float("-inf") + ram_str = ( + f" RAM: {psutil.virtual_memory().percent:.0f}%" + if _PSUTIL_OK else "" + ) + print( + f" [{session_done:>4}/{n_trials}] " + f"Best Sharpe: {best:+.3f} | " + f"ETA: {eta_str}{ram_str}" + ) + + # ── Telegram heartbeat every 10% of SESSION trials ─────────────────── + if n_trials > 0: + milestone = (int(session_done / n_trials * 100) // 10) * 10 + if milestone > 0 and milestone not in heartbeat_pct: + heartbeat_pct.add(milestone) + best = study.best_value if study.best_trial else float("-inf") + send_telegram_msg( + f"Optimization {milestone}% complete\n" + f"Study: {study_name}\n" + f"Best Sharpe: {best:.3f} | " + f"Trials: {session_done}/{n_trials}" + ) + + return [_callback] + + +def _study_name( + base: str = "gold_regime_x", + broker: str = "standard", + tier: str = "small", + tf: str = "H1", +) -> str: + """Deterministic study name — must be identical across sessions to resume.""" + parts = [base, tier, broker] + if tf.upper() != "H1": + parts.append(tf.upper()) + return "_".join(parts) + + +def run_optimization( + n_trials: int = 250, + balance: float = 15.0, + broker: str = "standard", + tf: str = "H1", + n_jobs: int = 1, +) -> optuna.Study: + """Run (or resume) an Optuna study and return it when complete. + + The SQLite study.db persists every completed trial to disk. If the run + is interrupted at any point (Ctrl+C, crash, power cut), simply re-run the + same command — Optuna will load_if_exists=True and pick up exactly where + it left off. + + To start fresh: delete models/study.db manually. + + Args: + n_trials: Additional trials to run THIS session. Optuna adds them + on top of any already-completed trials in the DB. + n_jobs: Parallel trial workers (Python threads). Default 1. + XGBoost already uses all CPU cores internally, so n_jobs>2 + can cause core contention and actually slow things down. + For true multi-process parallelism, open multiple terminals + and run the same command simultaneously — they all share the + same study.db safely via SQLite locking. + """ + tier = _get_tier(balance) + name = _study_name(broker=broker, tier=tier, tf=tf) + + os.makedirs("models", exist_ok=True) + + study = optuna.create_study( + study_name=name, + storage=STUDY_DB, + direction="maximize", + load_if_exists=True, # crash-safe resume + pruner=optuna.pruners.MedianPruner(), + ) + + already_done = len([ + t for t in study.trials + if t.state == optuna.trial.TrialState.COMPLETE + ]) + if already_done > 0: + pct = already_done / (already_done + n_trials) * 100 + print( + f"\nFailsafe: {already_done} completed trials found in study.db " + f"({pct:.0f}% of total {already_done + n_trials}). " + f"Resuming from trial #{already_done + 1}...\n" + ) + else: + print(f"\nStarting new study '{name}' — {n_trials} trials.\n") + + logger.info( + "Optimization: tf=%s broker=%s tier=%s balance=$%.0f " + "n_trials=%d n_jobs=%d study=%s already_done=%d", + tf, broker, tier, balance, n_trials, n_jobs, name, already_done, + ) + + # Suppress Optuna's verbose per-trial INFO logs; our callback handles output + optuna.logging.set_verbosity(optuna.logging.WARNING) + + study.optimize( + make_objective(balance=balance, broker=broker, tf=tf), + n_trials=n_trials, + n_jobs=n_jobs, + show_progress_bar=(n_jobs == 1), # tqdm bar misleads with threads + callbacks=_make_callbacks(n_trials, name, already_done=already_done), + ) + + best = study.best_value + total_done = len([ + t for t in study.trials + if t.state == optuna.trial.TrialState.COMPLETE + ]) + logger.info( + "Optimization done: best_score=%.3f total_trials=%d params=%s", + best, total_done, study.best_params, + ) + + # ── Completion banner ───────────────────────────────────────────────────── + pad = 46 + print( + "\n" + "************************************************************\n" + "* *\n" + "* OPTIMIZATION COMPLETE *\n" + f"* Study : {name:<{pad}}*\n" + f"* Trials: {total_done:<{pad}}*\n" + f"* Best : {best:<+{pad}.4f}*\n" + "* *\n" + "************************************************************\n" + ) + + from src.notifier import send_telegram_msg + send_telegram_msg( + f"Optimization 100% Complete!\n" + f"Study: {name}\n" + f"Trials: {total_done}\n" + f"Best OOS Sharpe: {best:.3f}\n" + + "\n".join( + f" {k}: {v}" + for k, v in study.best_params.items() + ) + ) + + return study + + +def get_best_params( + balance: float = 15.0, + broker: str = "standard", + tf: str = "H1", +) -> dict: + """Load the best hyperparameters from the persisted SQLite study.""" + tier = _get_tier(balance) + name = _study_name(broker=broker, tier=tier, tf=tf) + study = optuna.load_study(study_name=name, storage=STUDY_DB) + return study.best_params diff --git a/src/processor.py b/src/processor.py new file mode 100644 index 0000000..9cbcdf2 --- /dev/null +++ b/src/processor.py @@ -0,0 +1,157 @@ +import numpy as np +import pandas as pd +from pathlib import Path +from src.logger import setup_logger + +logger = setup_logger(__name__) + +# Per-timeframe configuration ───────────────────────────────────────────────── +# Kalman obs_cov controls smoothing: higher value = more smoothing (less trust +# in raw observations). M15 is ~4× noisier than H1; M5 uses a low obs_cov +# (0.05) to keep the filter very responsive for fast scalping regime detection. +# n_states_default: M5 uses 4 states to capture micro-noise; M15/H1 use 3. +TF_CONFIG = { + "H1": { + "raw_path": Path("data/raw/XAU_1h_data.csv"), + "processed_path": Path("data/processed/gold_h1_processed.parquet"), + "obs_cov_default": 1.0, + "trans_cov_default": 0.01, + "n_states_default": 3, + }, + "M15": { + "raw_path": Path("data/raw/XAU_15m_data.csv"), + "processed_path": Path("data/processed/gold_m15_processed.parquet"), + "obs_cov_default": 4.0, + "trans_cov_default": 0.01, + "n_states_default": 3, + }, + "M5": { + "raw_path": Path("data/raw/XAU_5m_data.csv"), + "processed_path": Path("data/processed/gold_m5_processed.parquet"), + "obs_cov_default": 0.05, # low value = responsive Kalman for scalping + "trans_cov_default": 0.01, + "n_states_default": 4, # 4 states to capture micro-regime noise + }, +} + +# Legacy aliases kept for callers that use the bare constants directly +RAW_PATH = TF_CONFIG["H1"]["raw_path"] +PROCESSED_PATH = TF_CONFIG["H1"]["processed_path"] + + +def load_raw_data(path: Path = RAW_PATH) -> pd.DataFrame: + if not path.exists(): + raise FileNotFoundError( + f"Raw data file not found: {path}\n" + "Export XAUUSD data from MT5 as CSV (semicolon-delimited) and place it at that path." + ) + df = pd.read_csv( + path, sep=";", parse_dates=["Date"], date_format="%Y.%m.%d %H:%M" + ) + df.set_index("Date", inplace=True) + df.sort_index(inplace=True) + logger.info("Loaded %d rows from %s", len(df), path) + return df + + +def filter_data(df: pd.DataFrame, years: int = 10) -> pd.DataFrame: + cutoff = df.index.max() - pd.DateOffset(years=years) + df = df[df.index >= cutoff] + before = len(df) + df = df[df["Volume"] > 5] + logger.info( + "Filtered to last %d years: %d rows (%d low-volume bars removed)", + years, len(df), before - len(df), + ) + return df + + +def compute_log_returns(close: pd.Series) -> pd.Series: + return np.log(close / close.shift(1)).rename("log_return") + + +def kalman_smooth(data: np.ndarray, obs_cov: float = 1.0, trans_cov: float = 0.01) -> np.ndarray: + mu = 0.0 + P = 1.0 + smoothed = np.zeros_like(data, dtype=np.float64) + for i in range(len(data)): + P = P + trans_cov + if np.isnan(data[i]): + smoothed[i] = mu + else: + K = P / (P + obs_cov) + mu = mu + K * (data[i] - mu) + P = (1 - K) * P + smoothed[i] = mu + return smoothed + + +def compute_volatility(log_returns: pd.Series, window: int = 20) -> pd.Series: + return log_returns.rolling(window=window).std().rename("volatility") + + +def compute_rsi(close: pd.Series, period: int = 14) -> pd.Series: + delta = close.diff() + gain = delta.clip(lower=0) + loss = -delta.clip(upper=0) + avg_gain = gain.ewm(alpha=1 / period, min_periods=period).mean() + avg_loss = loss.ewm(alpha=1 / period, min_periods=period).mean() + rs = avg_gain / avg_loss + return (100 - (100 / (1 + rs))).rename("rsi") + + +def compute_atr(df: pd.DataFrame, period: int = 14) -> pd.Series: + high_low = df["High"] - df["Low"] + high_close = (df["High"] - df["Close"].shift(1)).abs() + low_close = (df["Low"] - df["Close"].shift(1)).abs() + true_range = pd.concat([high_low, high_close, low_close], axis=1).max(axis=1) + atr = true_range.rolling(window=period).mean() + return (atr / df["Close"]).rename("atr_normalized") + + +def process_pipeline( + obs_cov: float = None, + trans_cov: float = None, + save: bool = True, + tf: str = "H1", +) -> pd.DataFrame: + """Run the full feature-engineering pipeline for the given timeframe. + + Args: + obs_cov: Kalman observation covariance. Defaults to TF-specific value + (1.0 for H1, 4.0 for M15). + trans_cov: Kalman transition covariance. Defaults to 0.01. + save: Persist processed parquet to disk. + tf: Timeframe — ``"H1"`` or ``"M15"``. + """ + tf = tf.upper() + if tf not in TF_CONFIG: + raise ValueError(f"Unknown timeframe '{tf}'. Choose from {list(TF_CONFIG)}") + + cfg = TF_CONFIG[tf] + obs_cov = obs_cov if obs_cov is not None else cfg["obs_cov_default"] + trans_cov = trans_cov if trans_cov is not None else cfg["trans_cov_default"] + + df = load_raw_data(cfg["raw_path"]) + df = filter_data(df) + + df["log_return"] = compute_log_returns(df["Close"]) + df["kalman_return"] = kalman_smooth(df["log_return"].values, obs_cov, trans_cov) + df["volatility"] = compute_volatility(df["log_return"]) + df["rsi"] = compute_rsi(df["Close"]) + df["rsi_slope"] = df["rsi"].diff() + df["atr_normalized"] = compute_atr(df) + + df.dropna(inplace=True) + logger.info( + "Pipeline [%s] complete: %d rows, columns: %s", + tf, len(df), list(df.columns), + ) + + if save: + out_path = cfg["processed_path"] + out_path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(out_path) + logger.info("Saved processed data to %s", out_path) + + return df diff --git a/src/remote_control.py b/src/remote_control.py new file mode 100644 index 0000000..f2109e4 --- /dev/null +++ b/src/remote_control.py @@ -0,0 +1,192 @@ +"""Telegram remote control panel for Gold Regime X. + +Runs a long-polling Telegram bot that accepts keyboard button commands from +the authorised user and dispatches them as subprocesses so the listener +thread is never blocked. + +Keyboard layout: + ┌──────────────────────┬─────────────────────┐ + │ 🚀 START TRADING │ 🛑 STOP TRADING │ + ├──────────────────────┼─────────────────────┤ + │ 📉 START OPTIMIZE │ 📊 BOT STATUS │ + │ (M5) │ │ + └──────────────────────┴─────────────────────┘ + +Required env vars (see .env.example): + TELEGRAM_BOT_TOKEN — BotFather token + TELEGRAM_CHAT_ID — Your chat ID (used for outbound heartbeat messages) + ALLOWED_USER_ID — Your numeric Telegram user ID (security gate) + LIVE_TF — Default TF for START TRADING (default: H1) + LIVE_BROKER — Default broker (default: headway_cent) + LIVE_BALANCE — Default balance (default: 15) + +Usage: + python main.py --mode listen +""" + +import os +import subprocess +import time + +import requests +from requests.exceptions import ReadTimeout + +from src.logger import setup_logger +from src.notifier import get_credentials, send_telegram_msg + +logger = setup_logger(__name__) + +# ── Telegram keyboard sent with every reply ──────────────────────────────────── +_KEYBOARD = { + "keyboard": [ + ["🚀 START TRADING", "🛑 STOP TRADING"], + ["📉 START OPTIMIZE (M5)", "📊 BOT STATUS"], + ], + "resize_keyboard": True, + "one_time_keyboard": False, +} + +# Tracks subprocesses launched by this session so we can terminate them +_procs: dict[str, subprocess.Popen] = {} + + +def _api(token: str, method: str, _req_timeout=(10, 15), **params) -> dict: + """Call a Telegram Bot API method; returns the parsed JSON response. + + ``_req_timeout`` is passed to requests as (connect_timeout, read_timeout) + and is intentionally NOT forwarded to Telegram — hence the underscore prefix. + Long-polling calls should supply a read timeout > the Telegram ``timeout`` + parameter to avoid spurious ReadTimeout exceptions. + """ + try: + r = requests.post( + f"https://api.telegram.org/bot{token}/{method}", + json=params, + timeout=_req_timeout, + ) + return r.json() + except ReadTimeout: + # Expected for getUpdates when no new messages arrive within the poll + # window — Telegram returns an empty result list, which is normal. + logger.debug("Telegram API (%s): long-poll returned empty (no new messages).", method) + return {} + except Exception as exc: + logger.warning("Telegram API (%s) failed: %s", method, exc) + return {} + + +def _reply(token: str, chat_id, text: str) -> None: + """Send a message with the persistent control keyboard attached.""" + _api( + token, "sendMessage", + chat_id=chat_id, + text=text, + reply_markup=_KEYBOARD, + parse_mode="HTML", + ) + + +def _proc_alive(key: str) -> bool: + """Return True if the subprocess stored under *key* is still running.""" + proc = _procs.get(key) + return proc is not None and proc.poll() is None + + +def _handle(token: str, chat_id, user_id: str, text: str) -> None: + """Authorise and dispatch a single inbound command.""" + allowed = os.getenv("ALLOWED_USER_ID", "") + if user_id != allowed: + _api(token, "sendMessage", chat_id=chat_id, text="Unauthorized.") + return + + tf = os.getenv("LIVE_TF", "H1") + broker = os.getenv("LIVE_BROKER", "headway_cent") + balance = os.getenv("LIVE_BALANCE", "15") + + if text == "🚀 START TRADING": + if _proc_alive("trading"): + _reply(token, chat_id, "Trading is already running.") + return + _procs["trading"] = subprocess.Popen([ + "python", "main.py", + "--mode", "live", "--account", "live", + "--tf", tf, "--broker", broker, "--balance", balance, + ]) + _reply(token, chat_id, + f"Trading started\nTF={tf} broker={broker} balance=${balance}") + + elif text == "🛑 STOP TRADING": + if _proc_alive("trading"): + _procs["trading"].terminate() + _reply(token, chat_id, "Trading stopped.") + else: + _reply(token, chat_id, "No active trading process found.") + + elif text == "📉 START OPTIMIZE (M5)": + if _proc_alive("optimizer"): + _reply(token, chat_id, "Optimization is already running.") + return + _procs["optimizer"] = subprocess.Popen([ + "python", "main.py", + "--mode", "optimize", "--tf", "M5", + "--broker", broker, "--balance", balance, "--trials", "500", + ]) + _reply(token, chat_id, + "M5 optimization started.\n" + "(Resumes from study.db if previous progress exists.)") + + elif text == "📊 BOT STATUS": + try: + from src.auditor import get_daily_report + report = get_daily_report(broker=broker) + except Exception as exc: + report = f"Status unavailable: {exc}" + _reply(token, chat_id, report) + + else: + _reply(token, chat_id, "Use the keyboard buttons below to control the bot.") + + +def run_listener() -> None: + """Poll Telegram for updates and dispatch commands until KeyboardInterrupt. + + Uses the getUpdates long-polling method (timeout=30s) so the bot reacts + within seconds without holding a persistent WebSocket connection. + """ + token, _ = get_credentials() + if not token: + raise RuntimeError( + "TELEGRAM_BOT_TOKEN not set. " + "Copy .env.example to .env and fill in your credentials." + ) + + logger.info("Remote control listener started. Waiting for commands...") + send_telegram_msg("Gold Regime X remote control is online.") + + offset = 0 + while True: + try: + # timeout=25: Telegram holds the connection up to 25s for new msgs. + # _req_timeout read must exceed that so requests doesn't give up first. + data = _api(token, "getUpdates", offset=offset, + timeout=25, _req_timeout=(10, 35)) + for update in data.get("result", []): + offset = update["update_id"] + 1 + msg = update.get("message", {}) + chat_id = msg.get("chat", {}).get("id") + user_id = str(msg.get("from", {}).get("id", "")) + text = msg.get("text", "").strip() + if chat_id and text: + logger.info( + "Inbound: user_id=%s chat_id=%s text=%r", + user_id, chat_id, text, + ) + _handle(token, chat_id, user_id, text) + + except KeyboardInterrupt: + logger.info("Remote control stopped.") + send_telegram_msg("Gold Regime X remote control is offline.") + break + except Exception as exc: + logger.error("Listener error: %s — retrying in 10s", exc) + time.sleep(10) diff --git a/src/risk_manager.py b/src/risk_manager.py new file mode 100644 index 0000000..20c6e5f --- /dev/null +++ b/src/risk_manager.py @@ -0,0 +1,149 @@ +from src.logger import setup_logger + +logger = setup_logger(__name__) + +# Spread and commission fractions of price per trade (normalized cost) +BROKER_CONFIGS = { + "standard": { + "spread_frac": 0.0002, + "commission_frac": 0.0001, + }, + "headway_cent": { + # XAUUSD on Headway Cent: ~$0.30 spread + ~$0.03/micro-lot commission + "spread_frac": 0.0002, + "commission_frac": 0.0002, + }, +} + +MIN_CAPITAL_USD = 15.0 +CENT_MULTIPLIER = 100 # Headway Cent: 1 USD displayed as 100 cents +SMALL_ACCOUNT_THRESHOLD = 50.0 # USD — boundary between small and growth tiers + + +class CentConverter: + """Lot sizing for Headway Cent accounts (micro-lot floor = 0.01).""" + + MIN_LOT = 0.01 + + def __init__(self, account_balance_usd: float): + self.account_balance_usd = max(float(account_balance_usd), MIN_CAPITAL_USD) + + @property + def displayed_balance(self) -> float: + return self.account_balance_usd * CENT_MULTIPLIER + + def calculate_lot(self, risk_pct: float, stop_distance_norm: float) -> float: + if stop_distance_norm <= 0: + return self.MIN_LOT + risk_amount = self.account_balance_usd * risk_pct + lot = risk_amount / stop_distance_norm + lot = max(self.MIN_LOT, (lot // self.MIN_LOT) * self.MIN_LOT) + return round(lot, 2) + + def __repr__(self) -> str: + return ( + f"CentConverter(balance=${self.account_balance_usd:.2f}, " + f"displayed={self.displayed_balance:.0f} cents)" + ) + + +class SessionManager: + """Legacy per-session limiter (kept for backwards compat). + + Prefer ``AdaptiveRiskManager`` for new code. + """ + + def __init__(self, account_size: float): + self.account_size = float(account_size) + # Delegate to AdaptiveRiskManager for consistent limits + arm = AdaptiveRiskManager(account_size) + base = arm.get_trade_limits() + self.max_trades = base["max_daily_trades"] + self.trades_this_session = 0 + + def can_trade(self) -> bool: + return self.trades_this_session < self.max_trades + + def log_trade(self): + self.trades_this_session += 1 + + def reset_session(self): + self.trades_this_session = 0 + + def __repr__(self) -> str: + return ( + f"SessionManager(account=${self.account_size:.0f}, " + f"max={self.max_trades}, used={self.trades_this_session})" + ) + + +class AdaptiveRiskManager: + """Dynamic trade limits and lot sizing per the Headway Cent spec. + + Account tiers + ───────────── + ≤ $50 USD — small account: + • max_daily_trades : 2 + • pos_per_trade : 1 (single position per signal) + • total daily pos : 2 + + > $50 USD — growth account: + • max_daily_trades : 3 in Bull/Bear, 2 in Chop (market-dependent) + • pos_per_trade : 2 (dual/hedging positions per signal) + • total daily pos : 4 or 6 + """ + + CHOP_STATE = 2 + + def __init__(self, balance: float): + self.balance = max(float(balance), MIN_CAPITAL_USD) + self.daily_trades = 0 + tier = "small" if self.balance <= SMALL_ACCOUNT_THRESHOLD else "growth" + logger.debug("AdaptiveRiskManager: balance=$%.2f tier=%s", self.balance, tier) + + @property + def is_small_account(self) -> bool: + return self.balance <= SMALL_ACCOUNT_THRESHOLD + + def get_trade_limits(self, market_state: int = None) -> dict: + """Return trade limits for the current balance and optional HMM state. + + Args: + market_state: Current HMM state index. If None, uses the most + permissive limit for the account tier. + """ + if self.is_small_account: + return {"max_daily_trades": 2, "pos_per_trade": 1, "total_daily_pos": 2} + + # Growth account: market-state-dependent + in_chop = (market_state == self.CHOP_STATE) if market_state is not None else False + max_daily = 2 if in_chop else 3 + pos_per_trade = 2 + return { + "max_daily_trades": max_daily, + "pos_per_trade": pos_per_trade, + "total_daily_pos": max_daily * pos_per_trade, + } + + def calculate_lot_size(self, stop_loss_pips: float) -> float: + """1% risk rule with Headway Cent pip-value adjustment.""" + risk_per_trade = self.balance * 0.01 + if stop_loss_pips <= 0: + return 0.01 + # Cent account pip value: stop_loss_pips × 0.01 + lot_size = risk_per_trade / (stop_loss_pips * 0.01) + return max(0.01, round(lot_size, 2)) + + def can_trade(self, market_state: int = None) -> bool: + limits = self.get_trade_limits(market_state) + return self.daily_trades < limits["max_daily_trades"] + + def log_trade(self): + self.daily_trades += 1 + + def reset_daily(self): + self.daily_trades = 0 + + def __repr__(self) -> str: + tier = "small" if self.is_small_account else "growth" + return f"AdaptiveRiskManager(balance=${self.balance:.0f}, tier={tier}, daily_trades={self.daily_trades})" diff --git a/src/validator.py b/src/validator.py new file mode 100644 index 0000000..8cd88e4 --- /dev/null +++ b/src/validator.py @@ -0,0 +1,199 @@ +"""Model Validation Gatekeeper. + +Loads recently synced MT5 data, applies the full feature pipeline, runs live +inference through the saved HMM + XGBoost models, and computes an annualised +Sharpe Ratio over the recent window. The result is classified as: + + pass (Sharpe >= 0.8) — model is stable; proceed to live. + warn (Sharpe 0.5–0.8) — borderline; proceed with extreme caution. + fail (Sharpe < 0.5) — model drift detected; retune before going live. + +NOTE: This validation Sharpe is computed on the most recent N months of live +market data. It is NOT equivalent to the OOS Sharpe from training, which uses +an 80/20 chronological split of the full 10-year dataset. Both metrics matter, +but they answer different questions. +""" + +import numpy as np +import pandas as pd +from pathlib import Path + +from src.logger import setup_logger +from src.processor import ( + TF_CONFIG, + compute_log_returns, + kalman_smooth, + compute_volatility, + compute_rsi, + compute_atr, +) +from src.engine_hmm import load_model as load_hmm, predict_states +from src.engine_xgb import load_xgb, prepare_features, get_predictions +from src.backtester import vectorized_backtest + +logger = setup_logger(__name__) + +SYNC_DATA_PATH = Path("data/processed/mt5_sync_data.csv") +SHARPE_PASS_THRESHOLD = 0.8 +SHARPE_WARN_THRESHOLD = 0.5 +MIN_TRADES_WARNING = 30 # warn (not fail) if fewer trades than this + + +# ───────────────────────────────────────────────────────────────────────────── +# Internal helpers +# ───────────────────────────────────────────────────────────────────────────── + +def _apply_features( + df: pd.DataFrame, + tf: str, + obs_cov: float, + trans_cov: float, +) -> pd.DataFrame: + """Apply the same feature steps as process_pipeline() to an arbitrary frame. + + This function mirrors processor.process_pipeline() exactly. If any + standalone function in processor.py changes its behaviour, this must be + updated to match — both must produce identical column values for the same + input data. + """ + cfg = TF_CONFIG[tf.upper()] + obs_cov = obs_cov if obs_cov is not None else cfg["obs_cov_default"] + trans_cov = trans_cov if trans_cov is not None else cfg["trans_cov_default"] + + df = df.copy() + df["log_return"] = compute_log_returns(df["Close"]) + df["kalman_return"] = kalman_smooth(df["log_return"].values, obs_cov, trans_cov) + df["volatility"] = compute_volatility(df["log_return"]) + df["rsi"] = compute_rsi(df["Close"]) + df["rsi_slope"] = df["rsi"].diff() + df["atr_normalized"] = compute_atr(df) + df.dropna(inplace=True) + return df + + +# ───────────────────────────────────────────────────────────────────────────── +# Primary entry point +# ───────────────────────────────────────────────────────────────────────────── + +def run_validation( + sync_data_path: Path = SYNC_DATA_PATH, + tf: str = "H1", + broker: str = "headway_cent", + account_size: float = 15.0, + obs_cov: float = None, + trans_cov: float = None, +) -> dict: + """Validate the saved models against recent synced MT5 data. + + Returns a dict: + sharpe (float) — annualised Sharpe over the validation window + n_trades (int) — number of trades fired by the model + win_rate (float) — fraction of winning trades + max_dd (float) — maximum drawdown (0–1) + status (str) — 'pass', 'warn', or 'fail' + message (str) — human-readable gate decision + + Raises: + FileNotFoundError if the sync CSV or model files are missing. + """ + sync_data_path = Path(sync_data_path) + if not sync_data_path.exists(): + raise FileNotFoundError( + f"Sync data not found at {sync_data_path}. " + "Run python main.py --mode sync_validate first." + ) + + # Attempt to load Optuna best params; fall back to TF defaults + try: + from src.optimizer import get_best_params + params = get_best_params(balance=account_size, broker=broker, tf=tf) + obs_cov = params.get("obs_cov") or obs_cov + trans_cov = params.get("trans_cov") or trans_cov + except Exception: + params = {} + + # Load and featurise data + df = pd.read_csv(sync_data_path, index_col="Date", parse_dates=True) + logger.info("Loaded %d bars from %s for validation", len(df), sync_data_path) + df = _apply_features(df, tf, obs_cov, trans_cov) + + # Load models + try: + model_hmm = load_hmm() + except FileNotFoundError: + raise FileNotFoundError( + "HMM model not found at models/hmm_model.pkl. " + "Run python main.py --mode train first." + ) + try: + model_xgb, _ = load_xgb() + except FileNotFoundError: + raise FileNotFoundError( + "XGB model not found at models/xgb_model.pkl. " + "Run python main.py --mode train first." + ) + + # Inference + states = predict_states(model_hmm, df) + X, _, df_aligned = prepare_features(df, states) + states_aligned = states[df.index.isin(df_aligned.index)] + _, probabilities = get_predictions(model_xgb, X) + + # Backtest the full synced window — no IS/OOS split + result = vectorized_backtest( + df_aligned, probabilities, states_aligned, + split_idx=None, + account_size=account_size, + broker=broker, + tf=tf, + ) + + sharpe = result["sharpe_ratio"] + n_trades = result["n_trades"] + win_rate = result["win_rate"] + max_dd = result["max_drawdown"] + + if n_trades < MIN_TRADES_WARNING: + logger.warning( + "Only %d trades in the validation window — Sharpe estimate may be " + "unreliable. Consider using a longer --period (e.g. '6m').", + n_trades, + ) + + # Gate decision + if sharpe >= SHARPE_PASS_THRESHOLD: + status = "pass" + message = ( + f"Recent-Period Sharpe {sharpe:.3f} >= {SHARPE_PASS_THRESHOLD}. " + "Model is stable — safe to go live." + ) + elif sharpe >= SHARPE_WARN_THRESHOLD: + status = "warn" + message = ( + f"Recent-Period Sharpe {sharpe:.3f} is borderline " + f"({SHARPE_WARN_THRESHOLD}–{SHARPE_PASS_THRESHOLD}). " + "Proceed with reduced position size or wait for a clearer regime." + ) + else: + status = "fail" + message = ( + f"Recent-Period Sharpe {sharpe:.3f} < {SHARPE_WARN_THRESHOLD}. " + "Market drift detected — DO NOT go live. " + "Run --mode optimize then --mode train to retune the model." + ) + + logger.info( + "Validation [%s]: status=%s sharpe=%.3f trades=%d wr=%.1f%% dd=%.1f%%", + tf, status, sharpe, n_trades, win_rate * 100, max_dd * 100, + ) + if status in ("warn", "fail"): + logger.warning("VALIDATION %s: %s", status.upper(), message) + + return { + "sharpe": sharpe, + "n_trades": n_trades, + "win_rate": win_rate, + "max_dd": max_dd, + "status": status, + "message": message, + } diff --git a/src/visualizer.py b/src/visualizer.py new file mode 100644 index 0000000..3c032bc --- /dev/null +++ b/src/visualizer.py @@ -0,0 +1,344 @@ +import numpy as np +import pandas as pd +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +from pathlib import Path +from src.logger import setup_logger + +logger = setup_logger(__name__) + +REPORT_DIR = Path("reports") +REGIME_COLORS = {0: "#2ecc71", 1: "#e74c3c", 2: "#f39c12", 3: "#9b59b6"} +REGIME_LABELS = {0: "Bull", 1: "Bear", 2: "Chop", 3: "Chop_High"} + + +def _tf_dir(tf: str = "H1") -> Path: + """Return the TF-specific report subdirectory, creating it if needed.""" + d = REPORT_DIR / tf.upper() + d.mkdir(parents=True, exist_ok=True) + return d + + +def plot_regime_overlay(df, hmm_states, state_names, tf="H1", save_path=None): + """Price chart with HMM regime shading.""" + save_path = save_path or _tf_dir(tf) / "1_regime_overlay.png" + + fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(18, 10), height_ratios=[3, 1], + sharex=True, gridspec_kw={"hspace": 0.05}) + + dates = df.index + close = df["Close"].values + + for state_id, label in state_names.items(): + mask = hmm_states == state_id + color = REGIME_COLORS.get(state_id, "#95a5a6") + ax1.fill_between(dates, close.min() * 0.98, close.max() * 1.02, + where=mask, alpha=0.15, color=color, label=label) + + ax1.plot(dates, close, color="#2c3e50", linewidth=0.5, alpha=0.9) + ax1.set_ylabel("XAUUSD Price (H1)", fontsize=11) + ax1.set_title("Gold Regime X \u2014 HMM Regime Detection Over Price", fontsize=14, fontweight="bold") + ax1.legend(loc="upper left", fontsize=10) + ax1.grid(True, alpha=0.3) + ax1.set_xlim(dates[0], dates[-1]) + + ax2.scatter(dates, hmm_states, c=[REGIME_COLORS.get(s, "#95a5a6") for s in hmm_states], + s=1, alpha=0.6) + ax2.set_ylabel("HMM State", fontsize=11) + ax2.set_yticks(list(state_names.keys())) + ax2.set_yticklabels(list(state_names.values())) + ax2.grid(True, alpha=0.3) + ax2.xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m")) + ax2.xaxis.set_major_locator(mdates.MonthLocator(interval=6)) + plt.xticks(rotation=45) + + fig.savefig(save_path, dpi=150, bbox_inches="tight") + plt.close(fig) + logger.info("Saved regime overlay chart to %s", save_path) + return str(save_path) + + +def plot_equity_curve(df, probabilities, hmm_states, split_idx=None, tf="H1", save_path=None): + """Equity curve, drawdown, and signal markers with train/test split.""" + save_path = save_path or _tf_dir(tf) / "2_equity_curve.png" + + from src.backtester import compute_signals, compute_position_sizes + + signals = compute_signals(probabilities, hmm_states) + sizes = compute_position_sizes(signals, df["atr_normalized"].values) + + log_returns = df["log_return"].values + next_returns = np.roll(log_returns, -1) + next_returns[-1] = 0.0 + strategy_returns = sizes * next_returns + + cumulative_strat = np.cumsum(strategy_returns) + cumulative_bh = np.cumsum(log_returns) + + running_max = np.maximum.accumulate(cumulative_strat) + drawdown = running_max - cumulative_strat + + dates = df.index + + fig, axes = plt.subplots(3, 1, figsize=(18, 12), height_ratios=[3, 1, 1], + sharex=True, gridspec_kw={"hspace": 0.08}) + + # Equity curves + ax1 = axes[0] + ax1.plot(dates, cumulative_strat, color="#2980b9", linewidth=1.2, label="Strategy (log return)") + ax1.plot(dates, cumulative_bh, color="#95a5a6", linewidth=0.8, alpha=0.7, label="Buy & Hold") + entry_mask = signals == 1 + ax1.scatter(dates[entry_mask], cumulative_strat[entry_mask], + c="#2ecc71", s=3, alpha=0.4, label=f"Long entries ({np.sum(entry_mask)})", zorder=3) + + # Train/test split line + if split_idx is not None and 0 < split_idx < len(dates): + split_date = dates[split_idx] + for ax in axes: + ax.axvline(x=split_date, color="#e74c3c", linewidth=1.5, linestyle="--", alpha=0.7) + ax1.axvspan(split_date, dates[-1], alpha=0.04, color="#e74c3c") + ax1.text(split_date, ax1.get_ylim()[1] * 0.95, " OOS \u2192", fontsize=10, + color="#e74c3c", fontweight="bold", va="top") + ax1.text(split_date, ax1.get_ylim()[1] * 0.95, "\u2190 Train ", fontsize=10, + color="#2980b9", fontweight="bold", va="top", ha="right") + + ax1.set_ylabel("Cumulative Log Return", fontsize=11) + ax1.set_title("Gold Regime X \u2014 Strategy Equity Curve vs Buy & Hold", fontsize=14, fontweight="bold") + ax1.legend(loc="upper left", fontsize=10) + ax1.grid(True, alpha=0.3) + ax1.axhline(y=0, color="black", linewidth=0.5, alpha=0.5) + + # Drawdown + ax2 = axes[1] + ax2.fill_between(dates, 0, -drawdown * 100, color="#e74c3c", alpha=0.5) + ax2.set_ylabel("Drawdown (%)", fontsize=11) + ax2.grid(True, alpha=0.3) + max_dd_idx = np.argmax(drawdown) + ax2.annotate(f"Max DD: {drawdown[max_dd_idx]*100:.1f}%", + xy=(dates[max_dd_idx], -drawdown[max_dd_idx] * 100), + fontsize=9, color="#c0392b", fontweight="bold") + + # XGB probability + ax3 = axes[2] + ax3.scatter(dates, probabilities, c=probabilities, cmap="RdYlGn", s=1, alpha=0.3, vmin=0.3, vmax=0.7) + ax3.axhline(y=0.65, color="#2ecc71", linewidth=1, linestyle="--", alpha=0.8, label="Long threshold (0.65)") + ax3.axhline(y=0.35, color="#e74c3c", linewidth=1, linestyle="--", alpha=0.8, label="Short threshold (0.35)") + ax3.axhline(y=0.5, color="#7f8c8d", linewidth=0.5, linestyle=":", alpha=0.5) + ax3.set_ylabel("XGB Probability", fontsize=11) + ax3.set_ylim(0.25, 0.75) + ax3.legend(loc="upper right", fontsize=9) + ax3.grid(True, alpha=0.3) + ax3.xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m")) + ax3.xaxis.set_major_locator(mdates.MonthLocator(interval=6)) + plt.xticks(rotation=45) + + fig.savefig(save_path, dpi=150, bbox_inches="tight") + plt.close(fig) + logger.info("Saved equity curve chart to %s", save_path) + return str(save_path) + + +def plot_feature_analysis(X, hmm_states, metrics, tf="H1", save_path=None): + """Feature importance, distributions per regime, and correlation.""" + save_path = save_path or _tf_dir(tf) / "3_feature_analysis.png" + + fig, axes = plt.subplots(2, 2, figsize=(16, 12)) + + ax = axes[0, 0] + importance = metrics.get("feature_importance", {}) + names = list(importance.keys()) + values = [float(v) for v in importance.values()] + colors = ["#2980b9", "#27ae60", "#e67e22", "#8e44ad"] + bars = ax.barh(names, values, color=colors[:len(names)]) + ax.set_xlabel("Importance (Gain)", fontsize=10) + ax.set_title("XGBoost Feature Importance", fontsize=12, fontweight="bold") + for bar, val in zip(bars, values): + ax.text(bar.get_width() + 0.005, bar.get_y() + bar.get_height() / 2, + f"{val:.3f}", va="center", fontsize=9) + ax.grid(True, alpha=0.3, axis="x") + + ax = axes[0, 1] + unique_states = sorted(np.unique(hmm_states)) + for s in unique_states: + mask = hmm_states == s + data = X.loc[mask, "rsi_slope"].dropna() + ax.hist(data, bins=80, alpha=0.5, label=REGIME_LABELS.get(s, str(s)), + color=REGIME_COLORS.get(s, "#95a5a6"), density=True) + ax.set_xlabel("RSI Slope", fontsize=10) + ax.set_ylabel("Density", fontsize=10) + ax.set_title("RSI Slope Distribution by Regime", fontsize=12, fontweight="bold") + ax.legend(fontsize=9) + ax.set_xlim(-10, 10) + ax.grid(True, alpha=0.3) + + ax = axes[1, 0] + for s in unique_states: + mask = hmm_states == s + data = X.loc[mask, "atr_normalized"].dropna() + ax.hist(data, bins=80, alpha=0.5, label=REGIME_LABELS.get(s, str(s)), + color=REGIME_COLORS.get(s, "#95a5a6"), density=True) + ax.set_xlabel("ATR Normalized", fontsize=10) + ax.set_ylabel("Density", fontsize=10) + ax.set_title("ATR (Normalized) Distribution by Regime", fontsize=12, fontweight="bold") + ax.legend(fontsize=9) + ax.grid(True, alpha=0.3) + + ax = axes[1, 1] + state_counts = pd.Series(hmm_states).value_counts().sort_index() + labels_pie = [REGIME_LABELS.get(s, str(s)) for s in state_counts.index] + colors_pie = [REGIME_COLORS.get(s, "#95a5a6") for s in state_counts.index] + wedges, texts, autotexts = ax.pie( + state_counts.values, labels=labels_pie, colors=colors_pie, + autopct="%1.1f%%", startangle=90, textprops={"fontsize": 10} + ) + ax.set_title("Regime Distribution", fontsize=12, fontweight="bold") + + plt.tight_layout() + fig.savefig(save_path, dpi=150, bbox_inches="tight") + plt.close(fig) + logger.info("Saved feature analysis chart to %s", save_path) + return str(save_path) + + +def plot_transition_matrix(model_hmm, state_names, tf="H1", save_path=None): + """HMM transition matrix heatmap.""" + save_path = save_path or _tf_dir(tf) / "4_transition_matrix.png" + + n = model_hmm.n_components + trans = model_hmm.transmat_ + + fig, ax = plt.subplots(figsize=(8, 6)) + im = ax.imshow(trans, cmap="YlOrRd", vmin=0, vmax=1, aspect="auto") + + labels = [state_names.get(i, str(i)) for i in range(n)] + ax.set_xticks(range(n)) + ax.set_xticklabels(labels, fontsize=11) + ax.set_yticks(range(n)) + ax.set_yticklabels(labels, fontsize=11) + ax.set_xlabel("To State", fontsize=12) + ax.set_ylabel("From State", fontsize=12) + ax.set_title("HMM Transition Matrix", fontsize=14, fontweight="bold") + + for i in range(n): + for j in range(n): + color = "white" if trans[i, j] > 0.5 else "black" + ax.text(j, i, f"{trans[i, j]:.3f}", ha="center", va="center", + fontsize=12, color=color, fontweight="bold") + + fig.colorbar(im, ax=ax, label="Transition Probability") + plt.tight_layout() + fig.savefig(save_path, dpi=150, bbox_inches="tight") + plt.close(fig) + logger.info("Saved transition matrix chart to %s", save_path) + return str(save_path) + + +def plot_summary_dashboard(result, params, tf="H1", save_path=None): + """Single-panel summary card with key metrics and params.""" + save_path = save_path or _tf_dir(tf) / "5_summary_dashboard.png" + + fig, ax = plt.subplots(figsize=(12, 8)) + ax.axis("off") + + # Title block + ax.text(0.50, 0.96, "GOLD REGIME X", fontsize=20, + fontweight="bold", transform=ax.transAxes, va="top", ha="center", color="#2c3e50") + ax.text(0.50, 0.90, f"Hybrid HMM + XGBoost | XAUUSD {tf.upper()}", fontsize=12, + transform=ax.transAxes, va="top", ha="center", color="#7f8c8d") + + # Divider line + ax.plot([0.05, 0.95], [0.86, 0.86], color="#bdc3c7", linewidth=1, transform=ax.transAxes) + + # Left column: Full-period metrics + metrics_data = [ + ["Sharpe Ratio", f"{result['sharpe_ratio']:.3f}"], + ["Max Drawdown", f"{result['max_drawdown']*100:.1f}%"], + ["Win Rate", f"{result['win_rate']*100:.1f}%"], + ["Total Return", f"{result['total_return']*100:.1f}%"], + ["Trade Count", f"{result['n_trades']}"], + ] + sharpe_color = "#2ecc71" if 1.0 <= result["sharpe_ratio"] <= 1.5 else "#e67e22" + dd_color = "#2ecc71" if 0.08 <= result["max_drawdown"] <= 0.15 else "#e67e22" + wr_color = "#2ecc71" if 0.50 <= result["win_rate"] <= 0.55 else "#e67e22" + metric_colors = [sharpe_color, dd_color, wr_color, "#3498db", "#3498db"] + + ax.text(0.05, 0.82, "Performance (Full Period)", fontsize=13, fontweight="bold", + transform=ax.transAxes, va="top", color="#2c3e50") + y = 0.74 + for i, (label, value) in enumerate(metrics_data): + ax.text(0.08, y, label, fontsize=12, transform=ax.transAxes, va="top", + fontfamily="monospace", color="#2c3e50") + ax.text(0.35, y, value, fontsize=12, transform=ax.transAxes, va="top", + fontfamily="monospace", fontweight="bold", color=metric_colors[i]) + y -= 0.085 + + # OOS metrics if available + if "oos_sharpe_ratio" in result: + y -= 0.02 + ax.text(0.05, y, "Out-of-Sample (Test 20%)", fontsize=13, fontweight="bold", + transform=ax.transAxes, va="top", color="#c0392b") + y -= 0.08 + oos_data = [ + ["Sharpe", f"{result['oos_sharpe_ratio']:.3f}"], + ["Max DD", f"{result['oos_max_drawdown']*100:.1f}%"], + ["Win Rate", f"{result['oos_win_rate']*100:.1f}%"], + ["Trades", f"{result['oos_n_trades']}"], + ] + for label, value in oos_data: + ax.text(0.08, y, label, fontsize=11, transform=ax.transAxes, va="top", + fontfamily="monospace", color="#7f8c8d") + ax.text(0.25, y, value, fontsize=11, transform=ax.transAxes, va="top", + fontfamily="monospace", fontweight="bold", color="#c0392b") + y -= 0.07 + + # Right column: Target ranges + Params + ax.text(0.55, 0.82, "Target Ranges", fontsize=13, fontweight="bold", + transform=ax.transAxes, va="top", color="#2c3e50") + targets = [ + "Sharpe: 1.0 - 1.5", + "Max DD: 8% - 15%", + "Win Rate: 50% - 55%", + ] + y_right = 0.74 + for t in targets: + ax.text(0.58, y_right, t, fontsize=11, transform=ax.transAxes, va="top", + fontfamily="monospace", color="#7f8c8d") + y_right -= 0.07 + + if params: + y_right -= 0.04 + ax.text(0.55, y_right, "Optimized Params", fontsize=13, fontweight="bold", + transform=ax.transAxes, va="top", color="#2c3e50") + y_right -= 0.08 + for k, v in params.items(): + val_str = f"{v:.4f}" if isinstance(v, float) else str(v) + ax.text(0.58, y_right, f"{k}: {val_str}", fontsize=9, transform=ax.transAxes, + va="top", fontfamily="monospace", color="#34495e") + y_right -= 0.055 + + # Border + fig.patch.set_facecolor("#fafafa") + rect = plt.Rectangle((0.02, 0.02), 0.96, 0.96, fill=False, edgecolor="#bdc3c7", + linewidth=2, transform=ax.transAxes) + ax.add_patch(rect) + + fig.savefig(save_path, dpi=150, bbox_inches="tight", facecolor="#fafafa") + plt.close(fig) + logger.info("Saved summary dashboard to %s", save_path) + return str(save_path) + + +def generate_full_report(df, hmm_states, state_names, model_hmm, + X, probabilities, metrics, result, params=None, + split_idx=None, tf="H1"): + """Generate all 5 charts into reports// and return list of file paths.""" + paths = [] + paths.append(plot_regime_overlay(df, hmm_states, state_names, tf=tf)) + paths.append(plot_equity_curve(df, probabilities, hmm_states, split_idx=split_idx, tf=tf)) + paths.append(plot_feature_analysis(X, hmm_states, metrics, tf=tf)) + paths.append(plot_transition_matrix(model_hmm, state_names, tf=tf)) + paths.append(plot_summary_dashboard(result, params or {}, tf=tf)) + logger.info("Full report [%s]: %d charts in %s", tf, len(paths), REPORT_DIR / tf.upper()) + return paths