mirror of
https://github.com/lucasmos/GoldRegime_X.git
synced 2026-11-02 06:26:15 +00:00
Initial commit: Gold Regime X hybrid HMM+XGBoost trading system
- Full pipeline: process → optimize → train → export → live - Optuna optimizer with OOS-only Sharpe scoring and prob_threshold in search space - MT5 live bridge with adaptive risk management and staged TPs - M5/M15/H1 timeframe support with per-TF Kalman, HMM, and XGBoost configs - Telegram remote control, guardian health monitor, and daily audit reports - MQL5 EA (v2.00) with ONNX inference and cent account support Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Luke Mosoti
@@ -0,0 +1,25 @@
|
||||
# Gold Regime X — Environment Variables
|
||||
# Copy this file to .env and fill in your credentials.
|
||||
# NEVER commit .env to git (it's in .gitignore).
|
||||
|
||||
# ── Telegram Bot ──────────────────────────────────────────────────────────────
|
||||
# 1. Open Telegram and search for @BotFather
|
||||
# 2. Send /newbot, follow the prompts, copy the token below
|
||||
TELEGRAM_BOT_TOKEN=8566397719:AAH7o9Lw7MQ3P6kEZDEYLjwwtTMOAvs0jNg
|
||||
|
||||
# 3. Send /start to your new bot, then visit:
|
||||
# https://api.telegram.org/bot<TOKEN>/getUpdates
|
||||
# Copy the "id" value from "chat" in the first result
|
||||
TELEGRAM_CHAT_ID=557964386
|
||||
|
||||
# ── Remote Control Security ───────────────────────────────────────────────────
|
||||
# Your personal Telegram numeric user ID (not username).
|
||||
# Only messages from this ID will be accepted by --mode listen.
|
||||
# Get it from: https://t.me/userinfobot or the getUpdates response above
|
||||
ALLOWED_USER_ID=557964386
|
||||
|
||||
# ── Live Trading Defaults for Remote Control ──────────────────────────────────
|
||||
# These values are used when you press "START TRADING" from Telegram.
|
||||
LIVE_TF=H1
|
||||
LIVE_BROKER=headway_cent
|
||||
LIVE_BALANCE=15
|
||||
@@ -0,0 +1 @@
|
||||
* !text !filter !merge !diff
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
# Secrets
|
||||
.env
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
dist/
|
||||
build/
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
# Data and models (large files)
|
||||
data/raw/
|
||||
data/processed/
|
||||
models/*.pkl
|
||||
models/*.joblib
|
||||
models/*.onnx
|
||||
models/study.db
|
||||
|
||||
# Reports
|
||||
reports/
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
|
||||
# Compiled MQL5 binary (source .mq5 is tracked)
|
||||
mql5/*.ex5
|
||||
|
||||
# IDE
|
||||
.vscode/settings.json
|
||||
.idea/
|
||||
+223
@@ -0,0 +1,223 @@
|
||||
# Gold Regime X — Live Trading Guide
|
||||
|
||||
This guide explains how to operate the Python MT5 Live Bridge for Gold Regime X.
|
||||
The bridge connects directly to your running MetaTrader5 terminal, validates the
|
||||
model against recent market data, and executes orders automatically on XAUUSD.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
| Requirement | Notes |
|
||||
|---|---|
|
||||
| Windows OS | MetaTrader5 Python package is Windows-only |
|
||||
| MT5 terminal running | Must be open and logged into your Headway account |
|
||||
| XAUUSD in Market Watch | Right-click Market Watch → Show All, or search XAUUSD |
|
||||
| Algorithmic trading enabled | MT5 → Tools → Options → Expert Advisors → Allow Algorithmic Trading |
|
||||
| Models trained | `models/hmm_model.pkl` and `models/xgb_model.pkl` must exist |
|
||||
| Python package | `pip install MetaTrader5>=5.0.45` |
|
||||
| **EA removed from chart** | The GoldRegimeX.mq5 EA and the Python bridge share `MAGIC_NUMBER = 123456`. Running both simultaneously will double-count daily trades. Remove the EA from the XAUUSD chart before starting the Python bridge. |
|
||||
|
||||
---
|
||||
|
||||
## MT5 Terminal Setup
|
||||
|
||||
1. Open MetaTrader5.
|
||||
2. Go to **Tools → Options → Expert Advisors**.
|
||||
3. Check **Allow Algorithmic Trading**.
|
||||
4. Check **Allow WebRequest for listed URL** and add `http://localhost` if using a local bridge.
|
||||
5. Log into your Headway account (Cent or Live).
|
||||
6. Add XAUUSD to the Market Watch if it is not already visible.
|
||||
|
||||
---
|
||||
|
||||
## Quick-Start Workflow
|
||||
|
||||
### Step 1 — Train the models (if not done already)
|
||||
|
||||
```
|
||||
python main.py --mode process --tf H1
|
||||
python main.py --mode optimize --trials 250 --broker headway_cent --balance 15 --tf H1
|
||||
python main.py --mode train --broker headway_cent --balance 15 --tf H1
|
||||
```
|
||||
|
||||
### Step 2 — Sync recent data and validate the model
|
||||
|
||||
Run this daily before going live to ensure the model is still healthy.
|
||||
|
||||
```
|
||||
python main.py --mode sync_validate --period 3m --broker headway_cent --balance 15 --tf H1
|
||||
```
|
||||
|
||||
| Validation Status | Meaning | Action |
|
||||
|---|---|---|
|
||||
| **PASS** (Sharpe ≥ 0.8) | Model is stable | Proceed to Step 3 |
|
||||
| **WARN** (Sharpe 0.5–0.8) | Borderline performance | Proceed with caution or re-optimise |
|
||||
| **FAIL** (Sharpe < 0.5) | Market regime drift | Run optimize + train before going live |
|
||||
|
||||
A FAIL status will exit with code 1 and block the script from continuing.
|
||||
|
||||
### Step 3 — Paper trade on demo (recommended first)
|
||||
|
||||
```
|
||||
python main.py --mode live --account demo --broker headway_cent --balance 15 --tf H1
|
||||
```
|
||||
|
||||
Demo mode logs every signal and simulates the full session-limit logic, but
|
||||
sends no real orders to MT5. Use this to verify signals appear as expected.
|
||||
|
||||
### Step 4 — Switch to live account
|
||||
|
||||
```
|
||||
python main.py --mode live --account live --broker headway_cent --balance 15 --tf H1
|
||||
```
|
||||
|
||||
You will be prompted to type `YES` to confirm before the loop starts.
|
||||
|
||||
---
|
||||
|
||||
## Command Reference
|
||||
|
||||
```
|
||||
python main.py --mode sync_validate [OPTIONS]
|
||||
--period 3m Lookback window (e.g. 3m, 6m, 12m)
|
||||
--tf H1 Timeframe (H1 or M15)
|
||||
--broker headway_cent
|
||||
--balance 15
|
||||
|
||||
python main.py --mode live [OPTIONS]
|
||||
--account demo|live demo = dry-run (no orders) live = real orders
|
||||
--tf H1 Must match the timeframe the models were trained on
|
||||
--broker headway_cent
|
||||
--balance 15 USD balance used for lot-sizing
|
||||
--period (not used in live mode)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## How to Monitor Live Trading
|
||||
|
||||
### Terminal View (MT5)
|
||||
|
||||
Open the **Trade** tab at the bottom of the MT5 terminal to see all open
|
||||
positions. Each position placed by the Python bridge will show:
|
||||
|
||||
- **Symbol**: XAUUSD
|
||||
- **Comment**: `GRX_BUY_p1of1_s0_p0.73` (direction / position index / HMM state / probability)
|
||||
- **Magic**: 123456
|
||||
|
||||
### Expert Logs
|
||||
|
||||
Go to the **Journal** or **Experts** tab in MT5. Errors from API calls
|
||||
(e.g. "Invalid S/L", "Not enough money") appear in red here.
|
||||
|
||||
### Python Log File
|
||||
|
||||
All signals, orders, and errors are written to `logs/goldregimex.log`.
|
||||
|
||||
```
|
||||
# Key log lines to watch for:
|
||||
SIGNAL BUY — signal fired, orders being placed
|
||||
Order filled — order confirmed by broker
|
||||
Session limit — daily cap enforced (no more trades today)
|
||||
Order failed — execution error (check MT5 Journal tab)
|
||||
Insufficient margin — reduce lot size or top up account
|
||||
High-vol deviation — elevated deviation used due to regime instability
|
||||
```
|
||||
|
||||
To follow the log in real time:
|
||||
|
||||
```
|
||||
# PowerShell
|
||||
Get-Content logs/goldregimex.log -Wait -Tail 30
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Risk Management Reference
|
||||
|
||||
| Account Balance | Max Trades/Day | Positions per Signal | Total Positions |
|
||||
|---|---|---|---|
|
||||
| ≤ $50 USD | 2 | 1 | 2 maximum |
|
||||
| > $50 USD (Bull/Bear) | 3 | 2 | 6 maximum |
|
||||
| > $50 USD (Chop) | 2 | 2 | 4 maximum |
|
||||
|
||||
**All trades use the 1% risk rule:**
|
||||
|
||||
```
|
||||
lot_size = (1% of USD balance) / (ATR(14) × 2.0)
|
||||
```
|
||||
|
||||
The stop-loss is placed at `entry ± ATR(14) × 2.0`. No take-profit is set —
|
||||
positions are managed by the next opposing signal or manual intervention.
|
||||
|
||||
**Cent Account Note**: On Headway Cent, the terminal displays $15 USD as
|
||||
`1500.00` in the balance field. The Python bridge detects `--broker headway_cent`
|
||||
and automatically divides by 100 for all internal calculations. If you see a
|
||||
balance of `1500.00` in MT5 and pass `--balance 15`, both will agree.
|
||||
|
||||
---
|
||||
|
||||
## Order Specifications
|
||||
|
||||
| Parameter | Value | Notes |
|
||||
|---|---|---|
|
||||
| Filling type | IOC (Immediate or Cancel) | Standard for Headway ECN |
|
||||
| Deviation (normal) | 20 points | $0.20 on XAUUSD |
|
||||
| Deviation (high-vol) | 50 points | When HMM self-transition prob < 0.70 |
|
||||
| Magic number | 123456 | Matches GoldRegimeX.mq5 |
|
||||
| SL | ATR × 2.0 below/above entry | Hard stop on every trade |
|
||||
| TP | None | No fixed take-profit |
|
||||
|
||||
IOC means: if the broker cannot fill the order within the deviation window,
|
||||
the order is cancelled entirely (never partially filled). This prevents
|
||||
unexpected partial exposure during gold news spikes.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Problem | Likely Cause | Fix |
|
||||
|---|---|---|
|
||||
| `ConnectionError: Could not connect to MT5` | Terminal not running | Open MT5 and log in first |
|
||||
| `FileNotFoundError: models/hmm_model.pkl` | Models not trained | Run `--mode train` |
|
||||
| Validation status FAIL every day | Model too old for current market | Run `--mode optimize` then `--mode train` |
|
||||
| `Order failed: retcode=10006` | No connection to broker | Check MT5 connection indicator (bottom-right) |
|
||||
| `Order failed: retcode=10015` | Price changed faster than deviation allows | Will retry on next bar; consider longer TF |
|
||||
| `Insufficient margin` repeated | Account below minimum for lot size | Top up account or reduce `--balance` value |
|
||||
| Signals appear but no trades fired | `dry_run=True` (demo mode) | Switch to `--account live` after confirming signals look correct |
|
||||
| Double positions appearing | EA still on chart | Remove GoldRegimeX.mq5 EA from XAUUSD chart |
|
||||
|
||||
---
|
||||
|
||||
## Emergency Stop
|
||||
|
||||
Press **Ctrl+C** in the terminal window. The loop handles `KeyboardInterrupt`
|
||||
gracefully: it logs the shutdown message and calls `mt5.shutdown()` to cleanly
|
||||
disconnect the Python package.
|
||||
|
||||
All open positions remain open in MT5 after the script exits. You must close
|
||||
them manually from the Trade tab or by placing an opposing order.
|
||||
|
||||
---
|
||||
|
||||
## Complete Workflow Example (Headway Cent, $15 balance, H1)
|
||||
|
||||
```bash
|
||||
# 1. First time setup
|
||||
python main.py --mode process --tf H1
|
||||
python main.py --mode optimize --trials 250 --broker headway_cent --balance 15 --tf H1
|
||||
python main.py --mode train --broker headway_cent --balance 15 --tf H1
|
||||
|
||||
# 2. Daily routine (before market open)
|
||||
python main.py --mode sync_validate --period 3m --broker headway_cent --balance 15 --tf H1
|
||||
# → PASS: continue
|
||||
# → WARN: continue with reduced size or check market conditions
|
||||
# → FAIL: run optimize + train again before going live
|
||||
|
||||
# 3. Paper trade for several days to verify signals
|
||||
python main.py --mode live --account demo --broker headway_cent --balance 15 --tf H1
|
||||
|
||||
# 4. Go live when satisfied
|
||||
python main.py --mode live --account live --broker headway_cent --balance 15 --tf H1
|
||||
```
|
||||
@@ -0,0 +1,134 @@
|
||||
# Gold Regime X — Remote Control & Telegram Setup
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Install the required packages:
|
||||
|
||||
```bash
|
||||
pip install requests python-dotenv psutil schedule
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 1 — Create a Telegram Bot
|
||||
|
||||
1. Open Telegram → search **@BotFather** → send `/newbot`
|
||||
2. Follow the prompts and copy the **Bot Token** (looks like `123456789:AAF-abc...`)
|
||||
3. Send **`/start`** to your new bot in Telegram
|
||||
4. Find your **Chat ID** by opening this URL in a browser:
|
||||
```
|
||||
https://api.telegram.org/bot<YOUR_TOKEN>/getUpdates
|
||||
```
|
||||
Look for `"chat": {"id": 123456789}` — that number is your Chat ID.
|
||||
5. Find your **User ID** (same as Chat ID for private chats, or use [@userinfobot](https://t.me/userinfobot))
|
||||
|
||||
---
|
||||
|
||||
## Step 2 — Configure .env
|
||||
|
||||
```bash
|
||||
copy .env.example .env
|
||||
```
|
||||
|
||||
Fill in the three required values:
|
||||
|
||||
```
|
||||
TELEGRAM_BOT_TOKEN=123456789:AAF-abc...
|
||||
TELEGRAM_CHAT_ID=123456789
|
||||
ALLOWED_USER_ID=123456789
|
||||
```
|
||||
|
||||
Optionally set the live trading defaults (used by the START TRADING button):
|
||||
|
||||
```
|
||||
LIVE_TF=H1
|
||||
LIVE_BROKER=headway_cent
|
||||
LIVE_BALANCE=15
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 3 — Start the Listener
|
||||
|
||||
On your laptop/PC (while MT5 is running):
|
||||
|
||||
```bash
|
||||
python main.py --mode listen --broker headway_cent --balance 15
|
||||
```
|
||||
|
||||
You will see a confirmation message in your Telegram chat and a keyboard appears.
|
||||
|
||||
---
|
||||
|
||||
## Telegram Keyboard Buttons
|
||||
|
||||
| Button | Action |
|
||||
|--------|--------|
|
||||
| 🚀 START TRADING | Launches the live execution loop (uses `.env` TF/broker/balance) |
|
||||
| 🛑 STOP TRADING | Terminates the running live loop process |
|
||||
| 📉 START OPTIMIZE (M5) | Starts M5 Optuna study (resumes from study.db if interrupted) |
|
||||
| 📊 BOT STATUS | Sends last 24h trade history with P&L, win rate, and floating positions |
|
||||
|
||||
---
|
||||
|
||||
## Additional Modes
|
||||
|
||||
### Guardian — Continuous health monitor
|
||||
```bash
|
||||
python main.py --mode guardian --tf M5,M15,H1 --period 3m --interval 3600 --broker headway_cent --balance 15
|
||||
```
|
||||
Checks rolling Sharpe for each TF every hour. Sends Telegram alert if Sharpe drops below 0.6.
|
||||
|
||||
### Audit — On-demand performance report
|
||||
```bash
|
||||
python main.py --mode audit --broker headway_cent --balance 15
|
||||
```
|
||||
Prints and sends the last 24h deal summary to Telegram.
|
||||
|
||||
---
|
||||
|
||||
## Nightly Auto-Report
|
||||
|
||||
When `--mode listen` is running, a background thread automatically sends the
|
||||
daily performance summary to Telegram at **23:55 UTC** every night.
|
||||
|
||||
No extra configuration needed — just keep the listener running.
|
||||
|
||||
---
|
||||
|
||||
## Security Notes
|
||||
|
||||
- The `ALLOWED_USER_ID` env var is the single security gate — only your Telegram
|
||||
ID can issue commands. Keep your `.env` file private.
|
||||
- Never commit `.env` to git. It is listed in `.gitignore`.
|
||||
- The listener uses Telegram's long-polling API (no public webhook needed).
|
||||
- All commands are launched as separate subprocesses so the bot stays responsive.
|
||||
|
||||
---
|
||||
|
||||
## Optimization Resume (Failsafe)
|
||||
|
||||
The optimizer saves every completed trial to `models/study.db`. If interrupted:
|
||||
|
||||
1. Press **Ctrl+C** in your terminal at any time — progress is saved
|
||||
2. Re-run the same optimize command:
|
||||
```bash
|
||||
python main.py --mode optimize --tf M5 --trials 500
|
||||
```
|
||||
3. You will see: `Failsafe: XX trials found in study.db. Resuming from trial #XX...`
|
||||
|
||||
To start a completely fresh study, delete `models/study.db` manually.
|
||||
|
||||
### Parallel Optimization (Advanced)
|
||||
|
||||
For true multi-process parallelism, open **multiple terminals** and run the same
|
||||
command in each — they all share `study.db` safely via SQLite locking:
|
||||
|
||||
```
|
||||
Terminal 1: python main.py --mode optimize --tf M5 --trials 500
|
||||
Terminal 2: python main.py --mode optimize --tf M5 --trials 500
|
||||
Terminal 3: python main.py --mode optimize --tf M5 --trials 500
|
||||
```
|
||||
|
||||
Each terminal runs an independent worker. Optuna coordinates via SQLite.
|
||||
This is more reliable than `--n_jobs` for CPU-bound objectives like HMM+XGBoost.
|
||||
@@ -0,0 +1,534 @@
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
# Load .env for Telegram credentials (silently ignored if python-dotenv not installed)
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from src.logger import setup_logger
|
||||
from src.processor import process_pipeline, TF_CONFIG, PROCESSED_PATH
|
||||
from src.engine_hmm import fit_hmm, save_model as save_hmm, load_model as load_hmm
|
||||
from src.engine_xgb import (
|
||||
prepare_features, train_xgb, get_predictions,
|
||||
export_onnx, save_xgb, load_xgb, ONNX_PATH, FEATURE_COLS,
|
||||
)
|
||||
from src.optimizer import run_optimization, get_best_params
|
||||
from src.backtester import vectorized_backtest
|
||||
from src.visualizer import generate_full_report
|
||||
from src.risk_manager import AdaptiveRiskManager
|
||||
|
||||
logger = setup_logger("main")
|
||||
|
||||
|
||||
def _resolve_balance(args) -> float:
|
||||
"""--balance takes precedence over --min_cap."""
|
||||
return args.balance if args.balance is not None else args.min_cap
|
||||
|
||||
|
||||
_M5_META_PATH = Path("models/m5_meta.json")
|
||||
_M5_EXPIRY_HOURS = 120 # 5 days
|
||||
|
||||
|
||||
def _check_m5_readiness(tf: str) -> bool:
|
||||
"""Return True if the M5 model is fresh enough for live trading.
|
||||
|
||||
The M5 timeframe is sensitive to microstructure changes, so the model
|
||||
must have been optimised within the last 5 days. A meta.json timestamp
|
||||
is written by cmd_optimize after each successful M5 study.
|
||||
"""
|
||||
if tf.upper() != "M5":
|
||||
return True
|
||||
if not _M5_META_PATH.exists():
|
||||
print(
|
||||
"\nERROR: M5 model meta-data not found.\n"
|
||||
"Run python main.py --mode optimize --tf M5 before live trading."
|
||||
)
|
||||
return False
|
||||
meta = json.loads(_M5_META_PATH.read_text())
|
||||
age_h = (time.time() - meta.get("timestamp", 0)) / 3600
|
||||
if age_h > _M5_EXPIRY_HOURS:
|
||||
print(
|
||||
f"\nWARNING: M5 model is {age_h:.0f} hours old (limit: {_M5_EXPIRY_HOURS}h / 5 days).\n"
|
||||
"REQUIRED ACTION: Run python main.py --mode optimize --tf M5 to refresh before live.\n"
|
||||
"\n Pre-flight checklist:\n"
|
||||
" 1. python main.py --mode sync_validate --period 3m --tf M5\n"
|
||||
" 2. python main.py --mode optimize --tf M5 --trials 500 --balance 15\n"
|
||||
" 3. python main.py --mode train --tf M5\n"
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _train_for_tf(tf: str, balance: float, broker: str, params: dict):
|
||||
"""Shared train logic for a single timeframe. Returns (result, model_hmm, states, X, metrics)."""
|
||||
df = process_pipeline(
|
||||
obs_cov=params.get("obs_cov"),
|
||||
trans_cov=params.get("trans_cov"),
|
||||
save=True,
|
||||
tf=tf,
|
||||
)
|
||||
model_hmm, states, state_map = fit_hmm(
|
||||
df, n_states=params.get("n_states", TF_CONFIG[tf].get("n_states_default", 3))
|
||||
)
|
||||
X, y, df_aligned = prepare_features(df, states)
|
||||
model_xgb, metrics = train_xgb(
|
||||
X, y,
|
||||
max_depth=params.get("max_depth", 4),
|
||||
learning_rate=params.get("learning_rate", 0.1),
|
||||
n_estimators=params.get("n_estimators", 200),
|
||||
subsample=params.get("subsample", 0.8),
|
||||
colsample_bytree=params.get("colsample_bytree", 0.8),
|
||||
min_child_weight=params.get("min_child_weight", 5),
|
||||
gamma=params.get("gamma", 1.0),
|
||||
reg_alpha=params.get("reg_alpha", 0.1),
|
||||
)
|
||||
_, probabilities = get_predictions(model_xgb, X)
|
||||
states_aligned = states[df.index.isin(df_aligned.index)]
|
||||
split_idx = metrics.get("split_idx")
|
||||
result = vectorized_backtest(
|
||||
df_aligned, probabilities, states_aligned,
|
||||
split_idx=split_idx,
|
||||
account_size=balance,
|
||||
broker=broker,
|
||||
tf=tf,
|
||||
)
|
||||
return result, model_hmm, state_map, model_xgb, metrics, df_aligned, states_aligned, X, probabilities
|
||||
|
||||
|
||||
def cmd_process(args):
|
||||
tfs = [t.strip().upper() for t in args.tf.split(",")]
|
||||
for tf in tfs:
|
||||
if tf not in TF_CONFIG:
|
||||
logger.error("Unknown timeframe '%s'. Valid: %s", tf, list(TF_CONFIG))
|
||||
continue
|
||||
try:
|
||||
df = process_pipeline(save=True, tf=tf)
|
||||
logger.info(
|
||||
"[%s] Done: %d bars | Range: %s to %s",
|
||||
tf, len(df), df.index.min(), df.index.max(),
|
||||
)
|
||||
except FileNotFoundError as e:
|
||||
logger.error(str(e))
|
||||
|
||||
|
||||
def cmd_optimize(args):
|
||||
balance = _resolve_balance(args)
|
||||
broker = args.broker
|
||||
tfs = [t.strip().upper() for t in args.tf.split(",")]
|
||||
for tf in tfs:
|
||||
logger.info("Optimizing [%s] broker=%s balance=$%.0f trials=%d", tf, broker, balance, args.trials)
|
||||
study = run_optimization(n_trials=args.trials, balance=balance, broker=broker, tf=tf, n_jobs=args.n_jobs)
|
||||
arm = AdaptiveRiskManager(balance)
|
||||
limits = arm.get_trade_limits()
|
||||
print(f"\n=== Best Result [{tf}] ===")
|
||||
print(f"Score: {study.best_value:.3f}")
|
||||
print(f"Broker: {broker}")
|
||||
print(f"Balance: ${balance:.0f} USD ({arm}")
|
||||
print(f"Session Limit: {limits['max_daily_trades']} trade(s)/day | {limits['pos_per_trade']} pos/trade")
|
||||
print("Best Params:")
|
||||
for k, v in study.best_params.items():
|
||||
print(f" {k}: {v}")
|
||||
if tf == "M5":
|
||||
_M5_META_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
_M5_META_PATH.write_text(json.dumps({
|
||||
"timestamp": time.time(),
|
||||
"tf": "M5",
|
||||
"best_score": study.best_value,
|
||||
}))
|
||||
print(
|
||||
"\nOptimization Complete. HMM States and XGBoost Weights have been updated. "
|
||||
"You are now cleared for M5 Live Trading for the next 5 days."
|
||||
)
|
||||
|
||||
|
||||
def cmd_train(args):
|
||||
balance = _resolve_balance(args)
|
||||
broker = args.broker
|
||||
tf = args.tf.upper()
|
||||
|
||||
try:
|
||||
params = get_best_params(balance=balance, broker=broker, tf=tf)
|
||||
logger.info("Using Optuna best params [%s/%s]: %s", tf, broker, params)
|
||||
except Exception:
|
||||
logger.warning("No Optuna study found for tf=%s broker=%s — using defaults", tf, broker)
|
||||
params = {}
|
||||
|
||||
result, model_hmm, state_map, model_xgb, metrics, *_ = _train_for_tf(tf, balance, broker, params)
|
||||
save_hmm(model_hmm)
|
||||
save_xgb(model_xgb, metrics)
|
||||
|
||||
arm = AdaptiveRiskManager(balance)
|
||||
limits = arm.get_trade_limits()
|
||||
print(f"\n=== Training Results [{tf}] ===")
|
||||
print(f"Broker: {broker} | Balance: ${balance:.0f} | Tier: {'small' if arm.is_small_account else 'growth'}")
|
||||
print(f"Session: {limits['max_daily_trades']}/day | Positions: {limits['pos_per_trade']}/trade")
|
||||
print(f"Sharpe: {result['sharpe_ratio']:.3f} | MaxDD: {result['max_drawdown']*100:.1f}% "
|
||||
f"| WR: {result['win_rate']*100:.1f}% | Trades: {result['n_trades']}")
|
||||
if "oos_sharpe_ratio" in result:
|
||||
print(f"\n--- In-Sample ---")
|
||||
print(f" Sharpe: {result['is_sharpe_ratio']:.3f} | DD: {result['is_max_drawdown']*100:.1f}% "
|
||||
f"| WR: {result['is_win_rate']*100:.1f}% | Trades: {result['is_n_trades']}")
|
||||
print(f"--- Out-of-Sample ---")
|
||||
print(f" Sharpe: {result['oos_sharpe_ratio']:.3f} | DD: {result['oos_max_drawdown']*100:.1f}% "
|
||||
f"| WR: {result['oos_win_rate']*100:.1f}% | Trades: {result['oos_n_trades']}")
|
||||
print(f"\nModels saved. Run --mode export to generate ONNX.")
|
||||
|
||||
|
||||
def cmd_compare(args):
|
||||
"""Train and backtest the specified timeframes, then show a side-by-side comparison."""
|
||||
balance = _resolve_balance(args)
|
||||
broker = args.broker
|
||||
tfs = [t.strip().upper() for t in args.tf.split(",")]
|
||||
results = {}
|
||||
|
||||
for tf in tfs:
|
||||
try:
|
||||
params = get_best_params(balance=balance, broker=broker, tf=tf)
|
||||
except Exception:
|
||||
params = {}
|
||||
|
||||
try:
|
||||
result, *_ = _train_for_tf(tf, balance, broker, params)
|
||||
results[tf] = result
|
||||
except FileNotFoundError as e:
|
||||
logger.error("Skipping %s: %s", tf, e)
|
||||
continue
|
||||
|
||||
if len(results) < 2:
|
||||
avail = ", ".join(tfs)
|
||||
print(f"\nOnly {len(results)} timeframe(s) available. Run --mode process --tf {avail} first.")
|
||||
if results:
|
||||
tf, r = next(iter(results.items()))
|
||||
print(f"\n[{tf}] Sharpe={r['sharpe_ratio']:.3f} | OOS={r.get('oos_sharpe_ratio',0):.3f} "
|
||||
f"| DD={r['max_drawdown']*100:.1f}% | Trades={r['n_trades']}")
|
||||
return
|
||||
|
||||
def _oos_sharpe(r):
|
||||
return r.get("oos_sharpe_ratio", r.get("sharpe_ratio", 0.0))
|
||||
|
||||
winner = max(results, key=lambda k: _oos_sharpe(results[k]))
|
||||
col_w = 12
|
||||
tf_list = list(results.keys())
|
||||
header = f"{'Metric':<20}" + "".join(f" {tf:>{col_w}}" for tf in tf_list)
|
||||
|
||||
print(f"\n{'='*50}")
|
||||
print(f" TIMEFRAME COMPARISON | Balance: ${balance:.0f} | Broker: {broker}")
|
||||
print(f"{'='*50}")
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
metrics_to_show = [
|
||||
("OOS Sharpe", "oos_sharpe_ratio", ".3f"),
|
||||
("IS Sharpe", "is_sharpe_ratio", ".3f"),
|
||||
("OOS Max DD", "oos_max_drawdown", ".1%"),
|
||||
("OOS Win Rate", "oos_win_rate", ".1%"),
|
||||
("OOS Trades", "oos_n_trades", "d"),
|
||||
("Full Sharpe", "sharpe_ratio", ".3f"),
|
||||
]
|
||||
for label, key, fmt in metrics_to_show:
|
||||
row = f"{label:<20}"
|
||||
for tf in tf_list:
|
||||
r = results[tf]
|
||||
val = r.get(key, r.get(key.replace("oos_", "").replace("is_", ""), 0))
|
||||
s = f"{val:{fmt}}" if fmt != "d" else str(int(val))
|
||||
row += f" {s:>{col_w}}"
|
||||
print(row)
|
||||
|
||||
for tf, r in results.items():
|
||||
logger.info(
|
||||
" %s Sharpe=%.3f (OOS=%.3f) | DD=%.1f%% | WR=%.1f%% | Trades=%d",
|
||||
tf, r["sharpe_ratio"], r.get("oos_sharpe_ratio", r["sharpe_ratio"]),
|
||||
r["max_drawdown"] * 100, r["win_rate"] * 100, r["n_trades"],
|
||||
)
|
||||
|
||||
print(f"\n-> Recommended timeframe: {winner} (higher OOS Sharpe)")
|
||||
print(f" Run: python main.py --mode train --tf {winner} --broker {broker} --balance {balance:.0f}")
|
||||
|
||||
|
||||
def cmd_export(args):
|
||||
try:
|
||||
model_xgb, _ = load_xgb()
|
||||
except FileNotFoundError:
|
||||
logger.error("No trained XGB model found. Run --mode train first.")
|
||||
sys.exit(1)
|
||||
export_onnx(model_xgb)
|
||||
print(f"\nONNX model exported to {ONNX_PATH}")
|
||||
print("Copy this file to your MT5 MQL5/Files/ directory.")
|
||||
|
||||
|
||||
def cmd_sync_validate(args):
|
||||
"""Download recent MT5 bars then run the model validation gatekeeper."""
|
||||
from src.mt5_sync import sync_mt5_data
|
||||
from src.validator import run_validation
|
||||
|
||||
balance = _resolve_balance(args)
|
||||
tf = args.tf.upper()
|
||||
|
||||
logger.info("Syncing MT5 data [%s] period=%s ...", tf, args.period)
|
||||
try:
|
||||
df = sync_mt5_data(tf=tf, period=args.period)
|
||||
logger.info(
|
||||
"Sync complete: %d bars %s -> %s",
|
||||
len(df), df.index.min(), df.index.max(),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("MT5 sync failed: %s", exc)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
result = run_validation(tf=tf, broker=args.broker, account_size=balance)
|
||||
except Exception as exc:
|
||||
logger.error("Validation error: %s", exc)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\n=== Validation Result [{tf}] ===")
|
||||
print(f" Recent Sharpe: {result['sharpe']:.3f}")
|
||||
print(f" Trades: {result['n_trades']}")
|
||||
print(f" Win Rate: {result['win_rate']*100:.1f}%")
|
||||
print(f" Max Drawdown: {result['max_dd']*100:.1f}%")
|
||||
print(f" Status: {result['status'].upper()}")
|
||||
print(f" {result['message']}")
|
||||
|
||||
if result["status"] == "fail":
|
||||
print(
|
||||
"\nABORTING: Validation failed. "
|
||||
"Retune with --mode optimize then --mode train before going live."
|
||||
)
|
||||
sys.exit(1)
|
||||
if result["status"] == "warn":
|
||||
print("\nWARNING: Borderline performance — consider re-optimising before live trading.")
|
||||
|
||||
|
||||
def cmd_live(args):
|
||||
"""Connect to MT5 and start the live signal execution loop."""
|
||||
from src.mt5_trader import run_live_loop
|
||||
|
||||
balance = _resolve_balance(args)
|
||||
tf = args.tf.upper()
|
||||
dry_run = (args.account == "demo")
|
||||
|
||||
if not _check_m5_readiness(tf):
|
||||
sys.exit(1)
|
||||
|
||||
if args.account == "live":
|
||||
print("\n" + "=" * 60)
|
||||
print(" WARNING: LIVE ACCOUNT — real money is at risk.")
|
||||
print(" Ensure --mode sync_validate passed before continuing.")
|
||||
print(" Ensure the GoldRegimeX EA is removed from the XAUUSD chart.")
|
||||
confirm = input(" Type YES to confirm live trading: ").strip()
|
||||
if confirm != "YES":
|
||||
print("Aborted.")
|
||||
sys.exit(0)
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
logger.info(
|
||||
"Starting live loop — TF=%s broker=%s balance=$%.0f dry_run=%s",
|
||||
tf, args.broker, balance, dry_run,
|
||||
)
|
||||
run_live_loop(tf=tf, broker=args.broker, account_size=balance, dry_run=dry_run)
|
||||
|
||||
|
||||
def cmd_report(args):
|
||||
balance = _resolve_balance(args)
|
||||
broker = args.broker
|
||||
tf = args.tf.upper()
|
||||
|
||||
try:
|
||||
params = get_best_params(balance=balance, broker=broker, tf=tf)
|
||||
except Exception:
|
||||
params = {}
|
||||
|
||||
df = process_pipeline(obs_cov=params.get("obs_cov"), trans_cov=params.get("trans_cov"),
|
||||
save=False, tf=tf)
|
||||
|
||||
try:
|
||||
model_hmm = load_hmm()
|
||||
from src.engine_hmm import predict_states, STATE_NAMES_3, STATE_NAMES_2, STATE_NAMES_4
|
||||
states = predict_states(model_hmm, df)
|
||||
n = model_hmm.n_components
|
||||
state_names = {2: STATE_NAMES_2, 3: STATE_NAMES_3, 4: STATE_NAMES_4}.get(n, STATE_NAMES_3)
|
||||
except Exception:
|
||||
model_hmm, states, state_names = fit_hmm(df, n_states=params.get("n_states", 3))
|
||||
|
||||
X, y, df_aligned = prepare_features(df, states)
|
||||
|
||||
try:
|
||||
model_xgb, metrics = load_xgb()
|
||||
except FileNotFoundError:
|
||||
model_xgb, metrics = train_xgb(
|
||||
X, y,
|
||||
max_depth=params.get("max_depth", 4),
|
||||
learning_rate=params.get("learning_rate", 0.1),
|
||||
n_estimators=params.get("n_estimators", 200),
|
||||
subsample=params.get("subsample", 0.8),
|
||||
colsample_bytree=params.get("colsample_bytree", 0.8),
|
||||
min_child_weight=params.get("min_child_weight", 5),
|
||||
gamma=params.get("gamma", 1.0),
|
||||
reg_alpha=params.get("reg_alpha", 0.1),
|
||||
)
|
||||
|
||||
if "feature_importance" not in metrics:
|
||||
metrics["feature_importance"] = dict(zip(FEATURE_COLS, model_xgb.feature_importances_))
|
||||
|
||||
_, probabilities = get_predictions(model_xgb, X)
|
||||
states_aligned = states[df.index.isin(df_aligned.index)]
|
||||
|
||||
# split_idx in saved metrics may come from a different TF's training run
|
||||
# (e.g. M15 split_idx ~186k used in an H1 report with only ~58k bars).
|
||||
# Recompute from current data when the stored value doesn't fit.
|
||||
split_idx = metrics.get("split_idx")
|
||||
if split_idx is None or not (0 < split_idx < len(X)):
|
||||
split_idx = int(len(X) * 0.8)
|
||||
|
||||
result = vectorized_backtest(
|
||||
df_aligned, probabilities, states_aligned,
|
||||
split_idx=split_idx, account_size=balance, broker=broker, tf=tf,
|
||||
)
|
||||
|
||||
arm = AdaptiveRiskManager(balance)
|
||||
limits = arm.get_trade_limits()
|
||||
display_params = dict(params)
|
||||
display_params.update({
|
||||
"broker": broker,
|
||||
"balance_usd": balance,
|
||||
"tier": "small" if arm.is_small_account else "growth",
|
||||
"session_limit": limits["max_daily_trades"],
|
||||
"pos_per_trade": limits["pos_per_trade"],
|
||||
"tf": tf,
|
||||
})
|
||||
|
||||
paths = generate_full_report(
|
||||
df_aligned, states_aligned, state_names, model_hmm,
|
||||
X, probabilities, metrics, result, display_params,
|
||||
split_idx=split_idx, tf=tf,
|
||||
)
|
||||
|
||||
print(f"\n=== Report Generated [{tf}] ===")
|
||||
for p in paths:
|
||||
print(f" {p}")
|
||||
print(f"\n5 charts saved to reports/{tf}/")
|
||||
|
||||
|
||||
def cmd_audit(args):
|
||||
"""Print (and optionally send) the daily MT5 performance report."""
|
||||
from src.auditor import get_daily_report
|
||||
from src.notifier import send_telegram_msg
|
||||
|
||||
balance = _resolve_balance(args)
|
||||
report = get_daily_report(broker=args.broker)
|
||||
print(report)
|
||||
sent = send_telegram_msg(report)
|
||||
if sent:
|
||||
print("\nReport also sent to Telegram.")
|
||||
else:
|
||||
print("\n(Telegram not configured — see .env.example)")
|
||||
|
||||
|
||||
def cmd_guardian(args):
|
||||
"""Start the multi-TF health monitor loop."""
|
||||
from src.guardian import run_guardian
|
||||
|
||||
balance = _resolve_balance(args)
|
||||
tfs = [t.strip().upper() for t in args.tf.split(",")]
|
||||
interval = getattr(args, "interval", 3600)
|
||||
|
||||
run_guardian(
|
||||
tfs=tfs,
|
||||
broker=args.broker,
|
||||
account_size=balance,
|
||||
period=args.period,
|
||||
interval_sec=interval,
|
||||
)
|
||||
|
||||
|
||||
def cmd_listen(args):
|
||||
"""Start the Telegram remote control listener + nightly report scheduler."""
|
||||
import threading
|
||||
from src.remote_control import run_listener
|
||||
from src.notifier import send_telegram_msg
|
||||
from src.auditor import get_daily_report
|
||||
|
||||
balance = _resolve_balance(args)
|
||||
|
||||
# ── Nightly audit scheduler (runs in background thread) ──────────────────
|
||||
def _run_scheduler():
|
||||
try:
|
||||
import schedule
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"schedule package not installed — nightly report disabled. "
|
||||
"Install with: pip install schedule"
|
||||
)
|
||||
return
|
||||
|
||||
def _send_nightly():
|
||||
report = get_daily_report(broker=args.broker)
|
||||
send_telegram_msg(f"<b>Nightly Report</b>\n{report}")
|
||||
logger.info("Nightly report sent via Telegram.")
|
||||
|
||||
schedule.every().day.at("23:55").do(_send_nightly)
|
||||
logger.info("Nightly report scheduled at 23:55 UTC.")
|
||||
while True:
|
||||
schedule.run_pending()
|
||||
time.sleep(60)
|
||||
|
||||
sched_thread = threading.Thread(target=_run_scheduler, daemon=True)
|
||||
sched_thread.start()
|
||||
|
||||
# ── Blocking Telegram listener ────────────────────────────────────────────
|
||||
run_listener()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Gold Regime X — Hybrid ML Trading System")
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=["process", "optimize", "train", "compare", "export", "report",
|
||||
"sync_validate", "live", "audit", "guardian", "listen"],
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument("--trials", type=int, default=250)
|
||||
parser.add_argument("--n_jobs", type=int, default=1,
|
||||
help="Parallel Optuna trial workers (default 1). "
|
||||
"See optimizer.py for caveats with n_jobs>1.")
|
||||
parser.add_argument("--interval", type=int, default=3600,
|
||||
help="Guardian check interval in seconds (default 3600 = 1h).")
|
||||
parser.add_argument("--min_cap", type=float, default=15.0,
|
||||
help="Account capital in USD (legacy, use --balance)")
|
||||
parser.add_argument("--balance", type=float, default=None,
|
||||
help="Account balance in USD — overrides --min_cap")
|
||||
parser.add_argument("--broker", type=str, default="standard",
|
||||
choices=["standard", "headway_cent"])
|
||||
parser.add_argument("--tf", type=str, default="H1",
|
||||
help="Timeframe: H1 | M15 | M15,H1 (process/compare accept comma list)")
|
||||
parser.add_argument("--period", type=str, default="3m",
|
||||
help="Lookback window for MT5 sync, e.g. '3m' '6m' '12m'.")
|
||||
parser.add_argument("--account", type=str, default="demo",
|
||||
choices=["live", "demo"],
|
||||
help="Account type for --mode live. 'live' requires confirmation.")
|
||||
|
||||
args = parser.parse_args()
|
||||
{
|
||||
"process": cmd_process,
|
||||
"optimize": cmd_optimize,
|
||||
"train": cmd_train,
|
||||
"compare": cmd_compare,
|
||||
"export": cmd_export,
|
||||
"report": cmd_report,
|
||||
"sync_validate": cmd_sync_validate,
|
||||
"live": cmd_live,
|
||||
"audit": cmd_audit,
|
||||
"guardian": cmd_guardian,
|
||||
"listen": cmd_listen,
|
||||
}[args.mode](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
{"timestamp": 1774801417.6893382, "tf": "M5", "best_score": 4.953615714300805}
|
||||
@@ -0,0 +1,299 @@
|
||||
//+------------------------------------------------------------------+
|
||||
//| GoldRegimeX.mq5 |
|
||||
//| Gold Regime X — Hybrid ML EA |
|
||||
//+------------------------------------------------------------------+
|
||||
#property copyright "Gold Regime X"
|
||||
#property version "2.00"
|
||||
#property strict
|
||||
|
||||
#resource "\\Files\\xgb_model.onnx" as uchar OnnxModel[]
|
||||
|
||||
//--- Trading inputs
|
||||
input double RiskPercent = 1.0; // Risk per trade (%)
|
||||
input double ProbThreshold = 0.65; // XGB probability threshold for Long
|
||||
input double ShortThreshold = 0.35; // XGB probability threshold for Short
|
||||
input int ATRPeriod = 14; // ATR period
|
||||
input double ATRMultiplier = 2.0; // ATR stop loss multiplier
|
||||
input int RSIPeriod = 14; // RSI period
|
||||
input int ChopState = 2; // HMM Chop state index (no trade)
|
||||
input int MagicNumber = 123456;
|
||||
input int NStates = 3; // Classes in trained model — must match n_states from Python (default 3)
|
||||
|
||||
//--- Timeframe input (M15 or H1)
|
||||
input ENUM_TIMEFRAMES SignalTimeframe = PERIOD_H1; // Timeframe for ONNX signals
|
||||
|
||||
//--- Account / session inputs
|
||||
// These are OVERRIDDEN at runtime by AdaptiveScaling logic below.
|
||||
// Set only as fallback defaults.
|
||||
input bool IsCentAccount = true; // Headway Cent micro-lot floor
|
||||
|
||||
//--- HMM proxy thresholds (update from Python training output)
|
||||
input double BullReturnThreshold = 0.0001;
|
||||
input double BearReturnThreshold = -0.0001;
|
||||
input double VolThreshold = 0.005;
|
||||
|
||||
//--- Handles
|
||||
long onnx_handle = INVALID_HANDLE;
|
||||
int atr_handle = INVALID_HANDLE;
|
||||
int rsi_handle = INVALID_HANDLE;
|
||||
|
||||
//--- Session state
|
||||
int session_trade_count = 0;
|
||||
datetime last_session_day = 0;
|
||||
|
||||
//+------------------------------------------------------------------+
|
||||
//| Adaptive scaling from Python spec |
|
||||
//| ≤ $50 USD : max_trades=2, pos_per_trade=1 |
|
||||
//| > $50 USD : max_trades=2|3 (Chop vs Bull/Bear), pos_per_trade=2 |
|
||||
//+------------------------------------------------------------------+
|
||||
int GetMaxTrades(double balance, int hmm_state)
|
||||
{
|
||||
if(balance <= 50.0) return 2;
|
||||
return (hmm_state == ChopState) ? 2 : 3;
|
||||
}
|
||||
|
||||
int GetPosPerTrade(double balance)
|
||||
{
|
||||
return (balance <= 50.0) ? 1 : 2;
|
||||
}
|
||||
|
||||
//+------------------------------------------------------------------+
|
||||
void CheckSessionReset()
|
||||
{
|
||||
MqlDateTime dt;
|
||||
TimeToStruct(TimeCurrent(), dt);
|
||||
datetime today = StringToTime(StringFormat("%04d.%02d.%02d 00:00",
|
||||
dt.year, dt.mon, dt.day));
|
||||
if(today != last_session_day)
|
||||
{
|
||||
if(last_session_day != 0)
|
||||
PrintFormat("New session [%s]: reset count (was %d)",
|
||||
TimeToString(today, TIME_DATE), session_trade_count);
|
||||
session_trade_count = 0;
|
||||
last_session_day = today;
|
||||
}
|
||||
}
|
||||
|
||||
//+------------------------------------------------------------------+
|
||||
int OnInit()
|
||||
{
|
||||
onnx_handle = OnnxCreateFromBuffer(OnnxModel, ONNX_DEFAULT);
|
||||
if(onnx_handle == INVALID_HANDLE)
|
||||
{
|
||||
Print("ONNX load failed: ", GetLastError());
|
||||
return INIT_FAILED;
|
||||
}
|
||||
long input_shape[] = {1, 4};
|
||||
long label_shape[] = {1};
|
||||
long prob_shape[] = {1, NStates};
|
||||
if(!OnnxSetInputShape(onnx_handle, 0, input_shape))
|
||||
{
|
||||
PrintFormat("ONNX input shape error (code %d)", GetLastError());
|
||||
return INIT_FAILED;
|
||||
}
|
||||
if(!OnnxSetOutputShape(onnx_handle, 0, label_shape))
|
||||
{
|
||||
PrintFormat("ONNX label output[0] shape error (code %d)", GetLastError());
|
||||
return INIT_FAILED;
|
||||
}
|
||||
if(!OnnxSetOutputShape(onnx_handle, 1, prob_shape))
|
||||
{
|
||||
PrintFormat("ONNX prob output[1] shape error — NStates=%d may not match trained model (code %d)",
|
||||
NStates, GetLastError());
|
||||
return INIT_FAILED;
|
||||
}
|
||||
|
||||
atr_handle = iATR(_Symbol, SignalTimeframe, ATRPeriod);
|
||||
rsi_handle = iRSI(_Symbol, SignalTimeframe, RSIPeriod, PRICE_CLOSE);
|
||||
if(atr_handle == INVALID_HANDLE || rsi_handle == INVALID_HANDLE)
|
||||
{
|
||||
Print("Indicator handle error");
|
||||
return INIT_FAILED;
|
||||
}
|
||||
|
||||
double bal = AccountInfoDouble(ACCOUNT_BALANCE);
|
||||
PrintFormat("GoldRegimeX v2.00 | TF=%s | Balance=%.2f | MaxTrades=%d | PosPerTrade=%d | Cent=%s",
|
||||
EnumToString(SignalTimeframe), bal,
|
||||
GetMaxTrades(bal, 0), GetPosPerTrade(bal),
|
||||
IsCentAccount ? "YES" : "NO");
|
||||
return INIT_SUCCEEDED;
|
||||
}
|
||||
|
||||
//+------------------------------------------------------------------+
|
||||
void OnDeinit(const int reason)
|
||||
{
|
||||
if(onnx_handle != INVALID_HANDLE) OnnxRelease(onnx_handle);
|
||||
if(atr_handle != INVALID_HANDLE) IndicatorRelease(atr_handle);
|
||||
if(rsi_handle != INVALID_HANDLE) IndicatorRelease(rsi_handle);
|
||||
}
|
||||
|
||||
//+------------------------------------------------------------------+
|
||||
double StdDev(double &arr[], int size)
|
||||
{
|
||||
double sum = 0, sq = 0;
|
||||
for(int i = 0; i < size; i++) { sum += arr[i]; sq += arr[i]*arr[i]; }
|
||||
double mean = sum / size;
|
||||
return MathSqrt(sq/size - mean*mean);
|
||||
}
|
||||
|
||||
int GetHMMState(double kalman_ret, double vol)
|
||||
{
|
||||
if(kalman_ret > BullReturnThreshold && vol < VolThreshold) return 0;
|
||||
if(kalman_ret < BearReturnThreshold) return 1;
|
||||
return 2;
|
||||
}
|
||||
|
||||
bool ComputeFeatures(float &features[], int &hmm_state_out)
|
||||
{
|
||||
ArrayResize(features, 4);
|
||||
double cl[];
|
||||
if(CopyClose(_Symbol, SignalTimeframe, 0, 22, cl) < 22) return false;
|
||||
|
||||
double rets[];
|
||||
ArrayResize(rets, 20);
|
||||
for(int i = 0; i < 20; i++) rets[i] = MathLog(cl[i+1] / cl[i]);
|
||||
|
||||
double vol = StdDev(rets, 20);
|
||||
double lr = MathLog(cl[21] / cl[20]);
|
||||
hmm_state_out = GetHMMState(lr, vol);
|
||||
features[0] = (float)hmm_state_out;
|
||||
|
||||
double rsi_buf[];
|
||||
if(CopyBuffer(rsi_handle, 0, 0, 2, rsi_buf) < 2) return false;
|
||||
features[1] = (float)(rsi_buf[1] - rsi_buf[0]);
|
||||
|
||||
double atr_buf[];
|
||||
if(CopyBuffer(atr_handle, 0, 0, 1, atr_buf) < 1) return false;
|
||||
features[2] = (float)(atr_buf[0] / cl[21]);
|
||||
features[3] = (float)MathLog(cl[20] / cl[19]);
|
||||
return true;
|
||||
}
|
||||
|
||||
//+------------------------------------------------------------------+
|
||||
//| Lot sizing with micro-lot floor for Headway Cent |
|
||||
//+------------------------------------------------------------------+
|
||||
double CalculateLotSize(double stop_pts, int pos_per_trade)
|
||||
{
|
||||
double balance = AccountInfoDouble(ACCOUNT_BALANCE);
|
||||
double risk_amt = balance * RiskPercent / 100.0 / pos_per_trade; // split across positions
|
||||
double tv = SymbolInfoDouble(_Symbol, SYMBOL_TRADE_TICK_VALUE);
|
||||
double ts = SymbolInfoDouble(_Symbol, SYMBOL_TRADE_TICK_SIZE);
|
||||
double min_lot = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_MIN);
|
||||
double max_lot = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_MAX);
|
||||
double lot_step = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_STEP);
|
||||
|
||||
if(IsCentAccount) min_lot = MathMax(min_lot, 0.01);
|
||||
if(tv == 0 || ts == 0 || stop_pts == 0) return min_lot;
|
||||
|
||||
double lot = risk_amt / (stop_pts / ts * tv);
|
||||
lot = MathFloor(lot / lot_step) * lot_step;
|
||||
return MathMax(min_lot, MathMin(max_lot, lot));
|
||||
}
|
||||
|
||||
bool HasOpenPosition()
|
||||
{
|
||||
for(int i = 0; i < PositionsTotal(); i++)
|
||||
if(PositionGetSymbol(i) == _Symbol &&
|
||||
PositionGetInteger(POSITION_MAGIC) == MagicNumber) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SendOrder(ENUM_ORDER_TYPE type, double price, double sl, double lot, string comment)
|
||||
{
|
||||
MqlTradeRequest req = {};
|
||||
MqlTradeResult res = {};
|
||||
req.action = TRADE_ACTION_DEAL;
|
||||
req.symbol = _Symbol;
|
||||
req.volume = lot;
|
||||
req.type = type;
|
||||
req.price = price;
|
||||
req.sl = sl;
|
||||
req.magic = MagicNumber;
|
||||
req.comment = comment;
|
||||
return OrderSend(req, res);
|
||||
}
|
||||
|
||||
//+------------------------------------------------------------------+
|
||||
void OnTick()
|
||||
{
|
||||
CheckSessionReset();
|
||||
|
||||
double balance = AccountInfoDouble(ACCOUNT_BALANCE);
|
||||
float features[];
|
||||
int hmm_state = 0;
|
||||
|
||||
// Compute features first so we know the HMM state for adaptive limits
|
||||
if(!ComputeFeatures(features, hmm_state)) return;
|
||||
|
||||
int max_trades = GetMaxTrades(balance, hmm_state);
|
||||
int pos_per_trade = GetPosPerTrade(balance);
|
||||
|
||||
// Enforce daily session limit
|
||||
if(session_trade_count >= max_trades) return;
|
||||
|
||||
// Only trade on new bar for the configured timeframe
|
||||
static datetime last_bar_time = 0;
|
||||
datetime cur_bar = iTime(_Symbol, SignalTimeframe, 0);
|
||||
if(cur_bar == last_bar_time) return;
|
||||
last_bar_time = cur_bar;
|
||||
|
||||
if(HasOpenPosition()) return;
|
||||
|
||||
// ONNX inference
|
||||
long labels[];
|
||||
float probs[];
|
||||
if(!OnnxRun(onnx_handle, ONNX_DEFAULT, features, labels, probs))
|
||||
{
|
||||
Print("ONNX run failed: ", GetLastError());
|
||||
return;
|
||||
}
|
||||
|
||||
// probs[0] = P(Bull/class-0), probs[1] = P(Bear/class-1)
|
||||
float bull_prob = probs[0];
|
||||
float bear_prob = probs[1];
|
||||
|
||||
double atr_buf[];
|
||||
CopyBuffer(atr_handle, 0, 0, 1, atr_buf);
|
||||
double stop_dist = atr_buf[0] * ATRMultiplier;
|
||||
double lot = CalculateLotSize(stop_dist, pos_per_trade);
|
||||
|
||||
double ask = SymbolInfoDouble(_Symbol, SYMBOL_ASK);
|
||||
double bid = SymbolInfoDouble(_Symbol, SYMBOL_BID);
|
||||
|
||||
// Long signal: high Bull probability
|
||||
if(bull_prob > ProbThreshold && hmm_state != ChopState)
|
||||
{
|
||||
double sl = NormalizeDouble(ask - stop_dist, _Digits);
|
||||
bool ok = false;
|
||||
for(int p = 0; p < pos_per_trade && session_trade_count < max_trades; p++)
|
||||
{
|
||||
if(SendOrder(ORDER_TYPE_BUY, ask, sl, lot,
|
||||
StringFormat("GRX Long p%d", p+1)))
|
||||
{
|
||||
session_trade_count++;
|
||||
ok = true;
|
||||
}
|
||||
}
|
||||
if(ok)
|
||||
PrintFormat("LONG ×%d | count=%d/%d | lot=%.2f | sl=%.2f | bull_prob=%.3f | state=%d | tf=%s",
|
||||
pos_per_trade, session_trade_count, max_trades,
|
||||
lot, sl, bull_prob, hmm_state, EnumToString(SignalTimeframe));
|
||||
}
|
||||
|
||||
// Short signal: high Bear probability (bear_prob > 1 - ShortThreshold, default 0.65)
|
||||
if(bear_prob > (1.0f - (float)ShortThreshold) && hmm_state != ChopState &&
|
||||
session_trade_count < max_trades)
|
||||
{
|
||||
double sl = NormalizeDouble(bid + stop_dist, _Digits);
|
||||
for(int p = 0; p < pos_per_trade && session_trade_count < max_trades; p++)
|
||||
{
|
||||
if(SendOrder(ORDER_TYPE_SELL, bid, sl, lot,
|
||||
StringFormat("GRX Short p%d", p+1)))
|
||||
session_trade_count++;
|
||||
}
|
||||
PrintFormat("SHORT ×%d | count=%d/%d | lot=%.2f | bear_prob=%.3f | state=%d | tf=%s",
|
||||
pos_per_trade, session_trade_count, max_trades,
|
||||
lot, bear_prob, hmm_state, EnumToString(SignalTimeframe));
|
||||
}
|
||||
}
|
||||
//+------------------------------------------------------------------+
|
||||
@@ -0,0 +1,14 @@
|
||||
numpy>=1.24
|
||||
pandas>=2.0
|
||||
hmmlearn>=0.3
|
||||
xgboost>=2.0
|
||||
optuna>=3.5
|
||||
scikit-learn>=1.3
|
||||
joblib>=1.3
|
||||
onnx>=1.15
|
||||
onnxmltools>=1.12
|
||||
skl2onnx>=1.16
|
||||
pyarrow>=14.0
|
||||
matplotlib>=3.8
|
||||
onnxconverter-common>=1.14
|
||||
metatrader5
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Daily performance auditor for Gold Regime X.
|
||||
|
||||
Fetches closed deal history from MT5 for the last 24 hours and formats
|
||||
a Telegram-ready HTML summary. Floating P&L from any currently open positions
|
||||
is included so you see the real-time risk alongside historical results.
|
||||
|
||||
Usage (via main.py):
|
||||
python main.py --mode audit --broker headway_cent
|
||||
|
||||
The report is also triggered automatically by:
|
||||
- The Telegram remote control "📊 BOT STATUS" button
|
||||
- The nightly scheduler inside --mode listen (fires at 23:55 UTC)
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from src.logger import setup_logger
|
||||
from src.risk_manager import CENT_MULTIPLIER
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def get_daily_report(broker: str = "headway_cent", days: int = 1) -> str:
|
||||
"""Fetch and format a performance summary for the last *days* days.
|
||||
|
||||
Returns an HTML-formatted string suitable for Telegram (parse_mode='HTML').
|
||||
Never raises — returns an error string if MT5 is unavailable.
|
||||
"""
|
||||
try:
|
||||
import MetaTrader5 as mt5
|
||||
except ImportError:
|
||||
return "MetaTrader5 package not installed — cannot fetch report."
|
||||
|
||||
to_date = datetime.now()
|
||||
from_date = to_date - timedelta(days=days)
|
||||
|
||||
history = mt5.history_deals_get(from_date, to_date)
|
||||
if history is None or len(history) == 0:
|
||||
period = "24 hours" if days == 1 else f"{days} days"
|
||||
return f"No trades closed in the last {period}."
|
||||
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
return "pandas not installed — cannot format report."
|
||||
|
||||
df = pd.DataFrame(list(history), columns=history[0]._asdict().keys())
|
||||
|
||||
# DEAL_ENTRY_OUT = closed exit deals (excludes balance deposits, etc.)
|
||||
df = df[df["entry"] == mt5.DEAL_ENTRY_OUT]
|
||||
if df.empty:
|
||||
return "No exit deals recorded in the last 24 hours."
|
||||
|
||||
# Normalise P&L: cent accounts report in cents, convert to USD
|
||||
divisor = float(CENT_MULTIPLIER) if broker == "headway_cent" else 1.0
|
||||
df["profit_usd"] = df["profit"] / divisor
|
||||
|
||||
total_trades = len(df)
|
||||
wins = int((df["profit"] > 0).sum())
|
||||
losses = total_trades - wins
|
||||
win_rate = (wins / total_trades * 100) if total_trades else 0.0
|
||||
total_pnl = float(df["profit_usd"].sum())
|
||||
best_trade = float(df["profit_usd"].max())
|
||||
worst_trade = float(df["profit_usd"].min())
|
||||
|
||||
# Floating P&L from any currently open positions
|
||||
positions = mt5.positions_get()
|
||||
floating_line = ""
|
||||
if positions:
|
||||
floating = sum(p.profit / divisor for p in positions)
|
||||
floating_line = f"Floating P&L: <code>{floating:+.2f} USD</code>\n"
|
||||
|
||||
icon_pnl = "+" if total_pnl >= 0 else ""
|
||||
period = "24h" if days == 1 else f"{days}d"
|
||||
stamp = datetime.now(timezone.utc).strftime("%H:%M UTC")
|
||||
|
||||
report = (
|
||||
f"<b>Gold Regime X — Daily Report ({period})</b>\n"
|
||||
f"{'—'*28}\n"
|
||||
f"Total PnL: <code>{icon_pnl}{total_pnl:.2f} USD</code>\n"
|
||||
f"{floating_line}"
|
||||
f"Total Trades: <b>{total_trades}</b> "
|
||||
f"(Wins: {wins} | Losses: {losses})\n"
|
||||
f"Win Rate: <b>{win_rate:.1f}%</b>\n"
|
||||
f"Best Trade: <code>+{best_trade:.2f} USD</code>\n"
|
||||
f"Worst Trade: <code>{worst_trade:.2f} USD</code>\n"
|
||||
f"{'—'*28}\n"
|
||||
f"<i>Generated at {stamp}</i>"
|
||||
)
|
||||
logger.info(
|
||||
"Daily report: pnl=%.2f trades=%d win_rate=%.1f%%",
|
||||
total_pnl, total_trades, win_rate,
|
||||
)
|
||||
return report
|
||||
@@ -0,0 +1,239 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from src.logger import setup_logger
|
||||
from src.risk_manager import BROKER_CONFIGS, AdaptiveRiskManager
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
ANNUALIZATION_FACTORS = {
|
||||
"H1": np.sqrt(252 * 24),
|
||||
"M15": np.sqrt(252 * 96), # 96 M15 bars per day
|
||||
"M5": np.sqrt(252 * 288), # 288 M5 bars per day
|
||||
}
|
||||
ANNUALIZATION_FACTOR = ANNUALIZATION_FACTORS["H1"] # default
|
||||
|
||||
RISK_PER_TRADE = 0.01
|
||||
PROB_THRESHOLD = 0.65
|
||||
CHOP_STATE = 2
|
||||
|
||||
|
||||
def compute_signals(probabilities, hmm_states, threshold=PROB_THRESHOLD):
|
||||
return ((probabilities > threshold) & (hmm_states != CHOP_STATE)).astype(int)
|
||||
|
||||
|
||||
def compute_position_sizes(
|
||||
signals,
|
||||
atr_normalized,
|
||||
atr_multiplier=2.0,
|
||||
risk_per_trade=RISK_PER_TRADE,
|
||||
pos_per_trade: int = 1,
|
||||
):
|
||||
"""Size positions using 1% risk / (2×ATR) with optional dual-position multiplier.
|
||||
|
||||
Args:
|
||||
pos_per_trade: 1 for small accounts (single), 2 for growth accounts (dual/hedging).
|
||||
"""
|
||||
stop_distance = atr_normalized * atr_multiplier
|
||||
stop_distance = np.where(stop_distance > 0, stop_distance, 1e-8)
|
||||
sizes = (risk_per_trade / stop_distance) * signals * pos_per_trade
|
||||
return sizes
|
||||
|
||||
|
||||
def apply_session_limits(
|
||||
signals: np.ndarray,
|
||||
dates,
|
||||
account_size: float = 15.0,
|
||||
hmm_states: np.ndarray = None,
|
||||
) -> np.ndarray:
|
||||
"""Cap signals to the adaptive daily limit based on account size and HMM state.
|
||||
|
||||
For growth accounts (> $50), the limit is market-state-dependent:
|
||||
- Bull / Bear: 3 signals/day
|
||||
- Chop: 2 signals/day
|
||||
|
||||
Uses the majority HMM state within each calendar day to decide the cap.
|
||||
"""
|
||||
arm = AdaptiveRiskManager(account_size)
|
||||
result = signals.copy()
|
||||
day_labels = pd.DatetimeIndex(dates).normalize().values
|
||||
|
||||
unique_days, day_starts = np.unique(day_labels, return_index=True)
|
||||
day_ends = np.append(day_starts[1:], len(signals))
|
||||
|
||||
for start, end in zip(day_starts, day_ends):
|
||||
# Market-state-aware limit for growth accounts
|
||||
if hmm_states is not None and not arm.is_small_account:
|
||||
day_states = hmm_states[start:end].astype(int)
|
||||
day_mode = int(np.bincount(day_states).argmax())
|
||||
limits = arm.get_trade_limits(day_mode)
|
||||
else:
|
||||
limits = arm.get_trade_limits()
|
||||
|
||||
max_daily = limits["max_daily_trades"]
|
||||
seg = result[start:end]
|
||||
trade_indices = np.where(seg == 1)[0]
|
||||
if len(trade_indices) > max_daily:
|
||||
seg[trade_indices[max_daily:]] = 0
|
||||
result[start:end] = seg
|
||||
|
||||
original = int(np.sum(signals))
|
||||
limited = int(np.sum(result))
|
||||
if original != limited:
|
||||
logger.debug(
|
||||
"Session limit (acct=$%.0f): %d -> %d trades",
|
||||
account_size, original, limited,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def compute_trade_costs(signals: np.ndarray, broker: str = "standard") -> np.ndarray:
|
||||
config = BROKER_CONFIGS.get(broker, BROKER_CONFIGS["standard"])
|
||||
cost_per_trade = config["spread_frac"] + config["commission_frac"]
|
||||
return signals * cost_per_trade
|
||||
|
||||
|
||||
def _compute_metrics(strategy_returns, signals, tf: str = "H1"):
|
||||
ann_factor = ANNUALIZATION_FACTORS.get(tf.upper(), ANNUALIZATION_FACTOR)
|
||||
mean_ret = np.mean(strategy_returns)
|
||||
std_ret = np.std(strategy_returns)
|
||||
sharpe = (mean_ret / std_ret) * ann_factor if std_ret > 0 else 0.0
|
||||
|
||||
cumulative = np.cumsum(strategy_returns)
|
||||
running_max = np.maximum.accumulate(cumulative)
|
||||
drawdowns = running_max - cumulative
|
||||
max_dd = float(np.max(drawdowns)) if len(drawdowns) > 0 else 0.0
|
||||
|
||||
active = signals == 1
|
||||
active_returns = strategy_returns[active[:len(strategy_returns)]]
|
||||
n_trades = int(np.sum(active))
|
||||
win_rate = float(np.sum(active_returns > 0) / len(active_returns)) if len(active_returns) > 0 else 0.0
|
||||
total_return = float(np.exp(np.sum(strategy_returns)) - 1)
|
||||
|
||||
return {
|
||||
"sharpe_ratio": float(sharpe),
|
||||
"max_drawdown": max_dd,
|
||||
"win_rate": win_rate,
|
||||
"total_return": total_return,
|
||||
"n_trades": n_trades,
|
||||
}
|
||||
|
||||
|
||||
def vectorized_backtest(
|
||||
df,
|
||||
probabilities,
|
||||
hmm_states,
|
||||
split_idx=None,
|
||||
account_size: float = 15.0,
|
||||
broker: str = "standard",
|
||||
tf: str = "H1",
|
||||
prob_threshold: float = None,
|
||||
):
|
||||
"""Vectorized backtest with adaptive session limits and broker costs.
|
||||
|
||||
Args:
|
||||
df: Processed DataFrame with ``log_return`` and ``atr_normalized``.
|
||||
probabilities: XGB probability array aligned with df.
|
||||
hmm_states: HMM state array aligned with df.
|
||||
split_idx: Bar index for IS/OOS split (None = full-period only).
|
||||
account_size: Account balance in USD; drives adaptive session limits.
|
||||
broker: Broker profile key from ``BROKER_CONFIGS``.
|
||||
tf: Timeframe string used for correct annualization ("H1" or "M15").
|
||||
prob_threshold: XGBoost probability threshold for signal generation.
|
||||
Defaults to PROB_THRESHOLD (0.65) if not supplied.
|
||||
"""
|
||||
log_returns = df["log_return"].values
|
||||
atr_norm = df["atr_normalized"].values
|
||||
|
||||
raw_signals = compute_signals(
|
||||
probabilities, hmm_states,
|
||||
threshold=prob_threshold if prob_threshold is not None else PROB_THRESHOLD,
|
||||
)
|
||||
signals = apply_session_limits(raw_signals, df.index, account_size, hmm_states)
|
||||
|
||||
# Determine pos_per_trade from the adaptive risk manager
|
||||
arm = AdaptiveRiskManager(account_size)
|
||||
base_limits = arm.get_trade_limits()
|
||||
pos_per_trade = base_limits["pos_per_trade"]
|
||||
|
||||
sizes = compute_position_sizes(signals, atr_norm, pos_per_trade=pos_per_trade)
|
||||
|
||||
next_returns = np.roll(log_returns, -1)
|
||||
next_returns[-1] = 0.0
|
||||
gross_returns = sizes * next_returns
|
||||
costs = compute_trade_costs(signals, broker=broker)
|
||||
strategy_returns = gross_returns - costs
|
||||
|
||||
result = _compute_metrics(strategy_returns, signals, tf=tf)
|
||||
result.update({
|
||||
"account_size": account_size,
|
||||
"broker": broker,
|
||||
"tf": tf,
|
||||
"session_max_trades": arm.get_trade_limits()["max_daily_trades"],
|
||||
"pos_per_trade": pos_per_trade,
|
||||
})
|
||||
|
||||
if split_idx is not None and 0 < split_idx < len(strategy_returns):
|
||||
is_m = _compute_metrics(strategy_returns[:split_idx], signals[:split_idx], tf)
|
||||
oos_m = _compute_metrics(strategy_returns[split_idx:], signals[split_idx:], tf)
|
||||
result["split_idx"] = split_idx
|
||||
for k, v in is_m.items():
|
||||
result[f"is_{k}"] = v
|
||||
for k, v in oos_m.items():
|
||||
result[f"oos_{k}"] = v
|
||||
logger.info(
|
||||
"Backtest IS [%s]: Sharpe=%.3f | MaxDD=%.3f | WinRate=%.3f | Trades=%d",
|
||||
tf, is_m["sharpe_ratio"], is_m["max_drawdown"], is_m["win_rate"], is_m["n_trades"],
|
||||
)
|
||||
logger.info(
|
||||
"Backtest OOS [%s]: Sharpe=%.3f | MaxDD=%.3f | WinRate=%.3f | Trades=%d",
|
||||
tf, oos_m["sharpe_ratio"], oos_m["max_drawdown"], oos_m["win_rate"], oos_m["n_trades"],
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Backtest [%s]: Sharpe=%.3f | MaxDD=%.3f | WR=%.3f | Return=%.1f%% | Trades=%d",
|
||||
tf, result["sharpe_ratio"], result["max_drawdown"], result["win_rate"],
|
||||
result["total_return"] * 100, result["n_trades"],
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"%s: account=$%.2f | tier=%s | session_limit=%d/day | pos_per_trade=%d | costs=%.4f/trade",
|
||||
broker,
|
||||
account_size,
|
||||
"small" if arm.is_small_account else "growth",
|
||||
arm.get_trade_limits()["max_daily_trades"],
|
||||
pos_per_trade,
|
||||
BROKER_CONFIGS.get(broker, {}).get("spread_frac", 0) + BROKER_CONFIGS.get(broker, {}).get("commission_frac", 0),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def compare_timeframes(m15_result: dict, h1_result: dict) -> tuple[dict, str]:
|
||||
"""Compare M15 vs H1 backtest results and recommend the better timeframe.
|
||||
|
||||
Comparison is based on the OOS Sharpe ratio when available, otherwise
|
||||
full-period Sharpe.
|
||||
|
||||
Returns:
|
||||
results: Dict mapping tf label to its result dict.
|
||||
winner: ``"M15"`` or ``"H1"``, whichever has the higher Sharpe.
|
||||
"""
|
||||
results = {"M15": m15_result, "H1": h1_result}
|
||||
|
||||
def _oos_sharpe(r):
|
||||
return r.get("oos_sharpe_ratio", r.get("sharpe_ratio", 0.0))
|
||||
|
||||
winner = max(results, key=lambda k: _oos_sharpe(results[k]))
|
||||
|
||||
logger.info("Timeframe comparison:")
|
||||
for tf, r in results.items():
|
||||
logger.info(
|
||||
" %s Sharpe=%.3f (OOS=%.3f) | DD=%.1f%% | WR=%.1f%% | Trades=%d",
|
||||
tf,
|
||||
r["sharpe_ratio"],
|
||||
r.get("oos_sharpe_ratio", r["sharpe_ratio"]),
|
||||
r["max_drawdown"] * 100,
|
||||
r["win_rate"] * 100,
|
||||
r["n_trades"],
|
||||
)
|
||||
logger.info("Winner: %s", winner)
|
||||
return results, winner
|
||||
@@ -0,0 +1,131 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import joblib
|
||||
from pathlib import Path
|
||||
from hmmlearn.hmm import GaussianHMM
|
||||
from src.logger import setup_logger, log_regime_transition
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
MODEL_PATH = Path("models/hmm_model.pkl")
|
||||
STATE_NAMES_3 = {0: "Bull", 1: "Bear", 2: "Chop"}
|
||||
STATE_NAMES_2 = {0: "Bull", 1: "Bear"}
|
||||
STATE_NAMES_4 = {0: "Bull", 1: "Bear", 2: "Chop_Low", 3: "Chop_High"}
|
||||
|
||||
|
||||
def _sort_states(model: GaussianHMM, raw_states: np.ndarray, n_states: int):
|
||||
mean_returns = model.means_[:, 0]
|
||||
sorted_idx = np.argsort(mean_returns)
|
||||
|
||||
if n_states == 2:
|
||||
remap = {int(sorted_idx[0]): 1, int(sorted_idx[1]): 0}
|
||||
state_names = STATE_NAMES_2
|
||||
elif n_states == 3:
|
||||
remap = {
|
||||
int(sorted_idx[0]): 1, # lowest mean -> Bear
|
||||
int(sorted_idx[1]): 2, # middle -> Chop
|
||||
int(sorted_idx[2]): 0, # highest -> Bull
|
||||
}
|
||||
state_names = STATE_NAMES_3
|
||||
else:
|
||||
remap = {int(sorted_idx[i]): i for i in range(n_states)}
|
||||
remap[int(sorted_idx[0])] = 1 # Bear
|
||||
remap[int(sorted_idx[-1])] = 0 # Bull
|
||||
for j in range(1, n_states - 1):
|
||||
remap[int(sorted_idx[j])] = j + 1
|
||||
state_names = STATE_NAMES_4
|
||||
|
||||
remapped = np.array([remap[s] for s in raw_states])
|
||||
|
||||
perm = [0] * n_states
|
||||
for old, new in remap.items():
|
||||
perm[new] = old
|
||||
model.means_ = model.means_[perm]
|
||||
model.covars_ = model.covars_[perm]
|
||||
model.transmat_ = model.transmat_[np.ix_(perm, perm)]
|
||||
model.startprob_ = model.startprob_[perm]
|
||||
|
||||
return remapped, state_names
|
||||
|
||||
|
||||
def _log_transition_matrix(model: GaussianHMM, state_names: dict):
|
||||
n = model.n_components
|
||||
logger.info("Transition Matrix:")
|
||||
header = " " + " ".join(f"{state_names.get(j, str(j)):>8}" for j in range(n))
|
||||
logger.info(header)
|
||||
for i in range(n):
|
||||
row = f"{state_names.get(i, str(i)):>8} " + " ".join(
|
||||
f"{model.transmat_[i, j]:8.4f}" for j in range(n)
|
||||
)
|
||||
logger.info(row)
|
||||
if model.transmat_[i, i] < 0.85:
|
||||
logger.warning(
|
||||
"State %s persistence %.3f < 0.85 -- consider higher Kalman smoothing",
|
||||
state_names.get(i, str(i)), model.transmat_[i, i],
|
||||
)
|
||||
|
||||
logger.info("State means (kalman_return, volatility):")
|
||||
for i in range(n):
|
||||
logger.info(
|
||||
" %s: return=%.6f, vol=%.6f",
|
||||
state_names.get(i, str(i)), model.means_[i, 0], model.means_[i, 1],
|
||||
)
|
||||
|
||||
|
||||
def fit_hmm(
|
||||
df: pd.DataFrame,
|
||||
n_states: int = 3,
|
||||
n_iter: int = 200,
|
||||
random_state: int = 42,
|
||||
):
|
||||
X = df[["kalman_return", "volatility"]].values
|
||||
model = GaussianHMM(
|
||||
n_components=n_states,
|
||||
covariance_type="full",
|
||||
n_iter=n_iter,
|
||||
random_state=random_state,
|
||||
verbose=False,
|
||||
)
|
||||
model.fit(X)
|
||||
raw_states = model.predict(X)
|
||||
|
||||
states, state_names = _sort_states(model, raw_states, n_states)
|
||||
_log_transition_matrix(model, state_names)
|
||||
|
||||
transitions = np.where(np.diff(states) != 0)[0] + 1
|
||||
logger.info("Total regime transitions: %d", len(transitions))
|
||||
for idx in transitions[:10]:
|
||||
log_regime_transition(
|
||||
logger, df.index[idx], states[idx - 1], states[idx], state_names
|
||||
)
|
||||
|
||||
return model, states, state_names
|
||||
|
||||
|
||||
def predict_states(model: GaussianHMM, df: pd.DataFrame) -> np.ndarray:
|
||||
X = df[["kalman_return", "volatility"]].values
|
||||
raw = model.predict(X)
|
||||
mean_returns = model.means_[:, 0]
|
||||
sorted_idx = np.argsort(mean_returns)
|
||||
n = model.n_components
|
||||
if n == 2:
|
||||
remap = {int(sorted_idx[0]): 1, int(sorted_idx[1]): 0}
|
||||
elif n == 3:
|
||||
remap = {int(sorted_idx[0]): 1, int(sorted_idx[1]): 2, int(sorted_idx[2]): 0}
|
||||
else:
|
||||
remap = {int(sorted_idx[0]): 1, int(sorted_idx[-1]): 0}
|
||||
for j in range(1, n - 1):
|
||||
remap[int(sorted_idx[j])] = j + 1
|
||||
return np.array([remap.get(s, s) for s in raw])
|
||||
|
||||
|
||||
def save_model(model: GaussianHMM, path: Path = MODEL_PATH):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
joblib.dump(model, path)
|
||||
logger.info("HMM model saved to %s", path)
|
||||
|
||||
|
||||
def load_model(path: Path = MODEL_PATH) -> GaussianHMM:
|
||||
model = joblib.load(path)
|
||||
logger.info("HMM model loaded from %s", path)
|
||||
return model
|
||||
@@ -0,0 +1,190 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import xgboost as xgb
|
||||
import joblib
|
||||
import onnx
|
||||
from pathlib import Path
|
||||
from sklearn.metrics import accuracy_score
|
||||
from src.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
ONNX_PATH = Path("models/xgb_model.onnx")
|
||||
XGB_PKL_PATH = Path("models/xgb_model.pkl")
|
||||
|
||||
FEATURE_COLS = ["hmm_state", "rsi_slope", "atr_normalized", "prev_log_return"]
|
||||
|
||||
|
||||
def prepare_features(df: pd.DataFrame, hmm_states: np.ndarray):
|
||||
df = df.copy()
|
||||
df["hmm_state"] = hmm_states
|
||||
df["prev_log_return"] = df["log_return"].shift(1)
|
||||
y = (df["log_return"].shift(-1) > 0).astype(int).rename("target")
|
||||
|
||||
X = df[FEATURE_COLS]
|
||||
valid = X.notna().all(axis=1) & y.notna()
|
||||
X = X[valid]
|
||||
y = y[valid]
|
||||
df_aligned = df.loc[X.index]
|
||||
|
||||
logger.info("Features prepared: %d samples, %d features", len(X), len(FEATURE_COLS))
|
||||
return X, y, df_aligned
|
||||
|
||||
|
||||
def train_xgb(
|
||||
X: pd.DataFrame,
|
||||
y: pd.Series,
|
||||
max_depth: int = 4,
|
||||
learning_rate: float = 0.1,
|
||||
n_estimators: int = 200,
|
||||
subsample: float = 0.8,
|
||||
min_child_weight: int = 5,
|
||||
gamma: float = 1.0,
|
||||
reg_alpha: float = 0.1,
|
||||
colsample_bytree: float = 0.8,
|
||||
train_ratio: float = 0.8,
|
||||
):
|
||||
split_idx = int(len(X) * train_ratio)
|
||||
X_train, X_test = X.iloc[:split_idx], X.iloc[split_idx:]
|
||||
y_train, y_test = y.iloc[:split_idx], y.iloc[split_idx:]
|
||||
|
||||
model = xgb.XGBClassifier(
|
||||
max_depth=max_depth,
|
||||
learning_rate=learning_rate,
|
||||
n_estimators=n_estimators,
|
||||
subsample=subsample,
|
||||
min_child_weight=min_child_weight,
|
||||
gamma=gamma,
|
||||
reg_alpha=reg_alpha,
|
||||
colsample_bytree=colsample_bytree,
|
||||
objective="binary:logistic",
|
||||
eval_metric="logloss",
|
||||
random_state=42,
|
||||
n_jobs=-1,
|
||||
verbosity=0,
|
||||
)
|
||||
model.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
|
||||
|
||||
train_acc = accuracy_score(y_train, model.predict(X_train))
|
||||
test_acc = accuracy_score(y_test, model.predict(X_test))
|
||||
importance = dict(zip(FEATURE_COLS, model.feature_importances_))
|
||||
|
||||
logger.info("XGB Train Acc: %.4f | Test Acc: %.4f", train_acc, test_acc)
|
||||
logger.info("Feature importance: %s", importance)
|
||||
|
||||
metrics = {
|
||||
"train_accuracy": train_acc,
|
||||
"test_accuracy": test_acc,
|
||||
"feature_importance": importance,
|
||||
"split_idx": split_idx,
|
||||
}
|
||||
return model, metrics
|
||||
|
||||
|
||||
def get_predictions(model: xgb.XGBClassifier, X: pd.DataFrame):
|
||||
predictions = model.predict(X)
|
||||
probabilities = model.predict_proba(X)[:, 1]
|
||||
return predictions, probabilities
|
||||
|
||||
|
||||
def save_xgb(model: xgb.XGBClassifier, metrics: dict = None, path: Path = XGB_PKL_PATH):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
joblib.dump({"model": model, "metrics": metrics or {}}, path)
|
||||
logger.info("XGB model saved to %s", path)
|
||||
|
||||
|
||||
def load_xgb(path: Path = XGB_PKL_PATH):
|
||||
data = joblib.load(path)
|
||||
if isinstance(data, dict) and "model" in data:
|
||||
model = data["model"]
|
||||
metrics = data.get("metrics", {})
|
||||
else:
|
||||
# Backwards compat: old saves stored just the model
|
||||
model = data
|
||||
metrics = {
|
||||
"feature_importance": dict(zip(FEATURE_COLS, model.feature_importances_)),
|
||||
}
|
||||
return model, metrics
|
||||
|
||||
|
||||
def _strip_zipmap(onnx_model):
|
||||
"""Remove ZipMap node from ONNX graph, exposing the raw float probability tensor.
|
||||
|
||||
onnxmltools converts XGBoost probabilities as ZipMap (sequence of maps).
|
||||
MT5's OnnxRun expects a plain float32 tensor. This surgery replaces the
|
||||
ZipMap output with the raw tensor it is wrapping, making the model fully
|
||||
compatible with MT5's OnnxSetOutputShape / OnnxRun API.
|
||||
"""
|
||||
import onnx as _onnx
|
||||
|
||||
graph = onnx_model.graph
|
||||
|
||||
float_tensor_name = None
|
||||
zipmap_output_name = None
|
||||
nodes_to_keep = []
|
||||
|
||||
for node in graph.node:
|
||||
if node.op_type == "ZipMap":
|
||||
float_tensor_name = node.input[0]
|
||||
zipmap_output_name = node.output[0]
|
||||
else:
|
||||
nodes_to_keep.append(node)
|
||||
|
||||
if float_tensor_name is None:
|
||||
return onnx_model # no ZipMap present — nothing to do
|
||||
|
||||
new_outputs = []
|
||||
for output in graph.output:
|
||||
if output.name == zipmap_output_name:
|
||||
new_outputs.append(
|
||||
_onnx.helper.make_tensor_value_info(
|
||||
float_tensor_name,
|
||||
_onnx.TensorProto.FLOAT,
|
||||
None, # shape inferred at runtime
|
||||
)
|
||||
)
|
||||
else:
|
||||
new_outputs.append(output)
|
||||
|
||||
new_graph = _onnx.helper.make_graph(
|
||||
nodes_to_keep,
|
||||
graph.name,
|
||||
list(graph.input),
|
||||
new_outputs,
|
||||
list(graph.initializer),
|
||||
)
|
||||
new_model = _onnx.helper.make_model(
|
||||
new_graph, opset_imports=onnx_model.opset_import
|
||||
)
|
||||
new_model.ir_version = onnx_model.ir_version
|
||||
return new_model
|
||||
|
||||
|
||||
def export_onnx(model: xgb.XGBClassifier, n_features: int = 4, path: Path = ONNX_PATH):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# onnxmltools requires feature names in 'f%d' pattern
|
||||
# Clone the model's booster with generic feature names
|
||||
import copy
|
||||
model_copy = copy.deepcopy(model)
|
||||
model_copy.get_booster().feature_names = [f"f{i}" for i in range(n_features)]
|
||||
|
||||
from onnxmltools import convert_xgboost
|
||||
from onnxmltools.convert.common.data_types import FloatTensorType
|
||||
|
||||
initial_type = [("input", FloatTensorType([None, n_features]))]
|
||||
onnx_model = convert_xgboost(model_copy, initial_types=initial_type)
|
||||
onnx_model = _strip_zipmap(onnx_model) # expose float tensor output for MT5
|
||||
|
||||
onnx.save_model(onnx_model, str(path))
|
||||
onnx.checker.check_model(onnx_model)
|
||||
|
||||
inputs = [i.name for i in onnx_model.graph.input]
|
||||
outputs = [o.name for o in onnx_model.graph.output]
|
||||
n_classes = model.n_classes_
|
||||
logger.info(
|
||||
"ONNX exported to %s | inputs: %s | outputs: %s | n_classes=%d",
|
||||
path, inputs, outputs, n_classes,
|
||||
)
|
||||
print(f"\n ONNX export OK — n_classes={n_classes}. "
|
||||
f"Set NStates={n_classes} in the MT5 EA inputs.\n")
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
"""Multi-timeframe health monitor for Gold Regime X.
|
||||
|
||||
Periodically re-validates HMM+XGBoost signal quality on the most recently
|
||||
synced MT5 data. If rolling Sharpe for any timeframe drops below the alert
|
||||
threshold a Telegram notification is fired immediately.
|
||||
|
||||
Usage (via main.py):
|
||||
python main.py --mode guardian --tf M5,M15,H1 --period 3m --interval 3600
|
||||
(defaults: all three TFs, last 3 months of data, check every hour)
|
||||
|
||||
The check re-uses the same validator.run_validation() that gates live trading,
|
||||
so the health score is directly comparable to sync_validate output.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
from src.logger import setup_logger
|
||||
from src.notifier import send_telegram_msg
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
SHARPE_ALERT_THRESHOLD = 0.6 # Telegram alert fires below this
|
||||
SHARPE_CRITICAL = 0.4 # escalated "CRITICAL" label
|
||||
|
||||
|
||||
def run_guardian(
|
||||
tfs: list,
|
||||
broker: str,
|
||||
account_size: float,
|
||||
period: str = "3m",
|
||||
interval_sec: int = 3600,
|
||||
) -> None:
|
||||
"""Monitor timeframe health continuously; alert via Telegram on degradation.
|
||||
|
||||
Args:
|
||||
tfs: List of timeframe strings, e.g. ["M5", "M15", "H1"].
|
||||
broker: Broker key used for backtest cost simulation.
|
||||
account_size: USD balance for lot-sizing in the backtest.
|
||||
period: MT5 lookback period for synced data ('3m', '6m', etc.).
|
||||
interval_sec: Seconds between health checks. Default 3600 (1 hour).
|
||||
"""
|
||||
from src.validator import run_validation
|
||||
|
||||
logger.info(
|
||||
"Guardian started — TFs=%s interval=%ds alert_threshold=%.1f",
|
||||
tfs, interval_sec, SHARPE_ALERT_THRESHOLD,
|
||||
)
|
||||
send_telegram_msg(
|
||||
f"<b>Guardian online</b>\n"
|
||||
f"Monitoring: <b>{', '.join(tfs)}</b>\n"
|
||||
f"Check interval: every {interval_sec // 60} min"
|
||||
)
|
||||
|
||||
cycle = 0
|
||||
while True:
|
||||
cycle += 1
|
||||
lines = [f"<b>Guardian Health Check #{cycle}</b>"]
|
||||
alerts = []
|
||||
|
||||
for tf in tfs:
|
||||
try:
|
||||
result = run_validation(
|
||||
tf=tf,
|
||||
broker=broker,
|
||||
account_size=account_size,
|
||||
period=period,
|
||||
)
|
||||
sharpe = result["sharpe"]
|
||||
status = result["status"]
|
||||
trades = result["n_trades"]
|
||||
|
||||
if sharpe < SHARPE_CRITICAL:
|
||||
icon = "CRITICAL"
|
||||
elif sharpe < SHARPE_ALERT_THRESHOLD:
|
||||
icon = "WARN"
|
||||
elif status == "pass":
|
||||
icon = "OK"
|
||||
else:
|
||||
icon = "WARN"
|
||||
|
||||
lines.append(
|
||||
f" [{tf}] Sharpe={sharpe:.3f} "
|
||||
f"Trades={trades} [{icon}]"
|
||||
)
|
||||
logger.info(
|
||||
"Guardian [%s]: sharpe=%.3f status=%s trades=%d",
|
||||
tf, sharpe, status, trades,
|
||||
)
|
||||
|
||||
if sharpe < SHARPE_ALERT_THRESHOLD:
|
||||
level = "CRITICAL" if sharpe < SHARPE_CRITICAL else "WARNING"
|
||||
alerts.append(
|
||||
f"<b>GUARDIAN {level} — [{tf}]</b>\n"
|
||||
f"Rolling Sharpe: <b>{sharpe:.3f}</b> "
|
||||
f"(threshold: {SHARPE_ALERT_THRESHOLD})\n"
|
||||
f"Action: python main.py --mode optimize --tf {tf}"
|
||||
)
|
||||
except Exception as exc:
|
||||
msg = f" [{tf}] Check error: {exc}"
|
||||
lines.append(msg)
|
||||
logger.error("Guardian [%s] check failed: %s", tf, exc)
|
||||
|
||||
# Send periodic digest and any alerts
|
||||
send_telegram_msg("\n".join(lines))
|
||||
for alert in alerts:
|
||||
send_telegram_msg(alert)
|
||||
|
||||
logger.info(
|
||||
"Guardian cycle #%d complete — next in %ds.", cycle, interval_sec
|
||||
)
|
||||
time.sleep(interval_sec)
|
||||
@@ -0,0 +1,38 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
LOG_DIR = Path("logs")
|
||||
|
||||
|
||||
def setup_logger(name: str, level: int = logging.INFO) -> logging.Logger:
|
||||
LOG_DIR.mkdir(exist_ok=True)
|
||||
logger = logging.getLogger(name)
|
||||
if logger.handlers:
|
||||
return logger
|
||||
logger.setLevel(level)
|
||||
fmt = logging.Formatter("[%(asctime)s %(levelname)s] %(name)s: %(message)s")
|
||||
sh = logging.StreamHandler()
|
||||
sh.setLevel(level)
|
||||
sh.setFormatter(fmt)
|
||||
logger.addHandler(sh)
|
||||
fh = logging.FileHandler(LOG_DIR / "goldregimex.log", mode="a")
|
||||
fh.setLevel(level)
|
||||
fh.setFormatter(fmt)
|
||||
logger.addHandler(fh)
|
||||
return logger
|
||||
|
||||
|
||||
def log_regime_transition(logger, timestamp, old_state, new_state, state_names):
|
||||
logger.info(
|
||||
"REGIME CHANGE at %s: %s -> %s",
|
||||
timestamp,
|
||||
state_names.get(old_state, str(old_state)),
|
||||
state_names.get(new_state, str(new_state)),
|
||||
)
|
||||
|
||||
|
||||
def log_trade_signal(logger, timestamp, direction, probability, hmm_state):
|
||||
logger.info(
|
||||
"SIGNAL at %s: %s (prob=%.3f, hmm_state=%d)",
|
||||
timestamp, direction, probability, hmm_state,
|
||||
)
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
"""MT5 Data Sync — downloads recent OHLCV bars from the MetaTrader5 terminal.
|
||||
|
||||
The MT5 terminal must already be running and logged into the desired account
|
||||
before calling any function here. The MetaTrader5 Python package is imported
|
||||
lazily so this module can be imported on machines that do not have MT5 installed
|
||||
(e.g. a CI environment that only runs the backtest pipeline).
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from src.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
SYNC_OUTPUT_PATH = Path("data/processed/mt5_sync_data.csv")
|
||||
DEFAULT_SYMBOL = "XAUUSD"
|
||||
|
||||
# Lazy MT5 timeframe map — populated on first call to _get_tf_map()
|
||||
_MT5_TF_MAP: dict | None = None
|
||||
|
||||
|
||||
def _get_tf_map() -> dict:
|
||||
global _MT5_TF_MAP
|
||||
if _MT5_TF_MAP is None:
|
||||
import MetaTrader5 as mt5
|
||||
_MT5_TF_MAP = {
|
||||
"M5": mt5.TIMEFRAME_M5,
|
||||
"M15": mt5.TIMEFRAME_M15,
|
||||
"H1": mt5.TIMEFRAME_H1,
|
||||
}
|
||||
return _MT5_TF_MAP
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Public helpers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def parse_period(period_str: str) -> int:
|
||||
"""Convert a period string such as ``'3m'`` to a month count integer."""
|
||||
period_str = period_str.strip().lower()
|
||||
if period_str.endswith("m") and period_str[:-1].isdigit():
|
||||
return int(period_str[:-1])
|
||||
raise ValueError(
|
||||
f"Unrecognised period format: '{period_str}'. "
|
||||
"Expected a digit followed by 'm', e.g. '3m', '6m', '12m'."
|
||||
)
|
||||
|
||||
|
||||
def connect_mt5(login: int = None, password: str = None, server: str = None) -> bool:
|
||||
"""Initialise the MT5 package and optionally log in programmatically.
|
||||
|
||||
If *login* is None the function relies on the account that is already
|
||||
active in the terminal. Returns ``True`` on success.
|
||||
"""
|
||||
import MetaTrader5 as mt5
|
||||
if not mt5.initialize():
|
||||
logger.error("MT5 initialize() failed: %s", mt5.last_error())
|
||||
return False
|
||||
if login is not None:
|
||||
if not mt5.login(login, password=password, server=server):
|
||||
logger.error("MT5 login(%d) failed: %s", login, mt5.last_error())
|
||||
mt5.shutdown()
|
||||
return False
|
||||
info = mt5.account_info()
|
||||
if info:
|
||||
logger.info(
|
||||
"MT5 connected — login=%d server=%s balance=%.2f %s",
|
||||
info.login, info.server, info.balance, info.currency,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def disconnect_mt5() -> None:
|
||||
"""Shut down the MT5 Python connection (safe to call when not connected)."""
|
||||
try:
|
||||
import MetaTrader5 as mt5
|
||||
mt5.shutdown()
|
||||
logger.debug("MT5 disconnected.")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def fetch_bars(symbol: str, tf: str, months: int) -> pd.DataFrame:
|
||||
"""Download completed OHLCV bars for *symbol* on *tf* going back *months*.
|
||||
|
||||
The currently open (incomplete) bar is always excluded.
|
||||
|
||||
Returns a DataFrame with columns ``Open, High, Low, Close, Volume`` and a
|
||||
UTC DatetimeIndex named ``Date`` — matching the convention expected by the
|
||||
standalone feature-engineering functions in ``processor.py``.
|
||||
"""
|
||||
import MetaTrader5 as mt5
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
tf_map = _get_tf_map()
|
||||
tf_key = tf.upper()
|
||||
if tf_key not in tf_map:
|
||||
raise ValueError(f"Unknown timeframe '{tf}'. Supported: {list(tf_map)}")
|
||||
|
||||
date_from = datetime.utcnow() - relativedelta(months=months)
|
||||
date_to = datetime.utcnow()
|
||||
|
||||
rates = mt5.copy_rates_range(symbol, tf_map[tf_key], date_from, date_to)
|
||||
if rates is None or len(rates) == 0:
|
||||
raise RuntimeError(
|
||||
f"MT5 returned no data for {symbol} {tf}: {mt5.last_error()}\n"
|
||||
"Ensure the symbol is in Market Watch and the terminal is connected."
|
||||
)
|
||||
|
||||
df = pd.DataFrame(rates)
|
||||
df["time"] = pd.to_datetime(df["time"], unit="s", utc=True)
|
||||
df = (
|
||||
df.rename(columns={
|
||||
"time": "Date",
|
||||
"open": "Open",
|
||||
"high": "High",
|
||||
"low": "Low",
|
||||
"close": "Close",
|
||||
"tick_volume": "Volume",
|
||||
})
|
||||
[["Date", "Open", "High", "Low", "Close", "Volume"]]
|
||||
)
|
||||
df.set_index("Date", inplace=True)
|
||||
df.sort_index(inplace=True)
|
||||
df = df.iloc[:-1] # drop the currently open bar
|
||||
|
||||
logger.info(
|
||||
"Fetched %d %s bars for %s: %s -> %s",
|
||||
len(df), tf_key, symbol, df.index.min(), df.index.max(),
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Primary entry point
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def sync_mt5_data(
|
||||
symbol: str = DEFAULT_SYMBOL,
|
||||
tf: str = "H1",
|
||||
period: str = "3m",
|
||||
output_path: Path = SYNC_OUTPUT_PATH,
|
||||
) -> pd.DataFrame:
|
||||
"""Connect to MT5, fetch recent bars, save a CSV, then disconnect.
|
||||
|
||||
Raises ``ConnectionError`` when the MT5 terminal cannot be reached.
|
||||
"""
|
||||
if not connect_mt5():
|
||||
raise ConnectionError(
|
||||
"Could not connect to MetaTrader5 terminal. "
|
||||
"Ensure MT5 is running and logged into your account."
|
||||
)
|
||||
try:
|
||||
months = parse_period(period)
|
||||
df = fetch_bars(symbol, tf.upper(), months)
|
||||
output_path = Path(output_path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_csv(output_path)
|
||||
logger.info("Saved %d bars -> %s", len(df), output_path)
|
||||
return df
|
||||
finally:
|
||||
disconnect_mt5()
|
||||
@@ -0,0 +1,721 @@
|
||||
"""MT5 Live Execution Engine.
|
||||
|
||||
Provides account telemetry, feature engineering parity with the training
|
||||
pipeline, margin validation, and live order placement through the
|
||||
MetaTrader5 Python package.
|
||||
|
||||
Usage (via main.py):
|
||||
python main.py --mode live --account demo --broker headway_cent --balance 15 --tf H1
|
||||
|
||||
IMPORTANT: Remove the GoldRegimeX.mq5 EA from the XAUUSD chart before running
|
||||
this script. Both the EA and the Python bridge use MAGIC_NUMBER = 123456 and
|
||||
their session counters are independent — running both simultaneously will
|
||||
double-count trades.
|
||||
"""
|
||||
|
||||
import time
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime, timezone
|
||||
from src.notifier import send_telegram_msg
|
||||
|
||||
from src.logger import setup_logger
|
||||
from src.processor import (
|
||||
TF_CONFIG,
|
||||
compute_log_returns,
|
||||
kalman_smooth,
|
||||
compute_volatility,
|
||||
compute_rsi,
|
||||
compute_atr,
|
||||
)
|
||||
from src.engine_hmm import load_model as load_hmm, predict_states
|
||||
from src.engine_xgb import load_xgb
|
||||
from src.risk_manager import AdaptiveRiskManager, CENT_MULTIPLIER
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Constants
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
DEFAULT_SYMBOL = "XAUUSD"
|
||||
MAGIC_NUMBER = 123456 # must match GoldRegimeX.mq5 MagicNumber
|
||||
CHOP_STATE = 2 # HMM Chop state index
|
||||
ATR_MULTIPLIER = 2.0 # SL = entry ± ATR × 2.0 (all TFs)
|
||||
DEFAULT_DEVIATION = 20 # fallback deviation for check_margin / send_market_order
|
||||
N_BARS_WARMUP = 200 # bars fetched for Kalman/HMM warm-up
|
||||
POLL_INTERVAL_SEC = 5 # seconds between bar-change checks
|
||||
HIGH_VOL_SELF_TRANS_THRESHOLD = 0.70 # self-transition prob below this -> elevated deviation
|
||||
MIN_SPREAD_RATIO = 1.5 # TP1 must be at least 1.5× spread to be viable
|
||||
|
||||
# ── Per-timeframe signal thresholds and order parameters ──────────────────────
|
||||
TF_PROB_THRESHOLD = {"M5": 0.70, "M15": 0.65, "H1": 0.65}
|
||||
TF_SHORT_THRESHOLD = {"M5": 0.30, "M15": 0.35, "H1": 0.35}
|
||||
TF_DEFAULT_DEV = {"M5": 30, "M15": 20, "H1": 20}
|
||||
TF_HIGH_VOL_DEV = {"M5": 50, "M15": 50, "H1": 50}
|
||||
|
||||
# ── State-aware multi-stage TP multipliers (relative to SL distance) ──────────
|
||||
# Bull / Bear: TP1 quick partial, TP2 runner.
|
||||
# Chop: tighter single TP — position 2 (runner) is skipped.
|
||||
# Format: {regime: [tp1_mult, tp2_mult]}; single-element list = one TP, close all.
|
||||
TF_TP_CONFIG = {
|
||||
"M5": {"trending": [1.0, 3.0], "chop": [0.8]},
|
||||
"M15": {"trending": [1.5, 3.0], "chop": [1.5]},
|
||||
"H1": {"trending": [1.5, 3.0], "chop": [1.5]},
|
||||
}
|
||||
|
||||
# Lazy MT5 timeframe map
|
||||
_MT5_TF_MAP: dict | None = None
|
||||
|
||||
|
||||
def _get_tf_map() -> dict:
|
||||
global _MT5_TF_MAP
|
||||
if _MT5_TF_MAP is None:
|
||||
import MetaTrader5 as mt5
|
||||
_MT5_TF_MAP = {
|
||||
"M5": mt5.TIMEFRAME_M5,
|
||||
"M15": mt5.TIMEFRAME_M15,
|
||||
"H1": mt5.TIMEFRAME_H1,
|
||||
}
|
||||
return _MT5_TF_MAP
|
||||
|
||||
|
||||
def _tp_multipliers(tf: str, hmm_state: int) -> list[float]:
|
||||
"""Return the TP multiplier list for this TF and HMM state."""
|
||||
cfg = TF_TP_CONFIG.get(tf.upper(), TF_TP_CONFIG["H1"])
|
||||
return cfg["chop"] if hmm_state == CHOP_STATE else cfg["trending"]
|
||||
|
||||
|
||||
def _normalise_balance(raw_balance: float, broker: str) -> float:
|
||||
"""Convert MT5 raw balance to USD for AdaptiveRiskManager.
|
||||
|
||||
On Headway Cent accounts the terminal displays balance in cents
|
||||
(e.g. $15 USD shows as 1500.00). The --broker headway_cent flag triggers
|
||||
the divide-by-100 normalisation.
|
||||
"""
|
||||
if broker == "headway_cent":
|
||||
return raw_balance / CENT_MULTIPLIER
|
||||
return raw_balance
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Account queries
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_account_telemetry() -> dict:
|
||||
"""Return a snapshot of the current MT5 account state.
|
||||
|
||||
Raises ``RuntimeError`` if not connected to MT5.
|
||||
"""
|
||||
import MetaTrader5 as mt5
|
||||
info = mt5.account_info()
|
||||
if info is None:
|
||||
raise RuntimeError(
|
||||
f"mt5.account_info() returned None: {mt5.last_error()}. "
|
||||
"Is the terminal connected?"
|
||||
)
|
||||
telemetry = {
|
||||
"balance": info.balance,
|
||||
"equity": info.equity,
|
||||
"margin": info.margin,
|
||||
"free_margin": info.margin_free,
|
||||
"margin_level": info.margin_level,
|
||||
"currency": info.currency,
|
||||
"login": info.login,
|
||||
"server": info.server,
|
||||
"company": info.company,
|
||||
"trade_mode": info.trade_mode,
|
||||
}
|
||||
logger.debug(
|
||||
"Telemetry: balance=%.2f free_margin=%.2f equity=%.2f",
|
||||
telemetry["balance"], telemetry["free_margin"], telemetry["equity"],
|
||||
)
|
||||
return telemetry
|
||||
|
||||
|
||||
def display_account_info(trading_balance: float = None) -> None:
|
||||
"""Print a formatted account dashboard to stdout."""
|
||||
import MetaTrader5 as mt5
|
||||
t = get_account_telemetry()
|
||||
is_demo = t["trade_mode"] == mt5.ACCOUNT_TRADE_MODE_DEMO
|
||||
print("=" * 50)
|
||||
print(f" GOLD REGIME X — LIVE BRIDGE")
|
||||
print("=" * 50)
|
||||
print(f" Broker: {t['company']}")
|
||||
print(f" Login: {t['login']}")
|
||||
print(f" Server: {t['server']}")
|
||||
print(f" Mode: {'DEMO' if is_demo else 'LIVE'}")
|
||||
print(f" MT5 Balance: {t['balance']:.2f} {t['currency']}")
|
||||
if trading_balance is not None:
|
||||
print(f" Risk Balance:{trading_balance:.2f} USD (used for lot sizing)")
|
||||
print(f" Equity: {t['equity']:.2f} {t['currency']}")
|
||||
print(f" Free Margin: {t['free_margin']:.2f} {t['currency']}")
|
||||
print("=" * 50)
|
||||
|
||||
|
||||
def has_open_position(symbol: str = DEFAULT_SYMBOL, magic: int = MAGIC_NUMBER) -> bool:
|
||||
"""Return True if there is at least one open position for *symbol* with *magic*."""
|
||||
import MetaTrader5 as mt5
|
||||
positions = mt5.positions_get(symbol=symbol)
|
||||
if positions is None:
|
||||
logger.debug("positions_get returned None: %s", mt5.last_error())
|
||||
return False
|
||||
return any(p.magic == magic for p in positions)
|
||||
|
||||
|
||||
def check_margin(symbol: str, lot: float, order_type: int, price: float) -> bool:
|
||||
"""Pre-flight margin check via mt5.order_check().
|
||||
|
||||
Returns True if the order passes margin validation without submitting it.
|
||||
"""
|
||||
import MetaTrader5 as mt5
|
||||
request = {
|
||||
"action": mt5.TRADE_ACTION_DEAL,
|
||||
"symbol": symbol,
|
||||
"volume": float(lot),
|
||||
"type": order_type,
|
||||
"price": float(price),
|
||||
"deviation": DEFAULT_DEVIATION,
|
||||
"magic": MAGIC_NUMBER,
|
||||
"type_filling": mt5.ORDER_FILLING_IOC,
|
||||
}
|
||||
check = mt5.order_check(request)
|
||||
if check is None:
|
||||
logger.warning("order_check returned None: %s", mt5.last_error())
|
||||
return False
|
||||
ok = check.retcode == 0
|
||||
logger.debug("Margin check: retcode=%d margin=%.2f free_margin=%.2f",
|
||||
check.retcode, check.margin, check.margin_free)
|
||||
return ok
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Signal derivation
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def compute_deviation(model_hmm, current_state: int, tf: str = "H1") -> int:
|
||||
"""Select order deviation based on regime stability and timeframe.
|
||||
|
||||
A low self-transition probability on the current HMM state means the regime
|
||||
is likely to flip next bar — a proxy for elevated market volatility.
|
||||
Deviation is widened to avoid requotes in those conditions.
|
||||
M5 uses a higher base deviation (30) than M15/H1 (20) for scalping fills.
|
||||
"""
|
||||
self_prob = model_hmm.transmat_[current_state, current_state]
|
||||
base_dev = TF_DEFAULT_DEV.get(tf.upper(), 20)
|
||||
high_vol_dev = TF_HIGH_VOL_DEV.get(tf.upper(), 50)
|
||||
if self_prob < HIGH_VOL_SELF_TRANS_THRESHOLD:
|
||||
logger.debug(
|
||||
"High-vol deviation: state=%d self_trans=%.3f < %.2f — using %d pts",
|
||||
current_state, self_prob, HIGH_VOL_SELF_TRANS_THRESHOLD, high_vol_dev,
|
||||
)
|
||||
return high_vol_dev
|
||||
return base_dev
|
||||
|
||||
|
||||
def _move_sl_to_breakeven(ticket: int, entry_price: float, mt5) -> None:
|
||||
"""Modify an open position's SL to the entry price (break-even)."""
|
||||
positions = mt5.positions_get(ticket=ticket)
|
||||
if not positions:
|
||||
return
|
||||
pos = positions[0]
|
||||
request = {
|
||||
"action": mt5.TRADE_ACTION_SLTP,
|
||||
"position": ticket,
|
||||
"sl": round(entry_price, 2),
|
||||
"tp": pos.tp,
|
||||
}
|
||||
res = mt5.order_send(request)
|
||||
if res and res.retcode == mt5.TRADE_RETCODE_DONE:
|
||||
logger.info("Break-even SL set: ticket=%d sl=%.2f", ticket, entry_price)
|
||||
else:
|
||||
logger.warning("Break-even SL failed: ticket=%d retcode=%s",
|
||||
ticket, res.retcode if res else "None")
|
||||
|
||||
|
||||
def _close_position(ticket: int, mt5) -> None:
|
||||
"""Close a specific open position at market price."""
|
||||
positions = mt5.positions_get(ticket=ticket)
|
||||
if not positions:
|
||||
return
|
||||
pos = positions[0]
|
||||
close_type = mt5.ORDER_TYPE_SELL if pos.type == mt5.ORDER_TYPE_BUY else mt5.ORDER_TYPE_BUY
|
||||
tick = mt5.symbol_info_tick(pos.symbol)
|
||||
price = tick.bid if close_type == mt5.ORDER_TYPE_SELL else tick.ask
|
||||
request = {
|
||||
"action": mt5.TRADE_ACTION_DEAL,
|
||||
"position": ticket,
|
||||
"symbol": pos.symbol,
|
||||
"volume": pos.volume,
|
||||
"type": close_type,
|
||||
"price": price,
|
||||
"deviation": 20,
|
||||
"magic": MAGIC_NUMBER,
|
||||
"comment": "GRX_close_chop",
|
||||
"type_filling": mt5.ORDER_FILLING_IOC,
|
||||
}
|
||||
res = mt5.order_send(request)
|
||||
if res and res.retcode == mt5.TRADE_RETCODE_DONE:
|
||||
logger.info("Position closed (Chop exit): ticket=%d", ticket)
|
||||
else:
|
||||
logger.warning("Position close failed: ticket=%d retcode=%s",
|
||||
ticket, res.retcode if res else "None")
|
||||
|
||||
|
||||
def _build_live_df(
|
||||
symbol: str,
|
||||
tf_mt5: int,
|
||||
n_bars: int,
|
||||
obs_cov: float,
|
||||
trans_cov: float,
|
||||
) -> pd.DataFrame:
|
||||
"""Fetch the last *n_bars* completed bars and return a featurised DataFrame.
|
||||
|
||||
The currently open (incomplete) bar is excluded via offset=1.
|
||||
Feature sequence mirrors processor.process_pipeline() exactly.
|
||||
"""
|
||||
import MetaTrader5 as mt5
|
||||
rates = mt5.copy_rates_from_pos(symbol, tf_mt5, 1, n_bars)
|
||||
if rates is None or len(rates) == 0:
|
||||
raise RuntimeError(
|
||||
f"copy_rates_from_pos returned no data: {mt5.last_error()}"
|
||||
)
|
||||
df = pd.DataFrame(rates)
|
||||
df["time"] = pd.to_datetime(df["time"], unit="s", utc=True)
|
||||
df = (
|
||||
df.rename(columns={
|
||||
"time": "Date",
|
||||
"open": "Open",
|
||||
"high": "High",
|
||||
"low": "Low",
|
||||
"close": "Close",
|
||||
"tick_volume": "Volume",
|
||||
})
|
||||
[["Date", "Open", "High", "Low", "Close", "Volume"]]
|
||||
)
|
||||
df.set_index("Date", inplace=True)
|
||||
df.sort_index(inplace=True)
|
||||
|
||||
df["log_return"] = compute_log_returns(df["Close"])
|
||||
df["kalman_return"] = kalman_smooth(df["log_return"].values, obs_cov, trans_cov)
|
||||
df["volatility"] = compute_volatility(df["log_return"])
|
||||
df["rsi"] = compute_rsi(df["Close"])
|
||||
df["rsi_slope"] = df["rsi"].diff()
|
||||
df["atr_normalized"] = compute_atr(df)
|
||||
df.dropna(inplace=True)
|
||||
return df
|
||||
|
||||
|
||||
def compute_live_features(
|
||||
symbol: str,
|
||||
tf_mt5: int,
|
||||
model_hmm,
|
||||
obs_cov: float,
|
||||
trans_cov: float,
|
||||
):
|
||||
"""Build the current-bar feature vector for XGBoost inference.
|
||||
|
||||
Returns:
|
||||
features (np.ndarray shape (1,4)): [hmm_state, rsi_slope, atr_normalized, prev_log_return]
|
||||
hmm_state (int): current HMM regime index
|
||||
atr_price (float): ATR in price terms for SL calculation
|
||||
|
||||
NOTE: We build the 4-element feature vector manually (bypassing
|
||||
prepare_features()) to avoid the last-row NaN drop that strips the signal
|
||||
bar from training feature matrices. Column order must match FEATURE_COLS.
|
||||
"""
|
||||
df = _build_live_df(symbol, tf_mt5, N_BARS_WARMUP, obs_cov, trans_cov)
|
||||
states = predict_states(model_hmm, df)
|
||||
|
||||
current_state = int(states[-1])
|
||||
rsi_slope = float(df["rsi_slope"].iloc[-1])
|
||||
atr_normalized = float(df["atr_normalized"].iloc[-1])
|
||||
prev_log_return = float(df["log_return"].iloc[-2]) # bar N-1
|
||||
|
||||
# FEATURE_COLS = ["hmm_state", "rsi_slope", "atr_normalized", "prev_log_return"]
|
||||
features = np.array(
|
||||
[[float(current_state), rsi_slope, atr_normalized, prev_log_return]],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
atr_price = atr_normalized * float(df["Close"].iloc[-1])
|
||||
return features, current_state, atr_price
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Order execution
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def send_market_order(
|
||||
symbol: str = DEFAULT_SYMBOL,
|
||||
order_type: int = None,
|
||||
lot: float = 0.01,
|
||||
sl: float = 0.0,
|
||||
tp: float = 0.0,
|
||||
deviation: int = DEFAULT_DEVIATION,
|
||||
magic: int = MAGIC_NUMBER,
|
||||
comment: str = "GRX_Python",
|
||||
) -> dict:
|
||||
"""Send an IOC market order to the MT5 terminal.
|
||||
|
||||
Uses ORDER_FILLING_IOC (Immediate or Cancel) — standard for ECN/Cent
|
||||
brokers. If the fill price is outside the deviation window the order is
|
||||
cancelled rather than partially filled, preventing unintended exposure.
|
||||
|
||||
Returns a dict with keys: retcode, order, comment, success.
|
||||
Does NOT raise on failure — inspect result['success'].
|
||||
"""
|
||||
import MetaTrader5 as mt5
|
||||
tick = mt5.symbol_info_tick(symbol)
|
||||
if tick is None:
|
||||
logger.error("Cannot get tick for %s: %s", symbol, mt5.last_error())
|
||||
return {"retcode": -1, "order": 0, "comment": "No tick data", "success": False}
|
||||
|
||||
price = tick.ask if order_type == mt5.ORDER_TYPE_BUY else tick.bid
|
||||
|
||||
request = {
|
||||
"action": mt5.TRADE_ACTION_DEAL,
|
||||
"symbol": symbol,
|
||||
"volume": float(lot),
|
||||
"type": order_type,
|
||||
"price": float(price),
|
||||
"sl": float(sl),
|
||||
"tp": float(tp),
|
||||
"deviation": int(deviation),
|
||||
"magic": int(magic),
|
||||
"comment": comment,
|
||||
"type_time": mt5.ORDER_TIME_GTC,
|
||||
"type_filling": mt5.ORDER_FILLING_IOC,
|
||||
}
|
||||
|
||||
result = mt5.order_send(request)
|
||||
if result is None:
|
||||
logger.error("order_send returned None: %s", mt5.last_error())
|
||||
return {"retcode": -1, "order": 0, "comment": str(mt5.last_error()), "success": False}
|
||||
|
||||
success = result.retcode == mt5.TRADE_RETCODE_DONE
|
||||
if success:
|
||||
logger.info(
|
||||
"Order filled: #%d %s lot=%.2f price=%.2f sl=%.2f tp=%.2f dev=%d",
|
||||
result.order,
|
||||
"BUY" if order_type == mt5.ORDER_TYPE_BUY else "SELL",
|
||||
lot, price, sl, tp, deviation,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Order rejected: retcode=%d comment=%s",
|
||||
result.retcode, result.comment,
|
||||
)
|
||||
return {
|
||||
"retcode": result.retcode,
|
||||
"order": result.order,
|
||||
"comment": result.comment,
|
||||
"success": success,
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Live execution loop
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def run_live_loop(
|
||||
tf: str = "H1",
|
||||
broker: str = "headway_cent",
|
||||
account_size: float = None,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""Connect to MT5 and run the signal → order loop until interrupted.
|
||||
|
||||
Fires on each newly completed bar. Position sizing, session limits, and
|
||||
margin validation are enforced before every order.
|
||||
|
||||
Args:
|
||||
tf: Timeframe to trade — "H1" or "M15".
|
||||
broker: Broker config key from risk_manager.BROKER_CONFIGS.
|
||||
account_size: USD balance used for lot-sizing. If None, reads from MT5
|
||||
and normalises for cent accounts automatically.
|
||||
dry_run: When True, signals are logged but no orders are placed.
|
||||
Use with --account demo for paper-trading simulation.
|
||||
"""
|
||||
import MetaTrader5 as mt5
|
||||
from src.mt5_sync import connect_mt5, disconnect_mt5
|
||||
|
||||
if not connect_mt5():
|
||||
raise ConnectionError("Could not connect to MT5 terminal.")
|
||||
|
||||
try:
|
||||
_run_loop_inner(tf, broker, account_size, dry_run, mt5)
|
||||
finally:
|
||||
disconnect_mt5()
|
||||
logger.info("Live loop terminated. MT5 disconnected.")
|
||||
|
||||
|
||||
def _run_loop_inner(tf: str, broker: str, account_size: float, dry_run: bool, mt5) -> None:
|
||||
"""Inner loop extracted to allow clean finally / disconnect in run_live_loop."""
|
||||
tf_mt5 = _get_tf_map()[tf.upper()]
|
||||
|
||||
# Resolve live balance
|
||||
telemetry = get_account_telemetry()
|
||||
if account_size is None:
|
||||
account_size = _normalise_balance(telemetry["balance"], broker)
|
||||
logger.info(
|
||||
"Balance auto-detected: MT5 raw=%.2f USD normalised=%.2f",
|
||||
telemetry["balance"], account_size,
|
||||
)
|
||||
|
||||
display_account_info(trading_balance=account_size)
|
||||
|
||||
# Load models
|
||||
try:
|
||||
model_hmm = load_hmm()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError("HMM model not found. Run --mode train first.")
|
||||
try:
|
||||
model_xgb, _ = load_xgb()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError("XGB model not found. Run --mode train first.")
|
||||
|
||||
# Resolve Kalman params from Optuna study or TF defaults
|
||||
try:
|
||||
from src.optimizer import get_best_params
|
||||
params = get_best_params(balance=account_size, broker=broker, tf=tf)
|
||||
obs_cov = params.get("obs_cov")
|
||||
trans_cov = params.get("trans_cov")
|
||||
prob_threshold_opt = params.get("prob_threshold")
|
||||
except Exception:
|
||||
params = {}
|
||||
obs_cov = None
|
||||
trans_cov = None
|
||||
prob_threshold_opt = None
|
||||
|
||||
cfg_tf = TF_CONFIG[tf.upper()]
|
||||
obs_cov = obs_cov if obs_cov is not None else cfg_tf["obs_cov_default"]
|
||||
trans_cov = trans_cov if trans_cov is not None else cfg_tf["trans_cov_default"]
|
||||
|
||||
# Use optimized prob threshold if available; otherwise fall back to TF hardcoded values.
|
||||
# Short threshold is symmetric: 1 - prob_threshold.
|
||||
if prob_threshold_opt is not None:
|
||||
prob_threshold = prob_threshold_opt
|
||||
short_threshold = 1.0 - prob_threshold_opt
|
||||
else:
|
||||
prob_threshold = TF_PROB_THRESHOLD.get(tf.upper(), 0.65)
|
||||
short_threshold = TF_SHORT_THRESHOLD.get(tf.upper(), 0.35)
|
||||
|
||||
# Session state (persists across bars within a day)
|
||||
daily_trades = 0
|
||||
last_bar_time = None
|
||||
current_day = None
|
||||
# Tracks the tickets and entry context of the most recently placed signal
|
||||
# so the break-even SL logic can fire when TP1 closes position 1.
|
||||
signal_tracker = {"tickets": [], "entry_price": 0.0,
|
||||
"direction": None, "tp1_hit": False}
|
||||
|
||||
arm = AdaptiveRiskManager(account_size)
|
||||
logger.info(
|
||||
"Live loop started — TF=%s broker=%s balance=$%.2f dry_run=%s %s",
|
||||
tf, broker, account_size, dry_run, arm,
|
||||
)
|
||||
logger.info(
|
||||
"Signal thresholds — BUY>%.3f SELL<%.3f (source: %s)",
|
||||
prob_threshold, short_threshold,
|
||||
"optuna" if prob_threshold_opt is not None else "hardcoded",
|
||||
)
|
||||
|
||||
while True:
|
||||
try:
|
||||
# ── 1. Daily reset at UTC midnight ────────────────────────────
|
||||
today = datetime.now(timezone.utc).date()
|
||||
if today != current_day:
|
||||
daily_trades = 0
|
||||
current_day = today
|
||||
logger.info("New UTC day %s — session counter reset.", today)
|
||||
|
||||
# ── 2. Bar-change detection ───────────────────────────────────
|
||||
bars = mt5.copy_rates_from_pos(DEFAULT_SYMBOL, tf_mt5, 1, 1)
|
||||
if bars is None or len(bars) == 0:
|
||||
time.sleep(POLL_INTERVAL_SEC)
|
||||
continue
|
||||
bar_time = bars[0]["time"]
|
||||
if bar_time == last_bar_time:
|
||||
time.sleep(POLL_INTERVAL_SEC)
|
||||
continue
|
||||
last_bar_time = bar_time
|
||||
|
||||
# ── 3. Refresh telemetry for margin/display (risk sizing always
|
||||
# uses the caller-supplied account_size, not the live MT5
|
||||
# balance, because demo accounts carry arbitrary balances).
|
||||
try:
|
||||
telemetry = get_account_telemetry()
|
||||
except Exception:
|
||||
telemetry = {}
|
||||
|
||||
arm = AdaptiveRiskManager(account_size)
|
||||
arm.daily_trades = daily_trades # restore session count
|
||||
|
||||
# ── 4. Compute live features ──────────────────────────────────
|
||||
features, hmm_state, atr_price = compute_live_features(
|
||||
DEFAULT_SYMBOL, tf_mt5, model_hmm, obs_cov, trans_cov,
|
||||
)
|
||||
|
||||
# ── 5. Session limit check ────────────────────────────────────
|
||||
if not arm.can_trade(hmm_state):
|
||||
logger.info(
|
||||
"Session limit reached (state=%d daily=%d/%d). Skipping bar.",
|
||||
hmm_state, arm.daily_trades,
|
||||
arm.get_trade_limits(hmm_state)["max_daily_trades"],
|
||||
)
|
||||
time.sleep(POLL_INTERVAL_SEC)
|
||||
continue
|
||||
|
||||
# ── 6. Skip if position already open ─────────────────────────
|
||||
if has_open_position(DEFAULT_SYMBOL, MAGIC_NUMBER):
|
||||
logger.debug("Open position exists — skipping bar.")
|
||||
time.sleep(POLL_INTERVAL_SEC)
|
||||
continue
|
||||
|
||||
bar_str = datetime.fromtimestamp(bar_time, timezone.utc).strftime("%Y-%m-%d %H:%M")
|
||||
state_lbl = {0: "Bull", 1: "Bear", 2: "Chop"}.get(hmm_state, str(hmm_state))
|
||||
max_trades_today = arm.get_trade_limits(hmm_state)["max_daily_trades"]
|
||||
prob = float(model_xgb.predict_proba(features)[:, 1][0])
|
||||
logger.info(
|
||||
"Bar %s | state=%s | prob=%.3f | trades=%d/%d",
|
||||
bar_str, hmm_state, prob,
|
||||
arm.daily_trades, max_trades_today,
|
||||
)
|
||||
send_telegram_msg(
|
||||
f"📊 <b>Bar</b> {bar_str} [{tf}]\n"
|
||||
f"Regime: <b>{state_lbl}</b> ({hmm_state}) | "
|
||||
f"Prob: <b>{prob:.3f}</b>\n"
|
||||
f"Trades today: <b>{arm.daily_trades}/{max_trades_today}</b>"
|
||||
+ (" 🔴 <i>DRY RUN</i>" if dry_run else "")
|
||||
)
|
||||
|
||||
# ── 7b. Break-even SL and Chop-exit position management ───────
|
||||
if not dry_run and signal_tracker["tickets"]:
|
||||
open_tickets = {
|
||||
p.ticket
|
||||
for p in (mt5.positions_get(symbol=DEFAULT_SYMBOL) or [])
|
||||
if p.magic == MAGIC_NUMBER
|
||||
}
|
||||
active = [t for t in signal_tracker["tickets"] if t in open_tickets]
|
||||
# TP1 hit: first position closed → move runner's SL to break-even
|
||||
if len(active) < len(signal_tracker["tickets"]) and not signal_tracker["tp1_hit"]:
|
||||
signal_tracker["tp1_hit"] = True
|
||||
for ticket in active:
|
||||
_move_sl_to_breakeven(
|
||||
ticket, signal_tracker["entry_price"], mt5,
|
||||
)
|
||||
# Regime shifted to Chop while runner is active → close immediately
|
||||
if hmm_state == CHOP_STATE and active:
|
||||
for ticket in active:
|
||||
_close_position(ticket, mt5)
|
||||
logger.info("Runner(s) closed: Chop state shift while position open.")
|
||||
active = []
|
||||
signal_tracker["tickets"] = active
|
||||
|
||||
# ── 8. Signal routing (TF-specific thresholds) ───────────────
|
||||
if prob > prob_threshold and hmm_state != CHOP_STATE:
|
||||
direction = "BUY"
|
||||
order_type = mt5.ORDER_TYPE_BUY
|
||||
elif prob < short_threshold and hmm_state != CHOP_STATE:
|
||||
direction = "SELL"
|
||||
order_type = mt5.ORDER_TYPE_SELL
|
||||
else:
|
||||
logger.debug("No signal (prob=%.3f state=%d). Holding.", prob, hmm_state)
|
||||
time.sleep(POLL_INTERVAL_SEC)
|
||||
continue
|
||||
|
||||
# ── 9. Position sizing ────────────────────────────────────────
|
||||
tp_mults = _tp_multipliers(tf, hmm_state)
|
||||
limits = arm.get_trade_limits(hmm_state)
|
||||
pos_per_trade = min(limits["pos_per_trade"], len(tp_mults))
|
||||
sl_distance = max(atr_price * ATR_MULTIPLIER, 0.01)
|
||||
lot_total = arm.calculate_lot_size(stop_loss_pips=sl_distance)
|
||||
lot_per_pos = max(0.01, round(lot_total / pos_per_trade, 2))
|
||||
|
||||
# ── 10. Deviation (TF-specific + regime stability) ────────────
|
||||
deviation = compute_deviation(model_hmm, hmm_state, tf)
|
||||
|
||||
# ── 11. SL / TP prices ────────────────────────────────────────
|
||||
tick = mt5.symbol_info_tick(DEFAULT_SYMBOL)
|
||||
if direction == "BUY":
|
||||
entry_price = tick.ask
|
||||
sl_price = round(entry_price - sl_distance, 2)
|
||||
tp_levels = [round(entry_price + sl_distance * m, 2) for m in tp_mults]
|
||||
else:
|
||||
entry_price = tick.bid
|
||||
sl_price = round(entry_price + sl_distance, 2)
|
||||
tp_levels = [round(entry_price - sl_distance * m, 2) for m in tp_mults]
|
||||
|
||||
# ── 11b. Spread viability guard (M5 scalping only) ───────────
|
||||
if tf.upper() == "M5":
|
||||
sym_info = mt5.symbol_info(DEFAULT_SYMBOL)
|
||||
spread_price = (sym_info.spread * sym_info.point) if sym_info else 0.0
|
||||
tp1_distance = sl_distance * tp_mults[0]
|
||||
if spread_price > 0 and tp1_distance < spread_price * MIN_SPREAD_RATIO:
|
||||
logger.warning(
|
||||
"Unviable M5 trade: TP1=%.4f < %.1fx spread=%.4f. Skipping.",
|
||||
tp1_distance, MIN_SPREAD_RATIO, spread_price,
|
||||
)
|
||||
time.sleep(POLL_INTERVAL_SEC)
|
||||
continue
|
||||
|
||||
# ── 12. Margin check ──────────────────────────────────────────
|
||||
if not check_margin(DEFAULT_SYMBOL, lot_per_pos, order_type, entry_price):
|
||||
logger.warning(
|
||||
"Insufficient margin for %s lot=%.2f. Skipping. Free margin: %.2f",
|
||||
direction, lot_per_pos, telemetry.get("free_margin", 0),
|
||||
)
|
||||
time.sleep(POLL_INTERVAL_SEC)
|
||||
continue
|
||||
|
||||
# ── 13. Send order(s) with state-aware staged TPs ─────────────
|
||||
signal_tracker = {"tickets": [], "entry_price": entry_price,
|
||||
"direction": direction, "tp1_hit": False}
|
||||
logger.info(
|
||||
"SIGNAL %s | state=%d | prob=%.3f | lot×%d=%.2f | sl=%.2f | tp=%s | dev=%d",
|
||||
direction, hmm_state, prob, pos_per_trade, lot_per_pos,
|
||||
sl_price, "/".join(f"{t:.2f}" for t in tp_levels), deviation,
|
||||
)
|
||||
for p in range(pos_per_trade):
|
||||
tp_price = tp_levels[p]
|
||||
comment = f"GRX_{direction}_p{p+1}of{pos_per_trade}_s{hmm_state}_tp{p+1}"
|
||||
if dry_run:
|
||||
logger.info(
|
||||
"[DRY RUN] Would send: %s lot=%.2f sl=%.2f tp=%.2f dev=%d comment=%s",
|
||||
direction, lot_per_pos, sl_price, tp_price, deviation, comment,
|
||||
)
|
||||
else:
|
||||
result = send_market_order(
|
||||
symbol=DEFAULT_SYMBOL,
|
||||
order_type=order_type,
|
||||
lot=lot_per_pos,
|
||||
sl=sl_price,
|
||||
tp=tp_price,
|
||||
deviation=deviation,
|
||||
magic=MAGIC_NUMBER,
|
||||
comment=comment,
|
||||
)
|
||||
if result["success"]:
|
||||
signal_tracker["tickets"].append(result["order"])
|
||||
daily_trades += 1
|
||||
arm.log_trade()
|
||||
else:
|
||||
logger.error(
|
||||
"Order %d/%d failed — retcode=%d %s",
|
||||
p + 1, pos_per_trade,
|
||||
result["retcode"], result["comment"],
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
daily_trades += 1 # count dry-run signals for session limit testing
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("KeyboardInterrupt — stopping live loop.")
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Unhandled loop error: %s — sleeping 30 s before retry.",
|
||||
exc, exc_info=True,
|
||||
)
|
||||
time.sleep(30)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Telegram notification helper for Gold Regime X.
|
||||
|
||||
Reads credentials from environment variables (or .env file if python-dotenv
|
||||
is installed). All public functions degrade gracefully when credentials are
|
||||
missing — the rest of the system continues to operate without Telegram.
|
||||
|
||||
Environment variables required in .env:
|
||||
TELEGRAM_BOT_TOKEN — BotFather token (e.g. 123456789:AAF-xxx…)
|
||||
TELEGRAM_CHAT_ID — Your personal chat ID (see README_REMOTE.md)
|
||||
|
||||
Install:
|
||||
pip install requests python-dotenv
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
|
||||
from src.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Load .env on import if python-dotenv is available (silently ignored otherwise)
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
_TELEGRAM_API = "https://api.telegram.org/bot{token}/sendMessage"
|
||||
|
||||
|
||||
def get_credentials() -> tuple:
|
||||
"""Return (bot_token, chat_id) from environment variables."""
|
||||
return os.getenv("TELEGRAM_BOT_TOKEN"), os.getenv("TELEGRAM_CHAT_ID")
|
||||
|
||||
|
||||
def send_telegram_msg(message: str, parse_mode: str = "HTML") -> bool:
|
||||
"""Send a message to your configured Telegram chat.
|
||||
|
||||
Returns True on success, False on any failure (never raises).
|
||||
The ``parse_mode`` parameter accepts 'HTML' or 'Markdown'.
|
||||
Use HTML tags like <b>bold</b>, <code>monospace</code>, <i>italic</i>.
|
||||
"""
|
||||
token, chat_id = get_credentials()
|
||||
if not token or not chat_id:
|
||||
logger.debug("Telegram credentials not set — skipping notification.")
|
||||
return False
|
||||
|
||||
payload = {
|
||||
"chat_id": chat_id,
|
||||
"text": message,
|
||||
"parse_mode": parse_mode,
|
||||
}
|
||||
try:
|
||||
r = requests.post(
|
||||
_TELEGRAM_API.format(token=token),
|
||||
data=payload,
|
||||
timeout=10,
|
||||
)
|
||||
if not r.ok:
|
||||
logger.warning("Telegram API error %d: %s", r.status_code, r.text[:200])
|
||||
return False
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Telegram send failed: %s", exc)
|
||||
return False
|
||||
@@ -0,0 +1,356 @@
|
||||
"""Optuna hyperparameter optimiser for Gold Regime X.
|
||||
|
||||
Key design decisions:
|
||||
- SQLite study.db with load_if_exists=True: crash-safe resume at any point
|
||||
- Scores on OOS Sharpe ONLY to prevent IS data leakage / overfitting
|
||||
- Per-trial gc.collect() to prevent RAM accumulation over long M5 runs
|
||||
- Optional n_jobs>1: Optuna uses threads; note that XGBoost independently
|
||||
uses multiple cores, so n_jobs>2 typically yields diminishing returns
|
||||
unless you have 16+ cores. Default n_jobs=1 is safest.
|
||||
- psutil RAM guard: if RAM usage exceeds 90%, new trials pause 30s to let
|
||||
the OS breathe before dispatching more work.
|
||||
- Progress dashboard every 5 trials with ETA.
|
||||
- Telegram heartbeat every 10% of requested trials (if credentials set).
|
||||
"""
|
||||
|
||||
import gc
|
||||
import os
|
||||
import time
|
||||
|
||||
import optuna
|
||||
|
||||
from src.processor import process_pipeline
|
||||
from src.engine_hmm import fit_hmm
|
||||
from src.engine_xgb import prepare_features, train_xgb, get_predictions
|
||||
from src.backtester import vectorized_backtest
|
||||
from src.risk_manager import SMALL_ACCOUNT_THRESHOLD
|
||||
from src.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Optional psutil for RAM guard — degrades gracefully if not installed
|
||||
try:
|
||||
import psutil
|
||||
_PSUTIL_OK = True
|
||||
except ImportError:
|
||||
_PSUTIL_OK = False
|
||||
logger.debug("psutil not installed — RAM guard disabled.")
|
||||
|
||||
STUDY_DB = "sqlite:///models/study.db"
|
||||
DD_HARD_LIMIT = 0.15
|
||||
MIN_OOS_TRADES = 50
|
||||
RAM_HIGH_PCT = 90 # pause new trials when used RAM exceeds this %
|
||||
RAM_PAUSE_SEC = 30 # seconds to sleep when RAM is low
|
||||
|
||||
# Study tier objectives --------------------------------------------------------
|
||||
# Small accounts ($15–$50): extreme drawdown protection — Sharpe - DD×5.0
|
||||
# Growth accounts (>$50): balanced with higher frequency — Sharpe - DD×3.0
|
||||
TIER_CONFIGS = {
|
||||
"small": {"dd_penalty": 5.0, "dd_limit": 0.10},
|
||||
"growth": {"dd_penalty": 3.0, "dd_limit": 0.15},
|
||||
}
|
||||
|
||||
|
||||
def _get_tier(balance: float) -> str:
|
||||
return "small" if balance <= SMALL_ACCOUNT_THRESHOLD else "growth"
|
||||
|
||||
|
||||
def _score_result(result: dict, tier: str, broker: str) -> float:
|
||||
cfg = TIER_CONFIGS[tier]
|
||||
if result["max_drawdown"] > cfg["dd_limit"]:
|
||||
return -10.0
|
||||
if broker == "headway_cent":
|
||||
return result["sharpe_ratio"] - result["max_drawdown"] * cfg["dd_penalty"]
|
||||
return result["sharpe_ratio"]
|
||||
|
||||
|
||||
def make_objective(balance: float = 15.0, broker: str = "standard", tf: str = "H1"):
|
||||
"""Return an Optuna objective function for the given account / TF context."""
|
||||
tier = _get_tier(balance)
|
||||
|
||||
def objective(trial: optuna.Trial) -> float:
|
||||
obs_cov = trial.suggest_float("obs_cov", 0.1, 5.0, log=True)
|
||||
trans_cov = trial.suggest_float("trans_cov", 0.001, 0.1, log=True)
|
||||
n_states = trial.suggest_int("n_states", 2, 4)
|
||||
prob_threshold = trial.suggest_float("prob_threshold", 0.52, 0.68)
|
||||
|
||||
max_depth = trial.suggest_int("max_depth", 3, 5)
|
||||
learning_rate = trial.suggest_float("learning_rate", 0.01, 0.3, log=True)
|
||||
n_estimators = trial.suggest_int("n_estimators", 100, 500, step=50)
|
||||
subsample = trial.suggest_float("subsample", 0.6, 1.0)
|
||||
colsample_bytree = trial.suggest_float("colsample_bytree", 0.5, 1.0)
|
||||
min_child_weight = trial.suggest_int("min_child_weight", 1, 15)
|
||||
gamma = trial.suggest_float("gamma", 0.01, 5.0, log=True)
|
||||
reg_alpha = trial.suggest_float("reg_alpha", 0.01, 10.0, log=True)
|
||||
|
||||
try:
|
||||
df = process_pipeline(
|
||||
obs_cov=obs_cov, trans_cov=trans_cov, save=False, tf=tf
|
||||
)
|
||||
_hmm, states, _ = fit_hmm(df, n_states=n_states)
|
||||
X, y, df_aligned = prepare_features(df, states)
|
||||
model_xgb, metrics = train_xgb(
|
||||
X, y,
|
||||
max_depth=max_depth,
|
||||
learning_rate=learning_rate,
|
||||
n_estimators=n_estimators,
|
||||
subsample=subsample,
|
||||
colsample_bytree=colsample_bytree,
|
||||
min_child_weight=min_child_weight,
|
||||
gamma=gamma,
|
||||
reg_alpha=reg_alpha,
|
||||
)
|
||||
_, probabilities = get_predictions(model_xgb, X)
|
||||
|
||||
states_aligned = states[df.index.isin(df_aligned.index)]
|
||||
split_idx = metrics.get("split_idx")
|
||||
result = vectorized_backtest(
|
||||
df_aligned, probabilities, states_aligned,
|
||||
split_idx=split_idx,
|
||||
account_size=balance,
|
||||
broker=broker,
|
||||
tf=tf,
|
||||
prob_threshold=prob_threshold,
|
||||
)
|
||||
|
||||
# Score on OOS only to prevent IS data leakage
|
||||
if split_idx and "oos_sharpe_ratio" in result:
|
||||
if result.get("oos_n_trades", 0) < MIN_OOS_TRADES:
|
||||
return -10.0
|
||||
oos_result = {
|
||||
"sharpe_ratio": result["oos_sharpe_ratio"],
|
||||
"max_drawdown": result["oos_max_drawdown"],
|
||||
}
|
||||
score = _score_result(oos_result, tier, broker)
|
||||
else:
|
||||
score = _score_result(result, tier, broker)
|
||||
|
||||
logger.info(
|
||||
"Trial %d [%s/%s tier=%s $%.0f]: score=%.3f "
|
||||
"IS=%.3f OOS=%.3f OOS_DD=%.1f%% trades=%d",
|
||||
trial.number, tf, broker, tier, balance,
|
||||
score,
|
||||
result.get("is_sharpe_ratio", result["sharpe_ratio"]),
|
||||
result.get("oos_sharpe_ratio", result["sharpe_ratio"]),
|
||||
result.get("oos_max_drawdown", result["max_drawdown"]) * 100,
|
||||
result.get("oos_n_trades", result["n_trades"]),
|
||||
)
|
||||
return score
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Trial %d failed: %s", trial.number, e)
|
||||
return -10.0
|
||||
|
||||
finally:
|
||||
# Release HMM/XGB objects and aligned arrays to prevent RAM creep
|
||||
# over long M5 runs (~750K bars × 500 trials)
|
||||
gc.collect()
|
||||
|
||||
return objective
|
||||
|
||||
|
||||
def _make_callbacks(n_trials: int, study_name: str, already_done: int = 0) -> list:
|
||||
"""Build Optuna callbacks for progress reporting, RAM guard, and Telegram.
|
||||
|
||||
Args:
|
||||
n_trials: Number of trials requested for THIS session.
|
||||
study_name: Optuna study name (for Telegram messages).
|
||||
already_done: Completed trials in the DB before this session started.
|
||||
All progress percentages are relative to session trials only,
|
||||
so a resumed run shows [1/500] not [373/500].
|
||||
"""
|
||||
from src.notifier import send_telegram_msg
|
||||
|
||||
start_time = [time.time()] # mutable list for closure mutation
|
||||
heartbeat_pct = set() # tracks which 10% milestones were pinged
|
||||
|
||||
def _callback(study: optuna.Study, _trial: optuna.trial.FrozenTrial) -> None:
|
||||
total_done = len([
|
||||
t for t in study.trials
|
||||
if t.state == optuna.trial.TrialState.COMPLETE
|
||||
])
|
||||
# session_done = trials completed in THIS session only
|
||||
session_done = max(0, total_done - already_done)
|
||||
|
||||
if session_done <= 0:
|
||||
return
|
||||
|
||||
# ── RAM guard: pause before dispatching the next trial ───────────────
|
||||
if _PSUTIL_OK:
|
||||
mem = psutil.virtual_memory()
|
||||
if mem.percent >= RAM_HIGH_PCT:
|
||||
logger.warning(
|
||||
"RAM at %.0f%% (>= %d%%) — pausing %ds.",
|
||||
mem.percent, RAM_HIGH_PCT, RAM_PAUSE_SEC,
|
||||
)
|
||||
time.sleep(RAM_PAUSE_SEC)
|
||||
|
||||
# ── Progress dashboard every 5 session trials ─────────────────────────
|
||||
if session_done % 5 == 0:
|
||||
elapsed = time.time() - start_time[0]
|
||||
rate = elapsed / session_done
|
||||
remaining = max(0, n_trials - session_done)
|
||||
eta_sec = remaining * rate
|
||||
if eta_sec >= 3600:
|
||||
eta_str = f"{int(eta_sec // 3600)}h {int((eta_sec % 3600) // 60)}m"
|
||||
else:
|
||||
eta_str = f"{int(eta_sec // 60)}m {int(eta_sec % 60)}s"
|
||||
best = study.best_value if study.best_trial else float("-inf")
|
||||
ram_str = (
|
||||
f" RAM: {psutil.virtual_memory().percent:.0f}%"
|
||||
if _PSUTIL_OK else ""
|
||||
)
|
||||
print(
|
||||
f" [{session_done:>4}/{n_trials}] "
|
||||
f"Best Sharpe: {best:+.3f} | "
|
||||
f"ETA: {eta_str}{ram_str}"
|
||||
)
|
||||
|
||||
# ── Telegram heartbeat every 10% of SESSION trials ───────────────────
|
||||
if n_trials > 0:
|
||||
milestone = (int(session_done / n_trials * 100) // 10) * 10
|
||||
if milestone > 0 and milestone not in heartbeat_pct:
|
||||
heartbeat_pct.add(milestone)
|
||||
best = study.best_value if study.best_trial else float("-inf")
|
||||
send_telegram_msg(
|
||||
f"Optimization <b>{milestone}%</b> complete\n"
|
||||
f"Study: <code>{study_name}</code>\n"
|
||||
f"Best Sharpe: <b>{best:.3f}</b> | "
|
||||
f"Trials: {session_done}/{n_trials}"
|
||||
)
|
||||
|
||||
return [_callback]
|
||||
|
||||
|
||||
def _study_name(
|
||||
base: str = "gold_regime_x",
|
||||
broker: str = "standard",
|
||||
tier: str = "small",
|
||||
tf: str = "H1",
|
||||
) -> str:
|
||||
"""Deterministic study name — must be identical across sessions to resume."""
|
||||
parts = [base, tier, broker]
|
||||
if tf.upper() != "H1":
|
||||
parts.append(tf.upper())
|
||||
return "_".join(parts)
|
||||
|
||||
|
||||
def run_optimization(
|
||||
n_trials: int = 250,
|
||||
balance: float = 15.0,
|
||||
broker: str = "standard",
|
||||
tf: str = "H1",
|
||||
n_jobs: int = 1,
|
||||
) -> optuna.Study:
|
||||
"""Run (or resume) an Optuna study and return it when complete.
|
||||
|
||||
The SQLite study.db persists every completed trial to disk. If the run
|
||||
is interrupted at any point (Ctrl+C, crash, power cut), simply re-run the
|
||||
same command — Optuna will load_if_exists=True and pick up exactly where
|
||||
it left off.
|
||||
|
||||
To start fresh: delete models/study.db manually.
|
||||
|
||||
Args:
|
||||
n_trials: Additional trials to run THIS session. Optuna adds them
|
||||
on top of any already-completed trials in the DB.
|
||||
n_jobs: Parallel trial workers (Python threads). Default 1.
|
||||
XGBoost already uses all CPU cores internally, so n_jobs>2
|
||||
can cause core contention and actually slow things down.
|
||||
For true multi-process parallelism, open multiple terminals
|
||||
and run the same command simultaneously — they all share the
|
||||
same study.db safely via SQLite locking.
|
||||
"""
|
||||
tier = _get_tier(balance)
|
||||
name = _study_name(broker=broker, tier=tier, tf=tf)
|
||||
|
||||
os.makedirs("models", exist_ok=True)
|
||||
|
||||
study = optuna.create_study(
|
||||
study_name=name,
|
||||
storage=STUDY_DB,
|
||||
direction="maximize",
|
||||
load_if_exists=True, # crash-safe resume
|
||||
pruner=optuna.pruners.MedianPruner(),
|
||||
)
|
||||
|
||||
already_done = len([
|
||||
t for t in study.trials
|
||||
if t.state == optuna.trial.TrialState.COMPLETE
|
||||
])
|
||||
if already_done > 0:
|
||||
pct = already_done / (already_done + n_trials) * 100
|
||||
print(
|
||||
f"\nFailsafe: {already_done} completed trials found in study.db "
|
||||
f"({pct:.0f}% of total {already_done + n_trials}). "
|
||||
f"Resuming from trial #{already_done + 1}...\n"
|
||||
)
|
||||
else:
|
||||
print(f"\nStarting new study '{name}' — {n_trials} trials.\n")
|
||||
|
||||
logger.info(
|
||||
"Optimization: tf=%s broker=%s tier=%s balance=$%.0f "
|
||||
"n_trials=%d n_jobs=%d study=%s already_done=%d",
|
||||
tf, broker, tier, balance, n_trials, n_jobs, name, already_done,
|
||||
)
|
||||
|
||||
# Suppress Optuna's verbose per-trial INFO logs; our callback handles output
|
||||
optuna.logging.set_verbosity(optuna.logging.WARNING)
|
||||
|
||||
study.optimize(
|
||||
make_objective(balance=balance, broker=broker, tf=tf),
|
||||
n_trials=n_trials,
|
||||
n_jobs=n_jobs,
|
||||
show_progress_bar=(n_jobs == 1), # tqdm bar misleads with threads
|
||||
callbacks=_make_callbacks(n_trials, name, already_done=already_done),
|
||||
)
|
||||
|
||||
best = study.best_value
|
||||
total_done = len([
|
||||
t for t in study.trials
|
||||
if t.state == optuna.trial.TrialState.COMPLETE
|
||||
])
|
||||
logger.info(
|
||||
"Optimization done: best_score=%.3f total_trials=%d params=%s",
|
||||
best, total_done, study.best_params,
|
||||
)
|
||||
|
||||
# ── Completion banner ─────────────────────────────────────────────────────
|
||||
pad = 46
|
||||
print(
|
||||
"\n"
|
||||
"************************************************************\n"
|
||||
"* *\n"
|
||||
"* OPTIMIZATION COMPLETE *\n"
|
||||
f"* Study : {name:<{pad}}*\n"
|
||||
f"* Trials: {total_done:<{pad}}*\n"
|
||||
f"* Best : {best:<+{pad}.4f}*\n"
|
||||
"* *\n"
|
||||
"************************************************************\n"
|
||||
)
|
||||
|
||||
from src.notifier import send_telegram_msg
|
||||
send_telegram_msg(
|
||||
f"<b>Optimization 100% Complete!</b>\n"
|
||||
f"Study: <code>{name}</code>\n"
|
||||
f"Trials: <b>{total_done}</b>\n"
|
||||
f"Best OOS Sharpe: <b>{best:.3f}</b>\n"
|
||||
+ "\n".join(
|
||||
f" <code>{k}</code>: {v}"
|
||||
for k, v in study.best_params.items()
|
||||
)
|
||||
)
|
||||
|
||||
return study
|
||||
|
||||
|
||||
def get_best_params(
|
||||
balance: float = 15.0,
|
||||
broker: str = "standard",
|
||||
tf: str = "H1",
|
||||
) -> dict:
|
||||
"""Load the best hyperparameters from the persisted SQLite study."""
|
||||
tier = _get_tier(balance)
|
||||
name = _study_name(broker=broker, tier=tier, tf=tf)
|
||||
study = optuna.load_study(study_name=name, storage=STUDY_DB)
|
||||
return study.best_params
|
||||
@@ -0,0 +1,157 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from src.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Per-timeframe configuration ─────────────────────────────────────────────────
|
||||
# Kalman obs_cov controls smoothing: higher value = more smoothing (less trust
|
||||
# in raw observations). M15 is ~4× noisier than H1; M5 uses a low obs_cov
|
||||
# (0.05) to keep the filter very responsive for fast scalping regime detection.
|
||||
# n_states_default: M5 uses 4 states to capture micro-noise; M15/H1 use 3.
|
||||
TF_CONFIG = {
|
||||
"H1": {
|
||||
"raw_path": Path("data/raw/XAU_1h_data.csv"),
|
||||
"processed_path": Path("data/processed/gold_h1_processed.parquet"),
|
||||
"obs_cov_default": 1.0,
|
||||
"trans_cov_default": 0.01,
|
||||
"n_states_default": 3,
|
||||
},
|
||||
"M15": {
|
||||
"raw_path": Path("data/raw/XAU_15m_data.csv"),
|
||||
"processed_path": Path("data/processed/gold_m15_processed.parquet"),
|
||||
"obs_cov_default": 4.0,
|
||||
"trans_cov_default": 0.01,
|
||||
"n_states_default": 3,
|
||||
},
|
||||
"M5": {
|
||||
"raw_path": Path("data/raw/XAU_5m_data.csv"),
|
||||
"processed_path": Path("data/processed/gold_m5_processed.parquet"),
|
||||
"obs_cov_default": 0.05, # low value = responsive Kalman for scalping
|
||||
"trans_cov_default": 0.01,
|
||||
"n_states_default": 4, # 4 states to capture micro-regime noise
|
||||
},
|
||||
}
|
||||
|
||||
# Legacy aliases kept for callers that use the bare constants directly
|
||||
RAW_PATH = TF_CONFIG["H1"]["raw_path"]
|
||||
PROCESSED_PATH = TF_CONFIG["H1"]["processed_path"]
|
||||
|
||||
|
||||
def load_raw_data(path: Path = RAW_PATH) -> pd.DataFrame:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Raw data file not found: {path}\n"
|
||||
"Export XAUUSD data from MT5 as CSV (semicolon-delimited) and place it at that path."
|
||||
)
|
||||
df = pd.read_csv(
|
||||
path, sep=";", parse_dates=["Date"], date_format="%Y.%m.%d %H:%M"
|
||||
)
|
||||
df.set_index("Date", inplace=True)
|
||||
df.sort_index(inplace=True)
|
||||
logger.info("Loaded %d rows from %s", len(df), path)
|
||||
return df
|
||||
|
||||
|
||||
def filter_data(df: pd.DataFrame, years: int = 10) -> pd.DataFrame:
|
||||
cutoff = df.index.max() - pd.DateOffset(years=years)
|
||||
df = df[df.index >= cutoff]
|
||||
before = len(df)
|
||||
df = df[df["Volume"] > 5]
|
||||
logger.info(
|
||||
"Filtered to last %d years: %d rows (%d low-volume bars removed)",
|
||||
years, len(df), before - len(df),
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def compute_log_returns(close: pd.Series) -> pd.Series:
|
||||
return np.log(close / close.shift(1)).rename("log_return")
|
||||
|
||||
|
||||
def kalman_smooth(data: np.ndarray, obs_cov: float = 1.0, trans_cov: float = 0.01) -> np.ndarray:
|
||||
mu = 0.0
|
||||
P = 1.0
|
||||
smoothed = np.zeros_like(data, dtype=np.float64)
|
||||
for i in range(len(data)):
|
||||
P = P + trans_cov
|
||||
if np.isnan(data[i]):
|
||||
smoothed[i] = mu
|
||||
else:
|
||||
K = P / (P + obs_cov)
|
||||
mu = mu + K * (data[i] - mu)
|
||||
P = (1 - K) * P
|
||||
smoothed[i] = mu
|
||||
return smoothed
|
||||
|
||||
|
||||
def compute_volatility(log_returns: pd.Series, window: int = 20) -> pd.Series:
|
||||
return log_returns.rolling(window=window).std().rename("volatility")
|
||||
|
||||
|
||||
def compute_rsi(close: pd.Series, period: int = 14) -> pd.Series:
|
||||
delta = close.diff()
|
||||
gain = delta.clip(lower=0)
|
||||
loss = -delta.clip(upper=0)
|
||||
avg_gain = gain.ewm(alpha=1 / period, min_periods=period).mean()
|
||||
avg_loss = loss.ewm(alpha=1 / period, min_periods=period).mean()
|
||||
rs = avg_gain / avg_loss
|
||||
return (100 - (100 / (1 + rs))).rename("rsi")
|
||||
|
||||
|
||||
def compute_atr(df: pd.DataFrame, period: int = 14) -> pd.Series:
|
||||
high_low = df["High"] - df["Low"]
|
||||
high_close = (df["High"] - df["Close"].shift(1)).abs()
|
||||
low_close = (df["Low"] - df["Close"].shift(1)).abs()
|
||||
true_range = pd.concat([high_low, high_close, low_close], axis=1).max(axis=1)
|
||||
atr = true_range.rolling(window=period).mean()
|
||||
return (atr / df["Close"]).rename("atr_normalized")
|
||||
|
||||
|
||||
def process_pipeline(
|
||||
obs_cov: float = None,
|
||||
trans_cov: float = None,
|
||||
save: bool = True,
|
||||
tf: str = "H1",
|
||||
) -> pd.DataFrame:
|
||||
"""Run the full feature-engineering pipeline for the given timeframe.
|
||||
|
||||
Args:
|
||||
obs_cov: Kalman observation covariance. Defaults to TF-specific value
|
||||
(1.0 for H1, 4.0 for M15).
|
||||
trans_cov: Kalman transition covariance. Defaults to 0.01.
|
||||
save: Persist processed parquet to disk.
|
||||
tf: Timeframe — ``"H1"`` or ``"M15"``.
|
||||
"""
|
||||
tf = tf.upper()
|
||||
if tf not in TF_CONFIG:
|
||||
raise ValueError(f"Unknown timeframe '{tf}'. Choose from {list(TF_CONFIG)}")
|
||||
|
||||
cfg = TF_CONFIG[tf]
|
||||
obs_cov = obs_cov if obs_cov is not None else cfg["obs_cov_default"]
|
||||
trans_cov = trans_cov if trans_cov is not None else cfg["trans_cov_default"]
|
||||
|
||||
df = load_raw_data(cfg["raw_path"])
|
||||
df = filter_data(df)
|
||||
|
||||
df["log_return"] = compute_log_returns(df["Close"])
|
||||
df["kalman_return"] = kalman_smooth(df["log_return"].values, obs_cov, trans_cov)
|
||||
df["volatility"] = compute_volatility(df["log_return"])
|
||||
df["rsi"] = compute_rsi(df["Close"])
|
||||
df["rsi_slope"] = df["rsi"].diff()
|
||||
df["atr_normalized"] = compute_atr(df)
|
||||
|
||||
df.dropna(inplace=True)
|
||||
logger.info(
|
||||
"Pipeline [%s] complete: %d rows, columns: %s",
|
||||
tf, len(df), list(df.columns),
|
||||
)
|
||||
|
||||
if save:
|
||||
out_path = cfg["processed_path"]
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_parquet(out_path)
|
||||
logger.info("Saved processed data to %s", out_path)
|
||||
|
||||
return df
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Telegram remote control panel for Gold Regime X.
|
||||
|
||||
Runs a long-polling Telegram bot that accepts keyboard button commands from
|
||||
the authorised user and dispatches them as subprocesses so the listener
|
||||
thread is never blocked.
|
||||
|
||||
Keyboard layout:
|
||||
┌──────────────────────┬─────────────────────┐
|
||||
│ 🚀 START TRADING │ 🛑 STOP TRADING │
|
||||
├──────────────────────┼─────────────────────┤
|
||||
│ 📉 START OPTIMIZE │ 📊 BOT STATUS │
|
||||
│ (M5) │ │
|
||||
└──────────────────────┴─────────────────────┘
|
||||
|
||||
Required env vars (see .env.example):
|
||||
TELEGRAM_BOT_TOKEN — BotFather token
|
||||
TELEGRAM_CHAT_ID — Your chat ID (used for outbound heartbeat messages)
|
||||
ALLOWED_USER_ID — Your numeric Telegram user ID (security gate)
|
||||
LIVE_TF — Default TF for START TRADING (default: H1)
|
||||
LIVE_BROKER — Default broker (default: headway_cent)
|
||||
LIVE_BALANCE — Default balance (default: 15)
|
||||
|
||||
Usage:
|
||||
python main.py --mode listen
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import requests
|
||||
from requests.exceptions import ReadTimeout
|
||||
|
||||
from src.logger import setup_logger
|
||||
from src.notifier import get_credentials, send_telegram_msg
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# ── Telegram keyboard sent with every reply ────────────────────────────────────
|
||||
_KEYBOARD = {
|
||||
"keyboard": [
|
||||
["🚀 START TRADING", "🛑 STOP TRADING"],
|
||||
["📉 START OPTIMIZE (M5)", "📊 BOT STATUS"],
|
||||
],
|
||||
"resize_keyboard": True,
|
||||
"one_time_keyboard": False,
|
||||
}
|
||||
|
||||
# Tracks subprocesses launched by this session so we can terminate them
|
||||
_procs: dict[str, subprocess.Popen] = {}
|
||||
|
||||
|
||||
def _api(token: str, method: str, _req_timeout=(10, 15), **params) -> dict:
|
||||
"""Call a Telegram Bot API method; returns the parsed JSON response.
|
||||
|
||||
``_req_timeout`` is passed to requests as (connect_timeout, read_timeout)
|
||||
and is intentionally NOT forwarded to Telegram — hence the underscore prefix.
|
||||
Long-polling calls should supply a read timeout > the Telegram ``timeout``
|
||||
parameter to avoid spurious ReadTimeout exceptions.
|
||||
"""
|
||||
try:
|
||||
r = requests.post(
|
||||
f"https://api.telegram.org/bot{token}/{method}",
|
||||
json=params,
|
||||
timeout=_req_timeout,
|
||||
)
|
||||
return r.json()
|
||||
except ReadTimeout:
|
||||
# Expected for getUpdates when no new messages arrive within the poll
|
||||
# window — Telegram returns an empty result list, which is normal.
|
||||
logger.debug("Telegram API (%s): long-poll returned empty (no new messages).", method)
|
||||
return {}
|
||||
except Exception as exc:
|
||||
logger.warning("Telegram API (%s) failed: %s", method, exc)
|
||||
return {}
|
||||
|
||||
|
||||
def _reply(token: str, chat_id, text: str) -> None:
|
||||
"""Send a message with the persistent control keyboard attached."""
|
||||
_api(
|
||||
token, "sendMessage",
|
||||
chat_id=chat_id,
|
||||
text=text,
|
||||
reply_markup=_KEYBOARD,
|
||||
parse_mode="HTML",
|
||||
)
|
||||
|
||||
|
||||
def _proc_alive(key: str) -> bool:
|
||||
"""Return True if the subprocess stored under *key* is still running."""
|
||||
proc = _procs.get(key)
|
||||
return proc is not None and proc.poll() is None
|
||||
|
||||
|
||||
def _handle(token: str, chat_id, user_id: str, text: str) -> None:
|
||||
"""Authorise and dispatch a single inbound command."""
|
||||
allowed = os.getenv("ALLOWED_USER_ID", "")
|
||||
if user_id != allowed:
|
||||
_api(token, "sendMessage", chat_id=chat_id, text="Unauthorized.")
|
||||
return
|
||||
|
||||
tf = os.getenv("LIVE_TF", "H1")
|
||||
broker = os.getenv("LIVE_BROKER", "headway_cent")
|
||||
balance = os.getenv("LIVE_BALANCE", "15")
|
||||
|
||||
if text == "🚀 START TRADING":
|
||||
if _proc_alive("trading"):
|
||||
_reply(token, chat_id, "Trading is already running.")
|
||||
return
|
||||
_procs["trading"] = subprocess.Popen([
|
||||
"python", "main.py",
|
||||
"--mode", "live", "--account", "live",
|
||||
"--tf", tf, "--broker", broker, "--balance", balance,
|
||||
])
|
||||
_reply(token, chat_id,
|
||||
f"<b>Trading started</b>\nTF={tf} broker={broker} balance=${balance}")
|
||||
|
||||
elif text == "🛑 STOP TRADING":
|
||||
if _proc_alive("trading"):
|
||||
_procs["trading"].terminate()
|
||||
_reply(token, chat_id, "<b>Trading stopped.</b>")
|
||||
else:
|
||||
_reply(token, chat_id, "No active trading process found.")
|
||||
|
||||
elif text == "📉 START OPTIMIZE (M5)":
|
||||
if _proc_alive("optimizer"):
|
||||
_reply(token, chat_id, "Optimization is already running.")
|
||||
return
|
||||
_procs["optimizer"] = subprocess.Popen([
|
||||
"python", "main.py",
|
||||
"--mode", "optimize", "--tf", "M5",
|
||||
"--broker", broker, "--balance", balance, "--trials", "500",
|
||||
])
|
||||
_reply(token, chat_id,
|
||||
"M5 optimization started.\n"
|
||||
"(Resumes from study.db if previous progress exists.)")
|
||||
|
||||
elif text == "📊 BOT STATUS":
|
||||
try:
|
||||
from src.auditor import get_daily_report
|
||||
report = get_daily_report(broker=broker)
|
||||
except Exception as exc:
|
||||
report = f"Status unavailable: {exc}"
|
||||
_reply(token, chat_id, report)
|
||||
|
||||
else:
|
||||
_reply(token, chat_id, "Use the keyboard buttons below to control the bot.")
|
||||
|
||||
|
||||
def run_listener() -> None:
|
||||
"""Poll Telegram for updates and dispatch commands until KeyboardInterrupt.
|
||||
|
||||
Uses the getUpdates long-polling method (timeout=30s) so the bot reacts
|
||||
within seconds without holding a persistent WebSocket connection.
|
||||
"""
|
||||
token, _ = get_credentials()
|
||||
if not token:
|
||||
raise RuntimeError(
|
||||
"TELEGRAM_BOT_TOKEN not set. "
|
||||
"Copy .env.example to .env and fill in your credentials."
|
||||
)
|
||||
|
||||
logger.info("Remote control listener started. Waiting for commands...")
|
||||
send_telegram_msg("<b>Gold Regime X</b> remote control is <b>online</b>.")
|
||||
|
||||
offset = 0
|
||||
while True:
|
||||
try:
|
||||
# timeout=25: Telegram holds the connection up to 25s for new msgs.
|
||||
# _req_timeout read must exceed that so requests doesn't give up first.
|
||||
data = _api(token, "getUpdates", offset=offset,
|
||||
timeout=25, _req_timeout=(10, 35))
|
||||
for update in data.get("result", []):
|
||||
offset = update["update_id"] + 1
|
||||
msg = update.get("message", {})
|
||||
chat_id = msg.get("chat", {}).get("id")
|
||||
user_id = str(msg.get("from", {}).get("id", ""))
|
||||
text = msg.get("text", "").strip()
|
||||
if chat_id and text:
|
||||
logger.info(
|
||||
"Inbound: user_id=%s chat_id=%s text=%r",
|
||||
user_id, chat_id, text,
|
||||
)
|
||||
_handle(token, chat_id, user_id, text)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Remote control stopped.")
|
||||
send_telegram_msg("Gold Regime X remote control is <b>offline</b>.")
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.error("Listener error: %s — retrying in 10s", exc)
|
||||
time.sleep(10)
|
||||
@@ -0,0 +1,149 @@
|
||||
from src.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Spread and commission fractions of price per trade (normalized cost)
|
||||
BROKER_CONFIGS = {
|
||||
"standard": {
|
||||
"spread_frac": 0.0002,
|
||||
"commission_frac": 0.0001,
|
||||
},
|
||||
"headway_cent": {
|
||||
# XAUUSD on Headway Cent: ~$0.30 spread + ~$0.03/micro-lot commission
|
||||
"spread_frac": 0.0002,
|
||||
"commission_frac": 0.0002,
|
||||
},
|
||||
}
|
||||
|
||||
MIN_CAPITAL_USD = 15.0
|
||||
CENT_MULTIPLIER = 100 # Headway Cent: 1 USD displayed as 100 cents
|
||||
SMALL_ACCOUNT_THRESHOLD = 50.0 # USD — boundary between small and growth tiers
|
||||
|
||||
|
||||
class CentConverter:
|
||||
"""Lot sizing for Headway Cent accounts (micro-lot floor = 0.01)."""
|
||||
|
||||
MIN_LOT = 0.01
|
||||
|
||||
def __init__(self, account_balance_usd: float):
|
||||
self.account_balance_usd = max(float(account_balance_usd), MIN_CAPITAL_USD)
|
||||
|
||||
@property
|
||||
def displayed_balance(self) -> float:
|
||||
return self.account_balance_usd * CENT_MULTIPLIER
|
||||
|
||||
def calculate_lot(self, risk_pct: float, stop_distance_norm: float) -> float:
|
||||
if stop_distance_norm <= 0:
|
||||
return self.MIN_LOT
|
||||
risk_amount = self.account_balance_usd * risk_pct
|
||||
lot = risk_amount / stop_distance_norm
|
||||
lot = max(self.MIN_LOT, (lot // self.MIN_LOT) * self.MIN_LOT)
|
||||
return round(lot, 2)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"CentConverter(balance=${self.account_balance_usd:.2f}, "
|
||||
f"displayed={self.displayed_balance:.0f} cents)"
|
||||
)
|
||||
|
||||
|
||||
class SessionManager:
|
||||
"""Legacy per-session limiter (kept for backwards compat).
|
||||
|
||||
Prefer ``AdaptiveRiskManager`` for new code.
|
||||
"""
|
||||
|
||||
def __init__(self, account_size: float):
|
||||
self.account_size = float(account_size)
|
||||
# Delegate to AdaptiveRiskManager for consistent limits
|
||||
arm = AdaptiveRiskManager(account_size)
|
||||
base = arm.get_trade_limits()
|
||||
self.max_trades = base["max_daily_trades"]
|
||||
self.trades_this_session = 0
|
||||
|
||||
def can_trade(self) -> bool:
|
||||
return self.trades_this_session < self.max_trades
|
||||
|
||||
def log_trade(self):
|
||||
self.trades_this_session += 1
|
||||
|
||||
def reset_session(self):
|
||||
self.trades_this_session = 0
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"SessionManager(account=${self.account_size:.0f}, "
|
||||
f"max={self.max_trades}, used={self.trades_this_session})"
|
||||
)
|
||||
|
||||
|
||||
class AdaptiveRiskManager:
|
||||
"""Dynamic trade limits and lot sizing per the Headway Cent spec.
|
||||
|
||||
Account tiers
|
||||
─────────────
|
||||
≤ $50 USD — small account:
|
||||
• max_daily_trades : 2
|
||||
• pos_per_trade : 1 (single position per signal)
|
||||
• total daily pos : 2
|
||||
|
||||
> $50 USD — growth account:
|
||||
• max_daily_trades : 3 in Bull/Bear, 2 in Chop (market-dependent)
|
||||
• pos_per_trade : 2 (dual/hedging positions per signal)
|
||||
• total daily pos : 4 or 6
|
||||
"""
|
||||
|
||||
CHOP_STATE = 2
|
||||
|
||||
def __init__(self, balance: float):
|
||||
self.balance = max(float(balance), MIN_CAPITAL_USD)
|
||||
self.daily_trades = 0
|
||||
tier = "small" if self.balance <= SMALL_ACCOUNT_THRESHOLD else "growth"
|
||||
logger.debug("AdaptiveRiskManager: balance=$%.2f tier=%s", self.balance, tier)
|
||||
|
||||
@property
|
||||
def is_small_account(self) -> bool:
|
||||
return self.balance <= SMALL_ACCOUNT_THRESHOLD
|
||||
|
||||
def get_trade_limits(self, market_state: int = None) -> dict:
|
||||
"""Return trade limits for the current balance and optional HMM state.
|
||||
|
||||
Args:
|
||||
market_state: Current HMM state index. If None, uses the most
|
||||
permissive limit for the account tier.
|
||||
"""
|
||||
if self.is_small_account:
|
||||
return {"max_daily_trades": 2, "pos_per_trade": 1, "total_daily_pos": 2}
|
||||
|
||||
# Growth account: market-state-dependent
|
||||
in_chop = (market_state == self.CHOP_STATE) if market_state is not None else False
|
||||
max_daily = 2 if in_chop else 3
|
||||
pos_per_trade = 2
|
||||
return {
|
||||
"max_daily_trades": max_daily,
|
||||
"pos_per_trade": pos_per_trade,
|
||||
"total_daily_pos": max_daily * pos_per_trade,
|
||||
}
|
||||
|
||||
def calculate_lot_size(self, stop_loss_pips: float) -> float:
|
||||
"""1% risk rule with Headway Cent pip-value adjustment."""
|
||||
risk_per_trade = self.balance * 0.01
|
||||
if stop_loss_pips <= 0:
|
||||
return 0.01
|
||||
# Cent account pip value: stop_loss_pips × 0.01
|
||||
lot_size = risk_per_trade / (stop_loss_pips * 0.01)
|
||||
return max(0.01, round(lot_size, 2))
|
||||
|
||||
def can_trade(self, market_state: int = None) -> bool:
|
||||
limits = self.get_trade_limits(market_state)
|
||||
return self.daily_trades < limits["max_daily_trades"]
|
||||
|
||||
def log_trade(self):
|
||||
self.daily_trades += 1
|
||||
|
||||
def reset_daily(self):
|
||||
self.daily_trades = 0
|
||||
|
||||
def __repr__(self) -> str:
|
||||
tier = "small" if self.is_small_account else "growth"
|
||||
return f"AdaptiveRiskManager(balance=${self.balance:.0f}, tier={tier}, daily_trades={self.daily_trades})"
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Model Validation Gatekeeper.
|
||||
|
||||
Loads recently synced MT5 data, applies the full feature pipeline, runs live
|
||||
inference through the saved HMM + XGBoost models, and computes an annualised
|
||||
Sharpe Ratio over the recent window. The result is classified as:
|
||||
|
||||
pass (Sharpe >= 0.8) — model is stable; proceed to live.
|
||||
warn (Sharpe 0.5–0.8) — borderline; proceed with extreme caution.
|
||||
fail (Sharpe < 0.5) — model drift detected; retune before going live.
|
||||
|
||||
NOTE: This validation Sharpe is computed on the most recent N months of live
|
||||
market data. It is NOT equivalent to the OOS Sharpe from training, which uses
|
||||
an 80/20 chronological split of the full 10-year dataset. Both metrics matter,
|
||||
but they answer different questions.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
from src.logger import setup_logger
|
||||
from src.processor import (
|
||||
TF_CONFIG,
|
||||
compute_log_returns,
|
||||
kalman_smooth,
|
||||
compute_volatility,
|
||||
compute_rsi,
|
||||
compute_atr,
|
||||
)
|
||||
from src.engine_hmm import load_model as load_hmm, predict_states
|
||||
from src.engine_xgb import load_xgb, prepare_features, get_predictions
|
||||
from src.backtester import vectorized_backtest
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
SYNC_DATA_PATH = Path("data/processed/mt5_sync_data.csv")
|
||||
SHARPE_PASS_THRESHOLD = 0.8
|
||||
SHARPE_WARN_THRESHOLD = 0.5
|
||||
MIN_TRADES_WARNING = 30 # warn (not fail) if fewer trades than this
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Internal helpers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _apply_features(
|
||||
df: pd.DataFrame,
|
||||
tf: str,
|
||||
obs_cov: float,
|
||||
trans_cov: float,
|
||||
) -> pd.DataFrame:
|
||||
"""Apply the same feature steps as process_pipeline() to an arbitrary frame.
|
||||
|
||||
This function mirrors processor.process_pipeline() exactly. If any
|
||||
standalone function in processor.py changes its behaviour, this must be
|
||||
updated to match — both must produce identical column values for the same
|
||||
input data.
|
||||
"""
|
||||
cfg = TF_CONFIG[tf.upper()]
|
||||
obs_cov = obs_cov if obs_cov is not None else cfg["obs_cov_default"]
|
||||
trans_cov = trans_cov if trans_cov is not None else cfg["trans_cov_default"]
|
||||
|
||||
df = df.copy()
|
||||
df["log_return"] = compute_log_returns(df["Close"])
|
||||
df["kalman_return"] = kalman_smooth(df["log_return"].values, obs_cov, trans_cov)
|
||||
df["volatility"] = compute_volatility(df["log_return"])
|
||||
df["rsi"] = compute_rsi(df["Close"])
|
||||
df["rsi_slope"] = df["rsi"].diff()
|
||||
df["atr_normalized"] = compute_atr(df)
|
||||
df.dropna(inplace=True)
|
||||
return df
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Primary entry point
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def run_validation(
|
||||
sync_data_path: Path = SYNC_DATA_PATH,
|
||||
tf: str = "H1",
|
||||
broker: str = "headway_cent",
|
||||
account_size: float = 15.0,
|
||||
obs_cov: float = None,
|
||||
trans_cov: float = None,
|
||||
) -> dict:
|
||||
"""Validate the saved models against recent synced MT5 data.
|
||||
|
||||
Returns a dict:
|
||||
sharpe (float) — annualised Sharpe over the validation window
|
||||
n_trades (int) — number of trades fired by the model
|
||||
win_rate (float) — fraction of winning trades
|
||||
max_dd (float) — maximum drawdown (0–1)
|
||||
status (str) — 'pass', 'warn', or 'fail'
|
||||
message (str) — human-readable gate decision
|
||||
|
||||
Raises:
|
||||
FileNotFoundError if the sync CSV or model files are missing.
|
||||
"""
|
||||
sync_data_path = Path(sync_data_path)
|
||||
if not sync_data_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Sync data not found at {sync_data_path}. "
|
||||
"Run python main.py --mode sync_validate first."
|
||||
)
|
||||
|
||||
# Attempt to load Optuna best params; fall back to TF defaults
|
||||
try:
|
||||
from src.optimizer import get_best_params
|
||||
params = get_best_params(balance=account_size, broker=broker, tf=tf)
|
||||
obs_cov = params.get("obs_cov") or obs_cov
|
||||
trans_cov = params.get("trans_cov") or trans_cov
|
||||
except Exception:
|
||||
params = {}
|
||||
|
||||
# Load and featurise data
|
||||
df = pd.read_csv(sync_data_path, index_col="Date", parse_dates=True)
|
||||
logger.info("Loaded %d bars from %s for validation", len(df), sync_data_path)
|
||||
df = _apply_features(df, tf, obs_cov, trans_cov)
|
||||
|
||||
# Load models
|
||||
try:
|
||||
model_hmm = load_hmm()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(
|
||||
"HMM model not found at models/hmm_model.pkl. "
|
||||
"Run python main.py --mode train first."
|
||||
)
|
||||
try:
|
||||
model_xgb, _ = load_xgb()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(
|
||||
"XGB model not found at models/xgb_model.pkl. "
|
||||
"Run python main.py --mode train first."
|
||||
)
|
||||
|
||||
# Inference
|
||||
states = predict_states(model_hmm, df)
|
||||
X, _, df_aligned = prepare_features(df, states)
|
||||
states_aligned = states[df.index.isin(df_aligned.index)]
|
||||
_, probabilities = get_predictions(model_xgb, X)
|
||||
|
||||
# Backtest the full synced window — no IS/OOS split
|
||||
result = vectorized_backtest(
|
||||
df_aligned, probabilities, states_aligned,
|
||||
split_idx=None,
|
||||
account_size=account_size,
|
||||
broker=broker,
|
||||
tf=tf,
|
||||
)
|
||||
|
||||
sharpe = result["sharpe_ratio"]
|
||||
n_trades = result["n_trades"]
|
||||
win_rate = result["win_rate"]
|
||||
max_dd = result["max_drawdown"]
|
||||
|
||||
if n_trades < MIN_TRADES_WARNING:
|
||||
logger.warning(
|
||||
"Only %d trades in the validation window — Sharpe estimate may be "
|
||||
"unreliable. Consider using a longer --period (e.g. '6m').",
|
||||
n_trades,
|
||||
)
|
||||
|
||||
# Gate decision
|
||||
if sharpe >= SHARPE_PASS_THRESHOLD:
|
||||
status = "pass"
|
||||
message = (
|
||||
f"Recent-Period Sharpe {sharpe:.3f} >= {SHARPE_PASS_THRESHOLD}. "
|
||||
"Model is stable — safe to go live."
|
||||
)
|
||||
elif sharpe >= SHARPE_WARN_THRESHOLD:
|
||||
status = "warn"
|
||||
message = (
|
||||
f"Recent-Period Sharpe {sharpe:.3f} is borderline "
|
||||
f"({SHARPE_WARN_THRESHOLD}–{SHARPE_PASS_THRESHOLD}). "
|
||||
"Proceed with reduced position size or wait for a clearer regime."
|
||||
)
|
||||
else:
|
||||
status = "fail"
|
||||
message = (
|
||||
f"Recent-Period Sharpe {sharpe:.3f} < {SHARPE_WARN_THRESHOLD}. "
|
||||
"Market drift detected — DO NOT go live. "
|
||||
"Run --mode optimize then --mode train to retune the model."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Validation [%s]: status=%s sharpe=%.3f trades=%d wr=%.1f%% dd=%.1f%%",
|
||||
tf, status, sharpe, n_trades, win_rate * 100, max_dd * 100,
|
||||
)
|
||||
if status in ("warn", "fail"):
|
||||
logger.warning("VALIDATION %s: %s", status.upper(), message)
|
||||
|
||||
return {
|
||||
"sharpe": sharpe,
|
||||
"n_trades": n_trades,
|
||||
"win_rate": win_rate,
|
||||
"max_dd": max_dd,
|
||||
"status": status,
|
||||
"message": message,
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.dates as mdates
|
||||
from pathlib import Path
|
||||
from src.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
REPORT_DIR = Path("reports")
|
||||
REGIME_COLORS = {0: "#2ecc71", 1: "#e74c3c", 2: "#f39c12", 3: "#9b59b6"}
|
||||
REGIME_LABELS = {0: "Bull", 1: "Bear", 2: "Chop", 3: "Chop_High"}
|
||||
|
||||
|
||||
def _tf_dir(tf: str = "H1") -> Path:
|
||||
"""Return the TF-specific report subdirectory, creating it if needed."""
|
||||
d = REPORT_DIR / tf.upper()
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def plot_regime_overlay(df, hmm_states, state_names, tf="H1", save_path=None):
|
||||
"""Price chart with HMM regime shading."""
|
||||
save_path = save_path or _tf_dir(tf) / "1_regime_overlay.png"
|
||||
|
||||
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(18, 10), height_ratios=[3, 1],
|
||||
sharex=True, gridspec_kw={"hspace": 0.05})
|
||||
|
||||
dates = df.index
|
||||
close = df["Close"].values
|
||||
|
||||
for state_id, label in state_names.items():
|
||||
mask = hmm_states == state_id
|
||||
color = REGIME_COLORS.get(state_id, "#95a5a6")
|
||||
ax1.fill_between(dates, close.min() * 0.98, close.max() * 1.02,
|
||||
where=mask, alpha=0.15, color=color, label=label)
|
||||
|
||||
ax1.plot(dates, close, color="#2c3e50", linewidth=0.5, alpha=0.9)
|
||||
ax1.set_ylabel("XAUUSD Price (H1)", fontsize=11)
|
||||
ax1.set_title("Gold Regime X \u2014 HMM Regime Detection Over Price", fontsize=14, fontweight="bold")
|
||||
ax1.legend(loc="upper left", fontsize=10)
|
||||
ax1.grid(True, alpha=0.3)
|
||||
ax1.set_xlim(dates[0], dates[-1])
|
||||
|
||||
ax2.scatter(dates, hmm_states, c=[REGIME_COLORS.get(s, "#95a5a6") for s in hmm_states],
|
||||
s=1, alpha=0.6)
|
||||
ax2.set_ylabel("HMM State", fontsize=11)
|
||||
ax2.set_yticks(list(state_names.keys()))
|
||||
ax2.set_yticklabels(list(state_names.values()))
|
||||
ax2.grid(True, alpha=0.3)
|
||||
ax2.xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m"))
|
||||
ax2.xaxis.set_major_locator(mdates.MonthLocator(interval=6))
|
||||
plt.xticks(rotation=45)
|
||||
|
||||
fig.savefig(save_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
logger.info("Saved regime overlay chart to %s", save_path)
|
||||
return str(save_path)
|
||||
|
||||
|
||||
def plot_equity_curve(df, probabilities, hmm_states, split_idx=None, tf="H1", save_path=None):
|
||||
"""Equity curve, drawdown, and signal markers with train/test split."""
|
||||
save_path = save_path or _tf_dir(tf) / "2_equity_curve.png"
|
||||
|
||||
from src.backtester import compute_signals, compute_position_sizes
|
||||
|
||||
signals = compute_signals(probabilities, hmm_states)
|
||||
sizes = compute_position_sizes(signals, df["atr_normalized"].values)
|
||||
|
||||
log_returns = df["log_return"].values
|
||||
next_returns = np.roll(log_returns, -1)
|
||||
next_returns[-1] = 0.0
|
||||
strategy_returns = sizes * next_returns
|
||||
|
||||
cumulative_strat = np.cumsum(strategy_returns)
|
||||
cumulative_bh = np.cumsum(log_returns)
|
||||
|
||||
running_max = np.maximum.accumulate(cumulative_strat)
|
||||
drawdown = running_max - cumulative_strat
|
||||
|
||||
dates = df.index
|
||||
|
||||
fig, axes = plt.subplots(3, 1, figsize=(18, 12), height_ratios=[3, 1, 1],
|
||||
sharex=True, gridspec_kw={"hspace": 0.08})
|
||||
|
||||
# Equity curves
|
||||
ax1 = axes[0]
|
||||
ax1.plot(dates, cumulative_strat, color="#2980b9", linewidth=1.2, label="Strategy (log return)")
|
||||
ax1.plot(dates, cumulative_bh, color="#95a5a6", linewidth=0.8, alpha=0.7, label="Buy & Hold")
|
||||
entry_mask = signals == 1
|
||||
ax1.scatter(dates[entry_mask], cumulative_strat[entry_mask],
|
||||
c="#2ecc71", s=3, alpha=0.4, label=f"Long entries ({np.sum(entry_mask)})", zorder=3)
|
||||
|
||||
# Train/test split line
|
||||
if split_idx is not None and 0 < split_idx < len(dates):
|
||||
split_date = dates[split_idx]
|
||||
for ax in axes:
|
||||
ax.axvline(x=split_date, color="#e74c3c", linewidth=1.5, linestyle="--", alpha=0.7)
|
||||
ax1.axvspan(split_date, dates[-1], alpha=0.04, color="#e74c3c")
|
||||
ax1.text(split_date, ax1.get_ylim()[1] * 0.95, " OOS \u2192", fontsize=10,
|
||||
color="#e74c3c", fontweight="bold", va="top")
|
||||
ax1.text(split_date, ax1.get_ylim()[1] * 0.95, "\u2190 Train ", fontsize=10,
|
||||
color="#2980b9", fontweight="bold", va="top", ha="right")
|
||||
|
||||
ax1.set_ylabel("Cumulative Log Return", fontsize=11)
|
||||
ax1.set_title("Gold Regime X \u2014 Strategy Equity Curve vs Buy & Hold", fontsize=14, fontweight="bold")
|
||||
ax1.legend(loc="upper left", fontsize=10)
|
||||
ax1.grid(True, alpha=0.3)
|
||||
ax1.axhline(y=0, color="black", linewidth=0.5, alpha=0.5)
|
||||
|
||||
# Drawdown
|
||||
ax2 = axes[1]
|
||||
ax2.fill_between(dates, 0, -drawdown * 100, color="#e74c3c", alpha=0.5)
|
||||
ax2.set_ylabel("Drawdown (%)", fontsize=11)
|
||||
ax2.grid(True, alpha=0.3)
|
||||
max_dd_idx = np.argmax(drawdown)
|
||||
ax2.annotate(f"Max DD: {drawdown[max_dd_idx]*100:.1f}%",
|
||||
xy=(dates[max_dd_idx], -drawdown[max_dd_idx] * 100),
|
||||
fontsize=9, color="#c0392b", fontweight="bold")
|
||||
|
||||
# XGB probability
|
||||
ax3 = axes[2]
|
||||
ax3.scatter(dates, probabilities, c=probabilities, cmap="RdYlGn", s=1, alpha=0.3, vmin=0.3, vmax=0.7)
|
||||
ax3.axhline(y=0.65, color="#2ecc71", linewidth=1, linestyle="--", alpha=0.8, label="Long threshold (0.65)")
|
||||
ax3.axhline(y=0.35, color="#e74c3c", linewidth=1, linestyle="--", alpha=0.8, label="Short threshold (0.35)")
|
||||
ax3.axhline(y=0.5, color="#7f8c8d", linewidth=0.5, linestyle=":", alpha=0.5)
|
||||
ax3.set_ylabel("XGB Probability", fontsize=11)
|
||||
ax3.set_ylim(0.25, 0.75)
|
||||
ax3.legend(loc="upper right", fontsize=9)
|
||||
ax3.grid(True, alpha=0.3)
|
||||
ax3.xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m"))
|
||||
ax3.xaxis.set_major_locator(mdates.MonthLocator(interval=6))
|
||||
plt.xticks(rotation=45)
|
||||
|
||||
fig.savefig(save_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
logger.info("Saved equity curve chart to %s", save_path)
|
||||
return str(save_path)
|
||||
|
||||
|
||||
def plot_feature_analysis(X, hmm_states, metrics, tf="H1", save_path=None):
|
||||
"""Feature importance, distributions per regime, and correlation."""
|
||||
save_path = save_path or _tf_dir(tf) / "3_feature_analysis.png"
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(16, 12))
|
||||
|
||||
ax = axes[0, 0]
|
||||
importance = metrics.get("feature_importance", {})
|
||||
names = list(importance.keys())
|
||||
values = [float(v) for v in importance.values()]
|
||||
colors = ["#2980b9", "#27ae60", "#e67e22", "#8e44ad"]
|
||||
bars = ax.barh(names, values, color=colors[:len(names)])
|
||||
ax.set_xlabel("Importance (Gain)", fontsize=10)
|
||||
ax.set_title("XGBoost Feature Importance", fontsize=12, fontweight="bold")
|
||||
for bar, val in zip(bars, values):
|
||||
ax.text(bar.get_width() + 0.005, bar.get_y() + bar.get_height() / 2,
|
||||
f"{val:.3f}", va="center", fontsize=9)
|
||||
ax.grid(True, alpha=0.3, axis="x")
|
||||
|
||||
ax = axes[0, 1]
|
||||
unique_states = sorted(np.unique(hmm_states))
|
||||
for s in unique_states:
|
||||
mask = hmm_states == s
|
||||
data = X.loc[mask, "rsi_slope"].dropna()
|
||||
ax.hist(data, bins=80, alpha=0.5, label=REGIME_LABELS.get(s, str(s)),
|
||||
color=REGIME_COLORS.get(s, "#95a5a6"), density=True)
|
||||
ax.set_xlabel("RSI Slope", fontsize=10)
|
||||
ax.set_ylabel("Density", fontsize=10)
|
||||
ax.set_title("RSI Slope Distribution by Regime", fontsize=12, fontweight="bold")
|
||||
ax.legend(fontsize=9)
|
||||
ax.set_xlim(-10, 10)
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
ax = axes[1, 0]
|
||||
for s in unique_states:
|
||||
mask = hmm_states == s
|
||||
data = X.loc[mask, "atr_normalized"].dropna()
|
||||
ax.hist(data, bins=80, alpha=0.5, label=REGIME_LABELS.get(s, str(s)),
|
||||
color=REGIME_COLORS.get(s, "#95a5a6"), density=True)
|
||||
ax.set_xlabel("ATR Normalized", fontsize=10)
|
||||
ax.set_ylabel("Density", fontsize=10)
|
||||
ax.set_title("ATR (Normalized) Distribution by Regime", fontsize=12, fontweight="bold")
|
||||
ax.legend(fontsize=9)
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
ax = axes[1, 1]
|
||||
state_counts = pd.Series(hmm_states).value_counts().sort_index()
|
||||
labels_pie = [REGIME_LABELS.get(s, str(s)) for s in state_counts.index]
|
||||
colors_pie = [REGIME_COLORS.get(s, "#95a5a6") for s in state_counts.index]
|
||||
wedges, texts, autotexts = ax.pie(
|
||||
state_counts.values, labels=labels_pie, colors=colors_pie,
|
||||
autopct="%1.1f%%", startangle=90, textprops={"fontsize": 10}
|
||||
)
|
||||
ax.set_title("Regime Distribution", fontsize=12, fontweight="bold")
|
||||
|
||||
plt.tight_layout()
|
||||
fig.savefig(save_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
logger.info("Saved feature analysis chart to %s", save_path)
|
||||
return str(save_path)
|
||||
|
||||
|
||||
def plot_transition_matrix(model_hmm, state_names, tf="H1", save_path=None):
|
||||
"""HMM transition matrix heatmap."""
|
||||
save_path = save_path or _tf_dir(tf) / "4_transition_matrix.png"
|
||||
|
||||
n = model_hmm.n_components
|
||||
trans = model_hmm.transmat_
|
||||
|
||||
fig, ax = plt.subplots(figsize=(8, 6))
|
||||
im = ax.imshow(trans, cmap="YlOrRd", vmin=0, vmax=1, aspect="auto")
|
||||
|
||||
labels = [state_names.get(i, str(i)) for i in range(n)]
|
||||
ax.set_xticks(range(n))
|
||||
ax.set_xticklabels(labels, fontsize=11)
|
||||
ax.set_yticks(range(n))
|
||||
ax.set_yticklabels(labels, fontsize=11)
|
||||
ax.set_xlabel("To State", fontsize=12)
|
||||
ax.set_ylabel("From State", fontsize=12)
|
||||
ax.set_title("HMM Transition Matrix", fontsize=14, fontweight="bold")
|
||||
|
||||
for i in range(n):
|
||||
for j in range(n):
|
||||
color = "white" if trans[i, j] > 0.5 else "black"
|
||||
ax.text(j, i, f"{trans[i, j]:.3f}", ha="center", va="center",
|
||||
fontsize=12, color=color, fontweight="bold")
|
||||
|
||||
fig.colorbar(im, ax=ax, label="Transition Probability")
|
||||
plt.tight_layout()
|
||||
fig.savefig(save_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
logger.info("Saved transition matrix chart to %s", save_path)
|
||||
return str(save_path)
|
||||
|
||||
|
||||
def plot_summary_dashboard(result, params, tf="H1", save_path=None):
|
||||
"""Single-panel summary card with key metrics and params."""
|
||||
save_path = save_path or _tf_dir(tf) / "5_summary_dashboard.png"
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 8))
|
||||
ax.axis("off")
|
||||
|
||||
# Title block
|
||||
ax.text(0.50, 0.96, "GOLD REGIME X", fontsize=20,
|
||||
fontweight="bold", transform=ax.transAxes, va="top", ha="center", color="#2c3e50")
|
||||
ax.text(0.50, 0.90, f"Hybrid HMM + XGBoost | XAUUSD {tf.upper()}", fontsize=12,
|
||||
transform=ax.transAxes, va="top", ha="center", color="#7f8c8d")
|
||||
|
||||
# Divider line
|
||||
ax.plot([0.05, 0.95], [0.86, 0.86], color="#bdc3c7", linewidth=1, transform=ax.transAxes)
|
||||
|
||||
# Left column: Full-period metrics
|
||||
metrics_data = [
|
||||
["Sharpe Ratio", f"{result['sharpe_ratio']:.3f}"],
|
||||
["Max Drawdown", f"{result['max_drawdown']*100:.1f}%"],
|
||||
["Win Rate", f"{result['win_rate']*100:.1f}%"],
|
||||
["Total Return", f"{result['total_return']*100:.1f}%"],
|
||||
["Trade Count", f"{result['n_trades']}"],
|
||||
]
|
||||
sharpe_color = "#2ecc71" if 1.0 <= result["sharpe_ratio"] <= 1.5 else "#e67e22"
|
||||
dd_color = "#2ecc71" if 0.08 <= result["max_drawdown"] <= 0.15 else "#e67e22"
|
||||
wr_color = "#2ecc71" if 0.50 <= result["win_rate"] <= 0.55 else "#e67e22"
|
||||
metric_colors = [sharpe_color, dd_color, wr_color, "#3498db", "#3498db"]
|
||||
|
||||
ax.text(0.05, 0.82, "Performance (Full Period)", fontsize=13, fontweight="bold",
|
||||
transform=ax.transAxes, va="top", color="#2c3e50")
|
||||
y = 0.74
|
||||
for i, (label, value) in enumerate(metrics_data):
|
||||
ax.text(0.08, y, label, fontsize=12, transform=ax.transAxes, va="top",
|
||||
fontfamily="monospace", color="#2c3e50")
|
||||
ax.text(0.35, y, value, fontsize=12, transform=ax.transAxes, va="top",
|
||||
fontfamily="monospace", fontweight="bold", color=metric_colors[i])
|
||||
y -= 0.085
|
||||
|
||||
# OOS metrics if available
|
||||
if "oos_sharpe_ratio" in result:
|
||||
y -= 0.02
|
||||
ax.text(0.05, y, "Out-of-Sample (Test 20%)", fontsize=13, fontweight="bold",
|
||||
transform=ax.transAxes, va="top", color="#c0392b")
|
||||
y -= 0.08
|
||||
oos_data = [
|
||||
["Sharpe", f"{result['oos_sharpe_ratio']:.3f}"],
|
||||
["Max DD", f"{result['oos_max_drawdown']*100:.1f}%"],
|
||||
["Win Rate", f"{result['oos_win_rate']*100:.1f}%"],
|
||||
["Trades", f"{result['oos_n_trades']}"],
|
||||
]
|
||||
for label, value in oos_data:
|
||||
ax.text(0.08, y, label, fontsize=11, transform=ax.transAxes, va="top",
|
||||
fontfamily="monospace", color="#7f8c8d")
|
||||
ax.text(0.25, y, value, fontsize=11, transform=ax.transAxes, va="top",
|
||||
fontfamily="monospace", fontweight="bold", color="#c0392b")
|
||||
y -= 0.07
|
||||
|
||||
# Right column: Target ranges + Params
|
||||
ax.text(0.55, 0.82, "Target Ranges", fontsize=13, fontweight="bold",
|
||||
transform=ax.transAxes, va="top", color="#2c3e50")
|
||||
targets = [
|
||||
"Sharpe: 1.0 - 1.5",
|
||||
"Max DD: 8% - 15%",
|
||||
"Win Rate: 50% - 55%",
|
||||
]
|
||||
y_right = 0.74
|
||||
for t in targets:
|
||||
ax.text(0.58, y_right, t, fontsize=11, transform=ax.transAxes, va="top",
|
||||
fontfamily="monospace", color="#7f8c8d")
|
||||
y_right -= 0.07
|
||||
|
||||
if params:
|
||||
y_right -= 0.04
|
||||
ax.text(0.55, y_right, "Optimized Params", fontsize=13, fontweight="bold",
|
||||
transform=ax.transAxes, va="top", color="#2c3e50")
|
||||
y_right -= 0.08
|
||||
for k, v in params.items():
|
||||
val_str = f"{v:.4f}" if isinstance(v, float) else str(v)
|
||||
ax.text(0.58, y_right, f"{k}: {val_str}", fontsize=9, transform=ax.transAxes,
|
||||
va="top", fontfamily="monospace", color="#34495e")
|
||||
y_right -= 0.055
|
||||
|
||||
# Border
|
||||
fig.patch.set_facecolor("#fafafa")
|
||||
rect = plt.Rectangle((0.02, 0.02), 0.96, 0.96, fill=False, edgecolor="#bdc3c7",
|
||||
linewidth=2, transform=ax.transAxes)
|
||||
ax.add_patch(rect)
|
||||
|
||||
fig.savefig(save_path, dpi=150, bbox_inches="tight", facecolor="#fafafa")
|
||||
plt.close(fig)
|
||||
logger.info("Saved summary dashboard to %s", save_path)
|
||||
return str(save_path)
|
||||
|
||||
|
||||
def generate_full_report(df, hmm_states, state_names, model_hmm,
|
||||
X, probabilities, metrics, result, params=None,
|
||||
split_idx=None, tf="H1"):
|
||||
"""Generate all 5 charts into reports/<TF>/ and return list of file paths."""
|
||||
paths = []
|
||||
paths.append(plot_regime_overlay(df, hmm_states, state_names, tf=tf))
|
||||
paths.append(plot_equity_curve(df, probabilities, hmm_states, split_idx=split_idx, tf=tf))
|
||||
paths.append(plot_feature_analysis(X, hmm_states, metrics, tf=tf))
|
||||
paths.append(plot_transition_matrix(model_hmm, state_names, tf=tf))
|
||||
paths.append(plot_summary_dashboard(result, params or {}, tf=tf))
|
||||
logger.info("Full report [%s]: %d charts in %s", tf, len(paths), REPORT_DIR / tf.upper())
|
||||
return paths
|
||||
Reference in New Issue
Block a user