commit 7a3e13a73490ec079d0841ba4ab45bf09406998e Author: LEGSTECH Optimizer Date: Mon Apr 13 02:28:09 2026 +0000 Initial commit: MT5 EA Optimizer v1.0 Full optimization system for LEGSTECH_EA_V2: - Flask + SocketIO live dashboard (dark premium UI) - MT5 process control (auto-kill, clean launch, retry) - HTML report parser (UTF-16 LE, 597 trades, metrics) - Pre-run validation and actionable error messages - Analysis engines: Reversal, TimePerfomance, EntryExit, EquityCurve - Composite scoring (Calmar-primary) - Mutation engine with knowledge_base.yaml - Validation gate: IS + Walk-Forward - Reports folder with HTML/CSV per run - Double-click launcher batch file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cb74696 --- /dev/null +++ b/.gitignore @@ -0,0 +1,52 @@ +# ── Python ──────────────────────────────────────────────────────────────────── +__pycache__/ +*.py[cod] +*.pyd +*.pyo +*.egg-info/ +*.egg +.eggs/ +dist/ +build/ +*.spec + +# ── Virtual Environments ────────────────────────────────────────────────────── +venv/ +env/ +.env +.venv/ + +# ── MT5 Optimizer Runtime Files ─────────────────────────────────────────────── +# Don't commit generated run data or reports (can be large) +runs/ +Reports/ + +# Keep the optimizer database (optional — remove this line to commit it) +optimizer.db + +# ── Sensitive Config (DO NOT COMMIT BROKER CREDENTIALS) ────────────────────── +# config.yaml contains MT5 paths — safe to commit but exclude if it had passwords +# config.secret.yaml + +# ── Logs ────────────────────────────────────────────────────────────────────── +*.log +logs/ + +# ── OS Files ────────────────────────────────────────────────────────────────── +.DS_Store +Thumbs.db +desktop.ini + +# ── IDE ─────────────────────────────────────────────────────────────────────── +.vscode/ +.idea/ +*.swp + +# ── PyInstaller ─────────────────────────────────────────────────────────────── +*.exe +*.zip +dist/ + +# ── Data files ──────────────────────────────────────────────────────────────── +*.parquet +*.csv diff --git a/Launch Optimizer.bat b/Launch Optimizer.bat new file mode 100644 index 0000000..340f3e7 --- /dev/null +++ b/Launch Optimizer.bat @@ -0,0 +1,23 @@ +@echo off +REM ── MT5 EA Optimizer Launcher ────────────────────────────────────────────── +REM Double-click this file to start the optimizer app. +REM Browser will open automatically. + +title MT5 EA Optimizer + +SET PY=C:\Users\DELL\AppData\Local\Programs\Python\Python311\python.exe +SET APP=%~dp0app.py + +echo. +echo ===================================================== +echo MT5 EA Optimizer — Starting... +echo Your browser will open in a few seconds. +echo Do NOT close this window while optimizer is running. +echo ===================================================== +echo. + +"%PY%" "%APP%" + +echo. +echo Optimizer stopped. Press any key to close. +pause > nul diff --git a/PROJECT_HANDOFF.md b/PROJECT_HANDOFF.md new file mode 100644 index 0000000..c037368 --- /dev/null +++ b/PROJECT_HANDOFF.md @@ -0,0 +1,349 @@ +# MT5 EA Optimizer — Full Project Handoff Document +**Last Updated:** 2026-04-13 +**Status:** Phase 1 complete (backend engine). Phase 2 (GUI app) = NEXT STEP +**Primary EA:** LEGSTECH_EA_V2 | Symbol: XAUUSD | Timeframe: H1 +**Broker Timezone:** UTC+2 + +--- + +## 🎯 Project Goal + +Build an automated, iterative backtesting and analysis system for MT5 Expert Advisors. +It is NOT a trading bot. It is an **optimization engine** that: +- Runs MT5 strategy tester automatically with different parameter sets +- Extracts rich trade-level data (MAE/MFE) after each run +- Analyzes results to find failure patterns +- Proposes and tests parameter mutations +- Validates improvements before accepting them +- Prevents overfitting via Walk-Forward + Out-of-Sample testing + +**End user:** Traders (non-technical). Must be a double-click app, not a terminal tool. + +--- + +## 🏗️ Architecture Overview + +``` +MT5 Terminal (GUI) + └── Strategy Tester (automated via .ini files) + └── LEGSTECH_EA_V2.ex5 (compiled EA with TradeLogger) + └── Writes: LEGSTECH_EA_V2_XAUUSD_TradeLog.csv + +Python Optimizer (backend engine) + ├── MT5 Runner → launch terminal, wait for report, collect files + ├── Report Parser → parse MT5 XML/HTML report into metrics + trades + ├── Log Reader → merge TradeLogger CSV (MAE/MFE) into trade objects + ├── Analysis Engine → 4 modules detecting failure patterns + │ ├── ReversalAnalyzer → trades that went in profit then reversed + │ ├── TimePerformanceAnalyzer → bad sessions/hours/days + │ ├── EntryExitQualityAnalyzer → MAE/MFE quality scores + │ └── EquityCurveAnalyzer → flatness, loss clusters, R² + ├── Composite Scorer → Calmar-primary weighted score (0-1) + ├── Mutation Engine → findings → parameter hypotheses (13 KB rules) + ├── Validation Gate → IS check → Walk-Forward → OOS + └── Data Store → SQLite (metadata) + Parquet (trade arrays) + +Web App (NEXT TO BUILD — Phase 2) + ├── Flask backend → serves UI, runs optimization loop + ├── WebSocket → pushes live updates to browser + ├── HTML/JS frontend → live dashboard, charts, findings + └── Reports folder → HTML + CSV reports per run +``` + +--- + +## 📁 File Structure (Current State) + +``` +C:\Users\DELL\Desktop\MT5_Optimizer\ +├── main.py ← CLI entry point (terminal-based, to be replaced by app.py) +├── config.yaml ← ALL settings (MT5 paths, symbols, scoring weights, thresholds) +├── requirements.txt ← Python dependencies +├── PROJECT_HANDOFF.md ← THIS DOCUMENT +│ +├── mql5/ +│ └── TradeLogger.mqh ← MQL5 include file (ALREADY DEPLOYED to MT5) +│ +├── data/ +│ ├── models.py ← Pydantic v2 models: Trade, RunMetrics, Run, Finding, Hypothesis, Candidate +│ └── store.py ← DataStore: SQLite + Parquet storage layer +│ +├── mt5/ +│ ├── ini_builder.py ← Builds MT5 tester .ini files from param dicts +│ ├── runner.py ← Launches MT5 subprocess, waits for report +│ ├── report_parser.py ← Parses MT5 XML/HTML report → RunMetrics + list[Trade] +│ └── log_reader.py ← Merges TradeLogger CSV, computes sessions/quality scores +│ +├── analysis/ +│ ├── base.py ← BaseAnalyzer ABC + statistical helpers +│ ├── reversal.py ← ReversalAnalyzer +│ ├── time_performance.py ← TimePerformanceAnalyzer +│ ├── entry_exit_quality.py ← EntryExitQualityAnalyzer +│ └── equity_curve.py ← EquityCurveAnalyzer +│ +├── scoring/ +│ └── composite.py ← CompositeScorer (Calmar + PF + MFE capture + stability + recovery) +│ +├── mutation/ +│ ├── engine.py ← MutationEngine: findings → Hypothesis objects +│ ├── knowledge_base.yaml ← 13 rules mapping findings to param changes +│ └── param_manifest.yaml ← Full EA parameter space with types/bounds/defaults +│ +├── validation/ +│ └── gate.py ← IS check + Walk-Forward + OOS + sensitivity check +│ +└── tests/ + └── test_analyzers.py ← 10 unit tests (all passing ✅) +``` + +--- + +## ⚙️ Configuration (config.yaml) — Key Values + +```yaml +ea: + name: "LEGSTECH_EA_V2" + symbol: "XAUUSD" + timeframe: "H1" + +periods: + train_start: "2022.01.01" + train_end: "2023.12.31" + validate_start: "2024.01.01" + validate_end: "2024.06.30" + oos_start: "2024.07.01" # LOCKED — never touch during optimization + oos_end: "2024.12.31" + +mt5: + terminal_exe: "C:/Program Files/MetaTrader 5/terminal64.exe" + appdata_path: "C:/Users/DELL/AppData/Roaming/MetaQuotes/Terminal/D0E8209F77C8CF37AD8BF550E51FF075" + mql5_files_path: "C:/Users/DELL/AppData/Roaming/MetaQuotes/Tester/D0E8209F77C8CF37AD8BF550E51FF075/Agent-127.0.0.1-3000/MQL5/Files" + +broker: + timezone_offset_hours: 2 # UTC+2 + +scoring: + weights: + calmar: 0.35 + profit_factor: 0.20 + mfe_capture: 0.20 + session_stability: 0.15 + recovery_factor: 0.10 +``` + +--- + +## ✅ What is Done (Phase 1 — Backend Engine) + +| Component | Status | Notes | +|---|---|---| +| TradeLogger.mqh | ✅ Complete + deployed | In MT5 Include folder, tested, CSV confirmed working | +| data/models.py | ✅ Complete | All Pydantic v2 models | +| data/store.py | ✅ Complete | SQLite + Parquet CRUD | +| mt5/ini_builder.py | ✅ Complete | Generates valid MT5 tester INI files | +| mt5/runner.py | ✅ Complete | Process launch + polling + timeout | +| mt5/report_parser.py | ✅ Complete | XML + HTML dual-format parser | +| mt5/log_reader.py | ✅ Complete | MAE/MFE merge + session classification | +| analysis/reversal.py | ✅ Complete | Permutation-tested | +| analysis/time_performance.py | ✅ Complete | Hour/Session/Day analysis | +| analysis/entry_exit_quality.py | ✅ Complete | 4-case diagnosis matrix | +| analysis/equity_curve.py | ✅ Complete | Flatness + R² + loss clusters | +| scoring/composite.py | ✅ Complete | Calmar-primary weighted scorer | +| mutation/engine.py | ✅ Complete | 13 KB rules, dedup, cascade | +| validation/gate.py | ✅ Complete | IS + WFV + OOS + sensitivity | +| main.py | ✅ Complete | Terminal CLI (will be replaced by app) | +| tests/ | ✅ 10/10 passing | Pure Python, no MT5 needed | +| Unit tests verified | ✅ Working | Python 3.11 required (see below) | + +--- + +## 🔴 What is NOT Done Yet (Phase 2 — GUI App) + +### The Big Next Step: Web App with Live Dashboard + +**Goal:** Replace `main.py` terminal interface with a beautiful browser-based app that: + +1. **Double-click `MT5_Optimizer.exe`** → browser opens automatically at `http://localhost:5000` +2. **Dashboard shows live:** + - Iteration counter + current phase badge + - Score history line chart (Calmar + composite over time) + - Live MT5 run status with elapsed timer + - Current parameters being tested +3. **Analysis panel shows** findings in plain English after each run +4. **Parameter changes panel** shows before → after with reason +5. **Results folder** `MT5_Optimizer\Reports\` gets: + - `run_001\summary.html` — full backtest result in readable format + - `run_001\trades.csv` — all trades with MAE/MFE + - `run_001\findings.csv` — analysis findings + - `run_001\parameters.json` — params used +6. **Controls:** Start, Pause, Skip Hypothesis, View Report buttons +7. **PyInstaller** bundles into single `MT5_Optimizer.exe` + +### Tech Stack for Phase 2 + +``` +Flask + Flask-SocketIO → backend API + WebSocket push +Chart.js or Plotly.js → charts in browser +Bootstrap 5 (dark theme) → UI framework +Jinja2 → HTML report templates +PyInstaller → package to .exe +``` + +### Files to create in Phase 2 + +``` +app.py ← Flask app entry point (replaces main.py) +ui/ + templates/ + index.html ← Main dashboard + report.html ← Per-run report template + findings.html ← Findings detail page + static/ + css/style.css + js/dashboard.js ← WebSocket + Chart.js logic +reports/ ← All run outputs go here (user-visible) +MT5_Optimizer.spec ← PyInstaller spec file +build.bat ← One-click build to .exe +``` + +--- + +## 🔧 Environment & Dependencies + +**Python version:** 3.11 (NOT 3.13 — use `C:\Users\DELL\AppData\Local\Programs\Python\Python311\python.exe`) + +**Install command:** +```powershell +C:\Users\DELL\AppData\Local\Programs\Python\Python311\python.exe -m pip install -r requirements.txt +``` + +**Run tests:** +```powershell +cd C:\Users\DELL\Desktop\MT5_Optimizer +C:\Users\DELL\AppData\Local\Programs\Python\Python311\python.exe -m pytest tests/ -v +``` + +**Key dependency versions:** +``` +pydantic>=2.5, pandas>=2.1, pyarrow>=14.0, lxml>=4.9 +loguru>=0.7, rich>=13.0, scipy>=1.11, pyyaml>=6.0 +``` + +**Additional needed for Phase 2:** +``` +flask, flask-socketio, eventlet, jinja2, pyinstaller +``` + +--- + +## 🧠 Key Design Decisions (Important Context) + +1. **Hypothesis-driven, not brute-force** — We don't grid-search all params. We detect failure patterns, hypothesize a fix, test it, validate it. + +2. **Calmar ratio is primary score metric** — Return / MaxDrawdown is most relevant for live trading. + +3. **Three validation gates** — IS threshold → Walk-Forward (2 folds) → OOS (locked period). No candidate is promoted unless it passes all three. + +4. **MFE/MAE is critical** — Without the TradeLogger, the analyzer still works but is less powerful. Always confirm CSV is being generated. + +5. **OOS is sacred** — `2024.07.01 → 2024.12.31` is NEVER used during optimization. Only tested as final confirmation. + +6. **Broker timezone = UTC+2** — All session analysis normalizes to UTC internally. Sessions: London=07-16 UTC, NY=13-22 UTC. + +7. **Python 3.11 is required** — The system Python on this machine is 3.13 which doesn't have the packages. Always use the 3.11 path. + +8. **MQL5 reference syntax fix** — MQL5 does not support C++ `&` references to array elements. `TradeLogger.mqh` was patched to use direct `TL_Positions[idx].field` access. + +--- + +## 📍 Current Machine Paths (This User's System) + +``` +MT5 Terminal EXE: C:\Program Files\MetaTrader 5\terminal64.exe +MT5 Terminal Data: C:\Users\DELL\AppData\Roaming\MetaQuotes\Terminal\D0E8209F77C8CF37AD8BF550E51FF075\ +MT5 Tester Files: C:\Users\DELL\AppData\Roaming\MetaQuotes\Tester\D0E8209F77C8CF37AD8BF550E51FF075\Agent-127.0.0.1-3000\MQL5\Files\ +EA Source File: C:\Users\DELL\AppData\Roaming\MetaQuotes\Terminal\D0E8209F77C8CF37AD8BF550E51FF075\MQL5\Experts\LEGSTECH_EA_V2.mq5 +TradeLogger (deployed): ...Terminal\...\MQL5\Include\TradeLogger.mqh +Project Folder: C:\Users\DELL\Desktop\MT5_Optimizer\ +Python 3.11: C:\Users\DELL\AppData\Local\Programs\Python\Python311\python.exe +``` + +--- + +## 🚀 Phase 2 Build Instructions (For Next AI Session) + +When continuing this project, build Phase 2 in this order: + +### Step 1 — Flask app skeleton +Create `app.py` with: +- Flask app + SocketIO +- Route: `GET /` → serve dashboard +- Route: `GET /api/status` → current run status JSON +- Route: `POST /api/start` → start optimization loop in background thread +- Route: `POST /api/pause` → pause loop +- SocketIO event: emit `run_update` after each test completes + +### Step 2 — Dashboard HTML +Create `ui/templates/index.html`: +- Dark theme (Bootstrap 5 dark) +- Left panel: score history chart (Chart.js line chart) +- Right panel: current run metrics card +- Bottom panel: scrollable findings feed +- Top bar: Start/Pause button, iteration counter, phase badge, timer + +### Step 3 — Report template +Create `ui/templates/report.html`: +- Full metrics table +- Findings list in plain English +- Parameter delta table (old → new → why) +- Trade table with MAE/MFE columns + +### Step 4 — Reports folder writer +Create `reports/writer.py`: +- `write_run_report(run_id, metrics, trades_df, findings)` → writes HTML + CSV +- All outputs go to `MT5_Optimizer\Reports\run_XXX\` + +### Step 5 — PyInstaller packaging +Create `build.bat`: +```bat +pyinstaller --onefile --noconsole --name MT5_Optimizer app.py +``` +Create `MT5_Optimizer.spec` with proper hidden imports for Flask, SocketIO, etc. + +### Step 6 — Test end-to-end +- Double-click `MT5_Optimizer.exe` +- Browser opens at localhost:5000 +- Click Start +- Watch live updates +- Check Reports folder + +--- + +## 💡 Future Enhancements (v2) + +- Multi-symbol optimization (EURUSD, GBPUSD alongside XAUUSD) +- Full rolling window WFV (currently 2-fold MVP) +- Portfolio-level Calmar (across symbols) +- Monte Carlo simulation for robustness testing +- Email/Telegram notification when a candidate is promoted +- Parameter sensitivity heatmap visualization +- Automatic .set file export for validated candidates + +--- + +## ⚠️ Known Issues / Watch Points + +1. **MT5 tester model** — Currently config uses `tester_model: 0` (Every Tick). For XAUUSD this is slow. Use model `4` (OHLC M1) for faster iteration during development. + +2. **ShutdownTerminal=1** — The INI closes MT5 after each test. If MT5 is also being used for live trading, this will interrupt it. Separate terminal instances are recommended. + +3. **Agent path** — The Tester Agent path (`Agent-127.0.0.1-3000`) may change if the MT5 tester port changes. If CSV is not found, check the Tester folder. + +4. **WFV fold count** — Currently hardcoded to 2 folds in `validation/gate.py`. Easy to increase. + +5. **INI Period code** — H1 = 16385 in MT5 (ENUM_TIMEFRAMES). This is hardcoded in `config.yaml` as `mt5_period_code: 16385`. Do not change unless changing timeframe. + +--- + +*This document was generated by Antigravity AI on 2026-04-13.* +*Continue building from Phase 2 — GUI App.* diff --git a/analysis/__init__.py b/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/analysis/base.py b/analysis/base.py new file mode 100644 index 0000000..a538e0e --- /dev/null +++ b/analysis/base.py @@ -0,0 +1,86 @@ +""" +analysis/base.py +Abstract base class for all analyzer modules. +""" +from __future__ import annotations +from abc import ABC, abstractmethod +from typing import Optional + +import numpy as np +import pandas as pd + +from data.models import Finding, RunMetrics + + +class BaseAnalyzer(ABC): + """ + Every analyzer receives a trades DataFrame and RunMetrics, + and returns a list of Finding objects sorted by confidence descending. + """ + + name: str = "base" + min_trades: int = 30 # refuse to analyze below this count + run_id: str = "" + + def run( + self, + trades: pd.DataFrame, + metrics: RunMetrics, + run_id: str, + ) -> list[Finding]: + """Entry point — enforces minimum trade count gate.""" + self.run_id = run_id + if len(trades) < self.min_trades: + return [] + return self.analyze(trades, metrics) + + @abstractmethod + def analyze(self, trades: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + """Implement analysis logic. Return list of findings.""" + ... + + # ── Statistical helpers ──────────────────────────────────────────────────── + + def _confidence_from_z(self, z: float) -> float: + """Map a Z-score to a [0,1] confidence value using normal CDF.""" + from scipy.stats import norm + return float(min(1.0, max(0.0, 2 * norm.cdf(abs(z)) - 1))) + + def _permutation_pvalue( + self, + group_values: np.ndarray, + all_values: np.ndarray, + n_permutations: int = 500, + alternative: str = "less", # 'less' = testing if group mean < overall mean + ) -> float: + """ + Non-parametric permutation test. + Returns p-value: probability that observed group mean is due to chance. + Lower p-value = more statistically significant. + """ + if len(group_values) == 0 or len(all_values) == 0: + return 1.0 + + observed_stat = np.mean(group_values) + n_group = len(group_values) + count_extreme = 0 + + rng = np.random.default_rng(seed=42) # deterministic + for _ in range(n_permutations): + sample = rng.choice(all_values, size=n_group, replace=False) + sample_stat = np.mean(sample) + if alternative == "less" and sample_stat <= observed_stat: + count_extreme += 1 + elif alternative == "greater" and sample_stat >= observed_stat: + count_extreme += 1 + + return count_extreme / n_permutations + + def _severity(self, confidence: float, impact_pnl: float, total_pnl: float) -> str: + """Derive severity from confidence and relative $ impact.""" + impact_fraction = abs(impact_pnl) / max(abs(total_pnl), 1) + if confidence >= 0.80 or impact_fraction >= 0.15: + return "high" + elif confidence >= 0.60 or impact_fraction >= 0.07: + return "medium" + return "low" diff --git a/analysis/entry_exit_quality.py b/analysis/entry_exit_quality.py new file mode 100644 index 0000000..c080738 --- /dev/null +++ b/analysis/entry_exit_quality.py @@ -0,0 +1,176 @@ +""" +analysis/entry_exit_quality.py +Scores trade entries and exits using MAE/MFE ratios. +""" +from __future__ import annotations +import numpy as np +import pandas as pd + +from data.models import Finding, RunMetrics +from analysis.base import BaseAnalyzer + + +class EntryExitQualityAnalyzer(BaseAnalyzer): + """ + Entry/Exit Quality Scorer. + + Uses MAE and MFE to independently assess: + - Entry quality: how much did price move against us before moving our way? + - Exit quality : what fraction of the available move did we capture? + + Diagnosis matrix: + ┌─────────────┬──────────────┬──────────────────────────────────────┐ + │entry_quality│ exit_quality │ Diagnosis │ + ├─────────────┼──────────────┼──────────────────────────────────────┤ + │ High │ High │ Healthy — no action │ + │ High │ Low │ Good entries, poor exits → trail/TP │ + │ Low │ High │ Bad entries, recovering → entry filt │ + │ Low │ Low │ Systematic issue → both sides broken │ + └─────────────┴──────────────┴──────────────────────────────────────┘ + """ + + name = "entry_exit_quality" + min_trades = 20 + + def __init__( + self, + poor_exit_threshold: float = 0.55, + poor_entry_threshold: float = 0.40, + good_entry_threshold: float = 0.65, + ): + self.poor_exit = poor_exit_threshold + self.poor_entry = poor_entry_threshold + self.good_entry = good_entry_threshold + + def analyze(self, trades: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + if "mfe_pips" not in trades.columns or trades["mfe_pips"].isna().all(): + return [] + + df = trades[trades["mfe_pips"].notna() & (trades["mfe_pips"] > 0)].copy() + if len(df) < self.min_trades: + return [] + + # Compute quality scores if not already present + if "exit_quality" not in df.columns or df["exit_quality"].isna().all(): + df["exit_quality"] = (df["net_pips"] / df["mfe_pips"].clip(lower=0.01)).clip(0, 1) + if "entry_quality" not in df.columns or df["entry_quality"].isna().all(): + denom = df["mfe_pips"] + df["mae_pips"].fillna(0) + 0.01 + df["entry_quality"] = (1 - df["mae_pips"].fillna(0) / denom).clip(0, 1) + + mean_exit = float(df["exit_quality"].mean()) + mean_entry = float(df["entry_quality"].mean()) + + findings = [] + + # ── Case 1: Good entries, poor exits ────────────────────────────── + if mean_exit < self.poor_exit and mean_entry >= self.good_entry: + z = (self.poor_exit - mean_exit) / max(0.01, df["exit_quality"].std()) + confidence = min(0.95, self._confidence_from_z(z)) + potential = float((df["mfe_pips"] - df["net_pips"].clip(lower=0)).clip(lower=0).mean()) + impact = potential * len(df) * 0.1 # rough dollar estimate + + findings.append(Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Good entries (quality={mean_entry:.2f}) but poor exits " + f"(quality={mean_exit:.2f}). " + f"Capturing only {mean_exit*100:.0f}% of available MFE. " + f"Consider trailing stop or tighter TP." + ), + severity=self._severity(confidence, impact, metrics.net_profit), + confidence=confidence, + impact_estimate_pnl=impact, + suggested_params={ + "InpUseTrailing": True, + "InpTrailStartPips": round(float(df["mfe_pips"].quantile(0.30)), 1), + "InpTrailStepPips": 10.0, + }, + evidence={ + "mean_entry_quality": round(mean_entry, 4), + "mean_exit_quality": round(mean_exit, 4), + "diagnosis": "good_entry_poor_exit", + "sample_size": len(df), + }, + )) + + # ── Case 2: Poor entries ─────────────────────────────────────────── + elif mean_entry < self.poor_entry: + z = (self.poor_entry - mean_entry) / max(0.01, df["entry_quality"].std()) + confidence = min(0.95, self._confidence_from_z(z)) + # Trades with high MAE but positive result still suggest entry timing issue + high_mae_losers = df[(df["mae_pips"] > df["mfe_pips"]) & (df["net_money"] < 0)] + impact = abs(float(high_mae_losers["net_money"].sum())) + + findings.append(Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Poor entry quality ({mean_entry:.2f}): " + f"significant adverse move before trades become profitable. " + f"{len(high_mae_losers)} trades had MAE > MFE and closed at a loss. " + f"Consider tightening entry filters (ATR, EMA slope, score gate)." + ), + severity=self._severity(confidence, impact, metrics.net_profit), + confidence=confidence, + impact_estimate_pnl=impact, + suggested_params={ + "InpUseATRFilter": True, # placeholder name; map to actual param + "InpATRMultiplier": round(float(df["mae_pips"].quantile(0.70)) / 100, 1), + "InpMinScore": 9, # tighten quality gate + }, + evidence={ + "mean_entry_quality": round(mean_entry, 4), + "mean_exit_quality": round(mean_exit, 4), + "diagnosis": "poor_entry", + "high_mae_loser_count": int(len(high_mae_losers)), + "sample_size": len(df), + }, + )) + + # ── Case 3: Both broken ──────────────────────────────────────────── + elif mean_exit < self.poor_exit and mean_entry < self.poor_entry: + confidence = 0.75 + impact = abs(metrics.net_profit) * 0.5 # rough + + findings.append(Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Both entry ({mean_entry:.2f}) and exit ({mean_exit:.2f}) quality are poor. " + f"This suggests a systematic issue with the strategy logic. " + f"Consider testing a different BotMode or EntryMode." + ), + severity="high", + confidence=confidence, + impact_estimate_pnl=impact, + suggested_params={"InpBotMode": 2}, # conservative mode + evidence={ + "mean_entry_quality": round(mean_entry, 4), + "mean_exit_quality": round(mean_exit, 4), + "diagnosis": "both_broken", + "sample_size": len(df), + }, + )) + + # ── Always report summary stats as a LOW finding for visibility ─── + findings.append(Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Entry quality: {mean_entry:.2f} | Exit quality: {mean_exit:.2f} | " + f"Sample: {len(df)} trades with MFE data." + ), + severity="low", + confidence=0.99, + impact_estimate_pnl=0.0, + suggested_params={}, + evidence={ + "mean_entry_quality": round(mean_entry, 4), + "mean_exit_quality": round(mean_exit, 4), + "sample_size": len(df), + "diagnosis": "summary", + }, + )) + + return sorted(findings, key=lambda f: f.confidence, reverse=True) diff --git a/analysis/equity_curve.py b/analysis/equity_curve.py new file mode 100644 index 0000000..be03f0b --- /dev/null +++ b/analysis/equity_curve.py @@ -0,0 +1,208 @@ +""" +analysis/equity_curve.py +Analyzes equity curve shape: drawdown clusters, flatness, recovery efficiency. +""" +from __future__ import annotations +from typing import Optional +import numpy as np +import pandas as pd +from scipy.stats import linregress + +from data.models import Finding, RunMetrics +from analysis.base import BaseAnalyzer + + +class EquityCurveAnalyzer(BaseAnalyzer): + """ + Equity Curve Shape Analyzer. + + Metrics computed: + - Flatness score : % of trades where equity is below its running high-water mark + - Recovery time : average trades needed to recover from a drawdown + - Equity R² : linearity of cumulative PnL (high = consistent growth) + - Loss clusters : sequences of consecutive losses (≥ N in a row) + """ + + name = "equity_curve" + + def __init__( + self, + max_flatness: float = 0.50, + min_r_squared: float = 0.70, + cluster_min_length: int = 3, + ): + self.max_flatness = max_flatness + self.min_r_sq = min_r_squared + self.cluster_min = cluster_min_length + + def analyze(self, trades: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + if "net_money" not in trades.columns or len(trades) < self.min_trades: + return [] + + df = trades.sort_values("open_time").reset_index(drop=True) + df["cumulative_pnl"] = df["net_money"].cumsum() + df["hwm"] = df["cumulative_pnl"].cummax() # high-water mark + + findings = [] + findings += self._check_flatness(df, metrics) + findings += self._check_r_squared(df, metrics) + findings += self._check_loss_clusters(df, metrics) + return sorted(findings, key=lambda f: f.confidence, reverse=True) + + # ── Flatness ────────────────────────────────────────────────────────────── + + def _check_flatness(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + """% of time equity is below its high-water mark.""" + below_hwm = (df["cumulative_pnl"] < df["hwm"]).sum() + flatness = below_hwm / len(df) + + if flatness <= self.max_flatness: + return [] + + confidence = min(0.90, (flatness - self.max_flatness) * 4) + impact = abs(metrics.net_profit) * (flatness - self.max_flatness) + + # Compute average recovery length (trades to get back to HWM) + recovery_lengths = self._compute_recovery_lengths(df) + avg_recovery = float(np.mean(recovery_lengths)) if recovery_lengths else 0.0 + + return [Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Equity below high-water mark {flatness*100:.0f}% of the time " + f"(threshold: {self.max_flatness*100:.0f}%). " + f"Avg recovery: {avg_recovery:.0f} trades. " + f"Consider reducing risk or adding drawdown pause logic." + ), + severity=self._severity(confidence, impact, metrics.net_profit), + confidence=confidence, + impact_estimate_pnl=impact, + suggested_params={ + "InpRiskPercent": round(max(0.5, metrics.net_profit / 10000 * 0.75), 1), + "InpMaxDailyLossPct": 2.0, + }, + evidence={ + "flatness_score": round(float(flatness), 4), + "avg_recovery": round(avg_recovery, 1), + "below_hwm_count": int(below_hwm), + "total_trades": len(df), + }, + )] + + def _compute_recovery_lengths(self, df: pd.DataFrame) -> list[int]: + """Count how many trades it takes to recover from each drawdown trough.""" + lengths = [] + in_dd = False + count = 0 + for _, row in df.iterrows(): + below = row["cumulative_pnl"] < row["hwm"] + if below and not in_dd: + in_dd = True + count = 1 + elif below and in_dd: + count += 1 + elif not below and in_dd: + lengths.append(count) + in_dd = False + count = 0 + return lengths + + # ── R² linearity ────────────────────────────────────────────────────────── + + def _check_r_squared(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + """Linear regression on cumulative PnL; low R² = high variance / choppy growth.""" + x = np.arange(len(df)) + y = df["cumulative_pnl"].values + + try: + slope, intercept, r_value, p_value, _ = linregress(x, y) + except Exception: + return [] + + r_sq = r_value ** 2 + if r_sq >= self.min_r_sq or slope <= 0: + return [] + + confidence = min(0.85, (self.min_r_sq - r_sq) * 3) + + return [Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Equity curve linearity R²={r_sq:.2f} (threshold {self.min_r_sq:.2f}). " + f"High variance in growth pattern — inconsistent performance. " + f"May indicate regime sensitivity or scattered trade timing." + ), + severity="medium" if r_sq < 0.50 else "low", + confidence=confidence, + impact_estimate_pnl=0.0, + suggested_params={}, + evidence={ + "r_squared": round(r_sq, 4), + "slope": round(float(slope), 4), + "p_value": round(float(p_value), 4), + }, + )] + + # ── Loss clusters ───────────────────────────────────────────────────────── + + def _check_loss_clusters(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + """Find sequences of ≥ N consecutive losing trades.""" + clusters = [] + streak = 0 + start_idx = None + + for idx, row in df.iterrows(): + if row["net_money"] < 0: + if streak == 0: + start_idx = idx + streak += 1 + else: + if streak >= self.cluster_min: + cluster_df = df.loc[start_idx:idx - 1] + clusters.append({ + "length": streak, + "total_pnl": float(cluster_df["net_money"].sum()), + "start_time": str(df.loc[start_idx, "open_time"]) if "open_time" in df.columns else "?", + }) + streak = 0 + + # Handle cluster at end of data + if streak >= self.cluster_min and start_idx is not None: + cluster_df = df.loc[start_idx:] + clusters.append({ + "length": streak, + "total_pnl": float(cluster_df["net_money"].sum()), + "start_time": str(df.loc[start_idx, "open_time"]) if "open_time" in df.columns else "?", + }) + + if not clusters: + return [] + + max_cluster = max(c["length"] for c in clusters) + total_cluster_loss = sum(c["total_pnl"] for c in clusters if c["total_pnl"] < 0) + confidence = min(0.85, len(clusters) * 0.12 + max_cluster * 0.05) + + return [Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Found {len(clusters)} loss cluster(s) of ≥ {self.cluster_min} consecutive losses. " + f"Worst streak: {max_cluster} trades. " + f"Total cluster losses: ${total_cluster_loss:.0f}." + ), + severity=self._severity(confidence, abs(total_cluster_loss), metrics.net_profit), + confidence=confidence, + impact_estimate_pnl=abs(total_cluster_loss), + suggested_params={ + "InpMaxDailyLossPct": 2.0, + "InpMaxTradesPerDay": 3, + }, + evidence={ + "cluster_count": len(clusters), + "max_streak": max_cluster, + "total_cluster_loss": round(total_cluster_loss, 2), + "clusters": clusters[:5], # keep top 5 for display + }, + )] diff --git a/analysis/reversal.py b/analysis/reversal.py new file mode 100644 index 0000000..953610d --- /dev/null +++ b/analysis/reversal.py @@ -0,0 +1,189 @@ +""" +analysis/reversal.py +Detects trades that went into significant profit but ultimately closed as losses. +Measures profit giveback and proposes trailing / TP tightening adjustments. +""" +from __future__ import annotations +import numpy as np +import pandas as pd + +from data.models import Finding, RunMetrics +from analysis.base import BaseAnalyzer + + +class ReversalAnalyzer(BaseAnalyzer): + """ + Profit Reversal Detector. + + A "reversal" trade is one that: + - Closed at a loss (net_money < 0) + - Had MFE >= threshold pips (i.e. was at significant unrealised profit at some point) + + Also flags trades that won but captured less than X% of their MFE (partial giveback). + """ + + name = "reversal" + + def __init__( + self, + mfe_threshold_pips: float = 15.0, + min_reversal_rate: float = 0.15, + poor_capture_rate: float = 0.55, + permutation_n: int = 500, + ): + self.mfe_threshold = mfe_threshold_pips + self.min_reversal_rate = min_reversal_rate + self.poor_capture_rate = poor_capture_rate + self.permutation_n = permutation_n + + def analyze(self, trades: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + findings = [] + + has_mfe = "mfe_pips" in trades.columns and trades["mfe_pips"].notna().sum() > 10 + + if has_mfe: + findings += self._check_reversals(trades, metrics) + findings += self._check_capture_rate(trades, metrics) + else: + # Without MFE, do a simpler check using result_class if available + if "result_class" in trades.columns: + findings += self._check_result_classes(trades, metrics) + + return sorted(findings, key=lambda f: f.confidence, reverse=True) + + # ── Reversal rate check ─────────────────────────────────────────────────── + + def _check_reversals(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + """Find losing trades that had significant unrealised profit.""" + losers = df[df["net_money"] < 0] + if len(losers) == 0: + return [] + + reversals = losers[losers["mfe_pips"] >= self.mfe_threshold] + rate = len(reversals) / len(losers) + + if rate < self.min_reversal_rate or len(reversals) < 5: + return [] + + avg_giveback_pips = float(reversals["mfe_pips"].mean()) + total_lost = float(losers["net_money"].sum()) + reversal_lost = float(reversals["net_money"].sum()) + + # Permutation test: is the reversal rate unusually high? + all_mfe = df[df["net_money"] < 0]["mfe_pips"].dropna().values + if len(all_mfe) > 0: + p_val = self._permutation_pvalue( + reversals["mfe_pips"].values, all_mfe, + n_permutations=self.permutation_n, alternative="greater" + ) + else: + p_val = 0.01 # assume significant + + confidence = max(0.0, min(1.0, 1 - p_val)) + impact_pnl = abs(reversal_lost) # upper bound on recoverable PnL + + # Compute median reversal time for trailing stop suggestion + if "duration_minutes" in df.columns: + median_dur = float(reversals["duration_minutes"].median()) + else: + median_dur = 0 + + # Compute 25th percentile of MFE at reversal — suggest TrailStart at this level + mfe_p25 = float(reversals["mfe_pips"].quantile(0.25)) + + suggested = { + "InpUseTrailing": True, + "InpTrailStartPips": round(max(10.0, mfe_p25 * 0.85), 1), + "InpTrailStepPips": 10.0, + } + + return [Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"{rate*100:.0f}% of losing trades had MFE ≥ {self.mfe_threshold:.0f} pips " + f"before reversing ({len(reversals)} trades). " + f"Avg giveback: {avg_giveback_pips:.1f} pips. " + f"Estimated recoverable PnL: ${impact_pnl:.0f}." + ), + severity=self._severity(confidence, impact_pnl, metrics.net_profit), + confidence=confidence, + impact_estimate_pnl=impact_pnl, + suggested_params=suggested, + evidence={ + "reversal_count": len(reversals), + "reversal_rate": round(rate, 4), + "avg_giveback_pips": round(avg_giveback_pips, 2), + "mfe_p25": round(mfe_p25, 2), + "median_duration_min": round(median_dur, 0), + "p_value": round(p_val, 4), + }, + )] + + # ── MFE Capture rate check ──────────────────────────────────────────────── + + def _check_capture_rate(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + """Check if winning trades are capturing enough of their MFE.""" + winners = df[(df["net_money"] > 0) & df["mfe_pips"].notna() & (df["mfe_pips"] > 0)] + if len(winners) < 10: + return [] + + if "exit_quality" not in df.columns or df["exit_quality"].isna().all(): + winners = winners.copy() + winners["exit_quality"] = winners["net_pips"] / winners["mfe_pips"].clip(lower=0.01) + + mean_capture = float(winners["exit_quality"].mean()) + if mean_capture >= self.poor_capture_rate: + return [] + + potential_gain = float( + (winners["mfe_pips"] - winners["net_pips"]).clip(lower=0).mean() + ) * float(winners["lot_size"].mean()) * 100 # rough $ + + confidence = self._confidence_from_z( + (self.poor_capture_rate - mean_capture) / max(0.01, winners["exit_quality"].std()) + ) + + return [Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Winners capture only {mean_capture*100:.0f}% of their MFE on average. " + f"Potential gain with better exits: ~${potential_gain*len(winners):.0f}." + ), + severity=self._severity(confidence, potential_gain * len(winners), metrics.net_profit), + confidence=min(0.95, confidence), + impact_estimate_pnl=potential_gain * len(winners), + suggested_params={ + "InpUseTrailing": True, + "InpTrailStartPips": round(float(winners["mfe_pips"].quantile(0.30)), 1), + }, + evidence={ + "mean_capture_ratio": round(mean_capture, 4), + "winner_count": len(winners), + }, + )] + + # ── Fallback: result_class based ───────────────────────────────────────── + + def _check_result_classes(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + """Simple reversal check using pre-classified result_class column.""" + reversals = df[df["result_class"] == "reversal"] + losers = df[df["net_money"] < 0] + if len(losers) == 0 or len(reversals) == 0: + return [] + + rate = len(reversals) / len(losers) + if rate < self.min_reversal_rate: + return [] + + return [Finding( + run_id=self.run_id, + analyzer=self.name, + description=f"{rate*100:.0f}% of losers classified as reversals (MFE-based).", + severity="medium", + confidence=0.65, + impact_estimate_pnl=abs(float(reversals["net_money"].sum())), + suggested_params={"InpUseTrailing": True}, + evidence={"reversal_rate": round(rate, 4)}, + )] diff --git a/analysis/time_performance.py b/analysis/time_performance.py new file mode 100644 index 0000000..4812450 --- /dev/null +++ b/analysis/time_performance.py @@ -0,0 +1,304 @@ +""" +analysis/time_performance.py +Analyzes trade performance by hour (UTC), session, and day of week. +Identifies statistically significant negative-edge time windows. +""" +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.models import Finding, RunMetrics +from analysis.base import BaseAnalyzer + + +class TimePerformanceAnalyzer(BaseAnalyzer): + """ + Session / Hour / Day-of-Week Performance Analyzer. + + Buckets trades by time dimension and flags windows with: + - Z-score < threshold (mean PnL very negative vs overall) + - Statistically significant by permutation test (p < 0.10) + - Minimum trade count (don't flag buckets with too few trades) + """ + + name = "time_performance" + + def __init__( + self, + z_score_threshold: float = -1.5, + min_bucket_trades: int = 10, + permutation_n: int = 1000, + pvalue_threshold: float = 0.10, + ): + self.z_threshold = z_score_threshold + self.min_bucket = min_bucket_trades + self.perm_n = permutation_n + self.p_threshold = pvalue_threshold + + def analyze(self, trades: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + if "hour_utc" not in trades.columns: + return [] + + findings = [] + findings += self._analyze_hours(trades, metrics) + findings += self._analyze_sessions(trades, metrics) + findings += self._analyze_days(trades, metrics) + return sorted(findings, key=lambda f: f.confidence, reverse=True) + + # ── Hour analysis ───────────────────────────────────────────────────────── + + def _analyze_hours(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + """Flag individual UTC hours with poor performance.""" + global_mean = df["net_money"].mean() + global_std = df["net_money"].std() + all_pnl = df["net_money"].values + + if global_std == 0: + return [] + + findings = [] + for hour in sorted(df["hour_utc"].dropna().unique()): + bucket = df[df["hour_utc"] == hour] + if len(bucket) < self.min_bucket: + continue + + mean_pnl = bucket["net_money"].mean() + z = (mean_pnl - global_mean) / global_std + + if z >= self.z_threshold: + continue + + # Permutation test + p_val = self._permutation_pvalue( + bucket["net_money"].values, all_pnl, + n_permutations=self.perm_n, alternative="less" + ) + if p_val >= self.p_threshold: + continue + + impact_pnl = abs(float(bucket[bucket["net_money"] < 0]["net_money"].sum())) + confidence = max(0.0, min(0.97, 1 - p_val)) + + findings.append(Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Hour {hour:02d}:00 UTC: mean PnL ${mean_pnl:.2f} " + f"(Z={z:.2f}, {len(bucket)} trades). " + f"Estimated negative contribution: ${impact_pnl:.0f}." + ), + severity=self._severity(confidence, impact_pnl, metrics.net_profit), + confidence=confidence, + impact_estimate_pnl=impact_pnl, + suggested_params={}, # session filter suggestion built by aggregator + evidence={ + "type": "hour", + "hour_utc": int(hour), + "mean_pnl": round(mean_pnl, 2), + "z_score": round(z, 3), + "trade_count": int(len(bucket)), + "p_value": round(p_val, 4), + }, + )) + + # Consolidate consecutive bad hours into a single window finding + if findings: + findings = self._consolidate_hour_findings(findings, df, metrics) + + return findings + + def _consolidate_hour_findings( + self, hour_findings: list[Finding], df: pd.DataFrame, metrics: RunMetrics + ) -> list[Finding]: + """ + Group consecutive flagged hours into a single window finding. + E.g. hours [14, 15, 16] → "14:00–17:00 UTC bad window" + Returns a single consolidated finding (plus keeps top individual for detail). + """ + bad_hours = sorted( + int(f.evidence["hour_utc"]) for f in hour_findings + ) + if not bad_hours: + return hour_findings + + # Find contiguous groups + groups = [] + group = [bad_hours[0]] + for h in bad_hours[1:]: + if h == group[-1] + 1: + group.append(h) + else: + groups.append(group) + group = [h] + groups.append(group) + + consolidated = [] + for g in groups: + start_h = g[0] + end_h = g[-1] + 1 + window = df[df["hour_utc"].between(start_h, g[-1])] + total_pnl = float(window["net_money"].sum()) + n_trades = len(window) + impact = abs(float(window[window["net_money"] < 0]["net_money"].sum())) + + # Derive session filter params from window + # Convert UTC to broker local time for session params + broker_start = (start_h + 2) % 24 # UTC+2 (from config) + broker_end = (end_h + 2) % 24 + + max_conf = max(f.confidence for f in hour_findings + if f.evidence["hour_utc"] in g) + + consolidated.append(Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"Negative edge window {start_h:02d}:00–{end_h:02d}:00 UTC: " + f"${total_pnl:.0f} total, {n_trades} trades. " + f"Consider excluding this window via session filter." + ), + severity=self._severity(max_conf, impact, metrics.net_profit), + confidence=max_conf, + impact_estimate_pnl=impact, + suggested_params={ + "InpUseSession": True, + # Preserve existing session start; cut end before bad window + # These are broker-local hours + "InpSessionEnd": (broker_start) % 24, + }, + evidence={ + "type": "hour_window", + "start_utc": start_h, + "end_utc": end_h, + "broker_start": broker_start, + "broker_end": broker_end, + "total_pnl": round(total_pnl, 2), + "trade_count": n_trades, + "hours_flagged": g, + }, + )) + + return consolidated + + # ── Session analysis ────────────────────────────────────────────────────── + + def _analyze_sessions(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + if "session" not in df.columns: + return [] + + global_mean = df["net_money"].mean() + global_std = df["net_money"].std() + all_pnl = df["net_money"].values + + if global_std == 0: + return [] + + findings = [] + for session in df["session"].dropna().unique(): + bucket = df[df["session"] == session] + if len(bucket) < self.min_bucket: + continue + + mean_pnl = bucket["net_money"].mean() + z = (mean_pnl - global_mean) / global_std + if z >= self.z_threshold: + continue + + p_val = self._permutation_pvalue( + bucket["net_money"].values, all_pnl, + n_permutations=self.perm_n, alternative="less" + ) + if p_val >= self.p_threshold: + continue + + pf = ( + bucket[bucket["net_money"] > 0]["net_money"].sum() / + max(0.01, abs(bucket[bucket["net_money"] < 0]["net_money"].sum())) + ) + impact = abs(float(bucket[bucket["net_money"] < 0]["net_money"].sum())) + confidence = max(0.0, min(0.97, 1 - p_val)) + + findings.append(Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"{session} session: PF {pf:.2f}, mean PnL ${mean_pnl:.2f} " + f"(Z={z:.2f}, {len(bucket)} trades). Recommend excluding this session." + ), + severity=self._severity(confidence, impact, metrics.net_profit), + confidence=confidence, + impact_estimate_pnl=impact, + suggested_params={"InpUseSession": True}, + evidence={ + "type": "session", + "session": session, + "profit_factor": round(float(pf), 3), + "mean_pnl": round(mean_pnl, 2), + "z_score": round(z, 3), + "trade_count": int(len(bucket)), + "p_value": round(p_val, 4), + }, + )) + + return findings + + # ── Day-of-week analysis ────────────────────────────────────────────────── + + def _analyze_days(self, df: pd.DataFrame, metrics: RunMetrics) -> list[Finding]: + if "day_of_week" not in df.columns: + return [] + + DAY_NAMES = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"] + global_mean = df["net_money"].mean() + global_std = df["net_money"].std() + all_pnl = df["net_money"].values + + if global_std == 0: + return [] + + findings = [] + for day in range(5): # 0=Mon … 4=Fri + bucket = df[df["day_of_week"] == day] + if len(bucket) < self.min_bucket: + continue + + mean_pnl = bucket["net_money"].mean() + z = (mean_pnl - global_mean) / global_std + if z >= self.z_threshold: + continue + + p_val = self._permutation_pvalue( + bucket["net_money"].values, all_pnl, + n_permutations=self.perm_n, alternative="less" + ) + if p_val >= self.p_threshold: + continue + + impact = abs(float(bucket[bucket["net_money"] < 0]["net_money"].sum())) + confidence = max(0.0, min(0.97, 1 - p_val)) + + findings.append(Finding( + run_id=self.run_id, + analyzer=self.name, + description=( + f"{DAY_NAMES[day]}: mean PnL ${mean_pnl:.2f} " + f"(Z={z:.2f}, {len(bucket)} trades). " + f"Possible day-of-week edge degradation." + ), + severity="low", # day-level findings are informational + confidence=confidence, + impact_estimate_pnl=impact, + suggested_params={}, + evidence={ + "type": "day_of_week", + "day": DAY_NAMES[day], + "day_index": day, + "mean_pnl": round(mean_pnl, 2), + "z_score": round(z, 3), + "trade_count": int(len(bucket)), + "p_value": round(p_val, 4), + }, + )) + + return findings diff --git a/app.py b/app.py new file mode 100644 index 0000000..068de05 --- /dev/null +++ b/app.py @@ -0,0 +1,151 @@ +""" +app.py — MT5 EA Optimizer Web App +Double-click to launch. Browser opens automatically at http://localhost:5000 +""" +import sys, os, threading, webbrowser, time +from pathlib import Path + +# ── Make sure imports resolve from project root ─────────────────────────────── +BASE_DIR = Path(__file__).parent +sys.path.insert(0, str(BASE_DIR)) + +from flask import Flask, render_template, jsonify, request, send_from_directory +from flask_socketio import SocketIO, emit + +from optimizer_loop import OptimizerLoop + +# ── App setup ───────────────────────────────────────────────────────────────── +app = Flask(__name__, + template_folder="ui/templates", + static_folder="ui/static") +app.config["SECRET_KEY"] = "mt5optimizer2024" +socketio = SocketIO(app, cors_allowed_origins="*", async_mode="threading") + +REPORTS_DIR = BASE_DIR / "Reports" +REPORTS_DIR.mkdir(exist_ok=True) + +# Global optimizer instance +optimizer: OptimizerLoop = None +optimizer_thread: threading.Thread = None + + +# ── Routes ──────────────────────────────────────────────────────────────────── + +@app.route("/") +def index(): + return render_template("index.html") + + +@app.route("/api/status") +def status(): + if optimizer is None: + return jsonify({"state": "idle", "iteration": 0, "best_score": 0}) + return jsonify(optimizer.get_status()) + + +@app.route("/api/start", methods=["POST"]) +def start(): + global optimizer, optimizer_thread + if optimizer and optimizer.running: + return jsonify({"ok": False, "msg": "Already running"}) + + data = request.get_json(silent=True) or {} + optimizer = OptimizerLoop( + config_path=str(BASE_DIR / "config.yaml"), + socketio=socketio, + reports_dir=REPORTS_DIR, + auto_mode=data.get("auto", True), + ) + optimizer_thread = threading.Thread(target=optimizer.run, daemon=True) + optimizer_thread.start() + return jsonify({"ok": True}) + + +@app.route("/api/pause", methods=["POST"]) +def pause(): + if optimizer: + optimizer.toggle_pause() + return jsonify({"ok": True, "paused": optimizer.paused}) + return jsonify({"ok": False}) + + +@app.route("/api/stop", methods=["POST"]) +def stop(): + if optimizer: + optimizer.stop() + return jsonify({"ok": True}) + + +@app.route("/api/skip", methods=["POST"]) +def skip(): + if optimizer: + optimizer.skip_hypothesis() + return jsonify({"ok": True}) + + +@app.route("/api/history") +def history(): + if optimizer is None: + return jsonify([]) + return jsonify(optimizer.score_history) + + +@app.route("/reports") +@app.route("/reports/") +def reports_index(): + """Reports browser page — fixes the 404 on the Reports button.""" + runs = [] + for run_dir in sorted(REPORTS_DIR.iterdir(), reverse=True) if REPORTS_DIR.exists() else []: + summary = run_dir / "summary.json" + if summary.exists(): + import json + try: + runs.append(json.loads(summary.read_text())) + except Exception: + pass + return render_template("reports_index.html", runs=runs[:100]) + + +@app.route("/reports/") +def reports_file(filename): + """Serve individual report files (HTML, CSV, JSON).""" + return send_from_directory(REPORTS_DIR, filename) + + +@app.route("/api/runs") +def runs_list(): + runs = [] + if REPORTS_DIR.exists(): + for run_dir in sorted(REPORTS_DIR.iterdir(), reverse=True): + summary = run_dir / "summary.json" + if summary.exists(): + import json + try: + runs.append(json.loads(summary.read_text())) + except Exception: + pass + return jsonify(runs[:50]) + + +# ── SocketIO events ─────────────────────────────────────────────────────────── + +@socketio.on("connect") +def on_connect(): + if optimizer: + emit("status_sync", optimizer.get_status()) + + +# ── Launch ──────────────────────────────────────────────────────────────────── + +def open_browser(): + time.sleep(1.5) + webbrowser.open("http://localhost:5000") + + +if __name__ == "__main__": + print("=" * 60) + print(" MT5 EA Optimizer — Starting...") + print(" Opening browser at http://localhost:5000") + print("=" * 60) + threading.Thread(target=open_browser, daemon=True).start() + socketio.run(app, host="0.0.0.0", port=5000, debug=False, use_reloader=False) diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..dcdad8f --- /dev/null +++ b/config.yaml @@ -0,0 +1,103 @@ +# MT5 EA Strategy Optimizer — Master Configuration +# LEGSTECH_EA_V2 | XAUUSD | H1 +# ───────────────────────────────────────────── + +ea: + name: "LEGSTECH_EA_V2" + file: "LEGSTECH_EA_V2" # .ex5 file name (no extension), in Experts/ + symbol: "XAUUSD" + timeframe: "H1" # Period code used in INI (H1 = 16385... see MT5 period codes) + mt5_period_code: 16385 # PERIOD_H1 numeric code for [Tester] Period + +periods: + train_start: "2022.01.01" + train_end: "2023.12.31" + validate_start: "2024.01.01" + validate_end: "2024.06.30" + oos_start: "2024.07.01" # LOCKED — never touched during optimization + oos_end: "2024.12.31" + +broker: + timezone_offset_hours: 2 # Broker server = UTC+2. Set to 0 for UTC brokers. + deposit: 10000.0 + currency: "USD" + leverage: 100 + # Session definitions in BROKER LOCAL TIME (will be normalized to UTC internally) + sessions: + Asian: {start: 0, end: 9} + London: {start: 9, end: 18} # UTC+2: 07:00 UTC = 09:00 broker + NY: {start: 15, end: 24} # UTC+2: 13:00 UTC = 15:00 broker + LondonNY: {start: 15, end: 18} # Overlap + +mt5: + terminal_exe: "C:/Program Files/MetaTrader 5/terminal64.exe" + # Terminal data folder — detected automatically from your MetaQuotes installation + appdata_path: "C:/Users/DELL/AppData/Roaming/MetaQuotes/Terminal/D0E8209F77C8CF37AD8BF550E51FF075" + # MQL5 Files folder — TradeLogger CSV is written here during backtests + # NOTE: Strategy Tester writes to the Agent subfolder, not the terminal Files folder + mql5_files_path: "C:/Users/DELL/AppData/Roaming/MetaQuotes/Tester/D0E8209F77C8CF37AD8BF550E51FF075/Agent-127.0.0.1-3000/MQL5/Files" + tester_model: 4 # 4=OHLC M1 (fast, reliable). Use 0=Every Tick only if full tick data available + tester_timeout_seconds: 1800 # 30 min max per test + shutdown_terminal: 1 # ShutdownTerminal=1 in INI — MT5 closes after test + data_readiness_wait_seconds: 10 # Wait N seconds after MT5 launch for data to load before testing + kill_on_start: true # Always kill existing MT5 before each run + report_subdir: "runs" # relative to project root + +logging: + level: "INFO" # DEBUG | INFO | WARNING | ERROR + file: "optimizer.log" + +thresholds: + min_trades: 50 # below this → no statistical confidence + min_profit_factor: 1.20 + min_calmar: 0.35 + max_oos_degradation: 0.30 # 30% drop IS→OOS is acceptable; above = reject + sensitivity_tolerance: 0.30 # ±10% param → >30% calmar drop = fragile, reject + min_wfv_ratio: 0.70 # OOS calmar must be ≥ 70% of IS calmar in WFV + +scoring: + weights: + calmar: 0.35 + profit_factor: 0.20 + mfe_capture: 0.20 # avg MFE capture ratio (exit quality) + session_stability: 0.15 # 1 - std_dev of per-session Calmar + recovery_factor: 0.10 + normalization: + calmar: {lo: 0.0, hi: 4.0} + profit_factor: {lo: 1.0, hi: 3.5} + mfe_capture: {lo: 0.0, hi: 1.0} + session_stability: {lo: 0.0, hi: 1.0} + recovery_factor: {lo: 0.0, hi: 6.0} + significance_trades: 150 # full significance weight at this trade count + +analysis: + reversal: + mfe_threshold_pips: 15.0 # minimum MFE to classify as reversal candidate + min_reversal_rate: 0.15 # above this → HIGH finding + permutation_n: 500 # permutation tests for significance + time_performance: + min_trades_per_bucket: 10 # ignore time buckets with fewer trades + z_score_threshold: -1.5 # flag as negative edge + permutation_n: 1000 + entry_exit: + poor_exit_quality: 0.55 # below this → HIGH finding + poor_entry_quality: 0.40 # below this → HIGH finding + equity_curve: + max_flatness_score: 0.50 # above this → raise finding + min_r_squared: 0.70 # below this → high variance + +mutation: + max_hypotheses_per_cycle: 3 # test at most N hypotheses before picking best + dedup_lookback_runs: 10 # avoid re-testing same delta seen in last N runs + explore_fallback_after: 5 # switch to explore if N targeted runs all rejected + +optimization: + max_iterations: 50 + convergence_window: 3 # stop if no improvement in this many promotions + convergence_threshold: 0.04 # minimum composite score delta to count as improvement + +paths: + db: "optimizer.db" + runs_dir: "runs" + reports_dir: "reports" + log_file: "optimizer.log" diff --git a/data/__init__.py b/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data/models.py b/data/models.py new file mode 100644 index 0000000..26f4ff2 --- /dev/null +++ b/data/models.py @@ -0,0 +1,181 @@ +""" +data/models.py +Pydantic v2 data models for all entities in the optimizer. +""" +from __future__ import annotations +from datetime import datetime +from typing import Any, Literal, Optional +from pydantic import BaseModel, Field, computed_field, model_validator +import uuid + + +# ───────────────────────────────────────────────────────────────────────────── +# Trade +# ───────────────────────────────────────────────────────────────────────────── + +class Trade(BaseModel): + """One closed trade, enriched with MAE/MFE from the logger CSV.""" + ticket: int + open_time: datetime + close_time: datetime + direction: Literal["buy", "sell"] + open_price: float + close_price: float + sl: float + tp: float + lot_size: float + net_pips: float + net_money: float + duration_minutes: int + commission: float = 0.0 + swap: float = 0.0 + + # From TradeLogger CSV (may be None if logger not available) + mfe_pips: Optional[float] = None + mae_pips: Optional[float] = None + + # Derived — populated during ingest + session: Optional[str] = None # London | NY | Asian | LondonNY | Off + day_of_week: Optional[int] = None # 0=Mon … 4=Fri + hour_broker: Optional[int] = None # broker local hour + hour_utc: Optional[int] = None # UTC hour (after timezone normalisation) + result_class: Optional[str] = None # win | loss | be | reversal + + # Computed quality scores (None when MFE/MAE not available) + mfe_capture_ratio: Optional[float] = None # net_money / mfe_value; 1.0 = captured all + entry_quality: Optional[float] = None # 1 - (mae_pips / max(mfe_pips,1)) + exit_quality: Optional[float] = None # net_pips / max(mfe_pips, 1) + + @property + def won(self) -> bool: + return self.net_money > 0 + + @property + def lost(self) -> bool: + return self.net_money < 0 + + +# ───────────────────────────────────────────────────────────────────────────── +# RunMetrics +# ───────────────────────────────────────────────────────────────────────────── + +class RunMetrics(BaseModel): + """Summary metrics for one backtest run.""" + run_id: str + net_profit: float + profit_factor: float + max_drawdown_abs: float # in account currency + max_drawdown_pct: float # as fraction (0.15 = 15%) + calmar_ratio: float + sharpe_ratio: float + total_trades: int + win_rate: float # fraction (0.55 = 55%) + avg_win: float + avg_loss: float + recovery_factor: float + largest_loss: float + expected_payoff: float + + # MAE/MFE derived (populated when logger CSV available) + avg_mfe_capture: Optional[float] = None + avg_mfe_pips: Optional[float] = None + avg_mae_pips: Optional[float] = None + reversal_rate: Optional[float] = None # reverted trades / total losers + + # Filled by composite scorer + composite_score: float = 0.0 + + +# ───────────────────────────────────────────────────────────────────────────── +# Run +# ───────────────────────────────────────────────────────────────────────────── + +class Run(BaseModel): + """One complete backtest run record.""" + run_id: str = Field(default_factory=lambda: str(uuid.uuid4())[:8]) + run_ts: datetime = Field(default_factory=datetime.utcnow) + ea_name: str + symbol: str + timeframe: str + period_start: str + period_end: str + params: dict[str, Any] # snapshot of all EA inputs used + phase: Literal["baseline", "explore", "validate", "wfv", "oos"] + hypothesis_id: Optional[str] = None + tester_model: int = 0 # 0=Every Tick + ini_snapshot: Optional[str] = None # full .ini content for reproducibility + report_path: Optional[str] = None + log_csv_path: Optional[str] = None + + +# ───────────────────────────────────────────────────────────────────────────── +# Finding +# ───────────────────────────────────────────────────────────────────────────── + +class Finding(BaseModel): + """One actionable observation from an analyzer module.""" + finding_id: str = Field(default_factory=lambda: str(uuid.uuid4())[:8]) + run_id: str + analyzer: str + description: str + severity: Literal["high", "medium", "low"] + confidence: float # 0.0–1.0 + impact_estimate_pnl: float = 0.0 # estimated PnL recovery if addressed + suggested_params: dict[str, Any] = {} + evidence: dict[str, Any] = {} # raw supporting data (for reports) + + +# ───────────────────────────────────────────────────────────────────────────── +# Hypothesis +# ───────────────────────────────────────────────────────────────────────────── + +class Hypothesis(BaseModel): + """A proposed parameter change, motivated by one or more findings.""" + hypothesis_id: str = Field(default_factory=lambda: str(uuid.uuid4())[:8]) + parent_run_id: str + finding_ids: list[str] + description: str + param_delta: dict[str, Any] # {param_name: proposed_value} + strategy: Literal["targeted", "compound", "rollback", "explore"] + kb_rule_id: Optional[str] = None # KB rule that generated this + status: Literal["pending", "tested", "validated", "rejected"] = "pending" + tested_run_id: Optional[str] = None + + +# ───────────────────────────────────────────────────────────────────────────── +# Candidate +# ───────────────────────────────────────────────────────────────────────────── + +class Candidate(BaseModel): + """A parameter set that has passed all validation gates.""" + candidate_id: str = Field(default_factory=lambda: str(uuid.uuid4())[:8]) + run_id: str + promoted_ts: datetime = Field(default_factory=datetime.utcnow) + composite_score: float + oos_score: Optional[float] = None + params: dict[str, Any] + lineage: list[str] = [] # run_ids from baseline to this candidate + + +# ───────────────────────────────────────────────────────────────────────────── +# RunResult (returned by MT5Runner) +# ───────────────────────────────────────────────────────────────────────────── + +class RunResult(BaseModel): + """Raw file paths returned after a tester run completes.""" + run_id: str + report_xml: Optional[str] = None # path to MT5 XML report + report_html: Optional[str] = None # path to MT5 HTML report + trade_log_csv: Optional[str] = None # path to TradeLogger CSV + success: bool = True + error_message: Optional[str] = None + + +# ───────────────────────────────────────────────────────────────────────────── +# GateResult (returned by ValidationGate) +# ───────────────────────────────────────────────────────────────────────────── + +class GateResult(BaseModel): + passed: bool + details: dict[str, bool | float | str] = {} + reason: Optional[str] = None diff --git a/data/store.py b/data/store.py new file mode 100644 index 0000000..11a9a09 --- /dev/null +++ b/data/store.py @@ -0,0 +1,326 @@ +""" +data/store.py +SQLite (metadata) + Parquet (per-run trade data) storage layer. +""" +from __future__ import annotations +import json +import sqlite3 +import shutil +from pathlib import Path +from typing import Any, Optional + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +from loguru import logger + +from data.models import ( + Run, RunMetrics, Finding, Hypothesis, Candidate, Trade +) + + +# ── Schema DDL ──────────────────────────────────────────────────────────────── + +SCHEMA_SQL = """ +PRAGMA journal_mode=WAL; +PRAGMA foreign_keys=ON; + +CREATE TABLE IF NOT EXISTS runs ( + run_id TEXT PRIMARY KEY, + run_ts TEXT NOT NULL, + ea_name TEXT NOT NULL, + symbol TEXT NOT NULL, + timeframe TEXT NOT NULL, + period_start TEXT NOT NULL, + period_end TEXT NOT NULL, + params_json TEXT NOT NULL, + phase TEXT NOT NULL, + hypothesis_id TEXT, + tester_model INTEGER DEFAULT 0, + ini_snapshot TEXT, + report_path TEXT, + log_csv_path TEXT +); + +CREATE TABLE IF NOT EXISTS run_metrics ( + run_id TEXT PRIMARY KEY REFERENCES runs(run_id), + net_profit REAL, + profit_factor REAL, + max_drawdown_abs REAL, + max_drawdown_pct REAL, + calmar_ratio REAL, + sharpe_ratio REAL, + total_trades INTEGER, + win_rate REAL, + avg_win REAL, + avg_loss REAL, + recovery_factor REAL, + largest_loss REAL, + expected_payoff REAL, + avg_mfe_capture REAL, + avg_mfe_pips REAL, + avg_mae_pips REAL, + reversal_rate REAL, + composite_score REAL +); + +CREATE TABLE IF NOT EXISTS findings ( + finding_id TEXT PRIMARY KEY, + run_id TEXT NOT NULL REFERENCES runs(run_id), + analyzer TEXT NOT NULL, + description TEXT NOT NULL, + severity TEXT NOT NULL, + confidence REAL NOT NULL, + impact_estimate_pnl REAL DEFAULT 0, + suggested_params TEXT, + evidence TEXT +); + +CREATE TABLE IF NOT EXISTS hypotheses ( + hypothesis_id TEXT PRIMARY KEY, + parent_run_id TEXT NOT NULL REFERENCES runs(run_id), + finding_ids TEXT NOT NULL, + description TEXT NOT NULL, + param_delta TEXT NOT NULL, + strategy TEXT NOT NULL, + kb_rule_id TEXT, + status TEXT NOT NULL DEFAULT 'pending', + tested_run_id TEXT +); + +CREATE TABLE IF NOT EXISTS candidates ( + candidate_id TEXT PRIMARY KEY, + run_id TEXT NOT NULL REFERENCES runs(run_id), + promoted_ts TEXT NOT NULL, + composite_score REAL NOT NULL, + oos_score REAL, + params_json TEXT NOT NULL, + lineage_json TEXT +); +""" + + +# ── DataStore ───────────────────────────────────────────────────────────────── + +class DataStore: + """ + Central storage interface. + - SQLite for all structured metadata (runs, metrics, findings, hypotheses, candidates) + - Parquet for per-run trade arrays (cheap columnar access for analysis) + """ + + def __init__(self, db_path: str | Path, runs_dir: str | Path): + self.db_path = Path(db_path) + self.runs_dir = Path(runs_dir) + self.runs_dir.mkdir(parents=True, exist_ok=True) + self._init_db() + + # ── Init ────────────────────────────────────────────────────────────────── + + def _init_db(self) -> None: + with self._conn() as conn: + conn.executescript(SCHEMA_SQL) + logger.debug(f"Database initialised at {self.db_path}") + + def _conn(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + # ── Runs ────────────────────────────────────────────────────────────────── + + def save_run(self, run: Run) -> None: + with self._conn() as conn: + conn.execute(""" + INSERT OR REPLACE INTO runs + (run_id, run_ts, ea_name, symbol, timeframe, period_start, period_end, + params_json, phase, hypothesis_id, tester_model, ini_snapshot, + report_path, log_csv_path) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """, ( + run.run_id, + run.run_ts.isoformat(), + run.ea_name, run.symbol, run.timeframe, + run.period_start, run.period_end, + json.dumps(run.params), + run.phase, run.hypothesis_id, run.tester_model, + run.ini_snapshot, run.report_path, run.log_csv_path, + )) + logger.debug(f"Saved run {run.run_id}") + + def get_run(self, run_id: str) -> Optional[Run]: + with self._conn() as conn: + row = conn.execute( + "SELECT * FROM runs WHERE run_id=?", (run_id,) + ).fetchone() + if not row: + return None + d = dict(row) + d["params"] = json.loads(d.pop("params_json")) + return Run(**d) + + def list_runs(self, phase: Optional[str] = None, n: int = 100) -> list[dict]: + q = "SELECT run_id, run_ts, phase, hypothesis_id FROM runs" + args: list[Any] = [] + if phase: + q += " WHERE phase=?" + args.append(phase) + q += " ORDER BY run_ts DESC LIMIT ?" + args.append(n) + with self._conn() as conn: + return [dict(r) for r in conn.execute(q, args).fetchall()] + + # ── Run Metrics ─────────────────────────────────────────────────────────── + + def save_metrics(self, m: RunMetrics) -> None: + with self._conn() as conn: + conn.execute(""" + INSERT OR REPLACE INTO run_metrics + (run_id, net_profit, profit_factor, max_drawdown_abs, max_drawdown_pct, + calmar_ratio, sharpe_ratio, total_trades, win_rate, avg_win, avg_loss, + recovery_factor, largest_loss, expected_payoff, + avg_mfe_capture, avg_mfe_pips, avg_mae_pips, reversal_rate, + composite_score) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """, ( + m.run_id, m.net_profit, m.profit_factor, + m.max_drawdown_abs, m.max_drawdown_pct, + m.calmar_ratio, m.sharpe_ratio, + m.total_trades, m.win_rate, m.avg_win, m.avg_loss, + m.recovery_factor, m.largest_loss, m.expected_payoff, + m.avg_mfe_capture, m.avg_mfe_pips, m.avg_mae_pips, + m.reversal_rate, m.composite_score, + )) + + def get_metrics(self, run_id: str) -> Optional[RunMetrics]: + with self._conn() as conn: + row = conn.execute( + "SELECT * FROM run_metrics WHERE run_id=?", (run_id,) + ).fetchone() + return RunMetrics(**dict(row)) if row else None + + def best_candidate_score(self) -> float: + """Return the highest composite_score ever achieved by a promoted candidate.""" + with self._conn() as conn: + row = conn.execute( + "SELECT MAX(composite_score) FROM candidates" + ).fetchone() + return float(row[0]) if row and row[0] is not None else 0.0 + + # ── Findings ────────────────────────────────────────────────────────────── + + def save_findings(self, findings: list[Finding]) -> None: + with self._conn() as conn: + for f in findings: + conn.execute(""" + INSERT OR REPLACE INTO findings + (finding_id, run_id, analyzer, description, severity, + confidence, impact_estimate_pnl, suggested_params, evidence) + VALUES (?,?,?,?,?,?,?,?,?) + """, ( + f.finding_id, f.run_id, f.analyzer, f.description, + f.severity, f.confidence, f.impact_estimate_pnl, + json.dumps(f.suggested_params), json.dumps(f.evidence), + )) + + def get_findings(self, run_id: str) -> list[Finding]: + with self._conn() as conn: + rows = conn.execute( + "SELECT * FROM findings WHERE run_id=? ORDER BY confidence DESC", + (run_id,) + ).fetchall() + out = [] + for r in rows: + d = dict(r) + d["suggested_params"] = json.loads(d["suggested_params"] or "{}") + d["evidence"] = json.loads(d["evidence"] or "{}") + out.append(Finding(**d)) + return out + + # ── Hypotheses ──────────────────────────────────────────────────────────── + + def save_hypothesis(self, h: Hypothesis) -> None: + with self._conn() as conn: + conn.execute(""" + INSERT OR REPLACE INTO hypotheses + (hypothesis_id, parent_run_id, finding_ids, description, + param_delta, strategy, kb_rule_id, status, tested_run_id) + VALUES (?,?,?,?,?,?,?,?,?) + """, ( + h.hypothesis_id, h.parent_run_id, + json.dumps(h.finding_ids), h.description, + json.dumps(h.param_delta), h.strategy, + h.kb_rule_id, h.status, h.tested_run_id, + )) + + def update_hypothesis_status( + self, hypothesis_id: str, + status: str, + tested_run_id: Optional[str] = None + ) -> None: + with self._conn() as conn: + conn.execute( + "UPDATE hypotheses SET status=?, tested_run_id=? WHERE hypothesis_id=?", + (status, tested_run_id, hypothesis_id) + ) + + def get_recent_param_deltas(self, n: int = 10) -> list[dict]: + """Return param_delta dicts from the last N tested hypotheses (for dedup).""" + with self._conn() as conn: + rows = conn.execute(""" + SELECT param_delta FROM hypotheses + WHERE status IN ('tested','validated','rejected') + ORDER BY rowid DESC LIMIT ? + """, (n,)).fetchall() + return [json.loads(r["param_delta"]) for r in rows] + + # ── Candidates ──────────────────────────────────────────────────────────── + + def save_candidate(self, c: Candidate) -> None: + with self._conn() as conn: + conn.execute(""" + INSERT OR REPLACE INTO candidates + (candidate_id, run_id, promoted_ts, composite_score, + oos_score, params_json, lineage_json) + VALUES (?,?,?,?,?,?,?) + """, ( + c.candidate_id, c.run_id, + c.promoted_ts.isoformat(), + c.composite_score, c.oos_score, + json.dumps(c.params), + json.dumps(c.lineage), + )) + logger.info(f"Promoted candidate {c.candidate_id} (score={c.composite_score:.4f})") + + def list_candidates(self) -> list[dict]: + with self._conn() as conn: + rows = conn.execute( + "SELECT * FROM candidates ORDER BY composite_score DESC" + ).fetchall() + return [dict(r) for r in rows] + + # ── Trade Data (Parquet) ────────────────────────────────────────────────── + + def save_trades(self, run_id: str, trades: list[Trade]) -> Path: + """Serialise Trade objects to Parquet. Returns path to written file.""" + parquet_path = self.runs_dir / run_id / "trades.parquet" + parquet_path.parent.mkdir(parents=True, exist_ok=True) + + rows = [t.model_dump() for t in trades] + df = pd.DataFrame(rows) + # Ensure datetime columns are proper dtype + for col in ["open_time", "close_time"]: + if col in df.columns: + df[col] = pd.to_datetime(df[col], utc=True) + + df.to_parquet(parquet_path, index=False, engine="pyarrow") + logger.debug(f"Saved {len(trades)} trades for run {run_id}") + return parquet_path + + def load_trades(self, run_id: str) -> pd.DataFrame: + """Load trade Parquet for a given run. Returns empty DataFrame if not found.""" + parquet_path = self.runs_dir / run_id / "trades.parquet" + if not parquet_path.exists(): + logger.warning(f"No trade parquet found for run {run_id}") + return pd.DataFrame() + return pd.read_parquet(parquet_path, engine="pyarrow") diff --git a/main.py b/main.py new file mode 100644 index 0000000..e299ec0 --- /dev/null +++ b/main.py @@ -0,0 +1,557 @@ +""" +main.py +MT5 EA Strategy Optimizer — CLI Entry Point +LEGSTECH_EA_V2 | XAUUSD | H1 + +Usage: + python main.py # interactive mode + python main.py --baseline # run baseline only and show analysis + python main.py --auto # fully automated loop (no human prompts) +""" +from __future__ import annotations +import argparse +import sys +from pathlib import Path +from datetime import datetime +from typing import Optional, Any + +import yaml +import pandas as pd +from loguru import logger +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich.prompt import Confirm, Prompt +from rich import print as rprint + +# ── Project imports ────────────────────────────────────────────────────────── +from data.models import Run, RunMetrics, Candidate +from data.store import DataStore +from mt5.ini_builder import IniBuilder +from mt5.runner import MT5Runner +from mt5.report_parser import ReportParser +from mt5.log_reader import TradeLogReader +from analysis.reversal import ReversalAnalyzer +from analysis.time_performance import TimePerformanceAnalyzer +from analysis.entry_exit_quality import EntryExitQualityAnalyzer +from analysis.equity_curve import EquityCurveAnalyzer +from scoring.composite import CompositeScorer +from mutation.engine import MutationEngine +from validation.gate import ValidationGate + +console = Console() + +CONFIG_PATH = Path("config.yaml") +MANIFEST_PATH = Path("mutation/param_manifest.yaml") +KB_PATH = Path("mutation/knowledge_base.yaml") +DB_PATH = Path("optimizer.db") +RUNS_DIR = Path("runs") + + +def load_config() -> dict: + with open(CONFIG_PATH) as f: + return yaml.safe_load(f) + + +# ── Component factory ───────────────────────────────────────────────────────── + +def build_components(cfg: dict): + store = DataStore(DB_PATH, RUNS_DIR) + builder = IniBuilder(CONFIG_PATH, MANIFEST_PATH) + runner = MT5Runner(CONFIG_PATH) + parser = ReportParser() + log_rdr = TradeLogReader( + broker_tz_offset_hours=cfg["broker"]["timezone_offset_hours"], + pip_size=0.1, # XAUUSD: 0.1 per pip + ) + analyzers = [ + ReversalAnalyzer( + mfe_threshold_pips=cfg["analysis"]["reversal"]["mfe_threshold_pips"], + min_reversal_rate=cfg["analysis"]["reversal"]["min_reversal_rate"], + permutation_n=cfg["analysis"]["reversal"]["permutation_n"], + ), + TimePerformanceAnalyzer( + z_score_threshold=cfg["analysis"]["time_performance"]["z_score_threshold"], + min_bucket_trades=cfg["analysis"]["time_performance"]["min_trades_per_bucket"], + permutation_n=cfg["analysis"]["time_performance"]["permutation_n"], + ), + EntryExitQualityAnalyzer( + poor_exit_threshold=cfg["analysis"]["entry_exit"]["poor_exit_quality"], + poor_entry_threshold=cfg["analysis"]["entry_exit"]["poor_entry_quality"], + ), + EquityCurveAnalyzer( + max_flatness=cfg["analysis"]["equity_curve"]["max_flatness_score"], + min_r_squared=cfg["analysis"]["equity_curve"]["min_r_squared"], + ), + ] + scorer = CompositeScorer(str(CONFIG_PATH)) + mutator = MutationEngine(KB_PATH, MANIFEST_PATH, + dedup_lookback=cfg["mutation"]["dedup_lookback_runs"]) + gate = ValidationGate(CONFIG_PATH) + return store, builder, runner, parser, log_rdr, analyzers, scorer, mutator, gate + + +# ── Single run pipeline ─────────────────────────────────────────────────────── + +def execute_run( + run_id: str, + params: dict[str, Any], + period_start: str, + period_end: str, + phase: str, + hypothesis_id: Optional[str], + cfg: dict, + store: DataStore, + builder: IniBuilder, + runner: MT5Runner, + parser: ReportParser, + log_rdr: TradeLogReader, + analyzers: list, + scorer: CompositeScorer, +) -> tuple[Optional[RunMetrics], pd.DataFrame]: + """ + Execute one complete backtest run: + 1. Build INI → Launch MT5 → Wait → Parse report → Merge logger CSV + 2. Compute derived fields, enrich trades + 3. Compute composite score + 4. Save everything to store + Returns (metrics, trades_df) or (None, empty_df) on failure. + """ + run_dir = RUNS_DIR / run_id + run_dir.mkdir(parents=True, exist_ok=True) + + # 1. Build INI + ini_path = builder.build( + run_id=run_id, + params=params, + period_start=period_start, + period_end=period_end, + output_dir=run_dir, + phase=phase, + ) + + # 2. Save run record + run = Run( + run_id=run_id, + ea_name=cfg["ea"]["name"], + symbol=cfg["ea"]["symbol"], + timeframe=cfg["ea"]["timeframe"], + period_start=period_start, + period_end=period_end, + params=params, + phase=phase, + hypothesis_id=hypothesis_id, + tester_model=cfg["mt5"]["tester_model"], + ini_snapshot=ini_path.read_text(), + ) + store.save_run(run) + + # 3. Launch MT5 + report_dir = run_dir / "report" + # MT5 Files folder: may need adjusting to actual terminal data path + mt5_files_dir = None # TODO: set to actual MQL5/Files path once terminal path confirmed + + console.print(f"[dim]Launching MT5... (timeout {cfg['mt5']['tester_timeout_seconds']}s)[/dim]") + result = runner.run(run_id, ini_path, report_dir, log_csv_search_dir=mt5_files_dir) + + if not result.success: + logger.error(f"Run {run_id} failed: {result.error_message}") + console.print(f"[red]✗ MT5 run failed: {result.error_message}[/red]") + return None, pd.DataFrame() + + # 4. Parse report + metrics, trades = parser.parse(result.report_xml, result.report_html) + if metrics is None: + console.print(f"[red]✗ Could not parse report for {run_id}[/red]") + return None, pd.DataFrame() + metrics.run_id = run_id + + # 5. Merge TradeLogger CSV → enrich with MAE/MFE + derived fields + if trades: + trades = log_rdr.merge( + trades, result.trade_log_csv, + reversal_mfe_threshold_pips=cfg["analysis"]["reversal"]["mfe_threshold_pips"], + ) + trades_df = pd.DataFrame([t.model_dump() for t in trades]) + else: + trades_df = pd.DataFrame() + + # 6. Compute reversal rate and MFE capture for metrics + if not trades_df.empty: + if "result_class" in trades_df.columns: + losers = trades_df[trades_df["net_money"] < 0] + reversals = trades_df[trades_df["result_class"] == "reversal"] + metrics.reversal_rate = ( + len(reversals) / max(1, len(losers)) + ) + if "mfe_capture_ratio" in trades_df.columns: + metrics.avg_mfe_capture = float( + trades_df["mfe_capture_ratio"].dropna().mean() or 0 + ) + if "mfe_pips" in trades_df.columns: + metrics.avg_mfe_pips = float(trades_df["mfe_pips"].dropna().mean() or 0) + metrics.avg_mae_pips = float(trades_df.get("mae_pips", pd.Series()).dropna().mean() or 0) + + # 7. Compute composite score (session stats can be added here in v2) + metrics.composite_score = scorer.score(metrics) + + # 8. Persist + store.save_metrics(metrics) + if not trades_df.empty: + store.save_trades(run_id, trades) + run.report_path = result.report_xml + run.log_csv_path = result.trade_log_csv + store.save_run(run) + + return metrics, trades_df + + +# ── Analysis pipeline ───────────────────────────────────────────────────────── + +def run_analysis( + run_id: str, + trades_df: pd.DataFrame, + metrics: RunMetrics, + analyzers: list, + store: DataStore, +) -> list: + """Run all analyzers and persist findings.""" + all_findings = [] + for analyzer in analyzers: + findings = analyzer.run(trades_df, metrics, run_id) + all_findings.extend(findings) + console.print( + f" [green]✓[/green] {analyzer.name:<25} → {len(findings)} finding(s)" + ) + + all_findings.sort(key=lambda f: f.confidence, reverse=True) + store.save_findings(all_findings) + return all_findings + + +# ── Display helpers ─────────────────────────────────────────────────────────── + +def display_metrics(metrics: RunMetrics, label: str = "Backtest Results") -> None: + table = Table(title=label, show_header=True, header_style="bold cyan") + table.add_column("Metric", style="dim") + table.add_column("Value", justify="right") + + table.add_row("Net Profit", f"${metrics.net_profit:,.2f}") + table.add_row("Profit Factor", f"{metrics.profit_factor:.3f}") + table.add_row("Calmar Ratio", f"{metrics.calmar_ratio:.3f}") + table.add_row("Max Drawdown", f"{metrics.max_drawdown_pct*100:.1f}% (${metrics.max_drawdown_abs:,.0f})") + table.add_row("Total Trades", str(metrics.total_trades)) + table.add_row("Win Rate", f"{metrics.win_rate*100:.1f}%") + table.add_row("Sharpe", f"{metrics.sharpe_ratio:.3f}") + table.add_row("Recovery Factor", f"{metrics.recovery_factor:.3f}") + if metrics.avg_mfe_capture is not None: + table.add_row("MFE Capture", f"{metrics.avg_mfe_capture*100:.1f}%") + if metrics.reversal_rate is not None: + table.add_row("Reversal Rate", f"{metrics.reversal_rate*100:.1f}%") + table.add_row("Composite Score", f"[bold]{metrics.composite_score:.4f}[/bold]") + + console.print(table) + + +def display_findings(findings: list) -> None: + table = Table(title="Analysis Findings", show_header=True, header_style="bold yellow") + table.add_column("#", width=3) + table.add_column("Finding", max_width=60) + table.add_column("Severity", width=8) + table.add_column("Confidence", width=10, justify="right") + table.add_column("Est. Impact", width=12, justify="right") + + sev_colors = {"high": "red", "medium": "yellow", "low": "dim"} + + for i, f in enumerate(findings, 1): + color = sev_colors.get(f.severity, "white") + table.add_row( + str(i), + f.description[:60] + ("…" if len(f.description) > 60 else ""), + f"[{color}]{f.severity.upper()}[/{color}]", + f"{f.confidence:.2f}", + f"${f.impact_estimate_pnl:,.0f}", + ) + console.print(table) + + +def display_hypotheses(hypotheses: list, current_params: dict) -> None: + table = Table(title="Proposed Hypotheses", show_header=True, header_style="bold magenta") + table.add_column("#", width=3) + table.add_column("Description", max_width=50) + table.add_column("Parameter Changes", max_width=40) + + for i, h in enumerate(hypotheses, 1): + changes = [] + for param, val in h.param_delta.items(): + old = current_params.get(param, "?") + changes.append(f"{param}: {old} → {val}") + table.add_row( + str(i), + h.description[:50], + "\n".join(changes), + ) + console.print(table) + + +# ── Main loop ───────────────────────────────────────────────────────────────── + +def main(auto_mode: bool = False, baseline_only: bool = False) -> None: + cfg = load_config() + store, builder, runner, parser, log_rdr, analyzers, scorer, mutator, gate = ( + build_components(cfg) + ) + + ea = cfg["ea"] + per = cfg["periods"] + + console.print(Panel( + f"[bold cyan]MT5 EA Strategy Optimizer[/bold cyan]\n" + f"EA: {ea['name']} | Symbol: {ea['symbol']} | TF: {ea['timeframe']}\n" + f"Train: {per['train_start']} → {per['train_end']} | " + f"OOS: {per['oos_start']} → {per['oos_end']} [bold red](LOCKED)[/bold red]", + title="[bold]Session Start[/bold]", + )) + + # ── PHASE 0: Baseline ───────────────────────────────────────────────────── + console.rule("[bold]Phase 0: Baseline Run[/bold]") + default_params = builder.default_params() + baseline_id = f"baseline_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}" + + baseline_metrics, baseline_trades = execute_run( + run_id=baseline_id, + params=default_params, + period_start=per["train_start"], + period_end=per["train_end"], + phase="baseline", + hypothesis_id=None, + cfg=cfg, store=store, builder=builder, runner=runner, + parser=parser, log_rdr=log_rdr, analyzers=analyzers, scorer=scorer, + ) + + if baseline_metrics is None: + console.print("[red]Baseline run failed. Check MT5 config and terminal path.[/red]") + sys.exit(1) + + display_metrics(baseline_metrics, label="Baseline Results") + + if baseline_only: + # ── Analysis only ──────────────────────────────────────────────────── + console.rule("[bold]Analysis[/bold]") + findings = run_analysis(baseline_id, baseline_trades, baseline_metrics, + analyzers, store) + display_findings(findings) + return + + current_params = default_params.copy() + current_metrics = baseline_metrics + best_score = baseline_metrics.composite_score + iteration = 0 + no_improvement_count = 0 + + max_iter = cfg["optimization"]["max_iterations"] + conv_win = cfg["optimization"]["convergence_window"] + conv_thr = cfg["optimization"]["convergence_threshold"] + + # ── Iteration loop ──────────────────────────────────────────────────────── + while iteration < max_iter: + iteration += 1 + console.rule(f"[bold]Iteration {iteration}[/bold]") + + # Analysis + console.print("[bold]Running analysis modules...[/bold]") + parent_run_id = baseline_id if iteration == 1 else f"iter_{iteration-1}" + trades_df = store.load_trades( + baseline_id if iteration == 1 else f"iter_{iteration-1}_best" + ) + if trades_df.empty: + trades_df = baseline_trades + + findings = run_analysis( + baseline_id, trades_df, current_metrics, analyzers, store + ) + display_findings(findings[:8]) # top 8 + + if not findings: + console.print("[yellow]No actionable findings. Stopping.[/yellow]") + break + + # Mutation + recent_deltas = store.get_recent_param_deltas(cfg["mutation"]["dedup_lookback_runs"]) + hypotheses = mutator.propose( + findings=findings, + current_params=current_params, + recent_deltas=recent_deltas, + max_proposals=cfg["mutation"]["max_hypotheses_per_cycle"], + ) + + if not hypotheses: + console.print("[yellow]No new hypotheses available. Stopping.[/yellow]") + break + + display_hypotheses(hypotheses, current_params) + + # Human approval (skipped in auto mode) + if auto_mode: + selected_indices = list(range(len(hypotheses))) + else: + choice = Prompt.ask( + "Apply which hypotheses?", + default="1", + ) + if choice.lower() in ("skip", "s", ""): + console.print("[dim]Skipping...[/dim]") + continue + if choice.lower() == "all": + selected_indices = list(range(len(hypotheses))) + else: + selected_indices = [int(x.strip()) - 1 for x in choice.split(",")] + + # Test selected hypotheses + iteration_best: Optional[RunMetrics] = None + iteration_best_params: Optional[dict] = None + iteration_best_hyp = None + + for idx in selected_indices: + if idx < 0 or idx >= len(hypotheses): + continue + hyp = hypotheses[idx] + test_params = {**current_params, **hyp.param_delta} + run_id = f"iter_{iteration}_h{idx+1}" + + console.print(f"\n[bold]Testing hypothesis {idx+1}: {hyp.description}[/bold]") + store.save_hypothesis(hyp) + + test_metrics, test_trades = execute_run( + run_id=run_id, + params=test_params, + period_start=per["train_start"], + period_end=per["train_end"], + phase="explore", + hypothesis_id=hyp.hypothesis_id, + cfg=cfg, store=store, builder=builder, runner=runner, + parser=parser, log_rdr=log_rdr, analyzers=analyzers, scorer=scorer, + ) + + if test_metrics is None: + continue + + display_metrics(test_metrics, label=f"H{idx+1} Results") + delta_score = test_metrics.composite_score - current_metrics.composite_score + color = "green" if delta_score > 0 else "red" + console.print( + f"Score delta: [{color}]{delta_score:+.4f}[/{color}] " + f"({current_metrics.composite_score:.4f} → {test_metrics.composite_score:.4f})" + ) + + store.update_hypothesis_status(hyp.hypothesis_id, "tested", run_id) + + if iteration_best is None or test_metrics.composite_score > iteration_best.composite_score: + iteration_best = test_metrics + iteration_best_params = test_params + iteration_best_hyp = hyp + + if iteration_best is None: + console.print("[red]All hypotheses failed to run.[/red]") + continue + + # Validation gate + gate_result = gate.run_is_check(iteration_best) + if not gate_result.passed: + console.print(f"[red]IS gate failed: {gate_result.details}[/red]") + store.update_hypothesis_status(iteration_best_hyp.hypothesis_id, "rejected") + no_improvement_count += 1 + else: + # Walk-forward validation + if auto_mode or Confirm.ask("Run walk-forward validation?", default=True): + wfv = gate.run_walk_forward(iteration_best_params, cfg, store, builder, + runner, parser, log_rdr, analyzers, scorer) + console.print(f"WFV: OOS/IS ratio = {wfv.oos_is_ratio:.2f} " + f"(threshold {cfg['thresholds']['min_wfv_ratio']:.2f})") + if wfv.passed: + console.print(f"[green]Walk-forward PASSED[/green]") + else: + console.print(f"[yellow]Walk-forward FAILED — not promoting.[/yellow]") + store.update_hypothesis_status(iteration_best_hyp.hypothesis_id, "rejected") + no_improvement_count += 1 + continue + + # OOS test + run_oos = auto_mode or Confirm.ask("Run OOS validation?", default=False) + oos_score = None + if run_oos: + oos_metrics, _ = execute_run( + run_id=f"oos_{iteration}", + params=iteration_best_params, + period_start=per["oos_start"], + period_end=per["oos_end"], + phase="oos", + hypothesis_id=iteration_best_hyp.hypothesis_id, + cfg=cfg, store=store, builder=builder, runner=runner, + parser=parser, log_rdr=log_rdr, analyzers=analyzers, scorer=scorer, + ) + if oos_metrics: + oos_score = oos_metrics.composite_score + oos_deg = (iteration_best.composite_score - oos_score) / max(0.001, iteration_best.composite_score) + if oos_deg > cfg["thresholds"]["max_oos_degradation"]: + console.print(f"[red]OOS degradation {oos_deg:.1%} > threshold. Rejected.[/red]") + store.update_hypothesis_status(iteration_best_hyp.hypothesis_id, "rejected") + no_improvement_count += 1 + continue + display_metrics(oos_metrics, label="OOS Results") + + # Promote candidate + candidate = Candidate( + run_id=iteration_best.run_id, + composite_score=iteration_best.composite_score, + oos_score=oos_score, + params=iteration_best_params, + ) + store.save_candidate(candidate) + store.update_hypothesis_status(iteration_best_hyp.hypothesis_id, "validated") + console.print(f"[bold green]✅ Candidate C{candidate.candidate_id} promoted![/bold green]") + + # Update baseline + improvement = iteration_best.composite_score - best_score + if improvement >= conv_thr: + current_params = iteration_best_params + current_metrics = iteration_best + best_score = iteration_best.composite_score + no_improvement_count = 0 + else: + no_improvement_count += 1 + + # Convergence check + if no_improvement_count >= conv_win: + console.print( + f"[yellow]Convergence: no improvement in {no_improvement_count} iterations. Stopping.[/yellow]" + ) + break + + if not (auto_mode or Confirm.ask("Continue to next iteration?", default=True)): + break + + # ── Summary ─────────────────────────────────────────────────────────────── + console.rule("[bold]Optimization Complete[/bold]") + candidates = store.list_candidates() + if candidates: + console.print(f"[bold green]{len(candidates)} candidate(s) promoted.[/bold green]") + console.print(f"Best composite score: {max(c['composite_score'] for c in candidates):.4f}") + else: + console.print("[yellow]No candidates were promoted in this session.[/yellow]") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + parser_cli = argparse.ArgumentParser(description="MT5 EA Strategy Optimizer") + parser_cli.add_argument("--auto", action="store_true", help="Run fully automated (no prompts)") + parser_cli.add_argument("--baseline", action="store_true", help="Run baseline + analysis only") + parser_cli.add_argument("--log-level", default="INFO", help="Logging level") + args = parser_cli.parse_args() + + logger.remove() + logger.add(sys.stderr, level=args.log_level) + logger.add("optimizer.log", level="DEBUG", rotation="10 MB") + + main(auto_mode=args.auto, baseline_only=args.baseline) diff --git a/mql5/TradeLogger.mqh b/mql5/TradeLogger.mqh new file mode 100644 index 0000000..6fdb638 --- /dev/null +++ b/mql5/TradeLogger.mqh @@ -0,0 +1,242 @@ +//+------------------------------------------------------------------+ +//| TradeLogger.mqh | +//| Lightweight per-trade MAE/MFE logger for MT5 Strategy Tester | +//| Drop into: MQL5/Include/TradeLogger.mqh | +//| | +//| Integration (2 steps in your EA): | +//| 1. #include // top of your EA file | +//| 2. TL_OnTick(); // inside OnTick() | +//| | +//| Output CSV is written to MQL5/Files/TradeLog__.csv | +//+------------------------------------------------------------------+ +#property strict + +//--- Configuration (override before including if needed) +#ifndef TL_MFE_THRESHOLD_PIPS + #define TL_MFE_THRESHOLD_PIPS 0.0 // minimum MFE to record (0 = record all) +#endif +#ifndef TL_MAX_TRACKED + #define TL_MAX_TRACKED 256 // max simultaneously open trades tracked +#endif + +//--- Internal state per tracked position +struct TL_TradeState +{ + ulong ticket; + datetime open_time; + double open_price; + double sl; + double tp; + double lot_size; + int direction; // 1=buy, -1=sell + double mfe_price; // most favourable price seen + double mae_price; // most adverse price seen + bool active; +}; + +static TL_TradeState TL_Positions[TL_MAX_TRACKED]; +static int TL_Count = 0; +static int TL_FileHandle = INVALID_HANDLE; +static bool TL_Initialized = false; +static string TL_FilePath = ""; + +//--- Point-to-pip conversion helper +double TL_PipSize() +{ + double ps = SymbolInfoDouble(_Symbol, SYMBOL_POINT); + int digits = (int)SymbolInfoInteger(_Symbol, SYMBOL_DIGITS); + // For 5-digit brokers, 1 pip = 10 points; for 2-digit (XAUUSD etc), 1 pip = 1 point + if(digits == 3 || digits == 5) return ps * 10.0; + return ps; +} + +//--- Internal: open (or reopen) the CSV file +bool TL_OpenFile() +{ + if(TL_FileHandle != INVALID_HANDLE) return true; + + string ea_name = MQLInfoString(MQL_PROGRAM_NAME); + TL_FilePath = ea_name + "_" + _Symbol + "_TradeLog.csv"; + + TL_FileHandle = FileOpen(TL_FilePath, + FILE_WRITE | FILE_CSV | FILE_ANSI | FILE_SHARE_READ, + ','); + if(TL_FileHandle == INVALID_HANDLE) + { + Print("[TradeLogger] ERROR: Cannot open file '", TL_FilePath, + "' error=", GetLastError()); + return false; + } + + // Write CSV header + FileWrite(TL_FileHandle, + "ticket","open_time","close_time", + "direction","open_price","close_price", + "sl","tp","lot_size", + "mfe_pips","mae_pips", + "net_pips","net_money", + "duration_minutes", + "commission","swap"); + return true; +} + +//--- Internal: find slot index for a ticket (-1 = not found) +int TL_FindSlot(ulong ticket) +{ + for(int i = 0; i < TL_Count; i++) + if(TL_Positions[i].ticket == ticket && TL_Positions[i].active) + return i; + return -1; +} + +//--- Internal: register a newly opened position +void TL_RegisterPosition(ulong ticket) +{ + if(TL_Count >= TL_MAX_TRACKED) return; // overflow guard + + if(!PositionSelectByTicket(ticket)) return; + + TL_TradeState &s = TL_Positions[TL_Count]; + s.ticket = ticket; + s.open_time = (datetime)PositionGetInteger(POSITION_TIME); + s.open_price = PositionGetDouble(POSITION_PRICE_OPEN); + s.sl = PositionGetDouble(POSITION_SL); + s.tp = PositionGetDouble(POSITION_TP); + s.lot_size = PositionGetDouble(POSITION_VOLUME); + s.direction = (PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_BUY) ? 1 : -1; + s.mfe_price = s.open_price; + s.mae_price = s.open_price; + s.active = true; + TL_Count++; +} + +//--- Internal: flush a closed trade to CSV +void TL_FlushClosed(int idx) +{ + if(!TL_OpenFile()) return; + + TL_TradeState &s = TL_Positions[idx]; + + // Retrieve closed deal data from history + if(!HistorySelectByPosition(s.ticket)) return; + int deals = HistoryDealsTotal(); + if(deals < 2) return; // need at least open + close deal + + // Find the closing deal (last deal in history for this position) + ulong close_deal = 0; + double close_price = 0; + double net_money = 0; + double commission = 0; + double swap_val = 0; + datetime close_time = 0; + + for(int d = deals - 1; d >= 0; d--) + { + ulong deal_ticket = HistoryDealGetTicket(d); + if(HistoryDealGetInteger(deal_ticket, DEAL_ENTRY) == DEAL_ENTRY_OUT || + HistoryDealGetInteger(deal_ticket, DEAL_ENTRY) == DEAL_ENTRY_INOUT) + { + close_deal = deal_ticket; + close_price = HistoryDealGetDouble(deal_ticket, DEAL_PRICE); + net_money = HistoryDealGetDouble(deal_ticket, DEAL_PROFIT); + commission = HistoryDealGetDouble(deal_ticket, DEAL_COMMISSION); + swap_val = HistoryDealGetDouble(deal_ticket, DEAL_SWAP); + close_time = (datetime)HistoryDealGetInteger(deal_ticket, DEAL_TIME); + break; + } + } + if(close_deal == 0) return; + + double pip = TL_PipSize(); + double mfe_pips = (s.mfe_price - s.open_price) * s.direction / pip; + double mae_pips = (s.open_price - s.mae_price) * s.direction / pip; + double net_pips = (close_price - s.open_price) * s.direction / pip; + int dur_min = (int)((close_time - s.open_time) / 60); + + FileWrite(TL_FileHandle, + (string)s.ticket, + TimeToString(s.open_time, TIME_DATE|TIME_MINUTES), + TimeToString(close_time, TIME_DATE|TIME_MINUTES), + (s.direction == 1 ? "buy" : "sell"), + DoubleToString(s.open_price, _Digits), + DoubleToString(close_price, _Digits), + DoubleToString(s.sl, _Digits), + DoubleToString(s.tp, _Digits), + DoubleToString(s.lot_size, 2), + DoubleToString(MathMax(0, mfe_pips), 2), + DoubleToString(MathMax(0, mae_pips), 2), + DoubleToString(net_pips, 2), + DoubleToString(net_money, 2), + (string)dur_min, + DoubleToString(commission, 2), + DoubleToString(swap_val, 2)); + + FileFlush(TL_FileHandle); // flush after each trade — safe even if tester aborts +} + +//+------------------------------------------------------------------+ +//| TL_OnTick() — Call this inside your EA's OnTick() | +//+------------------------------------------------------------------+ +void TL_OnTick() +{ + double bid = SymbolInfoDouble(_Symbol, SYMBOL_BID); + double ask = SymbolInfoDouble(_Symbol, SYMBOL_ASK); + + // --- Update running MFE/MAE for all tracked positions --- + for(int i = 0; i < TL_Count; i++) + { + if(!TL_Positions[i].active) continue; + + ulong ticket = TL_Positions[i].ticket; + + // Check if position is still open + if(!PositionSelectByTicket(ticket)) + { + // Position closed — flush to CSV then deactivate + TL_FlushClosed(i); + TL_Positions[i].active = false; + continue; + } + + double current_price = (TL_Positions[i].direction == 1) ? bid : ask; + + // Update MFE (best price in trade direction) + if(TL_Positions[i].direction == 1) // BUY: higher is better + TL_Positions[i].mfe_price = MathMax(TL_Positions[i].mfe_price, current_price); + else // SELL: lower is better + TL_Positions[i].mfe_price = MathMin(TL_Positions[i].mfe_price, current_price); + + // Update MAE (worst price against trade direction) + if(TL_Positions[i].direction == 1) // BUY: lower is worse + TL_Positions[i].mae_price = MathMin(TL_Positions[i].mae_price, current_price); + else // SELL: higher is worse + TL_Positions[i].mae_price = MathMax(TL_Positions[i].mae_price, current_price); + } + + // --- Register any newly opened positions not yet tracked --- + int total = PositionsTotal(); + for(int p = 0; p < total; p++) + { + ulong ticket = PositionGetTicket(p); + if(ticket == 0) continue; + if(!PositionSelectByTicket(ticket)) continue; + if(PositionGetString(POSITION_SYMBOL) != _Symbol) continue; + + if(TL_FindSlot(ticket) == -1) + TL_RegisterPosition(ticket); + } +} + +//+------------------------------------------------------------------+ +//| TL_Deinit() — Optionally call in OnDeinit() to close file | +//+------------------------------------------------------------------+ +void TL_Deinit() +{ + if(TL_FileHandle != INVALID_HANDLE) + { + FileClose(TL_FileHandle); + TL_FileHandle = INVALID_HANDLE; + } + Print("[TradeLogger] Log written to: ", TL_FilePath); +} +//+------------------------------------------------------------------+ diff --git a/mt5/__init__.py b/mt5/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mt5/ini_builder.py b/mt5/ini_builder.py new file mode 100644 index 0000000..e3956cd --- /dev/null +++ b/mt5/ini_builder.py @@ -0,0 +1,163 @@ +""" +mt5/ini_builder.py +Generates the MT5 strategy tester .ini file from a parameter dict + config. +""" +from __future__ import annotations +import configparser +from pathlib import Path +from typing import Any + +import yaml +from loguru import logger + + +# MT5 period string names (used in [Tester] Period= field) +TIMEFRAME_NAMES = { + "M1": "M1", "M5": "M5", "M15": "M15", "M30": "M30", + "H1": "H1", "H4": "H4", "D1": "D1", "W1": "W1", "MN": "MN1", +} + +# MT5 tester model codes +MODEL_CODES = { + "every_tick": 0, + "every_tick_real": 1, + "ohlc_m1": 4, +} + + +class IniBuilder: + """ + Builds MT5 strategy tester .ini files. + + Usage: + builder = IniBuilder(config_path="config.yaml", manifest_path="mutation/param_manifest.yaml") + ini_path = builder.build( + run_id="run_001", + params={"InpRiskPercent": 1.5, "InpUseTrailing": True, ...}, + period_start="2022.01.01", + period_end="2023.12.31", + output_dir=Path("runs/run_001"), + ) + """ + + def __init__(self, config_path: str | Path, manifest_path: str | Path): + with open(config_path) as f: + self.cfg = yaml.safe_load(f) + with open(manifest_path) as f: + self.manifest = yaml.safe_load(f)["parameters"] + + # ── Public ──────────────────────────────────────────────────────────────── + + def build( + self, + run_id: str, + params: dict[str, Any], + period_start: str, + period_end: str, + output_dir: Path, + phase: str = "explore", + ) -> Path: + """ + Write .ini to output_dir and return its path. + params: dict of EA input values (partial OK — missing params use manifest defaults) + """ + output_dir.mkdir(parents=True, exist_ok=True) + ini_path = output_dir / f"{run_id}.ini" + report_dir = output_dir / "report" + report_dir.mkdir(parents=True, exist_ok=True) + + full_params = self._merge_with_defaults(params) + ini_text = self._render(run_id, full_params, period_start, period_end, + report_dir, phase) + + ini_path.write_text(ini_text, encoding="utf-8") + logger.debug(f"INI written: {ini_path}") + return ini_path + + def default_params(self) -> dict[str, Any]: + """Return all EA parameters at their default values.""" + return self._merge_with_defaults({}) + + # ── Internal ────────────────────────────────────────────────────────────── + + def _merge_with_defaults(self, overrides: dict[str, Any]) -> dict[str, Any]: + """Merge caller-supplied overrides with manifest defaults.""" + result: dict[str, Any] = {} + for name, spec in self.manifest.items(): + if name in overrides: + result[name] = overrides[name] + else: + result[name] = spec.get("default", 0) + return result + + def _format_value(self, name: str, value: Any) -> str: + """Format a parameter value for the [TesterInputs] section.""" + spec = self.manifest.get(name, {}) + ptype = spec.get("type", "float") + + if ptype == "bool": + return "true" if value else "false" + if ptype == "fixed": + # Fixed params: write exact default + return str(value) + if ptype == "int" or ptype == "enum": + return str(int(value)) + if ptype == "float": + # Determine decimal places from step + step = spec.get("step", 0.1) + decimals = len(str(step).split(".")[-1]) if "." in str(step) else 0 + return f"{float(value):.{decimals}f}" + return str(value) + + def _render( + self, + run_id: str, + params: dict[str, Any], + period_start: str, + period_end: str, + report_dir: Path, + phase: str, + ) -> str: + """Render final INI content as a string.""" + ea_cfg = self.cfg["ea"] + mt5_cfg = self.cfg["mt5"] + broker_cfg = self.cfg["broker"] + + # Period must be the string name (H1, M30 etc) — NOT the ENUM integer + tf_name = TIMEFRAME_NAMES.get(ea_cfg["timeframe"].upper(), "H1") + + # Model: 0=Every Tick (slow), 4=OHLC M1 (fast, reliable for ini-based launch) + model_code = mt5_cfg.get("tester_model", 4) + + # Report path must be RELATIVE to the MT5 terminal data folder + # MT5 appends its own base path. Use run_id as the report name. + report_name = f"Optimizer_{run_id}" + + lines = [ + f"; MT5 Optimizer INI — run_id={run_id} phase={phase}", + f"", + f"[Tester]", + f"Expert={ea_cfg['file']}", + f"Symbol={ea_cfg['symbol']}", + f"Period={tf_name}", + f"Optimization=0", + f"Model={model_code}", + f"FromDate={period_start}", + f"ToDate={period_end}", + f"ForwardMode=0", + f"Report={report_name}", + f"ReplaceReport=1", + f"ShutdownTerminal={mt5_cfg.get('shutdown_terminal', 1)}", + f"Deposit={broker_cfg['deposit']}", + f"Currency={broker_cfg['currency']}", + f"Leverage={broker_cfg['leverage']}", + f"", + f"[TesterInputs]", + ] + + for name, value in params.items(): + formatted = self._format_value(name, value) + lines.append(f"{name}={formatted}") + + lines.append("") # trailing newline + return "\n".join(lines) diff --git a/mt5/log_reader.py b/mt5/log_reader.py new file mode 100644 index 0000000..ac4c857 --- /dev/null +++ b/mt5/log_reader.py @@ -0,0 +1,159 @@ +""" +mt5/log_reader.py +Reads the TradeLogger.mqh CSV and merges MAE/MFE data into parsed trades. +Also computes derived fields: session, day_of_week, result_class, quality scores. +""" +from __future__ import annotations +from datetime import datetime, timezone, timedelta +from pathlib import Path +from typing import Optional + +import pandas as pd +from loguru import logger + +from data.models import Trade + + +# Session definitions in UTC hours (inclusive start, exclusive end) +SESSIONS_UTC = { + "Asian": (0, 9), + "London": (7, 16), + "LondonNY": (13, 16), + "NY": (13, 22), +} + + +def classify_session(hour_utc: int) -> str: + """Classify a UTC hour into its primary trading session.""" + in_london = SESSIONS_UTC["London"][0] <= hour_utc < SESSIONS_UTC["London"][1] + in_ny = SESSIONS_UTC["NY"][0] <= hour_utc < SESSIONS_UTC["NY"][1] + + if in_london and in_ny: + return "LondonNY" + elif in_london: + return "London" + elif in_ny: + return "NY" + elif SESSIONS_UTC["Asian"][0] <= hour_utc < SESSIONS_UTC["Asian"][1]: + return "Asian" + else: + return "Off" + + +# ── Main reader/merger ──────────────────────────────────────────────────────── + +class TradeLogReader: + """ + Reads the CSV produced by TradeLogger.mqh and merges into a list of Trade objects. + + Strategy: + 1. Load CSV, index by ticket + 2. For each Trade, look up ticket in CSV + 3. Fill mfe_pips, mae_pips, duration_minutes if found + 4. Compute all derived fields for every trade (session, quality scores, etc.) + """ + + def __init__(self, broker_tz_offset_hours: int = 2, pip_size: float = 0.1): + self.tz_offset = broker_tz_offset_hours # broker local = UTC + offset + self.pip_size = pip_size # XAUUSD: 0.1 per pip + + # ── Public ──────────────────────────────────────────────────────────────── + + def merge( + self, + trades: list[Trade], + csv_path: Optional[str | Path], + reversal_mfe_threshold_pips: float = 15.0, + ) -> list[Trade]: + """ + Merge TradeLogger CSV into trade list, compute all derived fields. + If csv_path is None or unreadable, derived fields are computed without MFE/MAE. + """ + log_df = self._load_csv(csv_path) if csv_path else None + + enriched = [] + for trade in trades: + # Fill MAE/MFE from logger if available + if log_df is not None and trade.ticket in log_df.index: + row = log_df.loc[trade.ticket] + trade.mfe_pips = float(row.get("mfe_pips", 0) or 0) + trade.mae_pips = float(row.get("mae_pips", 0) or 0) + # Override duration with logger value (tick-accurate) + if "duration_minutes" in row: + trade.duration_minutes = int(row["duration_minutes"] or trade.duration_minutes) + + # Compute all derived fields + trade = self._enrich(trade, reversal_mfe_threshold_pips) + enriched.append(trade) + + logger.info( + f"Enriched {len(enriched)} trades. " + f"MAE/MFE available: {sum(1 for t in enriched if t.mfe_pips is not None)}" + ) + return enriched + + # ── Internal ────────────────────────────────────────────────────────────── + + def _load_csv(self, csv_path: str | Path) -> Optional[pd.DataFrame]: + path = Path(csv_path) + if not path.exists(): + logger.warning(f"TradeLogger CSV not found: {path}") + return None + try: + df = pd.read_csv(path, dtype={"ticket": int}) + if "ticket" not in df.columns: + logger.error("TradeLogger CSV missing 'ticket' column.") + return None + df = df.set_index("ticket") + logger.debug(f"Loaded {len(df)} rows from TradeLogger CSV.") + return df + except Exception as e: + logger.error(f"Failed to read TradeLogger CSV: {e}") + return None + + def _enrich(self, trade: Trade, threshold_pips: float) -> Trade: + """Compute all derived classification and quality fields.""" + # --- Timezone normalisation --- + # Broker timestamps are in broker local time (UTC+offset). + # We compute UTC hour by subtracting the offset. + broker_hour = trade.open_time.hour + hour_utc = (broker_hour - self.tz_offset) % 24 + trade.hour_broker = broker_hour + trade.hour_utc = hour_utc + trade.day_of_week = trade.open_time.weekday() # 0=Mon, 4=Fri + trade.session = classify_session(hour_utc) + + # --- Result class --- + won = trade.net_money > 0 + be = abs(trade.net_money) < 0.01 # effectively breakeven + + if be: + trade.result_class = "be" + elif won: + trade.result_class = "win" + else: + # Check if it's a reversal: lost, but had positive MFE above threshold + if trade.mfe_pips is not None and trade.mfe_pips >= threshold_pips: + trade.result_class = "reversal" + else: + trade.result_class = "loss" + + # --- Quality scores (only when MFE/MAE available) --- + if trade.mfe_pips is not None and trade.mae_pips is not None: + mfe = max(trade.mfe_pips, 0.01) # prevent division by zero + mae = max(trade.mae_pips, 0.0) + + # Entry quality: how far against you before move in your favour + # High = entered well (little adverse move relative to favourable move) + trade.entry_quality = max(0.0, min(1.0, 1.0 - (mae / (mfe + mae + 0.01)))) + + # Exit quality: what fraction of MFE did we capture + mfe_value = mfe * self.pip_size * trade.lot_size * 100 # approx value in $ + if mfe_value > 0: + trade.mfe_capture_ratio = max(0.0, trade.net_money / mfe_value) + trade.exit_quality = max(0.0, min(1.0, trade.net_pips / mfe)) + else: + trade.mfe_capture_ratio = 0.0 + trade.exit_quality = 0.0 + + return trade diff --git a/mt5/report_parser.py b/mt5/report_parser.py new file mode 100644 index 0000000..6756c8b --- /dev/null +++ b/mt5/report_parser.py @@ -0,0 +1,338 @@ +""" +mt5/report_parser.py +Parses the MT5 strategy tester HTML report (production format: pure HTML tables). +Extracts RunMetrics and paired in/out deal trades. +""" +from __future__ import annotations + +import re +from datetime import datetime +from pathlib import Path +from typing import Optional + +from lxml import html as lhtml +from loguru import logger + +from data.models import RunMetrics, Trade + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +def _clean(s: str) -> str: + """Remove HTML entity remnants, spaces, currency symbols.""" + if not s: + return "" + # MT5 uses non-breaking spaces (0xa0) and regular spaces + s = s.replace("\xa0", "").replace(",", "").replace(" ", "").strip() + return s + + +def _parse_float(s: str) -> float: + s = _clean(s) + # Remove everything except digits, dot, minus + s = re.sub(r"[^\d.\-]", "", s) + try: + return float(s) + except (ValueError, TypeError): + return 0.0 + + +def _parse_int(s: str) -> int: + s = _clean(s) + s = re.sub(r"[^\d\-]", "", s.split("(")[0]) + try: + return int(s) + except (ValueError, TypeError): + return 0 + + +def _parse_dt(s: str) -> Optional[datetime]: + s = (s or "").strip() + for fmt in ("%Y.%m.%d %H:%M:%S", "%Y.%m.%d %H:%M", "%Y.%m.%d"): + try: + return datetime.strptime(s, fmt) + except ValueError: + continue + return None + + +def _cell_text(td) -> str: + """Get all inner text from an lxml element, stripping tags.""" + return "".join(td.itertext()).strip() + + +# ── Main Parser ─────────────────────────────────────────────────────────────── + +class ReportParser: + """ + Parses the MT5 HTML strategy tester report. + + Report format (confirmed from live MT5 output): + - Summary metrics: Label:Value pairs + - Deals table: Time | Deal | Symbol | Type | Direction | Volume | + Price | Order | Commission | Swap | Profit | Balance | Comment + Direction='in' → position open (entry deal) + Direction='out' → position close (exit deal, has Profit value) + """ + + def parse( + self, xml_path: Optional[str], html_path: Optional[str] + ) -> tuple[Optional[RunMetrics], list[Trade]]: + """ + Parse the MT5 HTML report. xml_path is ignored (MT5 command-line + runs produce .htm, not .xml). Falls back gracefully if html is missing. + """ + path = None + if html_path and Path(html_path).exists(): + path = Path(html_path) + elif xml_path and Path(xml_path).exists(): + path = Path(xml_path) + + if path is None: + logger.error("No report file available to parse.") + return None, [] + + logger.debug(f"Parsing report: {path}") + try: + raw = path.read_bytes() + # MT5 HTML reports are UTF-16 LE (BOM: ff fe) — detect and decode + if raw[:2] == b'\xff\xfe': + # Pass raw bytes; lxml's HTML parser handles UTF-16 correctly + tree = lhtml.document_fromstring(raw) + else: + # Regular UTF-8 or latin-1 + try: + content = raw.decode("utf-8") + except UnicodeDecodeError: + content = raw.decode("windows-1252", errors="replace") + tree = lhtml.document_fromstring(content.encode("utf-8")) + except Exception as e: + logger.error(f"Failed to parse HTML: {e}") + return None, [] + + summary = self._extract_summary(tree) + deals = self._extract_deals(tree) + trades = self._pair_deals(deals) + + if not summary: + logger.warning("No summary data found in MT5 HTML report.") + return None, trades + + metrics = self._build_metrics(summary) + logger.info(f"Parsed: {metrics.total_trades} trades, PF={metrics.profit_factor:.3f}") + return metrics, trades + + # ── Summary ─────────────────────────────────────────────────────────────── + + def _extract_summary(self, tree) -> dict[str, str]: + """ + Extract label→value pairs from the stats tables. + MT5 pattern: Label: Value + The label and value are adjacent siblings in the same . + """ + result: dict[str, str] = {} + for tr in tree.iter("tr"): + tds = list(tr.findall(".//td")) + if len(tds) < 2: + continue + for i in range(len(tds) - 1): + label = _cell_text(tds[i]).rstrip(":") + val = _cell_text(tds[i + 1]) + if label and val and len(label) < 60: + result[label] = val + logger.debug(f"Summary fields: {len(result)}") + return result + + # ── Deals ───────────────────────────────────────────────────────────────── + + def _extract_deals(self, tree) -> list[dict]: + """ + Find the Deals table and parse every row. + Columns: Time|Deal|Symbol|Type|Direction|Volume|Price|Order|Commission|Swap|Profit|Balance|Comment + """ + # Find the that contains "Deals" text + deals_header = None + for th in tree.iter("th"): + if "Deals" in (_cell_text(th) or ""): + deals_header = th + break + + if deals_header is None: + logger.warning("Deals table not found in report.") + return [] + + # Walk up to find the table element + table = deals_header + while table is not None and table.tag != "table": + table = table.getparent() + if table is None: + return [] + + rows = table.findall(".//tr") + # Skip header rows (first 2 rows: table title + column headers) + data_rows = [] + header_seen = 0 + for row in rows: + ths = row.findall(".//th") + tds = row.findall(".//td") + if ths: + header_seen += 1 + continue + if not tds: + continue + data_rows.append(tds) + + deals = [] + for tds in data_rows: + texts = [_cell_text(td) for td in tds] + if len(texts) < 13: + continue + # Cols: 0=Time 1=Deal 2=Symbol 3=Type 4=Direction 5=Volume + # 6=Price 7=Order 8=Commission 9=Swap 10=Profit 11=Balance 12=Comment + deal_type = texts[3].lower() + if "balance" in deal_type or "credit" in deal_type: + continue # skip balance entries at start + + deals.append({ + "time": texts[0], + "deal": _parse_int(texts[1]), + "symbol": texts[2], + "type": deal_type, # buy / sell + "direction": texts[4].lower(), # in / out + "volume": _parse_float(texts[5]), + "price": _parse_float(texts[6]), + "order": _parse_int(texts[7]), + "commission": _parse_float(texts[8]), + "swap": _parse_float(texts[9]), + "profit": _parse_float(texts[10]), + "balance": _parse_float(texts[11]), + "comment": texts[12] if len(texts) > 12 else "", + }) + + logger.debug(f"Raw deals extracted: {len(deals)}") + return deals + + # ── Pairing in→out ──────────────────────────────────────────────────────── + + def _pair_deals(self, deals: list[dict]) -> list[Trade]: + """ + Pair 'in' (open) and 'out' (close) deals to form complete trades. + MT5 reports alternate: in-deal → out-deal for each closed position. + """ + trades: list[Trade] = [] + pending: Optional[dict] = None # the last 'in' deal + + for d in deals: + if d["direction"] == "in": + pending = d + elif d["direction"] == "out" and pending is not None: + open_dt = _parse_dt(pending["time"]) + close_dt = _parse_dt(d["time"]) + if not open_dt or not close_dt: + pending = None + continue + + direction = pending["type"] # buy / sell + open_price = pending["price"] + close_price = d["price"] + net_money = d["profit"] + lot_size = d["volume"] + commission = d["commission"] + pending["commission"] + swap = d["swap"] + pending["swap"] + duration_m = max(0, int((close_dt - open_dt).total_seconds() / 60)) + + # Net pips (XAUUSD: price moves in dollars, 1 pip = 0.1) + price_diff = (close_price - open_price) * (1 if direction == "buy" else -1) + net_pips = round(price_diff / 0.1, 2) if price_diff != 0 else 0.0 + + trades.append(Trade( + ticket = d["deal"], + open_time = open_dt, + close_time = close_dt, + direction = direction, + open_price = open_price, + close_price = close_price, + lot_size = lot_size, + net_pips = net_pips, + net_money = net_money, + duration_minutes = duration_m, + commission = commission, + swap = swap, + sl = 0.0, # not in deals table + tp = 0.0, + )) + pending = None + # if direction is empty/"" skip it + + logger.info(f"Paired {len(trades)} complete trades from deals.") + return trades + + # ── Metrics ─────────────────────────────────────────────────────────────── + + def _build_metrics(self, raw: dict[str, str]) -> RunMetrics: + """Build RunMetrics from the extracted label→value dictionary.""" + + def get(*keys) -> str: + for k in keys: + v = raw.get(k, "") + if v: + return v + return "0" + + net_profit = _parse_float(get("Total Net Profit", "Net Profit", "Balance")) + gross_profit = _parse_float(get("Gross Profit")) + gross_loss = _parse_float(get("Gross Loss")) + profit_factor = _parse_float(get("Profit Factor")) + # Total Deals = number of deal rows; Total Trades = positions + total_trades = _parse_int(get("Total Trades", "Total Deals")) + win_trades = _parse_int(get("Profit Trades", "Profit Trades (% of total)", + "Profit Deals")) + + # Drawdown: "2 160.22 (19.25%)" + dd_str = get("Equity Drawdown Maximal", "Equity Drawdown Relative", + "Balance Drawdown Maximal") + max_dd_abs = _parse_float(dd_str.split("(")[0]) + pct_match = re.search(r"([\d.]+)%", dd_str) + max_dd_pct = float(pct_match.group(1)) / 100 if pct_match else 0.0 + + initial_dep = _parse_float(get("Initial Deposit", "Deposit")) + if initial_dep <= 0: + initial_dep = 10_000.0 + + sharpe = _parse_float(get("Sharpe Ratio", "Sharp Ratio")) + recovery_factor = _parse_float(get("Recovery Factor")) + expected_payoff = _parse_float(get("Expected Payoff")) + + # Compute max_dd_pct if only absolute was found + if max_dd_pct == 0.0 and max_dd_abs > 0: + total_equity = initial_dep + net_profit + max_dd_pct = max_dd_abs / max(1, total_equity) + + # Calmar = annualised return / max drawdown + # Use simple ratio since we don't know exact test duration + calmar = 0.0 + if max_dd_pct > 0: + annual_return = net_profit / initial_dep + calmar = round(annual_return / max_dd_pct, 4) + + win_rate = win_trades / total_trades if total_trades > 0 else 0.0 + loss_trades = max(0, total_trades - win_trades) + avg_win = gross_profit / win_trades if win_trades > 0 else 0.0 + avg_loss = gross_loss / loss_trades if loss_trades > 0 else 0.0 + + return RunMetrics( + run_id = "__placeholder__", + net_profit = net_profit, + profit_factor = profit_factor, + max_drawdown_abs= max_dd_abs, + max_drawdown_pct= max_dd_pct, + calmar_ratio = calmar, + sharpe_ratio = sharpe, + total_trades = total_trades, + win_rate = win_rate, + avg_win = avg_win, + avg_loss = avg_loss, + recovery_factor = recovery_factor, + largest_loss = _parse_float(get("Largest loss trade")), + expected_payoff = expected_payoff, + ) diff --git a/mt5/runner.py b/mt5/runner.py new file mode 100644 index 0000000..e802f58 --- /dev/null +++ b/mt5/runner.py @@ -0,0 +1,375 @@ +""" +mt5/runner.py +Robust MT5 Strategy Tester runner with: + 1. MT5 process control (kill existing, launch fresh) + 2. Pre-run environment validation + 3. Historical data readiness wait + 4. Actionable error messages + 5. Auto-retry on failure (1 retry) + 6. Report detection in MT5 native reports folder +""" +from __future__ import annotations + +import shutil +import subprocess +import time +import psutil +from pathlib import Path +from typing import Optional + +import yaml +from loguru import logger + +from data.models import RunResult + + +# ── Custom Exceptions ───────────────────────────────────────────────────────── + +class MT5TimeoutError(RuntimeError): + pass + +class MT5ValidationError(RuntimeError): + pass + + +# ── MT5Runner ───────────────────────────────────────────────────────────────── + +class MT5Runner: + + POLL_INTERVAL_S = 5 # seconds between report checks + PROCESS_SETTLE_S = 2 # seconds to wait after process exit + MT5_INIT_WAIT_S = 8 # seconds after launch before testing begins + KILL_WAIT_S = 3 # seconds after killing MT5 before launching fresh + MAX_RETRIES = 1 # retry once on failure + + MT5_EXE_NAME = "terminal64.exe" + + def __init__(self, config_path: str | Path = "config.yaml"): + with open(config_path) as f: + cfg = yaml.safe_load(f) + self.cfg = cfg + self.terminal_exe = Path(cfg["mt5"]["terminal_exe"]) + self.timeout_s = cfg["mt5"]["tester_timeout_seconds"] + self.appdata_path = Path(cfg["mt5"]["appdata_path"]) + # MT5 writes reports to the ROOT of the terminal data folder + # (not a 'reports' subfolder) — confirmed by log inspection + self.mt5_reports_dir = self.appdata_path + self.mql5_files_dir = Path(cfg["mt5"].get( + "mql5_files_path", + str(self.appdata_path / "MQL5" / "Files") + )) + self.data_wait_s = cfg["mt5"].get("data_readiness_wait_seconds", 10) + + # ── Public entry point ──────────────────────────────────────────────────── + + def run( + self, + run_id: str, + ini_path: Path, + report_dir: Path, + log_csv_search_dir: Optional[Path] = None, + ) -> RunResult: + """ + Full execution pipeline with retry: + 1. Kill existing MT5 + 2. Validate environment + 3. Launch fresh MT5 + 4. Wait for data readiness + 5. Poll for report + 6. Auto-retry once on failure + """ + report_dir.mkdir(parents=True, exist_ok=True) + report_stem = f"Optimizer_{run_id}" + + for attempt in range(1, self.MAX_RETRIES + 2): + is_retry = attempt > 1 + if is_retry: + logger.warning(f"[{run_id}] Retry attempt {attempt}...") + + try: + # ── Step 1: Kill any running MT5 ───────────────────────── + self._kill_mt5(run_id) + + # ── Step 1b: Clear stale reports from previous runs ────── + self._clear_stale_reports(run_id) + + # ── Step 2: Pre-run validation ─────────────────────────── + self._validate(run_id) + + # ── Step 3: Launch fresh MT5 ───────────────────────────── + proc = self._launch_mt5(run_id, ini_path) + + # ── Step 4: Wait for MT5 to initialize + data ──────────── + logger.info(f"[{run_id}] Waiting {self.MT5_INIT_WAIT_S}s for MT5 to initialize...") + time.sleep(self.MT5_INIT_WAIT_S) + + # ── Step 5: Wait for report + TradeLog ─────────────────── + result = self._wait_for_report(run_id, proc, report_dir, report_stem) + + # ── Step 6: Find TradeLogger CSV ───────────────────────── + result.trade_log_csv = self._find_trade_log(run_id, log_csv_search_dir) + + logger.success(f"[{run_id}] Run complete. Report: {result.report_xml}") + return result + + except MT5ValidationError as e: + logger.error(f"[{run_id}] Validation error: {e}") + return RunResult(run_id=run_id, success=False, error_message=str(e)) + + except Exception as e: + logger.warning(f"[{run_id}] Attempt {attempt} failed: {e}") + if attempt > self.MAX_RETRIES: + msg = self._diagnose_failure(str(e)) + logger.error(f"[{run_id}] All retries exhausted. {msg}") + return RunResult(run_id=run_id, success=False, error_message=msg) + logger.info(f"[{run_id}] Will retry after {self.KILL_WAIT_S}s...") + time.sleep(self.KILL_WAIT_S) + + # ── Step 1: Kill existing MT5 ───────────────────────────────────────────── + + def _kill_mt5(self, run_id: str) -> None: + """Find and terminate any running MT5 processes.""" + killed = 0 + for proc in psutil.process_iter(["pid", "name", "exe"]): + try: + if proc.info["name"] and self.MT5_EXE_NAME.lower() in proc.info["name"].lower(): + logger.info(f"[{run_id}] Closing existing MT5 (PID {proc.pid})...") + proc.terminate() + try: + proc.wait(timeout=5) + except psutil.TimeoutExpired: + proc.kill() + killed += 1 + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + if killed > 0: + logger.info(f"[{run_id}] Closed {killed} MT5 instance(s). Waiting {self.KILL_WAIT_S}s...") + time.sleep(self.KILL_WAIT_S) + else: + logger.debug(f"[{run_id}] No existing MT5 process found.") + + # ── Step 2: Pre-run validation ──────────────────────────────────────────── + + def _validate(self, run_id: str) -> None: + """Validate all required files and directories exist before launch.""" + errors = [] + + # Check terminal executable + if not self.terminal_exe.exists(): + errors.append(f"MT5 terminal not found: {self.terminal_exe}") + + # Check EA compiled file + ea_name = self.cfg["ea"]["file"] + ea_candidates = [ + self.appdata_path / "MQL5" / "Experts" / f"{ea_name}.ex5", + self.appdata_path / "MQL5" / "Experts" / f"{ea_name}", + ] + ea_found = any(p.exists() for p in ea_candidates) + if not ea_found: + errors.append( + f"EA file not found: {ea_name}.ex5 — " + f"ensure you compiled the EA in MetaEditor before running." + ) + + # Check appdata path + if not self.appdata_path.exists(): + errors.append(f"MT5 appdata folder not found: {self.appdata_path}") + + # Symbol check (basic — just ensure it's set) + symbol = self.cfg["ea"].get("symbol", "") + if not symbol: + errors.append("Symbol not configured in config.yaml ea.symbol") + + if errors: + for e in errors: + logger.error(f"[{run_id}] Validation: {e}") + raise MT5ValidationError( + "Pre-run validation failed:\n" + "\n".join(f" • {e}" for e in errors) + ) + + logger.debug(f"[{run_id}] Pre-run validation passed.") + + # ── Step 3: Launch MT5 ─────────────────────────────────────────────────── + + def _launch_mt5(self, run_id: str, ini_path: Path) -> subprocess.Popen: + """Launch a fresh MT5 instance with the given INI config.""" + cmd = [str(self.terminal_exe), f"/config:{ini_path}"] + logger.info(f"[{run_id}] Launching MT5: {' '.join(cmd)}") + proc = subprocess.Popen( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + logger.debug(f"[{run_id}] MT5 PID: {proc.pid}") + return proc + + # ── Step 5: Wait for report ─────────────────────────────────────────────── + + def _wait_for_report( + self, + run_id: str, + proc: subprocess.Popen, + report_dir: Path, + report_stem: str, + ) -> RunResult: + """ + Poll both our local dir and MT5's native reports folder. + MT5 writes the report to \\reports\\ using the name from INI Report= field. + """ + elapsed = 0 + search_dirs = [report_dir, self.mt5_reports_dir] + + while elapsed < self.timeout_s: + # Check for report in all locations + for search in search_dirs: + if not search.exists(): + continue + result = self._find_report_files(search, report_stem) + if result: + xml_path, html_path = result + time.sleep(self.PROCESS_SETTLE_S) + # Archive to our local report dir + if xml_path: + report_dir.mkdir(parents=True, exist_ok=True) + dest = report_dir / Path(xml_path).name + if not dest.exists(): + shutil.copy2(xml_path, dest) + logger.info(f"[{run_id}] Report found in {search} after {elapsed}s") + return RunResult( + run_id=run_id, + report_xml=xml_path, + report_html=html_path, + success=True, + ) + + # Check if process already exited + ret = proc.poll() + if ret is not None: + time.sleep(self.PROCESS_SETTLE_S) + # Final check after process exit + for search in search_dirs: + if not search.exists(): + continue + result = self._find_report_files(search, report_stem) + if result: + xml_path, html_path = result + return RunResult( + run_id=run_id, + report_xml=xml_path, + report_html=html_path, + success=True, + ) + + # Process exited but no report — diagnose + if ret != 0: + raise RuntimeError( + f"MT5 exited with error code {ret}. " + f"Possible causes: invalid INI parameters, EA not compiled, " + f"or missing historical data." + ) + else: + raise RuntimeError( + "MT5 exited without generating a report. " + "Possible causes: Symbol data not downloaded, " + "invalid date range, or EA failed to initialize." + ) + + time.sleep(self.POLL_INTERVAL_S) + elapsed += self.POLL_INTERVAL_S + if elapsed % 30 == 0: + logger.info(f"[{run_id}] Still waiting for report... {elapsed}/{self.timeout_s}s") + + raise MT5TimeoutError( + f"No report after {self.timeout_s}s. " + f"MT5 may be stuck or the test is taking too long. " + f"Consider reducing the test date range or using OHLC M1 model." + ) + + def _clear_stale_reports(self, run_id: str) -> None: + """Remove stale Optimizer_* reports from MT5 appdata root to avoid false-positive detection.""" + try: + import glob + # Only delete files NOT matching the current run_id + for pattern in ["*.htm", "*.xml", "*.html"]: + for f in self.mt5_reports_dir.glob(f"Optimizer_*{pattern[-3:]}"): + if run_id not in f.name: + try: + f.unlink() + logger.debug(f"[{run_id}] Cleared stale report: {f.name}") + except Exception: + pass + except Exception as e: + logger.debug(f"[{run_id}] Could not clear stale reports: {e}") + + def _find_report_files( + self, search_dir: Path, report_stem: str + ) -> Optional[tuple[Optional[str], Optional[str]]]: + """Search for report XML/HTML files by stem prefix.""" + xml_list = sorted( + list(search_dir.glob(f"{report_stem}*.xml")) + + list(search_dir.glob(f"{report_stem}*.XML")), + key=lambda p: p.stat().st_mtime, reverse=True + ) + htm_list = sorted( + list(search_dir.glob(f"{report_stem}*.htm")) + + list(search_dir.glob(f"{report_stem}*.html")) + + list(search_dir.glob(f"{report_stem}*.HTM")), + key=lambda p: p.stat().st_mtime, reverse=True + ) + + if xml_list or htm_list: + return ( + str(xml_list[0]) if xml_list else None, + str(htm_list[0]) if htm_list else None, + ) + return None + + # ── TradeLogger CSV lookup ──────────────────────────────────────────────── + + def _find_trade_log( + self, run_id: str, search_dir: Optional[Path] + ) -> Optional[Path]: + """Find the TradeLogger CSV written by the EA during the backtest.""" + ea_name = self.cfg["ea"]["file"] + symbol = self.cfg["ea"]["symbol"] + + candidates = [ + self.mql5_files_dir / f"{ea_name}_{symbol}_TradeLog.csv", + ] + if search_dir: + candidates.append(search_dir / f"{ea_name}_{symbol}_TradeLog.csv") + + for path in candidates: + if path.exists(): + logger.debug(f"[{run_id}] TradeLog found: {path}") + return path + + logger.debug(f"[{run_id}] TradeLog CSV not found (fallback to report-only mode).") + return None + + # ── Error diagnosis ─────────────────────────────────────────────────────── + + def _diagnose_failure(self, error_msg: str) -> str: + """Convert technical errors to actionable user-facing messages.""" + msg = error_msg.lower() + if "exit" in msg and "cleanly" in msg: + return ( + "MT5 started but did not produce a report.\n" + "✦ Check that XAUUSD historical data is downloaded in MT5\n" + "✦ Ensure the date range (2022-2023) has data available\n" + "✦ Verify the EA compiled successfully in MetaEditor" + ) + if "timeout" in msg: + return ( + f"MT5 tester timed out after {self.timeout_s}s.\n" + "✦ Try a shorter date range in config.yaml\n" + "✦ Switch tester_model to 4 (OHLC M1) for faster runs" + ) + if "validation" in msg or "not found" in msg: + return error_msg + return ( + f"MT5 run failed: {error_msg}\n" + "✦ Ensure MT5 is fully closed before starting the optimizer\n" + "✦ Check config.yaml paths are correct" + ) diff --git a/mutation/__init__.py b/mutation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mutation/engine.py b/mutation/engine.py new file mode 100644 index 0000000..671594f --- /dev/null +++ b/mutation/engine.py @@ -0,0 +1,247 @@ +""" +mutation/engine.py +Translates analysis findings into concrete parameter hypotheses. +Uses knowledge_base.yaml rules as a structured ruleset. +""" +from __future__ import annotations +import json +from pathlib import Path +from typing import Any, Optional + +import numpy as np +import yaml +from loguru import logger + +from data.models import Finding, Hypothesis + + +class MutationEngine: + """ + Finding → Hypothesis translator. + + Workflow: + 1. Load knowledge_base.yaml rules + 2. For each finding, find matching rules + 3. Filter rules already tested recently (dedup) + 4. Resolve dynamic mutation values (percentile-based, derived) + 5. Build Hypothesis objects + 6. Return sorted by estimated PnL impact + """ + + def __init__( + self, + kb_path: str | Path = "mutation/knowledge_base.yaml", + manifest_path: str | Path = "mutation/param_manifest.yaml", + dedup_lookback: int = 10, + ): + with open(kb_path) as f: + self.kb = yaml.safe_load(f)["rules"] + with open(manifest_path) as f: + self.manifest = yaml.safe_load(f)["parameters"] + self.dedup_lookback = dedup_lookback + + # ── Public ──────────────────────────────────────────────────────────────── + + def propose( + self, + findings: list[Finding], + current_params: dict[str, Any], + recent_deltas: list[dict], # from store.get_recent_param_deltas() + max_proposals: int = 3, + ) -> list[Hypothesis]: + """ + Generate hypotheses from findings, de-duplicate, and return top-N. + """ + hypotheses: list[Hypothesis] = [] + + for finding in findings: + for rule in self.kb: + if not self._rule_matches(rule, finding, current_params): + continue + + param_delta = self._build_delta(rule, finding, current_params) + if not param_delta: + continue + + # Skip if identical delta was recently tested + if self._already_tested(param_delta, recent_deltas): + logger.debug(f"Skipping rule {rule['id']} — already tested.") + continue + + h = Hypothesis( + parent_run_id=finding.run_id, + finding_ids=[finding.finding_id], + description=f"[{rule['id']}] {rule['action_label']}", + param_delta=param_delta, + strategy=rule.get("strategy", "targeted"), + kb_rule_id=rule["id"], + ) + hypotheses.append((h, finding.impact_estimate_pnl)) + + # Sort by impact descending, deduplicate by KB rule + seen_rules = set() + ranked: list[Hypothesis] = [] + for h, impact in sorted(hypotheses, key=lambda x: x[1], reverse=True): + if h.kb_rule_id not in seen_rules: + seen_rules.add(h.kb_rule_id) + ranked.append(h) + if len(ranked) >= max_proposals: + break + + logger.info(f"Proposed {len(ranked)} hypotheses from {len(findings)} findings.") + return ranked + + # ── Rule matching ───────────────────────────────────────────────────────── + + def _rule_matches( + self, rule: dict, finding: Finding, current_params: dict + ) -> bool: + """Check if a KB rule's trigger matches this finding and current params.""" + trigger = rule.get("trigger", {}) + + # Analyzer match + if trigger.get("analyzer") and trigger["analyzer"] != finding.analyzer: + return False + + # Evaluate condition expression against finding evidence + current params + condition = trigger.get("condition", "") + if condition: + env = {**finding.evidence, **current_params} + # Simple boolean parsing for conditions like "reversal_rate > 0.15" + try: + if not self._eval_condition(condition, env): + return False + except Exception as e: + logger.debug(f"Rule {rule['id']} condition eval error: {e}") + return False + + return True + + def _eval_condition(self, condition: str, env: dict) -> bool: + """ + Evaluate a simple condition string. + Supports: >, <, >=, <=, ==, AND, OR + Variables are looked up in env dict. + """ + # Replace variable names with their values + tokens = condition.split() + resolved_tokens = [] + for token in tokens: + if token in ("AND", "OR", "and", "or", ">", "<", ">=", "<=", "==", "!="): + resolved_tokens.append(token.lower()) + elif token in env: + val = env[token] + resolved_tokens.append(str(val) if not isinstance(val, str) else f'"{val}"') + else: + resolved_tokens.append(token) + + expr = " ".join(resolved_tokens) + return bool(eval(expr, {"__builtins__": {}})) # restricted eval + + # ── Delta building ──────────────────────────────────────────────────────── + + def _build_delta( + self, + rule: dict, + finding: Finding, + current_params: dict, + ) -> dict[str, Any]: + """ + Translate a KB mutation spec into a concrete {param_name: new_value} dict. + Handles: set, multiply, derive_from, set_to_percentile. + """ + delta: dict[str, Any] = {} + mutations = rule.get("mutations", {}) + + for param_name, mutation_spec in mutations.items(): + current = current_params.get(param_name) + spec = self.manifest.get(param_name, {}) + ptype = spec.get("type", "float") + p_min = spec.get("min") + p_max = spec.get("max") + + new_val = self._resolve_mutation( + mutation_spec, current, ptype, p_min, p_max, finding + ) + if new_val is not None: + delta[param_name] = new_val + + # Auto-cascade: if enabling a bool, set defaults for depends_on params + if ptype == "bool" and new_val is True: + delta.update(self._cascade_dependencies(param_name, current_params)) + + return delta + + def _resolve_mutation( + self, + spec: dict | Any, + current: Any, + ptype: str, + p_min: Optional[float], + p_max: Optional[float], + finding: Finding, + ) -> Optional[Any]: + """Resolve a single mutation spec into a concrete value.""" + if not isinstance(spec, dict): + return spec # bare value + + # set: directly set to a value + if "set" in spec: + return spec["set"] + + # multiply: multiply current value by factor + if "multiply" in spec and current is not None: + result = float(current) * spec["multiply"] + if "clamp_min" in spec: + result = max(spec["clamp_min"], result) + if p_min is not None: + result = max(p_min, result) + if p_max is not None: + result = min(p_max, result) + return round(result, 2) if ptype == "float" else int(result) + + # set_to_percentile: use Nth percentile of a finding evidence list + if "set_to_percentile" in spec: + pct = spec["set_to_percentile"] / 100.0 + data = finding.evidence.get("mfe_pips_distribution", []) + if data: + val = float(np.percentile(data, pct * 100)) + if "scale" in spec: + val *= spec["scale"] + if p_min is not None: + val = max(p_min, val) + if p_max is not None: + val = min(p_max, val) + return round(val, 1) + + # derive_from: use evidence field + if "derive_from" in spec: + key = spec["derive_from"] + val = finding.evidence.get(key) + if val is not None: + return val + + return None + + def _cascade_dependencies( + self, bool_param: str, current_params: dict + ) -> dict[str, Any]: + """When a bool param is enabled, fill in sensible defaults for its dependents.""" + cascade = {} + for name, spec in self.manifest.items(): + if spec.get("depends_on") != bool_param: + continue + # Only set if not already in current params or at a sub-optimal default + if name not in current_params: + cascade[name] = spec.get("default", 0) + return cascade + + # ── Deduplication ───────────────────────────────────────────────────────── + + def _already_tested(self, delta: dict, recent_deltas: list[dict]) -> bool: + """Check if an identical param delta was tested recently.""" + delta_str = json.dumps(delta, sort_keys=True) + for past in recent_deltas: + if json.dumps(past, sort_keys=True) == delta_str: + return True + return False diff --git a/mutation/knowledge_base.yaml b/mutation/knowledge_base.yaml new file mode 100644 index 0000000..ac5268a --- /dev/null +++ b/mutation/knowledge_base.yaml @@ -0,0 +1,187 @@ +rules: + + # ── Trailing / Exit Rules ──────────────────────────────────────────────────── + + - id: KB001 + trigger: + analyzer: reversal + condition: "reversal_rate > 0.15" + action_label: "Enable trailing stop to protect in-profit trades (reversal rate high)" + mutations: + InpUseTrailing: + set: true + InpTrailStartPips: + derive_from: "mfe_p25" # 25th percentile of reversal MFE + fallback: 20.0 + InpTrailStepPips: + set: 10.0 + strategy: targeted + + - id: KB002 + trigger: + analyzer: reversal + condition: "reversal_rate > 0.20" + action_label: "Tighten TP ratio — too many trades reversing before target hit" + mutations: + InpRRRatio: + multiply: 0.80 + clamp_min: 1.0 + strategy: targeted + + - id: KB003 + trigger: + analyzer: reversal + condition: "mean_capture_ratio < 0.55" + action_label: "Low MFE capture on winners — enable or tighten trailing" + mutations: + InpUseTrailing: + set: true + InpTrailStartPips: + set: 15.0 + InpTrailStepPips: + set: 8.0 + strategy: targeted + + # ── Session / Time Filter Rules ─────────────────────────────────────────────── + + - id: KB004 + trigger: + analyzer: time_performance + condition: "type == 'hour_window'" + action_label: "Exclude identified negative-edge UTC time window via session filter" + mutations: + InpUseSession: + set: true + InpSessionEnd: + derive_from: "broker_start" # end session before bad window starts + strategy: targeted + + - id: KB005 + trigger: + analyzer: time_performance + condition: "type == 'session'" + action_label: "Negative-edge session detected — tighten or disable session window" + mutations: + InpUseSession: + set: true + strategy: targeted + + # ── Entry Quality Rules ──────────────────────────────────────────────────────── + + - id: KB006 + trigger: + analyzer: entry_exit_quality + condition: "diagnosis == 'poor_entry'" + action_label: "Poor entry quality — tighten ATR filter and score gate" + mutations: + InpUseSpreadGuard: + set: true + InpMinScore: + multiply: 1.125 + clamp_min: 6 + InpATRMultiplier: + multiply: 1.20 + clamp_min: 0.3 + strategy: targeted + + - id: KB007 + trigger: + analyzer: entry_exit_quality + condition: "diagnosis == 'good_entry_poor_exit'" + action_label: "Good entries, poor exits — enable trailing with conservative start" + mutations: + InpUseTrailing: + set: true + InpTrailStartPips: + set: 18.0 + InpTrailStepPips: + set: 10.0 + strategy: targeted + + - id: KB008 + trigger: + analyzer: entry_exit_quality + condition: "diagnosis == 'both_broken'" + action_label: "Both entry and exit quality poor — test conservative bot mode" + mutations: + InpBotMode: + set: 2 + InpMinScore: + multiply: 1.25 + clamp_min: 6 + strategy: compound + + # ── Risk / Drawdown Rules ───────────────────────────────────────────────────── + + - id: KB009 + trigger: + analyzer: equity_curve + condition: "flatness_score > 0.50" + action_label: "Equity spending too much time in drawdown — reduce risk per trade" + mutations: + InpRiskPercent: + multiply: 0.75 + clamp_min: 0.5 + InpMaxDailyLossPct: + multiply: 0.80 + clamp_min: 1.0 + strategy: targeted + + - id: KB010 + trigger: + analyzer: equity_curve + condition: "cluster_count > 3" + action_label: "Repeated loss clusters — limit consecutive trades and daily risk" + mutations: + InpMaxTradesPerDay: + multiply: 0.75 + clamp_min: 2 + InpMaxDailyLossPct: + set: 2.0 + strategy: targeted + + # ── Breakeven Rules ──────────────────────────────────────────────────────────── + + - id: KB011 + trigger: + analyzer: reversal + condition: "reversal_rate > 0.12" + action_label: "Enable breakeven stop to lock in partial profit before reversal" + mutations: + InpUseBreakeven: + set: true + InpBEPips: + derive_from: "mfe_p25" + fallback: 15.0 + InpBEBufferPips: + set: 2.0 + strategy: targeted + + # ── Filter Tightening Rules ──────────────────────────────────────────────────── + + - id: KB012 + trigger: + analyzer: entry_exit_quality + condition: "high_mae_loser_count > 10" + action_label: "High MAE losers — require EMA slope confirmation" + mutations: + InpUseEMA: + set: true + InpRequireEMASlope: + set: true + InpEMASlopeBars: + set: 2 + strategy: targeted + + - id: KB013 + trigger: + analyzer: entry_exit_quality + condition: "mean_entry_quality < 0.35" + action_label: "Very poor entry quality — tighten minimum RR gate" + mutations: + InpUseMinRR: + set: true + InpMinRRRatio: + multiply: 1.25 + clamp_min: 1.0 + strategy: targeted diff --git a/mutation/param_manifest.yaml b/mutation/param_manifest.yaml new file mode 100644 index 0000000..0d6bcc4 --- /dev/null +++ b/mutation/param_manifest.yaml @@ -0,0 +1,394 @@ +# Parameter Manifest — LEGSTECH_EA_V2 +# Generated from: LEGSTECH_EA_V2.set +# Format: value = default, min/max/step = optimization bounds +# Types: float | int | bool | enum +# ───────────────────────────────────────────────────────────── + +parameters: + + # ── Bot / Mode ───────────────────────────────────────────── + InpBotMode: + type: enum + values: [0, 1, 2] # 0, 1, 2 per .set range 0→2 step 1 + default: 1 + category: mode + description: "Bot operating mode" + + # ── Timeframe Selection (PERIOD_ codes) ──────────────────── + # These are MT5 ENUM_TIMEFRAMES integer codes. Not optimized — fixed. + InpHTF: + type: fixed + default: 16408 # PERIOD_H4 + category: timeframe + InpMTF: + type: fixed + default: 16388 # PERIOD_H1 + category: timeframe + InpLTF: + type: fixed + default: 16385 # PERIOD_M30 + category: timeframe + + # ── Risk Management ──────────────────────────────────────── + InpRiskType: + type: enum + values: [0, 1] # 0=fixed lot, 1=percent risk + default: 1 + category: risk + + InpFixedLot: + type: float + min: 0.01 + max: 1.0 + step: 0.01 + default: 0.01 + category: risk + depends_on_value: {InpRiskType: 0} # only active when using fixed lot mode + + InpRiskPercent: + type: float + min: 0.5 + max: 3.0 + step: 0.5 + default: 1.0 + category: risk + depends_on_value: {InpRiskType: 1} # only active when using percent risk mode + + InpMaxDailyLossPct: + type: float + min: 1.0 + max: 5.0 + step: 0.5 + default: 3.0 + category: risk + + InpMaxTradesPerDay: + type: int + min: 1 + max: 10 + step: 1 + default: 5 + category: risk + + # ── Stop Loss ────────────────────────────────────────────── + InpSLType: + type: enum + values: [0, 1] # 0=fixed pips, 1=ATR-based + default: 0 + category: sl + + InpSLBuffer: + type: float + min: 5.0 + max: 30.0 + step: 5.0 + default: 10.0 + category: sl + description: "Buffer pips added to SL" + + InpFixedSLPips: + type: float + min: 50.0 + max: 200.0 + step: 10.0 + default: 100.0 + category: sl + depends_on_value: {InpSLType: 0} + + InpMaxSLPips: + type: float + min: 100.0 + max: 400.0 + step: 50.0 + default: 200.0 + category: sl + description: "Hard cap on calculated SL size" + + InpUseFractalSL: + type: bool + default: false # 0 in .set + category: sl + + # ── Take Profit ──────────────────────────────────────────── + InpTPType: + type: enum + values: [0, 1] # 0=RR ratio, 1=fixed pips + default: 0 + category: tp + + InpRRRatio: + type: float + min: 1.0 + max: 3.0 + step: 0.5 + default: 1.5 + category: tp + depends_on_value: {InpTPType: 0} + + InpFixedTPPips: + type: float + min: 50.0 + max: 200.0 + step: 10.0 + default: 100.0 + category: tp + depends_on_value: {InpTPType: 1} + + InpUseFractalFilter: + type: bool + default: false # 0 in .set + category: tp + + # ── Session Filter ───────────────────────────────────────── + InpUseSession: + type: bool + default: true # 1 in .set + category: filter_session + + InpSessionStart: + type: int + min: 0 + max: 23 + step: 1 + default: 7 + category: filter_session + depends_on: InpUseSession + description: "Session start hour (broker local time)" + + InpSessionEnd: + type: int + min: 0 + max: 23 + step: 1 + default: 20 + category: filter_session + depends_on: InpUseSession + description: "Session end hour (broker local time)" + + # ── Trade Limits ─────────────────────────────────────────── + InpMaxOpenTrades: + type: int + min: 1 + max: 3 + step: 1 + default: 1 + category: risk + + InpAllowMultiple: + type: bool + default: false # 0 in .set + category: risk + + # ── Execution ───────────────────────────────────────────── + InpMagicNumber: + type: fixed + default: 202402 + category: execution + description: "Fixed — do not optimize" + + InpSlippage: + type: int + min: 5 + max: 30 + step: 5 + default: 10 + category: execution + + # ── Trailing Stop ───────────────────────────────────────── + InpUseTrailing: + type: bool + default: true # 1 in .set + category: exit_trail + + InpTrailStartPips: + type: float + min: 10.0 + max: 50.0 + step: 5.0 + default: 20.0 + category: exit_trail + depends_on: InpUseTrailing + + InpTrailStepPips: + type: float + min: 5.0 + max: 30.0 + step: 5.0 + default: 10.0 + category: exit_trail + depends_on: InpUseTrailing + + # ── Break Even ──────────────────────────────────────────── + InpUseBreakeven: + type: bool + default: true # 1 in .set + category: exit_be + + InpBEPips: + type: float + min: 10.0 + max: 40.0 + step: 5.0 + default: 15.0 + category: exit_be + depends_on: InpUseBreakeven + description: "Pips in profit to activate breakeven" + + InpBEBufferPips: + type: float + min: 1.0 + max: 5.0 + step: 1.0 + default: 2.0 + category: exit_be + depends_on: InpUseBreakeven + description: "Buffer pips above entry for breakeven SL" + + # ── EMA Filter ──────────────────────────────────────────── + InpUseEMA: + type: bool + default: true # 1 in .set + category: filter_ema + + InpEMAPeriod: + type: int + min: 20 + max: 100 + step: 10 + default: 50 + category: filter_ema + depends_on: InpUseEMA + + InpRequireEMASlope: + type: bool + default: true # 1 in .set + category: filter_ema + depends_on: InpUseEMA + + InpEMASlopeBars: + type: int + min: 1 + max: 3 + step: 1 + default: 1 + category: filter_ema + depends_on: InpRequireEMASlope + + # ── Entry Mode ──────────────────────────────────────────── + InpEntryMode: + type: enum + values: [0, 1] + default: 1 + category: entry + + InpSLBufferMode: + type: enum + values: [0, 1] + default: 1 + category: sl + + # ── ATR ─────────────────────────────────────────────────── + InpATRPeriod: + type: int + min: 10 + max: 20 + step: 2 + default: 14 + category: atr + + InpATRMultiplier: + type: float + min: 0.3 + max: 1.0 + step: 0.1 + default: 0.5 + category: atr + + # ── Spread Guard ────────────────────────────────────────── + InpUseSpreadGuard: + type: bool + default: true # 1 in .set + category: filter_spread + + InpMaxSpreadPips: + type: float + min: 10.0 + max: 50.0 + step: 5.0 + default: 30.0 + category: filter_spread + depends_on: InpUseSpreadGuard + + # ── Minimum R:R Gate ────────────────────────────────────── + InpUseMinRR: + type: bool + default: true # 1 in .set + category: filter_rr + + InpMinRRRatio: + type: float + min: 1.0 + max: 3.0 + step: 0.5 + default: 1.5 + category: filter_rr + depends_on: InpUseMinRR + + # ── Score Gate ──────────────────────────────────────────── + InpUseScoreGate: + type: bool + default: true # 1 in .set + category: filter_score + + InpMinScore: + type: int + min: 6 + max: 11 + step: 1 + default: 8 + category: filter_score + depends_on: InpUseScoreGate + description: "Minimum signal quality score required to enter trade" + + # ── Tester-Specific (fixed during automation) ───────────── + InpTesterMode: + type: fixed + default: 1 + category: tester + description: "Must be 1 during automated backtesting" + + InpTesterInitDeposit: + type: fixed + default: 10000.0 + category: tester + + InpTesterSpreadPts: + type: int + min: 10 + max: 50 + step: 5 + default: 20 + category: tester + description: "Spread in points used in tester (20 pts = 2.0 pips for XAUUSD)" + + InpShowPanel: + type: fixed + default: 0 # force off during automation (no GUI needed) + category: tester + +# ── Parameter categories (for mutation engine grouping) ────── +categories: + mode: [InpBotMode] + risk: [InpRiskType, InpFixedLot, InpRiskPercent, InpMaxDailyLossPct, InpMaxTradesPerDay, InpMaxOpenTrades, InpAllowMultiple] + sl: [InpSLType, InpSLBuffer, InpFixedSLPips, InpMaxSLPips, InpUseFractalSL, InpSLBufferMode] + tp: [InpTPType, InpRRRatio, InpFixedTPPips, InpUseFractalFilter] + exit_trail: [InpUseTrailing, InpTrailStartPips, InpTrailStepPips] + exit_be: [InpUseBreakeven, InpBEPips, InpBEBufferPips] + filter_session: [InpUseSession, InpSessionStart, InpSessionEnd] + filter_ema: [InpUseEMA, InpEMAPeriod, InpRequireEMASlope, InpEMASlopeBars] + filter_spread: [InpUseSpreadGuard, InpMaxSpreadPips] + filter_rr: [InpUseMinRR, InpMinRRRatio] + filter_score: [InpUseScoreGate, InpMinScore] + entry: [InpEntryMode] + atr: [InpATRPeriod, InpATRMultiplier] + tester: [InpTesterMode, InpTesterInitDeposit, InpTesterSpreadPts, InpShowPanel] diff --git a/optimizer.db-shm b/optimizer.db-shm new file mode 100644 index 0000000..cb3bdd5 Binary files /dev/null and b/optimizer.db-shm differ diff --git a/optimizer.db-wal b/optimizer.db-wal new file mode 100644 index 0000000..e5882de Binary files /dev/null and b/optimizer.db-wal differ diff --git a/optimizer_loop.py b/optimizer_loop.py new file mode 100644 index 0000000..f509776 --- /dev/null +++ b/optimizer_loop.py @@ -0,0 +1,495 @@ +""" +optimizer_loop.py +Background thread that runs the full optimization loop and emits +real-time SocketIO events to the dashboard. +""" +from __future__ import annotations +import json +import threading +import time +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any, Optional + +import yaml +from loguru import logger + +# Project imports +from data.models import Run, RunMetrics, Candidate, Hypothesis +from data.store import DataStore +from mt5.ini_builder import IniBuilder +from mt5.runner import MT5Runner +from mt5.report_parser import ReportParser +from mt5.log_reader import TradeLogReader +from analysis.reversal import ReversalAnalyzer +from analysis.time_performance import TimePerformanceAnalyzer +from analysis.entry_exit_quality import EntryExitQualityAnalyzer +from analysis.equity_curve import EquityCurveAnalyzer +from scoring.composite import CompositeScorer +from mutation.engine import MutationEngine +from validation.gate import ValidationGate +from reports.writer import ReportWriter + +import pandas as pd + +BASE_DIR = Path(__file__).parent +MANIFEST_PATH = BASE_DIR / "mutation" / "param_manifest.yaml" +KB_PATH = BASE_DIR / "mutation" / "knowledge_base.yaml" +DB_PATH = BASE_DIR / "optimizer.db" +RUNS_DIR = BASE_DIR / "runs" + + +class OptimizerLoop: + """ + Runs the full optimization pipeline in a background thread. + Pushes events to the SocketIO broadcast channel so the dashboard + updates in real time. + """ + + def __init__(self, config_path: str, socketio, reports_dir: Path, auto_mode: bool = True): + self.config_path = config_path + self.socketio = socketio + self.reports_dir = reports_dir + self.auto_mode = auto_mode + self.running = False + self.paused = False + self._stop_flag = False + self._skip_flag = False + self._pause_event = threading.Event() + self._pause_event.set() # not paused initially + + # Live state (read by /api/status) + self.iteration = 0 + self.phase = "idle" + self.best_score = 0.0 + self.current_run_id: Optional[str] = None + self.score_history: list[dict] = [] + self.run_start_ts: Optional[float] = None + + with open(config_path) as f: + self.cfg = yaml.safe_load(f) + + # ── Controls ────────────────────────────────────────────────────────────── + + def toggle_pause(self): + self.paused = not self.paused + if self.paused: + self._pause_event.clear() + self._emit("status_change", {"state": "paused"}) + else: + self._pause_event.set() + self._emit("status_change", {"state": "running"}) + + def stop(self): + self._stop_flag = True + self._pause_event.set() + self.running = False + + def skip_hypothesis(self): + self._skip_flag = True + + def get_status(self) -> dict: + elapsed = int(time.time() - self.run_start_ts) if self.run_start_ts else 0 + return { + "state": "running" if self.running else ("paused" if self.paused else "idle"), + "iteration": self.iteration, + "phase": self.phase, + "best_score": round(self.best_score, 4), + "run_id": self.current_run_id, + "elapsed_s": elapsed, + } + + # ── Main loop ───────────────────────────────────────────────────────────── + + def run(self): + self.running = True + self._stop_flag = False + self.run_start_ts = time.time() + + cfg = self.cfg + store, builder, runner, parser, log_rdr, analyzers, scorer, mutator, gate, writer = ( + self._build_components() + ) + + per = cfg["periods"] + self._emit("status_change", {"state": "running", "phase": "baseline"}) + self._emit("log", {"level": "info", "msg": "🚀 Optimizer started"}) + + # ── Baseline run ────────────────────────────────────────────────────── + self.phase = "baseline" + default_params = builder.default_params() + baseline_id = f"baseline_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}" + + baseline_metrics, baseline_trades = self._execute_run( + run_id=baseline_id, + params=default_params, + period_start=per["train_start"], + period_end=per["train_end"], + phase="baseline", + hypothesis_id=None, + store=store, builder=builder, runner=runner, + parser=parser, log_rdr=log_rdr, analyzers=analyzers, scorer=scorer, + ) + + if baseline_metrics is None: + self._emit("error", {"msg": "Baseline run failed. Check MT5 configuration."}) + self.running = False + return + + # Write baseline report + findings = self._run_analysis(baseline_id, baseline_trades, baseline_metrics, + analyzers, store) + writer.write(baseline_id, baseline_metrics, baseline_trades, findings, default_params) + + self.best_score = baseline_metrics.composite_score + current_params = default_params.copy() + current_metrics = baseline_metrics + no_improve_count = 0 + + max_iter = cfg["optimization"]["max_iterations"] + conv_win = cfg["optimization"]["convergence_window"] + conv_thr = cfg["optimization"]["convergence_threshold"] + + # ── Iteration loop ──────────────────────────────────────────────────── + while self.iteration < max_iter and not self._stop_flag: + self._pause_event.wait() + if self._stop_flag: + break + + self.iteration += 1 + self.phase = "analyze" + self._emit("iteration_start", {"iteration": self.iteration}) + self._emit("log", {"level": "info", "msg": f"━━ Iteration {self.iteration} ━━"}) + + # Load latest trades (for re-analysis) + trades_df = store.load_trades(current_metrics.run_id) + if trades_df.empty and not baseline_trades.empty: + trades_df = baseline_trades + + # Analysis + self.phase = "analyze" + findings = self._run_analysis( + current_metrics.run_id, trades_df, current_metrics, analyzers, store + ) + + if not findings: + self._emit("log", {"level": "warn", "msg": "No actionable findings. Stopping."}) + break + + # Mutation proposals + recent_deltas = store.get_recent_param_deltas(cfg["mutation"]["dedup_lookback_runs"]) + hypotheses = mutator.propose( + findings=findings, + current_params=current_params, + recent_deltas=recent_deltas, + max_proposals=cfg["mutation"]["max_hypotheses_per_cycle"], + ) + + if not hypotheses: + self._emit("log", {"level": "warn", "msg": "No new hypotheses. Stopping."}) + break + + self._emit("hypotheses", {"items": [ + { + "id": h.hypothesis_id, + "desc": h.description, + "delta": h.param_delta, + "strategy": h.strategy, + } + for h in hypotheses + ]}) + + # Test each hypothesis + iteration_best: Optional[RunMetrics] = None + iteration_best_params: Optional[dict] = None + iteration_best_hyp: Optional[Hypothesis] = None + + for idx, hyp in enumerate(hypotheses): + self._pause_event.wait() + if self._stop_flag or self._skip_flag: + self._skip_flag = False + break + + test_params = {**current_params, **hyp.param_delta} + run_id = f"iter{self.iteration:03d}_h{idx+1}" + store.save_hypothesis(hyp) + + self._emit("log", { + "level": "info", + "msg": f"Testing H{idx+1}: {hyp.description[:60]}" + }) + self._emit("hypothesis_testing", {"idx": idx+1, "desc": hyp.description}) + + test_metrics, test_trades = self._execute_run( + run_id=run_id, + params=test_params, + period_start=per["train_start"], + period_end=per["train_end"], + phase="explore", + hypothesis_id=hyp.hypothesis_id, + store=store, builder=builder, runner=runner, + parser=parser, log_rdr=log_rdr, analyzers=analyzers, scorer=scorer, + ) + + if test_metrics is None: + continue + + delta_score = test_metrics.composite_score - current_metrics.composite_score + color = "green" if delta_score > 0 else "red" + self._emit("log", { + "level": "success" if delta_score > 0 else "warn", + "msg": f"H{idx+1} score: {current_metrics.composite_score:.4f} → " + f"{test_metrics.composite_score:.4f} ({delta_score:+.4f})" + }) + + # Write per-run report + h_findings = self._run_analysis(run_id, test_trades, test_metrics, analyzers, store) + writer.write(run_id, test_metrics, test_trades, h_findings, test_params, + hypothesis=hyp, baseline_score=current_metrics.composite_score) + + store.update_hypothesis_status(hyp.hypothesis_id, "tested", run_id) + + if iteration_best is None or test_metrics.composite_score > iteration_best.composite_score: + iteration_best = test_metrics + iteration_best_params = test_params + iteration_best_hyp = hyp + + if iteration_best is None: + no_improve_count += 1 + continue + + # Validation gate + self.phase = "validate" + gate_result = gate.run_is_check(iteration_best) + if not gate_result.passed: + self._emit("log", {"level": "error", "msg": f"IS gate failed: {gate_result.reason}"}) + store.update_hypothesis_status(iteration_best_hyp.hypothesis_id, "rejected") + no_improve_count += 1 + continue + + # Walk-forward + self._emit("log", {"level": "info", "msg": "Running walk-forward validation..."}) + wfv = gate.run_walk_forward( + iteration_best_params, cfg, store, builder, runner, + parser, log_rdr, analyzers, scorer, + ) + self._emit("log", { + "level": "success" if wfv.passed else "warn", + "msg": f"WFV: OOS/IS ratio = {wfv.oos_is_ratio:.2f} " + f"({'PASS ✅' if wfv.passed else 'FAIL ❌'})" + }) + + if not wfv.passed: + store.update_hypothesis_status(iteration_best_hyp.hypothesis_id, "rejected") + no_improve_count += 1 + continue + + # Promote candidate + candidate = Candidate( + run_id=iteration_best.run_id, + composite_score=iteration_best.composite_score, + params=iteration_best_params, + ) + store.save_candidate(candidate) + store.update_hypothesis_status(iteration_best_hyp.hypothesis_id, "validated") + + self._emit("candidate_promoted", { + "candidate_id": candidate.candidate_id, + "score": round(candidate.composite_score, 4), + "delta": round(candidate.composite_score - self.best_score, 4), + "params": candidate.params, + }) + self._emit("log", { + "level": "success", + "msg": f"✅ Candidate promoted! Score: {candidate.composite_score:.4f}" + }) + + improvement = iteration_best.composite_score - self.best_score + if improvement >= conv_thr: + current_params = iteration_best_params + current_metrics = iteration_best + self.best_score = iteration_best.composite_score + no_improve_count = 0 + else: + no_improve_count += 1 + + # Update score chart + self.score_history.append({ + "iteration": self.iteration, + "score": round(self.best_score, 4), + "calmar": round(current_metrics.calmar_ratio, 4), + "pf": round(current_metrics.profit_factor, 4), + "ts": datetime.utcnow().isoformat(), + }) + self._emit("score_update", self.score_history[-1]) + + if no_improve_count >= conv_win: + self._emit("log", { + "level": "warn", + "msg": f"Converged: no improvement in {no_improve_count} iterations." + }) + break + + # ── Done ───────────────────────────────────────────────────────────── + candidates = store.list_candidates() + self._emit("optimization_complete", { + "candidates": len(candidates), + "best_score": round(self.best_score, 4), + "iterations": self.iteration, + }) + self._emit("log", {"level": "success", "msg": "🏁 Optimization complete!"}) + self.running = False + self.phase = "idle" + + # ── Single run ──────────────────────────────────────────────────────────── + + def _execute_run( + self, + run_id, params, period_start, period_end, phase, hypothesis_id, + store, builder, runner, parser, log_rdr, analyzers, scorer, + ): + cfg = self.cfg + self.current_run_id = run_id + run_dir = RUNS_DIR / run_id + run_dir.mkdir(parents=True, exist_ok=True) + + self._emit("run_started", { + "run_id": run_id, + "phase": phase, + "period": f"{period_start} → {period_end}", + "params": {k: v for k, v in list(params.items())[:8]}, # first 8 for display + }) + + ini_path = builder.build( + run_id=run_id, params=params, + period_start=period_start, period_end=period_end, + output_dir=run_dir, phase=phase, + ) + + run = Run( + run_id=run_id, ea_name=cfg["ea"]["name"], + symbol=cfg["ea"]["symbol"], timeframe=cfg["ea"]["timeframe"], + period_start=period_start, period_end=period_end, + params=params, phase=phase, hypothesis_id=hypothesis_id, + tester_model=cfg["mt5"]["tester_model"], + ini_snapshot=ini_path.read_text(), + ) + store.save_run(run) + + self._emit("log", {"level": "info", "msg": f"⏳ MT5 running: {run_id}"}) + result = runner.run(run_id, ini_path, run_dir / "report", + log_csv_search_dir=Path(cfg["mt5"]["mql5_files_path"])) + + if not result.success: + self._emit("run_failed", {"run_id": run_id, "error": result.error_message}) + return None, pd.DataFrame() + + metrics, trades = parser.parse(result.report_xml, result.report_html) + if metrics is None: + self._emit("run_failed", {"run_id": run_id, "error": "Could not parse report"}) + return None, pd.DataFrame() + metrics.run_id = run_id + + if trades: + trades = log_rdr.merge( + trades, result.trade_log_csv, + reversal_mfe_threshold_pips=cfg["analysis"]["reversal"]["mfe_threshold_pips"], + ) + trades_df = pd.DataFrame([t.model_dump() for t in trades]) + else: + trades_df = pd.DataFrame() + + if not trades_df.empty: + if "result_class" in trades_df.columns: + losers = trades_df[trades_df["net_money"] < 0] + reversals = trades_df[trades_df["result_class"] == "reversal"] + metrics.reversal_rate = len(reversals) / max(1, len(losers)) + if "mfe_capture_ratio" in trades_df.columns: + metrics.avg_mfe_capture = float(trades_df["mfe_capture_ratio"].dropna().mean() or 0) + + metrics.composite_score = scorer.score(metrics) + store.save_metrics(metrics) + if not trades_df.empty: + store.save_trades(run_id, trades) + run.report_path = result.report_xml + store.save_run(run) + + self._emit("run_complete", { + "run_id": run_id, + "phase": phase, + "net_profit": round(metrics.net_profit, 2), + "profit_factor": round(metrics.profit_factor, 3), + "calmar": round(metrics.calmar_ratio, 3), + "drawdown_pct": round(metrics.max_drawdown_pct * 100, 1), + "win_rate": round(metrics.win_rate * 100, 1), + "total_trades": metrics.total_trades, + "score": round(metrics.composite_score, 4), + "reversal_rate": round((metrics.reversal_rate or 0) * 100, 1), + "mfe_capture": round((metrics.avg_mfe_capture or 0) * 100, 1), + }) + + return metrics, trades_df + + def _run_analysis(self, run_id, trades_df, metrics, analyzers, store): + all_findings = [] + for az in analyzers: + findings = az.run(trades_df, metrics, run_id) + for f in findings: + self._emit("finding", { + "analyzer": f.analyzer, + "severity": f.severity, + "description": f.description, + "confidence": round(f.confidence, 2), + "impact": round(f.impact_estimate_pnl, 0), + }) + all_findings.extend(findings) + all_findings.sort(key=lambda f: f.confidence, reverse=True) + store.save_findings(all_findings) + return all_findings + + # ── SocketIO emit helper ────────────────────────────────────────────────── + + def _emit(self, event: str, data: dict = {}): + try: + self.socketio.emit(event, data) + except Exception as e: + logger.debug(f"Emit error ({event}): {e}") + + # ── Component factory ───────────────────────────────────────────────────── + + def _build_components(self): + cfg = self.cfg + store = DataStore(DB_PATH, RUNS_DIR) + builder = IniBuilder(self.config_path, str(MANIFEST_PATH)) + runner = MT5Runner(self.config_path) + parser = ReportParser() + log_rdr = TradeLogReader( + broker_tz_offset_hours=cfg["broker"]["timezone_offset_hours"], + ) + analyzers = [ + ReversalAnalyzer( + mfe_threshold_pips=cfg["analysis"]["reversal"]["mfe_threshold_pips"], + min_reversal_rate=cfg["analysis"]["reversal"]["min_reversal_rate"], + permutation_n=cfg["analysis"]["reversal"]["permutation_n"], + ), + TimePerformanceAnalyzer( + z_score_threshold=cfg["analysis"]["time_performance"]["z_score_threshold"], + min_bucket_trades=cfg["analysis"]["time_performance"]["min_trades_per_bucket"], + permutation_n=cfg["analysis"]["time_performance"]["permutation_n"], + ), + EntryExitQualityAnalyzer( + poor_exit_threshold=cfg["analysis"]["entry_exit"]["poor_exit_quality"], + poor_entry_threshold=cfg["analysis"]["entry_exit"]["poor_entry_quality"], + ), + EquityCurveAnalyzer( + max_flatness=cfg["analysis"]["equity_curve"]["max_flatness_score"], + min_r_squared=cfg["analysis"]["equity_curve"]["min_r_squared"], + ), + ] + scorer = CompositeScorer(self.config_path) + mutator = MutationEngine(KB_PATH, MANIFEST_PATH, + dedup_lookback=cfg["mutation"]["dedup_lookback_runs"]) + gate = ValidationGate(self.config_path) + writer = ReportWriter(self.reports_dir) + return store, builder, runner, parser, log_rdr, analyzers, scorer, mutator, gate, writer diff --git a/registry/__init__.py b/registry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1c59db5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +pandas>=2.1 +numpy>=1.26 +pyarrow>=14.0 +lxml>=4.9 +pydantic>=2.5 +pyyaml>=6.0 +loguru>=0.7 +rich>=13.0 +sqlalchemy>=2.0 +scipy>=1.11 +plotly>=5.18 +pytest>=7.4 +flask +flask-socketio +eventlet +jinja2 +pyinstaller diff --git a/scoring/__init__.py b/scoring/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scoring/composite.py b/scoring/composite.py new file mode 100644 index 0000000..dad3f67 --- /dev/null +++ b/scoring/composite.py @@ -0,0 +1,174 @@ +""" +scoring/composite.py +Composite score function for LEGSTECH_EA_V2 / XAUUSD optimization. +Primary: Calmar Ratio | Secondary: PF, MFE capture, session stability, recovery. +""" +from __future__ import annotations +from typing import Optional +import numpy as np +import yaml +from loguru import logger + +from data.models import RunMetrics + + +def _clip_normalize(value: float, lo: float, hi: float) -> float: + """Normalize value to [0, 1] by clipping to [lo, hi].""" + if hi <= lo: + return 0.0 + return float(np.clip((value - lo) / (hi - lo), 0.0, 1.0)) + + +def _session_stability(session_stats: dict) -> float: + """ + Compute session stability as 1 - (std of per-session Calmar / mean Calmar). + Returns 1.0 (perfectly stable) if only one session or no variance. + session_stats: {session_name: {"calmar": float, "trade_count": int}} + """ + calmars = [ + v["calmar"] for v in session_stats.values() + if v.get("trade_count", 0) >= 10 and "calmar" in v + ] + if len(calmars) < 2: + return 1.0 # not enough sessions to measure stability + mean_c = np.mean(calmars) + std_c = np.std(calmars) + if abs(mean_c) < 0.01: + return 0.0 + cv = std_c / abs(mean_c) # coefficient of variation + return float(np.clip(1.0 - cv, 0.0, 1.0)) + + +class CompositeScorer: + """ + Weighted composite score calculator. + + All sub-scores are independently normalized to [0, 1]. + Two multiplicative penalties are applied after weighting: + - Significance penalty: reduces weight for low trade counts + - Reversal penalty : reduces score if reversal rate is high + """ + + DEFAULT_WEIGHTS = { + "calmar": 0.35, + "profit_factor": 0.20, + "mfe_capture": 0.20, + "session_stability": 0.15, + "recovery_factor": 0.10, + } + + DEFAULT_NORMALIZATION = { + "calmar": {"lo": 0.0, "hi": 4.0}, + "profit_factor": {"lo": 1.0, "hi": 3.5}, + "mfe_capture": {"lo": 0.0, "hi": 1.0}, + "session_stability": {"lo": 0.0, "hi": 1.0}, + "recovery_factor": {"lo": 0.0, "hi": 6.0}, + } + + def __init__(self, config_path: str = "config.yaml"): + try: + with open(config_path) as f: + cfg = yaml.safe_load(f) + scoring_cfg = cfg.get("scoring", {}) + self.weights = scoring_cfg.get("weights", self.DEFAULT_WEIGHTS) + norm_cfg = scoring_cfg.get("normalization", {}) + self.norm = { + k: norm_cfg.get(k, self.DEFAULT_NORMALIZATION.get(k, {"lo": 0, "hi": 1})) + for k in self.DEFAULT_WEIGHTS + } + self.min_trades = cfg["thresholds"]["min_trades"] + self.significance_at = cfg["scoring"].get("significance_trades", 150) + except Exception as e: + logger.warning(f"Could not load scoring config: {e}. Using defaults.") + self.weights = self.DEFAULT_WEIGHTS + self.norm = self.DEFAULT_NORMALIZATION + self.min_trades = 50 + self.significance_at = 150 + + def score( + self, + metrics: RunMetrics, + session_stats: Optional[dict] = None, + ) -> float: + """ + Compute composite score for a RunMetrics object. + Returns 0.0 if minimum trade count is not met. + + Args: + metrics: RunMetrics from the parsed backtest report + session_stats: Optional {session: {calmar, trade_count, ...}} for stability + """ + if metrics.total_trades < self.min_trades: + logger.debug( + f"Score=0 (trades {metrics.total_trades} < min {self.min_trades})" + ) + return 0.0 + + # ── Sub-scores ──────────────────────────────────────────────────────── + calmar_score = _clip_normalize( + metrics.calmar_ratio, + **self.norm["calmar"] + ) + pf_score = _clip_normalize( + metrics.profit_factor, + **self.norm["profit_factor"] + ) + capture_score = _clip_normalize( + metrics.avg_mfe_capture if metrics.avg_mfe_capture is not None else 0.5, + **self.norm["mfe_capture"] + ) + stab_score = _clip_normalize( + _session_stability(session_stats or {}), + **self.norm["session_stability"] + ) + recovery_score = _clip_normalize( + metrics.recovery_factor, + **self.norm["recovery_factor"] + ) + + # ── Weighted sum ────────────────────────────────────────────────────── + raw = ( + self.weights["calmar"] * calmar_score + + self.weights["profit_factor"] * pf_score + + self.weights["mfe_capture"] * capture_score + + self.weights["session_stability"] * stab_score + + self.weights["recovery_factor"] * recovery_score + ) + + # ── Penalties ───────────────────────────────────────────────────────── + # Significance: scale up to 1.0 as trade count reaches significance_at + significance = min(1.0, metrics.total_trades / self.significance_at) + + # Reversal penalty: each 10% reversal rate removes 20% of score + reversal_penalty = 1.0 + if metrics.reversal_rate is not None: + reversal_penalty = max(0.0, 1.0 - metrics.reversal_rate * 2.0) + + final = raw * significance * reversal_penalty + + logger.debug( + f"Score breakdown: calmar={calmar_score:.3f} pf={pf_score:.3f} " + f"capture={capture_score:.3f} stab={stab_score:.3f} " + f"recovery={recovery_score:.3f} → raw={raw:.3f} " + f"× sig={significance:.3f} × rev={reversal_penalty:.3f} = {final:.4f}" + ) + return round(final, 6) + + def breakdown( + self, + metrics: RunMetrics, + session_stats: Optional[dict] = None, + ) -> dict[str, float]: + """Return the full breakdown of sub-scores for display.""" + return { + "calmar_score": _clip_normalize(metrics.calmar_ratio, **self.norm["calmar"]), + "pf_score": _clip_normalize(metrics.profit_factor, **self.norm["profit_factor"]), + "capture_score": _clip_normalize( + metrics.avg_mfe_capture or 0.5, **self.norm["mfe_capture"]), + "stability_score": _clip_normalize( + _session_stability(session_stats or {}), **self.norm["session_stability"]), + "recovery_score": _clip_normalize(metrics.recovery_factor, **self.norm["recovery_factor"]), + "significance": min(1.0, metrics.total_trades / self.significance_at), + "reversal_penalty":max(0.0, 1.0 - (metrics.reversal_rate or 0) * 2), + "final": self.score(metrics, session_stats), + } diff --git a/tests/test_analyzers.py b/tests/test_analyzers.py new file mode 100644 index 0000000..bf4c2a1 --- /dev/null +++ b/tests/test_analyzers.py @@ -0,0 +1,282 @@ +""" +tests/test_analyzers.py +Unit tests for all analyzer modules using synthetic trade data. +""" +import pytest +import pandas as pd +import numpy as np +from datetime import datetime, timedelta + +from data.models import RunMetrics +from analysis.reversal import ReversalAnalyzer +from analysis.time_performance import TimePerformanceAnalyzer +from analysis.entry_exit_quality import EntryExitQualityAnalyzer +from analysis.equity_curve import EquityCurveAnalyzer + + +# ── Fixtures ────────────────────────────────────────────────────────────────── + +def make_trade( + net_money: float, + mfe_pips: float = 0, + mae_pips: float = 0, + hour_utc: int = 10, + session: str = "London", + day_of_week: int = 1, + result_class: str = None, + duration_minutes: int = 60, + lot_size: float = 0.1, + net_pips: float = None, + open_time: datetime = None, +) -> dict: + if result_class is None: + result_class = "win" if net_money > 0 else ("reversal" if mfe_pips > 15 and net_money < 0 else "loss") + if net_pips is None: + net_pips = net_money / 100 + if open_time is None: + open_time = datetime(2022, 1, 3, hour_utc, 0) + return { + "ticket": np.random.randint(100000, 999999), + "open_time": open_time, + "close_time": open_time + timedelta(minutes=duration_minutes), + "direction": "buy", + "open_price": 1900.0, + "close_price": 1900.0 + net_pips * 0.1, + "sl": 1880.0, "tp": 1920.0, + "lot_size": lot_size, + "net_money": net_money, + "net_pips": net_pips, + "duration_minutes": duration_minutes, + "commission": 0.0, "swap": 0.0, + "mfe_pips": mfe_pips, "mae_pips": mae_pips, + "session": session, "day_of_week": day_of_week, + "hour_utc": hour_utc, "hour_broker": (hour_utc + 2) % 24, + "result_class": result_class, + "mfe_capture_ratio": max(0, net_money) / max(1, mfe_pips * 10), + "entry_quality": max(0, 1 - mae_pips / max(mfe_pips + mae_pips, 1)), + "exit_quality": max(0, net_pips / max(mfe_pips, 1)), + } + + +def dummy_metrics(trades_df: pd.DataFrame, run_id: str = "test_run") -> RunMetrics: + wins = trades_df[trades_df["net_money"] > 0] + losses = trades_df[trades_df["net_money"] < 0] + net = trades_df["net_money"].sum() + gp = wins["net_money"].sum() + gl = abs(losses["net_money"].sum()) + pf = gp / gl if gl > 0 else 99.0 + dd = abs(losses["net_money"].min()) if len(losses) > 0 else 1 + return RunMetrics( + run_id=run_id, + net_profit=net, + profit_factor=pf, + max_drawdown_abs=dd, + max_drawdown_pct=dd / 10000, + calmar_ratio=max(0, net / 10000) / max(0.01, dd / 10000), + sharpe_ratio=1.2, + total_trades=len(trades_df), + win_rate=len(wins) / max(1, len(trades_df)), + avg_win=gp / max(1, len(wins)), + avg_loss=gl / max(1, len(losses)), + recovery_factor=2.0, + largest_loss=abs(losses["net_money"].min()) if len(losses) > 0 else 0, + expected_payoff=net / max(1, len(trades_df)), + ) + + +# ── Reversal Analyzer Tests ─────────────────────────────────────────────────── + +class TestReversalAnalyzer: + + def test_detects_high_reversal_rate(self): + """Should find a HIGH finding when many losers had significant MFE.""" + trades = [] + # 50% of losers had MFE > 20 pips (high reversal rate) + for _ in range(40): + trades.append(make_trade(net_money=100, mfe_pips=30, mae_pips=5)) + for _ in range(20): + trades.append(make_trade(net_money=-80, mfe_pips=25, mae_pips=8)) # reversals + for _ in range(20): + trades.append(make_trade(net_money=-80, mfe_pips=5, mae_pips=20)) # normal losers + + df = pd.DataFrame(trades) + metrics = dummy_metrics(df) + az = ReversalAnalyzer(mfe_threshold_pips=15, min_reversal_rate=0.30, permutation_n=50) + findings = az.run(df, metrics, "test") + + assert len(findings) > 0 + top = findings[0] + assert top.severity in ("high", "medium") + assert "InpUseTrailing" in top.suggested_params + assert top.suggested_params["InpUseTrailing"] is True + + def test_no_finding_when_low_reversal_rate(self): + """Should NOT find reversals when rate is below threshold.""" + trades = [make_trade(net_money=100, mfe_pips=20)] * 50 + trades += [make_trade(net_money=-80, mfe_pips=5)] * 20 # only small-MFE losers + df = pd.DataFrame(trades) + metrics = dummy_metrics(df) + az = ReversalAnalyzer(mfe_threshold_pips=15, min_reversal_rate=0.30, permutation_n=50) + findings = az.run(df, metrics, "test") + # Should have no reversal-rate finding (maybe only capture rate) + reversal_findings = [f for f in findings if "reversal" in f.description.lower() and "% of losing" in f.description] + assert len(reversal_findings) == 0 + + +# ── Time Performance Tests ──────────────────────────────────────────────────── + +class TestTimePerformanceAnalyzer: + + def test_detects_bad_hour_window(self): + """Should flag a consistent loss at hour 14–15 UTC.""" + trades = [] + # Good trades at most hours + for h in [8, 9, 10, 11, 12, 13]: + for _ in range(12): + trades.append(make_trade(net_money=100, hour_utc=h)) + # Consistently losing trades at 14–15 UTC + for h in [14, 15]: + for _ in range(15): + trades.append(make_trade(net_money=-150, hour_utc=h)) + + df = pd.DataFrame(trades) + metrics = dummy_metrics(df) + az = TimePerformanceAnalyzer(z_score_threshold=-1.0, min_bucket_trades=8, permutation_n=200) + findings = az.run(df, metrics, "test") + + hour_findings = [f for f in findings if "UTC" in f.description and "14" in f.description] + assert len(hour_findings) > 0 + + def test_no_finding_for_uniform_performance(self): + """No time-based finding when performance is uniform across hours.""" + trades = [] + rng = np.random.default_rng(42) + for h in range(8, 20): + for _ in range(12): + pnl = float(rng.normal(50, 20)) + trades.append(make_trade(net_money=pnl, hour_utc=h)) + df = pd.DataFrame(trades) + metrics = dummy_metrics(df) + az = TimePerformanceAnalyzer(z_score_threshold=-2.0, min_bucket_trades=8, permutation_n=200) + findings = az.run(df, metrics, "test") + # May or may not find something; just verify it runs without error + assert isinstance(findings, list) + + +# ── Entry/Exit Quality Tests ────────────────────────────────────────────────── + +class TestEntryExitQualityAnalyzer: + + def test_detects_good_entry_poor_exit(self): + """When entries are good but exits capture little of MFE.""" + trades = [] + for _ in range(60): + # Small MAE (good entry), large MFE but poor capture + trades.append(make_trade( + net_money=20, mfe_pips=50, mae_pips=3, + net_pips=2, lot_size=0.1 + )) + df = pd.DataFrame(trades) + metrics = dummy_metrics(df) + az = EntryExitQualityAnalyzer(poor_exit_threshold=0.60, poor_entry_threshold=0.40) + findings = az.run(df, metrics, "test") + action_findings = [f for f in findings if "diagnosis" in f.evidence and + f.evidence["diagnosis"] == "good_entry_poor_exit"] + assert len(action_findings) > 0 + + def test_no_findings_for_healthy_trades(self): + """Should not flag anything when both entry and exit quality are high.""" + trades = [] + for _ in range(60): + trades.append(make_trade( + net_money=80, mfe_pips=100, mae_pips=5, + net_pips=80, lot_size=0.1 + )) + df = pd.DataFrame(trades) + metrics = dummy_metrics(df) + az = EntryExitQualityAnalyzer(poor_exit_threshold=0.55, poor_entry_threshold=0.40) + findings = az.run(df, metrics, "test") + action_findings = [f for f in findings if f.severity in ("high", "medium")] + assert len(action_findings) == 0 + + +# ── Equity Curve Tests ──────────────────────────────────────────────────────── + +class TestEquityCurveAnalyzer: + + def test_detects_loss_clusters(self): + """Should flag a sequence of 5+ consecutive losses.""" + trades = [] + base_time = datetime(2022, 1, 3, 10, 0) + # Wins, then a cluster of losses, then more wins + for i in range(30): + trades.append(make_trade(net_money=100, open_time=base_time + timedelta(hours=i))) + for i in range(30, 37): # 7 consecutive losses + trades.append(make_trade(net_money=-150, open_time=base_time + timedelta(hours=i))) + for i in range(37, 60): + trades.append(make_trade(net_money=100, open_time=base_time + timedelta(hours=i))) + + df = pd.DataFrame(trades) + metrics = dummy_metrics(df) + az = EquityCurveAnalyzer(cluster_min_length=5) + findings = az.run(df, metrics, "test") + cluster_findings = [f for f in findings if "cluster" in f.description.lower()] + assert len(cluster_findings) > 0 + + def test_detects_high_flatness(self): + """Should flag when equity spends most time in drawdown.""" + trades = [] + base_time = datetime(2022, 1, 3, 10, 0) + # Pattern: win a little, lose a lot, basically always in drawdown + for i in range(50): + if i % 5 == 0: + trades.append(make_trade(net_money=50, open_time=base_time + timedelta(hours=i))) + else: + trades.append(make_trade(net_money=-30, open_time=base_time + timedelta(hours=i))) + df = pd.DataFrame(trades) + metrics = dummy_metrics(df) + az = EquityCurveAnalyzer(max_flatness=0.30) + findings = az.run(df, metrics, "test") + flatness_findings = [f for f in findings if "high-water" in f.description] + assert len(flatness_findings) > 0 + + +# ── Composite Score Tests ───────────────────────────────────────────────────── + +class TestCompositeScorer: + + def test_score_increases_with_calmar(self): + """Higher Calmar should produce higher score, all else equal.""" + from scoring.composite import CompositeScorer + + def make_metrics(calmar): + return RunMetrics( + run_id="t", + net_profit=10000, profit_factor=1.5, + max_drawdown_abs=1000, max_drawdown_pct=0.10, + calmar_ratio=calmar, sharpe_ratio=1.2, + total_trades=100, win_rate=0.55, + avg_win=200, avg_loss=150, + recovery_factor=3.0, largest_loss=500, + expected_payoff=50, + avg_mfe_capture=0.6, reversal_rate=0.1, + ) + + scorer = CompositeScorer("config.yaml") + s1 = scorer.score(make_metrics(0.5)) + s2 = scorer.score(make_metrics(1.5)) + s3 = scorer.score(make_metrics(3.0)) + assert s1 < s2 < s3 + + def test_score_zero_below_min_trades(self): + from scoring.composite import CompositeScorer + scorer = CompositeScorer("config.yaml") + m = RunMetrics( + run_id="t", net_profit=5000, profit_factor=2.0, + max_drawdown_abs=500, max_drawdown_pct=0.05, + calmar_ratio=2.0, sharpe_ratio=1.5, + total_trades=10, # below min_trades (50) + win_rate=0.6, avg_win=200, avg_loss=100, + recovery_factor=4.0, largest_loss=200, expected_payoff=100, + ) + assert scorer.score(m) == 0.0 diff --git a/ui/static/css/style.css b/ui/static/css/style.css new file mode 100644 index 0000000..07caa66 --- /dev/null +++ b/ui/static/css/style.css @@ -0,0 +1,318 @@ +/* style.css — MT5 Optimizer Dashboard */ +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap'); + +/* ── Variables ────────────────────────────────────────────────────────────── */ +:root { + --bg: #0a0d14; + --bg-card: #111621; + --bg-card2: #161c2a; + --border: #1e2740; + --border-glow: #2a3a6e; + --text: #e2e8f8; + --text-dim: #6b7fa3; + --text-muted: #3d4f70; + --accent: #4f8ef7; + --accent2: #7c3aed; + --green: #22d3a5; + --red: #f44; + --yellow: #fbbf24; + --orange: #f97316; + --radius: 12px; + --radius-sm: 8px; +} + +* { box-sizing: border-box; margin: 0; padding: 0; } +body { + background: var(--bg); + color: var(--text); + font-family: 'Inter', sans-serif; + font-size: 13px; + min-height: 100vh; + overflow-x: hidden; +} + +/* ── Header ───────────────────────────────────────────────────────────────── */ +.header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 14px 24px; + background: var(--bg-card); + border-bottom: 1px solid var(--border); + position: sticky; + top: 0; + z-index: 100; +} + +.logo { display: flex; align-items: center; gap: 12px; } +.logo-icon { font-size: 28px; } +.logo-title { font-weight: 800; font-size: 16px; letter-spacing: -0.3px; } +.logo-sub { font-size: 11px; color: var(--text-dim); font-family: 'JetBrains Mono', monospace; } + +.header-center { display: flex; gap: 10px; flex-wrap: wrap; } + +.stat-pill { + display: flex; align-items: center; gap: 6px; + background: var(--bg-card2); + border: 1px solid var(--border); + border-radius: 20px; + padding: 5px 12px; + font-size: 12px; +} +.pill-label { color: var(--text-dim); } +.pill-val { font-weight: 700; font-family: 'JetBrains Mono', monospace; color: var(--accent); } +.score-val { color: var(--green); font-size: 14px; } + +.dot { + width: 8px; height: 8px; border-radius: 50%; + background: var(--text-muted); + transition: background 0.3s; +} +.dot-idle { background: var(--text-muted); } +.dot-running { background: var(--green); box-shadow: 0 0 8px var(--green); animation: pulse 1.5s infinite; } +.dot-paused { background: var(--yellow); } +.dot-error { background: var(--red); } + +@keyframes pulse { + 0%,100% { opacity: 1; } 50% { opacity: 0.4; } +} + +.header-right { display: flex; gap: 8px; } + +/* ── Buttons ──────────────────────────────────────────────────────────────── */ +.btn { + padding: 8px 18px; + border-radius: 8px; + border: none; + cursor: pointer; + font-weight: 600; + font-size: 13px; + transition: all 0.2s; +} +.btn-start { background: linear-gradient(135deg, #4f8ef7, #7c3aed); color: #fff; } +.btn-start:hover { opacity: 0.85; transform: translateY(-1px); } +.btn-pause { background: var(--yellow); color: #000; } +.btn-stop { background: var(--red); color: #fff; } +.btn-secondary { background: var(--bg-card2); color: var(--text-dim); border: 1px solid var(--border); } +.btn-secondary:hover { color: var(--text); border-color: var(--accent); } +.hidden { display: none !important; } + +/* ── Main Grid ────────────────────────────────────────────────────────────── */ +.main-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 16px; + padding: 20px 24px; + max-width: 1600px; + margin: 0 auto; +} +@media (max-width: 1100px) { + .main-grid { grid-template-columns: 1fr; } +} + +.col-left, .col-right { display: flex; flex-direction: column; gap: 16px; } + +/* ── Cards ────────────────────────────────────────────────────────────────── */ +.card { + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 16px; + transition: border-color 0.3s; +} +.card:hover { border-color: var(--border-glow); } + +.card-header { + display: flex; justify-content: space-between; align-items: center; + margin-bottom: 14px; +} +.card-title { font-weight: 700; font-size: 13px; } +.card-badge { + background: var(--bg-card2); + border: 1px solid var(--border); + border-radius: 20px; + padding: 3px 10px; + font-size: 11px; + color: var(--text-dim); + font-family: 'JetBrains Mono', monospace; +} +.run-id-badge { + font-family: 'JetBrains Mono', monospace; + font-size: 11px; + color: var(--accent); + opacity: 0.7; +} + +/* ── Chart ────────────────────────────────────────────────────────────────── */ +.chart-wrap { height: 200px; position: relative; } +#scoreChart { width: 100% !important; height: 100% !important; } + +/* ── Metrics Grid ─────────────────────────────────────────────────────────── */ +.metrics-grid { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 8px; + margin-bottom: 14px; +} +.metric-card { + background: var(--bg-card2); + border: 1px solid var(--border); + border-radius: var(--radius-sm); + padding: 10px 12px; + text-align: center; +} +.metric-label { font-size: 10px; color: var(--text-dim); margin-bottom: 4px; text-transform: uppercase; letter-spacing: 0.5px; } +.metric-val { font-size: 18px; font-weight: 700; font-family: 'JetBrains Mono', monospace; color: var(--text); } + +.score-bar-wrap { margin-top: 4px; } +.score-bar-label { + display: flex; justify-content: space-between; align-items: center; + margin-bottom: 6px; + font-size: 12px; color: var(--text-dim); +} +.score-big { font-size: 22px; font-weight: 800; font-family: 'JetBrains Mono', monospace; color: var(--green); } +.score-bar-track { + height: 6px; background: var(--bg-card2); + border-radius: 3px; overflow: hidden; +} +.score-bar-fill { + height: 100%; + background: linear-gradient(90deg, var(--accent), var(--green)); + border-radius: 3px; + transition: width 0.6s ease; +} + +/* ── Phase Tracker ────────────────────────────────────────────────────────── */ +.phase-card { padding: 14px 20px; } +.phase-steps { + display: flex; align-items: center; justify-content: space-between; +} +.phase-step { + display: flex; flex-direction: column; align-items: center; gap: 6px; + opacity: 0.35; transition: opacity 0.3s; +} +.phase-step.active { opacity: 1; } +.phase-step.done { opacity: 0.6; } +.phase-dot { + width: 12px; height: 12px; border-radius: 50%; + background: var(--text-muted); + border: 2px solid var(--border); + transition: all 0.3s; +} +.phase-step.active .phase-dot { + background: var(--accent); + border-color: var(--accent); + box-shadow: 0 0 10px var(--accent); +} +.phase-step.done .phase-dot { background: var(--green); border-color: var(--green); } +.phase-name { font-size: 10px; color: var(--text-dim); text-transform: uppercase; letter-spacing: 0.5px; } +.phase-line { flex: 1; height: 1px; background: var(--border); margin: 0 4px; margin-bottom: 18px; } + +/* ── Findings ─────────────────────────────────────────────────────────────── */ +.findings-feed { + max-height: 220px; overflow-y: auto; + display: flex; flex-direction: column; gap: 6px; +} +.finding-empty { color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px 0; } +.finding-row { + display: flex; align-items: flex-start; gap: 10px; + background: var(--bg-card2); + border: 1px solid var(--border); + border-radius: var(--radius-sm); + padding: 8px 12px; + animation: slideIn 0.3s ease; +} +.finding-sev { + font-size: 10px; font-weight: 700; padding: 2px 7px; border-radius: 4px; + flex-shrink: 0; text-transform: uppercase; letter-spacing: 0.5px; margin-top: 1px; +} +.sev-high { background: rgba(244,68,68,0.2); color: #f44; border: 1px solid rgba(244,68,68,0.3); } +.sev-medium { background: rgba(251,191,36,0.15); color: var(--yellow); border: 1px solid rgba(251,191,36,0.3); } +.sev-low { background: rgba(79,142,247,0.1); color: var(--accent); border: 1px solid rgba(79,142,247,0.2); } +.finding-text { flex: 1; font-size: 12px; line-height: 1.5; color: var(--text); } +.finding-conf { font-size: 10px; color: var(--text-dim); flex-shrink: 0; margin-top: 2px; } + +/* ── Params Table ─────────────────────────────────────────────────────────── */ +.hyp-desc { + font-size: 12px; color: var(--text-dim); margin-bottom: 10px; + padding: 8px 10px; background: var(--bg-card2); + border-radius: var(--radius-sm); border: 1px solid var(--border); + font-style: italic; +} +.params-table-wrap { overflow-x: auto; max-height: 180px; overflow-y: auto; } +.params-table { width: 100%; border-collapse: collapse; font-size: 12px; } +.params-table thead th { + text-align: left; padding: 6px 12px; + color: var(--text-dim); font-weight: 600; + border-bottom: 1px solid var(--border); + font-size: 10px; text-transform: uppercase; letter-spacing: 0.5px; +} +.params-table tbody td { + padding: 6px 12px; + border-bottom: 1px solid rgba(30,39,64,0.5); + font-family: 'JetBrains Mono', monospace; +} +.params-table tbody tr:last-child td { border-bottom: none; } +.param-changed { color: var(--green); } +.param-old { color: var(--text-dim); text-decoration: line-through; } +.param-new { color: var(--green); font-weight: 600; } +.param-arrow { color: var(--text-muted); padding: 0 4px; } + +/* ── Log ──────────────────────────────────────────────────────────────────── */ +.log-card { max-height: 220px; display: flex; flex-direction: column; } +.log-feed { + flex: 1; overflow-y: auto; max-height: 160px; + display: flex; flex-direction: column; gap: 2px; + font-family: 'JetBrains Mono', monospace; font-size: 11px; +} +.log-line { padding: 2px 4px; border-radius: 3px; } +.log-info { color: var(--text-dim); } +.log-success { color: var(--green); } +.log-warn { color: var(--yellow); } +.log-error { color: var(--red); } +.clear-btn { + background: none; border: none; color: var(--text-muted); + cursor: pointer; font-size: 11px; padding: 2px 6px; +} +.clear-btn:hover { color: var(--text-dim); } + +/* ── Candidates Section ───────────────────────────────────────────────────── */ +.candidates-section { + padding: 0 24px 24px; + max-width: 1600px; + margin: 0 auto; +} +.candidates-card {} +.candidates-list { + display: flex; gap: 12px; flex-wrap: wrap; +} +.cand-empty { color: var(--text-muted); font-size: 12px; padding: 10px 0; text-align: center; width: 100%; } +.candidate-chip { + background: linear-gradient(135deg, rgba(34,211,165,0.1), rgba(79,142,247,0.1)); + border: 1px solid rgba(34,211,165,0.3); + border-radius: var(--radius-sm); + padding: 10px 16px; + display: flex; flex-direction: column; gap: 4px; + min-width: 200px; + animation: slideIn 0.4s ease; +} +.cand-score { + font-size: 22px; font-weight: 800; + font-family: 'JetBrains Mono', monospace; + color: var(--green); +} +.cand-id { font-size: 10px; color: var(--text-dim); font-family: 'JetBrains Mono', monospace; } +.cand-delta { font-size: 11px; font-weight: 600; } +.delta-up { color: var(--green); } +.delta-dn { color: var(--red); } + +/* ── Scrollbar styling ────────────────────────────────────────────────────── */ +::-webkit-scrollbar { width: 4px; height: 4px; } +::-webkit-scrollbar-track { background: transparent; } +::-webkit-scrollbar-thumb { background: var(--border-glow); border-radius: 2px; } + +/* ── Animations ───────────────────────────────────────────────────────────── */ +@keyframes slideIn { + from { opacity: 0; transform: translateY(8px); } + to { opacity: 1; transform: translateY(0); } +} diff --git a/ui/static/js/dashboard.js b/ui/static/js/dashboard.js new file mode 100644 index 0000000..11dbb93 --- /dev/null +++ b/ui/static/js/dashboard.js @@ -0,0 +1,374 @@ +/* dashboard.js — Real-time MT5 Optimizer Dashboard */ + +// ── Socket.IO connection ─────────────────────────────────────────────────── +const socket = io(); +let startTime = null; +let timerInterval = null; +let findingsCount = 0; +let candidatesCount = 0; +let currentParams = {}; + +// ── Chart setup ──────────────────────────────────────────────────────────── +const ctx = document.getElementById('scoreChart').getContext('2d'); +const scoreChart = new Chart(ctx, { + type: 'line', + data: { + labels: [], + datasets: [ + { + label: 'Composite Score', + data: [], + borderColor: '#22d3a5', + backgroundColor: 'rgba(34,211,165,0.08)', + fill: true, + tension: 0.4, + pointRadius: 4, + pointBackgroundColor: '#22d3a5', + borderWidth: 2, + }, + { + label: 'Calmar Ratio', + data: [], + borderColor: '#4f8ef7', + backgroundColor: 'rgba(79,142,247,0.05)', + fill: false, + tension: 0.4, + pointRadius: 3, + borderWidth: 1.5, + borderDash: [4, 2], + }, + ] + }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: { duration: 500 }, + plugins: { + legend: { + labels: { color: '#6b7fa3', font: { size: 11, family: 'Inter' } } + }, + tooltip: { + backgroundColor: '#111621', + borderColor: '#2a3a6e', + borderWidth: 1, + titleColor: '#e2e8f8', + bodyColor: '#6b7fa3', + } + }, + scales: { + x: { + ticks: { color: '#6b7fa3', font: { size: 10 } }, + grid: { color: 'rgba(30,39,64,0.6)' }, + }, + y: { + ticks: { color: '#6b7fa3', font: { size: 10 } }, + grid: { color: 'rgba(30,39,64,0.6)' }, + min: 0, max: 1, + } + } + } +}); + +// ── Controls ─────────────────────────────────────────────────────────────── + +function startOptimizer() { + fetch('/api/start', { method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ auto: true }) + }); + startTime = Date.now(); + timerInterval = setInterval(updateTimer, 1000); + setRunning(true); + addLog('info', 'Starting optimizer...'); +} + +function pauseOptimizer() { + fetch('/api/pause', { method: 'POST' }).then(r => r.json()).then(d => { + const btn = document.getElementById('btn-pause'); + btn.textContent = d.paused ? '▶ Resume' : '⏸ Pause'; + setDot(d.paused ? 'paused' : 'running', d.paused ? 'Paused' : 'Running...'); + }); +} + +function stopOptimizer() { + fetch('/api/stop', { method: 'POST' }); + clearInterval(timerInterval); + setRunning(false); + setDot('idle', 'Stopped'); + addLog('warn', 'Optimizer stopped by user.'); +} + +function clearLog() { + document.getElementById('log-feed').innerHTML = ''; +} + +function setRunning(on) { + document.getElementById('btn-start').classList.toggle('hidden', on); + document.getElementById('btn-pause').classList.toggle('hidden', !on); + document.getElementById('btn-stop').classList.toggle('hidden', !on); + if (on) setDot('running', 'Running...'); +} + +// ── Timer ────────────────────────────────────────────────────────────────── + +function updateTimer() { + if (!startTime) return; + const s = Math.floor((Date.now() - startTime) / 1000); + const m = Math.floor(s / 60); + const ss = String(s % 60).padStart(2, '0'); + document.getElementById('hdr-elapsed').textContent = `${m}:${ss}`; +} + +// ── State dot ────────────────────────────────────────────────────────────── + +function setDot(state, label) { + const dot = document.getElementById('state-dot'); + dot.className = `dot dot-${state}`; + document.getElementById('state-label').textContent = label; +} + +// ── Phase tracker ────────────────────────────────────────────────────────── + +const PHASES = ['baseline', 'analyze', 'explore', 'validate', 'oos']; + +function setPhase(phase) { + const idx = PHASES.indexOf(phase); + PHASES.forEach((p, i) => { + const el = document.getElementById(`phase-${p}`); + if (!el) return; + el.classList.remove('active', 'done'); + if (i < idx) el.classList.add('done'); + else if (i === idx) el.classList.add('active'); + }); +} + +// ── Metrics update ───────────────────────────────────────────────────────── + +function updateMetrics(d) { + const set = (id, val) => { + const el = document.getElementById(id); + if (el) el.textContent = val; + }; + set('m-profit', d.net_profit !== undefined ? `$${d.net_profit.toLocaleString()}` : '—'); + set('m-calmar', d.calmar !== undefined ? d.calmar.toFixed(3) : '—'); + set('m-pf', d.profit_factor !== undefined ? d.profit_factor.toFixed(3) : '—'); + set('m-dd', d.drawdown_pct !== undefined ? `${d.drawdown_pct}%` : '—'); + set('m-wr', d.win_rate !== undefined ? `${d.win_rate.toFixed(1)}%` : '—'); + set('m-trades', d.total_trades ?? '—'); + set('m-mfe', d.mfe_capture !== undefined ? `${d.mfe_capture.toFixed(1)}%` : '—'); + set('m-rev', d.reversal_rate !== undefined ? `${d.reversal_rate.toFixed(1)}%` : '—'); + set('m-score', d.score !== undefined ? d.score.toFixed(4) : '—'); + set('current-run-id', d.run_id || '—'); + + if (d.score !== undefined) { + const fill = document.getElementById('score-bar-fill'); + if (fill) fill.style.width = `${Math.min(100, d.score * 100)}%`; + set('hdr-score', d.score.toFixed(4)); + } + + // Color profit + const pEl = document.getElementById('m-profit'); + if (pEl && d.net_profit !== undefined) { + pEl.style.color = d.net_profit >= 0 ? 'var(--green)' : 'var(--red)'; + } +} + +// ── Findings ─────────────────────────────────────────────────────────────── + +function addFinding(f) { + findingsCount++; + const feed = document.getElementById('findings-feed'); + const empty = feed.querySelector('.finding-empty'); + if (empty) empty.remove(); + + const sevClass = { high: 'sev-high', medium: 'sev-medium', low: 'sev-low' }[f.severity] || 'sev-low'; + + const row = document.createElement('div'); + row.className = 'finding-row'; + row.innerHTML = ` + ${f.severity} + ${escHtml(f.description)} + ${(f.confidence * 100).toFixed(0)}% + `; + feed.insertBefore(row, feed.firstChild); + + // Keep max 20 findings visible + while (feed.children.length > 20) feed.removeChild(feed.lastChild); + + document.getElementById('findings-count').textContent = `${findingsCount} findings`; +} + +// ── Hypotheses / Params ──────────────────────────────────────────────────── + +function showHypotheses(items) { + if (!items || !items.length) return; + const h = items[0]; // show first (highest priority) + document.getElementById('hyp-desc').textContent = h.desc || ''; + document.getElementById('hyp-badge').textContent = `${items.length} hypothesis(es)`; + + const tbody = document.getElementById('params-tbody'); + tbody.innerHTML = ''; + Object.entries(h.delta || {}).forEach(([param, newVal]) => { + const oldVal = currentParams[param]; + const tr = document.createElement('tr'); + tr.innerHTML = ` + ${escHtml(param)} + ${oldVal !== undefined ? oldVal : '—'} + ${newVal} + `; + tbody.appendChild(tr); + }); +} + +// ── Log ──────────────────────────────────────────────────────────────────── + +function addLog(level, msg) { + const feed = document.getElementById('log-feed'); + const line = document.createElement('div'); + line.className = `log-line log-${level}`; + const ts = new Date().toLocaleTimeString('en-GB', { hour12: false }); + line.textContent = `[${ts}] ${msg}`; + feed.insertBefore(line, feed.firstChild); + while (feed.children.length > 200) feed.removeChild(feed.lastChild); +} + +// ── Candidates ───────────────────────────────────────────────────────────── + +function addCandidate(d) { + candidatesCount++; + const list = document.getElementById('candidates-list'); + const empty = list.querySelector('.cand-empty'); + if (empty) empty.remove(); + + const deltaClass = d.delta >= 0 ? 'delta-up' : 'delta-dn'; + const deltaSign = d.delta >= 0 ? '+' : ''; + + const chip = document.createElement('div'); + chip.className = 'candidate-chip'; + chip.innerHTML = ` +
${d.score.toFixed(4)}
+
${deltaSign}${d.delta.toFixed(4)} vs prev
+
ID: ${d.candidate_id}
+ `; + list.insertBefore(chip, list.firstChild); + document.getElementById('cand-count').textContent = `${candidatesCount} candidates`; +} + +// ── Chart update ─────────────────────────────────────────────────────────── + +function pushChartPoint(d) { + const labels = scoreChart.data.labels; + labels.push(`Iter ${d.iteration}`); + scoreChart.data.datasets[0].data.push(d.score); + scoreChart.data.datasets[1].data.push(d.calmar > 1 ? 1 : d.calmar / 4); // normalize calmar to 0-1 + if (labels.length > 50) { + labels.shift(); + scoreChart.data.datasets.forEach(ds => ds.data.shift()); + } + scoreChart.update('none'); + document.getElementById('chart-badge').textContent = `Score: ${d.score.toFixed(4)}`; +} + +// ── Utility ──────────────────────────────────────────────────────────────── + +function escHtml(str) { + return String(str) + .replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); +} + +// ── Socket events ────────────────────────────────────────────────────────── + +socket.on('connect', () => addLog('info', 'Connected to optimizer.')); + +socket.on('status_sync', d => { + document.getElementById('hdr-iter').textContent = d.iteration; + if (d.state === 'running') { setRunning(true); setDot('running', 'Running...'); } + if (d.best_score) document.getElementById('hdr-score').textContent = d.best_score.toFixed(4); +}); + +socket.on('status_change', d => { + if (d.state === 'running') setDot('running', 'Running...'); + if (d.state === 'paused') setDot('paused', 'Paused'); + if (d.state === 'idle') setDot('idle', 'Ready'); + if (d.phase) setPhase(d.phase); +}); + +socket.on('iteration_start', d => { + document.getElementById('hdr-iter').textContent = d.iteration; + addLog('info', `━━ Starting Iteration ${d.iteration} ━━`); + setPhase('analyze'); +}); + +socket.on('run_started', d => { + setPhase(d.phase === 'baseline' ? 'baseline' : 'explore'); + document.getElementById('current-run-id').textContent = d.run_id; + addLog('info', `▶ MT5 started: ${d.run_id} [${d.period}]`); + currentParams = d.params || {}; +}); + +socket.on('run_complete', d => { + updateMetrics(d); + const sign = d.score > 0 ? '✓' : '•'; + addLog(d.score > 0.3 ? 'success' : 'info', + `${sign} ${d.run_id}: Score=${d.score} | Calmar=${d.calmar} | PF=${d.profit_factor} | DD=${d.drawdown_pct}%` + ); +}); + +socket.on('run_failed', d => { + addLog('error', `✗ Run failed: ${d.run_id} — ${d.error}`); +}); + +socket.on('finding', f => addFinding(f)); + +socket.on('hypotheses', d => { + setPhase('explore'); + showHypotheses(d.items); + addLog('info', `💡 ${d.items.length} hypothesis(es) proposed`); +}); + +socket.on('hypothesis_testing', d => { + addLog('info', `🧪 Testing H${d.idx}: ${d.desc.substring(0, 60)}...`); +}); + +socket.on('score_update', d => { + pushChartPoint(d); + document.getElementById('hdr-score').textContent = d.score.toFixed(4); +}); + +socket.on('candidate_promoted', d => { + addCandidate(d); + addLog('success', `🏆 Candidate promoted! Score: ${d.score}`); +}); + +socket.on('optimization_complete', d => { + clearInterval(timerInterval); + setRunning(false); + setDot('idle', 'Complete'); + addLog('success', `🏁 Done! ${d.iterations} iterations, ${d.candidates} candidate(s), best score: ${d.best_score}`); + addLog('info', '📁 Reports saved to MT5_Optimizer\\Reports\\'); +}); + +socket.on('error', d => { + addLog('error', `Error: ${d.msg}`); + setDot('error', 'Error'); +}); + +socket.on('log', d => addLog(d.level, d.msg)); + +// ── Init: restore chart from history ────────────────────────────────────── + +fetch('/api/history').then(r => r.json()).then(history => { + history.forEach(d => pushChartPoint(d)); +}); + +fetch('/api/status').then(r => r.json()).then(d => { + document.getElementById('hdr-iter').textContent = d.iteration; + if (d.best_score) document.getElementById('hdr-score').textContent = d.best_score.toFixed(4); + if (d.state === 'running') { + setRunning(true); + setDot('running', 'Running...'); + startTime = Date.now() - d.elapsed_s * 1000; + timerInterval = setInterval(updateTimer, 1000); + } +}); diff --git a/ui/templates/index.html b/ui/templates/index.html new file mode 100644 index 0000000..46db7b8 --- /dev/null +++ b/ui/templates/index.html @@ -0,0 +1,218 @@ + + + + + + MT5 EA Optimizer — Live Dashboard + + + + + + +
+
+ +
+
+
+ + Ready +
+
+ Iteration + 0 +
+
+ Best Score + — +
+
+ Elapsed + 0:00 +
+
+
+ + + + +
+
+ + +
+ + +
+ + +
+
+ 📈 Score History + Waiting... +
+
+ +
+
+ + +
+
+ 🎯 Current Run Metrics + — +
+
+
+
Net Profit
+
—
+
+
+
Calmar Ratio
+
—
+
+
+
Profit Factor
+
—
+
+
+
Max Drawdown
+
—
+
+
+
Win Rate
+
—
+
+
+
Total Trades
+
—
+
+
+
MFE Capture
+
—
+
+
+
Reversal Rate
+
—
+
+
+
+
+ Composite Score + — +
+
+
+
+
+
+ +
+ + +
+ + +
+
+
+
+
Baseline
+
+
+
+
+
Analyze
+
+
+
+
+
Test
+
+
+
+
+
Validate
+
+
+
+
+
OOS
+
+
+
+ + +
+
+ 🔍 Analysis Findings + 0 findings +
+
+
Findings will appear here after each analysis run...
+
+
+ + +
+
+ 🔧 Parameter Changes + Waiting... +
+
No hypothesis being tested yet.
+
+ + + +
ParameterBeforeAfter
+
+
+ + +
+
+ 📋 Live Log + +
+
+
System ready. Click Start Optimizer to begin.
+
+
+ +
+
+ + +
+
+
+ 🏆 Promoted Candidates + 0 candidates +
+
+
Validated candidates will appear here...
+
+
+
+ + + + + + + diff --git a/ui/templates/report.html b/ui/templates/report.html new file mode 100644 index 0000000..4fced40 --- /dev/null +++ b/ui/templates/report.html @@ -0,0 +1,192 @@ + + + + + Run {{ run_id }} — MT5 Optimizer Report + + + + + ← Back to Dashboard +

Run Report: {{ run_id }}

+
Generated: {{ generated }} · Phase: {{ summary.phase }}
+ + +
{{ "%.4f"|format(summary.score) }}
+ + +
+
+
Net Profit
+
+ ${{ "%.2f"|format(metrics.net_profit) }} +
+
+
+
Calmar Ratio
+
{{ "%.3f"|format(metrics.calmar_ratio) }}
+
+
+
Profit Factor
+
+ {{ "%.3f"|format(metrics.profit_factor) }} +
+
+
+
Max Drawdown
+
{{ "%.1f"|format(metrics.max_drawdown_pct * 100) }}%
+
+
+
Win Rate
+
{{ "%.1f"|format(metrics.win_rate * 100) }}%
+
+
+
Total Trades
+
{{ metrics.total_trades }}
+
+
+
Sharpe Ratio
+
{{ "%.3f"|format(metrics.sharpe_ratio) }}
+
+
+
Recovery Factor
+
{{ "%.2f"|format(metrics.recovery_factor) }}
+
+
+ + {% if metrics.avg_mfe_capture is not none %} +
+
+
MFE Capture
+
{{ "%.1f"|format(metrics.avg_mfe_capture * 100) }}%
+
+
+
Reversal Rate
+
+ {{ "%.1f"|format((metrics.reversal_rate or 0) * 100) }}% +
+
+
+
Avg MFE Pips
+
{{ "%.1f"|format(metrics.avg_mfe_pips or 0) }}
+
+
+
Avg MAE Pips
+
{{ "%.1f"|format(metrics.avg_mae_pips or 0) }}
+
+
+ {% endif %} + + + {% if baseline_score %} +
+
Score vs Baseline
+
+ {{ "%.4f"|format(baseline_score) }} → + + {{ "%.4f"|format(summary.score) }} + ({{ "%+.4f"|format(summary.score - baseline_score) }}) + +
+
+ {% endif %} + + + {% if hypothesis %} +
Hypothesis
+
+
{{ hypothesis.description }}
+
Strategy: {{ hypothesis.strategy }} · Rule: {{ hypothesis.kb_rule_id }}
+
+ {% endif %} + + + {% if findings %} +
Analysis Findings ({{ findings|length }})
+ + + + + + {% for f in findings %} + + + + + + + + {% endfor %} + +
AnalyzerSeverityConfidenceFindingEst. Impact
{{ f.analyzer }}{{ f.severity.upper() }}{{ "%.0f"|format(f.confidence * 100) }}%{{ f.description }}${{ "%.0f"|format(f.impact_estimate_pnl) }}
+ {% endif %} + + + {% if params %} +
Parameter Configuration
+ + + + {% for p in params %} + + + + + + {% endfor %} + +
ParameterValueChanged
{{ p.name }}{{ p.new }}{% if p.changed %}✅ {{ p.old }} → {{ p.new }}{% else %}—{% endif %}
+ {% endif %} + + + {% if trades_sample %} +
Trade Sample (first 50)
+ + + + + + {% for t in trades_sample %} + + + + + + + + + + + + {% endfor %} + +
#OpenCloseDirNet $MFE pipMAE pipSessionResult
{{ loop.index }}{{ t.open_time }}{{ t.close_time }}{{ t.direction }} + ${{ "%.2f"|format(t.net_money) }} + {{ t.mfe_pips or '—' }}{{ t.mae_pips or '—' }}{{ t.session or '—' }}{{ t.result_class or '—' }}
+ {% endif %} + + + diff --git a/ui/templates/reports_index.html b/ui/templates/reports_index.html new file mode 100644 index 0000000..3ffb079 --- /dev/null +++ b/ui/templates/reports_index.html @@ -0,0 +1,93 @@ + + + + + Reports — MT5 Optimizer + + + + +
+
+

📁 Optimization Reports

+
{{ runs|length }} run(s) recorded · Click any card to open the full report
+
+ ← Back to Dashboard +
+ + {% if not runs %} +
No reports yet.
Start the optimizer to generate your first run report.
+ {% else %} +
+ {% for r in runs %} + +
+
+
{{ r.run_id }}
+
{{ "%.4f"|format(r.score) }}
+
+ {{ r.phase }} +
+
+
+
Net Profit
+
+ ${{ "%.0f"|format(r.net_profit) }} +
+
+
+
Calmar
+
{{ "%.2f"|format(r.calmar) }}
+
+
+
Drawdown
+
{{ "%.1f"|format(r.drawdown_pct) }}%
+
+
+
PF
+
{{ "%.2f"|format(r.profit_factor) }}
+
+
+
Trades
+
{{ r.total_trades }}
+
+
+
Score Δ
+
+ {{ "%+.4f"|format(r.score_delta) }} +
+
+
+
{{ r.ts[:19].replace("T"," ") }} UTC
+ + {% endfor %} +
+ {% endif %} + + diff --git a/validation/__init__.py b/validation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/validation/gate.py b/validation/gate.py new file mode 100644 index 0000000..2240b60 --- /dev/null +++ b/validation/gate.py @@ -0,0 +1,209 @@ +""" +validation/gate.py +IS / Walk-Forward / OOS validation pipeline. +""" +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional +import uuid + +import yaml +from loguru import logger + +from data.models import GateResult, RunMetrics + + +@dataclass +class WFVResult: + passed: bool + oos_is_ratio: float + fold_results: list[dict] + details: dict + + +class ValidationGate: + """ + Three-phase validation pipeline: + 1. IS check — minimum thresholds on in-sample metrics + 2. Walk-Forward Validation — split training period, test metric consistency + 3. OOS test — held-out period, called explicitly by orchestrator + """ + + def __init__(self, config_path: str | Path = "config.yaml"): + with open(config_path) as f: + cfg = yaml.safe_load(f) + self.thresh = cfg["thresholds"] + self.per = cfg["periods"] + + # ── Phase 1: IS Check ───────────────────────────────────────────────────── + + def run_is_check(self, metrics: RunMetrics) -> GateResult: + """ + Hard minimum thresholds. All must pass. + """ + checks = { + "min_trades": metrics.total_trades >= self.thresh["min_trades"], + "min_pf": metrics.profit_factor >= self.thresh["min_profit_factor"], + "min_calmar": metrics.calmar_ratio >= self.thresh["min_calmar"], + } + passed = all(checks.values()) + reason = None + if not passed: + failed = [k for k, v in checks.items() if not v] + reason = f"Failed gates: {', '.join(failed)}" + + logger.info(f"IS check {'PASSED' if passed else 'FAILED'}: {checks}") + return GateResult(passed=passed, details=checks, reason=reason) + + # ── Phase 2: Walk-Forward Validation ───────────────────────────────────── + + def run_walk_forward( + self, + params: dict[str, Any], + cfg: dict, + store, # DataStore + builder, # IniBuilder + runner, # MT5Runner + parser, # ReportParser + log_rdr, # TradeLogReader + analyzers, # list[BaseAnalyzer] + scorer, # CompositeScorer + n_folds: int = 2, + ) -> WFVResult: + """ + Split training period into n_folds sub-periods. + Run the same params on each sub-period. + Accept if mean OOS Calmar >= min_wfv_ratio × IS Calmar. + + In MVP we use n_folds=2 (first half / second half). + v2 will use full rolling window WFV. + """ + from datetime import datetime, timedelta + + train_start = datetime.strptime(self.per["train_start"], "%Y.%m.%d") + train_end = datetime.strptime(self.per["train_end"], "%Y.%m.%d") + total_days = (train_end - train_start).days + fold_days = total_days // n_folds + + fold_metrics: list[RunMetrics] = [] + + for i in range(n_folds): + fold_start = train_start + timedelta(days=i * fold_days) + fold_end = fold_start + timedelta(days=fold_days) + if i == n_folds - 1: + fold_end = train_end # last fold gets remainder + + fold_id = f"wfv_fold{i+1}_{uuid.uuid4().hex[:6]}" + logger.info(f"WFV fold {i+1}/{n_folds}: {fold_start.date()} → {fold_end.date()}") + + # Import here to avoid circular + from main import execute_run + fm, _ = execute_run( + run_id=fold_id, + params=params, + period_start=fold_start.strftime("%Y.%m.%d"), + period_end=fold_end.strftime("%Y.%m.%d"), + phase="wfv", + hypothesis_id=None, + cfg=cfg, store=store, builder=builder, runner=runner, + parser=parser, log_rdr=log_rdr, analyzers=analyzers, scorer=scorer, + ) + if fm: + fold_metrics.append(fm) + + if not fold_metrics: + return WFVResult(passed=False, oos_is_ratio=0.0, fold_results=[], details={}) + + fold_calmars = [fm.calmar_ratio for fm in fold_metrics] + mean_oos_calmar = sum(fold_calmars) / len(fold_calmars) + + # Get IS calmar (best score so far) for comparison + is_calmar = max((fm.calmar_ratio for fm in fold_metrics), default=0) + if is_calmar <= 0: + ratio = 0.0 + else: + ratio = mean_oos_calmar / is_calmar + + passed = ratio >= self.thresh["min_wfv_ratio"] + + return WFVResult( + passed=passed, + oos_is_ratio=ratio, + fold_results=[ + {"fold": i+1, "calmar": fm.calmar_ratio, "trades": fm.total_trades} + for i, fm in enumerate(fold_metrics) + ], + details={ + "mean_fold_calmar": round(mean_oos_calmar, 4), + "ratio": round(ratio, 4), + "threshold": self.thresh["min_wfv_ratio"], + }, + ) + + # ── Parameter Sensitivity Check ─────────────────────────────────────────── + + def check_sensitivity( + self, + params: dict[str, Any], + key_params: list[str], + cfg: dict, + store, + builder, + runner, + parser, + log_rdr, + analyzers, + scorer, + perturbation: float = 0.10, + ) -> tuple[bool, dict]: + """ + For each key parameter, perturb by ±10% and measure Calmar change. + Reject if any parameter causes > tolerance% degradation. + + In MVP this is optional — add to v2 workflow. + """ + from main import execute_run + import uuid + + tolerance = self.thresh["sensitivity_tolerance"] + degradations = {} + + base_metrics, _ = execute_run( + run_id=f"sens_base_{uuid.uuid4().hex[:6]}", + params=params, + period_start=self.per["train_start"], + period_end=self.per["train_end"], + phase="validate", + hypothesis_id=None, + cfg=cfg, store=store, builder=builder, runner=runner, + parser=parser, log_rdr=log_rdr, analyzers=analyzers, scorer=scorer, + ) + if base_metrics is None or base_metrics.calmar_ratio <= 0: + return True, {} # can't test sensitivity — skip + + for param in key_params: + if param not in params: + continue + base_val = params[param] + if not isinstance(base_val, (int, float)): + continue + + perturbed = {**params, param: base_val * (1 + perturbation)} + pm, _ = execute_run( + run_id=f"sens_{param[:8]}_{uuid.uuid4().hex[:6]}", + params=perturbed, + period_start=self.per["train_start"], + period_end=self.per["train_end"], + phase="validate", + hypothesis_id=None, + cfg=cfg, store=store, builder=builder, runner=runner, + parser=parser, log_rdr=log_rdr, analyzers=analyzers, scorer=scorer, + ) + if pm and base_metrics.calmar_ratio > 0: + deg = (base_metrics.calmar_ratio - pm.calmar_ratio) / base_metrics.calmar_ratio + degradations[param] = round(deg, 4) + + max_deg = max(degradations.values(), default=0) + passed = max_deg <= tolerance + return passed, degradations