Initial commit: MT5 EA Optimizer v1.0
Full optimization system for LEGSTECH_EA_V2: - Flask + SocketIO live dashboard (dark premium UI) - MT5 process control (auto-kill, clean launch, retry) - HTML report parser (UTF-16 LE, 597 trades, metrics) - Pre-run validation and actionable error messages - Analysis engines: Reversal, TimePerfomance, EntryExit, EquityCurve - Composite scoring (Calmar-primary) - Mutation engine with knowledge_base.yaml - Validation gate: IS + Walk-Forward - Reports folder with HTML/CSV per run - Double-click launcher batch file
This commit is contained in:
LEGSTECH Optimizer
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
mt5/ini_builder.py
|
||||
Generates the MT5 strategy tester .ini file from a parameter dict + config.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import configparser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from loguru import logger
|
||||
|
||||
|
||||
# MT5 period string names (used in [Tester] Period= field)
|
||||
TIMEFRAME_NAMES = {
|
||||
"M1": "M1", "M5": "M5", "M15": "M15", "M30": "M30",
|
||||
"H1": "H1", "H4": "H4", "D1": "D1", "W1": "W1", "MN": "MN1",
|
||||
}
|
||||
|
||||
# MT5 tester model codes
|
||||
MODEL_CODES = {
|
||||
"every_tick": 0,
|
||||
"every_tick_real": 1,
|
||||
"ohlc_m1": 4,
|
||||
}
|
||||
|
||||
|
||||
class IniBuilder:
|
||||
"""
|
||||
Builds MT5 strategy tester .ini files.
|
||||
|
||||
Usage:
|
||||
builder = IniBuilder(config_path="config.yaml", manifest_path="mutation/param_manifest.yaml")
|
||||
ini_path = builder.build(
|
||||
run_id="run_001",
|
||||
params={"InpRiskPercent": 1.5, "InpUseTrailing": True, ...},
|
||||
period_start="2022.01.01",
|
||||
period_end="2023.12.31",
|
||||
output_dir=Path("runs/run_001"),
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(self, config_path: str | Path, manifest_path: str | Path):
|
||||
with open(config_path) as f:
|
||||
self.cfg = yaml.safe_load(f)
|
||||
with open(manifest_path) as f:
|
||||
self.manifest = yaml.safe_load(f)["parameters"]
|
||||
|
||||
# ── Public ────────────────────────────────────────────────────────────────
|
||||
|
||||
def build(
|
||||
self,
|
||||
run_id: str,
|
||||
params: dict[str, Any],
|
||||
period_start: str,
|
||||
period_end: str,
|
||||
output_dir: Path,
|
||||
phase: str = "explore",
|
||||
) -> Path:
|
||||
"""
|
||||
Write <run_id>.ini to output_dir and return its path.
|
||||
params: dict of EA input values (partial OK — missing params use manifest defaults)
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
ini_path = output_dir / f"{run_id}.ini"
|
||||
report_dir = output_dir / "report"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
full_params = self._merge_with_defaults(params)
|
||||
ini_text = self._render(run_id, full_params, period_start, period_end,
|
||||
report_dir, phase)
|
||||
|
||||
ini_path.write_text(ini_text, encoding="utf-8")
|
||||
logger.debug(f"INI written: {ini_path}")
|
||||
return ini_path
|
||||
|
||||
def default_params(self) -> dict[str, Any]:
|
||||
"""Return all EA parameters at their default values."""
|
||||
return self._merge_with_defaults({})
|
||||
|
||||
# ── Internal ──────────────────────────────────────────────────────────────
|
||||
|
||||
def _merge_with_defaults(self, overrides: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Merge caller-supplied overrides with manifest defaults."""
|
||||
result: dict[str, Any] = {}
|
||||
for name, spec in self.manifest.items():
|
||||
if name in overrides:
|
||||
result[name] = overrides[name]
|
||||
else:
|
||||
result[name] = spec.get("default", 0)
|
||||
return result
|
||||
|
||||
def _format_value(self, name: str, value: Any) -> str:
|
||||
"""Format a parameter value for the [TesterInputs] section."""
|
||||
spec = self.manifest.get(name, {})
|
||||
ptype = spec.get("type", "float")
|
||||
|
||||
if ptype == "bool":
|
||||
return "true" if value else "false"
|
||||
if ptype == "fixed":
|
||||
# Fixed params: write exact default
|
||||
return str(value)
|
||||
if ptype == "int" or ptype == "enum":
|
||||
return str(int(value))
|
||||
if ptype == "float":
|
||||
# Determine decimal places from step
|
||||
step = spec.get("step", 0.1)
|
||||
decimals = len(str(step).split(".")[-1]) if "." in str(step) else 0
|
||||
return f"{float(value):.{decimals}f}"
|
||||
return str(value)
|
||||
|
||||
def _render(
|
||||
self,
|
||||
run_id: str,
|
||||
params: dict[str, Any],
|
||||
period_start: str,
|
||||
period_end: str,
|
||||
report_dir: Path,
|
||||
phase: str,
|
||||
) -> str:
|
||||
"""Render final INI content as a string."""
|
||||
ea_cfg = self.cfg["ea"]
|
||||
mt5_cfg = self.cfg["mt5"]
|
||||
broker_cfg = self.cfg["broker"]
|
||||
|
||||
# Period must be the string name (H1, M30 etc) — NOT the ENUM integer
|
||||
tf_name = TIMEFRAME_NAMES.get(ea_cfg["timeframe"].upper(), "H1")
|
||||
|
||||
# Model: 0=Every Tick (slow), 4=OHLC M1 (fast, reliable for ini-based launch)
|
||||
model_code = mt5_cfg.get("tester_model", 4)
|
||||
|
||||
# Report path must be RELATIVE to the MT5 terminal data folder
|
||||
# MT5 appends its own base path. Use run_id as the report name.
|
||||
report_name = f"Optimizer_{run_id}"
|
||||
|
||||
lines = [
|
||||
f"; MT5 Optimizer INI — run_id={run_id} phase={phase}",
|
||||
f"",
|
||||
f"[Tester]",
|
||||
f"Expert={ea_cfg['file']}",
|
||||
f"Symbol={ea_cfg['symbol']}",
|
||||
f"Period={tf_name}",
|
||||
f"Optimization=0",
|
||||
f"Model={model_code}",
|
||||
f"FromDate={period_start}",
|
||||
f"ToDate={period_end}",
|
||||
f"ForwardMode=0",
|
||||
f"Report={report_name}",
|
||||
f"ReplaceReport=1",
|
||||
f"ShutdownTerminal={mt5_cfg.get('shutdown_terminal', 1)}",
|
||||
f"Deposit={broker_cfg['deposit']}",
|
||||
f"Currency={broker_cfg['currency']}",
|
||||
f"Leverage={broker_cfg['leverage']}",
|
||||
f"",
|
||||
f"[TesterInputs]",
|
||||
]
|
||||
|
||||
for name, value in params.items():
|
||||
formatted = self._format_value(name, value)
|
||||
lines.append(f"{name}={formatted}")
|
||||
|
||||
lines.append("") # trailing newline
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,159 @@
|
||||
"""
|
||||
mt5/log_reader.py
|
||||
Reads the TradeLogger.mqh CSV and merges MAE/MFE data into parsed trades.
|
||||
Also computes derived fields: session, day_of_week, result_class, quality scores.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import pandas as pd
|
||||
from loguru import logger
|
||||
|
||||
from data.models import Trade
|
||||
|
||||
|
||||
# Session definitions in UTC hours (inclusive start, exclusive end)
|
||||
SESSIONS_UTC = {
|
||||
"Asian": (0, 9),
|
||||
"London": (7, 16),
|
||||
"LondonNY": (13, 16),
|
||||
"NY": (13, 22),
|
||||
}
|
||||
|
||||
|
||||
def classify_session(hour_utc: int) -> str:
|
||||
"""Classify a UTC hour into its primary trading session."""
|
||||
in_london = SESSIONS_UTC["London"][0] <= hour_utc < SESSIONS_UTC["London"][1]
|
||||
in_ny = SESSIONS_UTC["NY"][0] <= hour_utc < SESSIONS_UTC["NY"][1]
|
||||
|
||||
if in_london and in_ny:
|
||||
return "LondonNY"
|
||||
elif in_london:
|
||||
return "London"
|
||||
elif in_ny:
|
||||
return "NY"
|
||||
elif SESSIONS_UTC["Asian"][0] <= hour_utc < SESSIONS_UTC["Asian"][1]:
|
||||
return "Asian"
|
||||
else:
|
||||
return "Off"
|
||||
|
||||
|
||||
# ── Main reader/merger ────────────────────────────────────────────────────────
|
||||
|
||||
class TradeLogReader:
|
||||
"""
|
||||
Reads the CSV produced by TradeLogger.mqh and merges into a list of Trade objects.
|
||||
|
||||
Strategy:
|
||||
1. Load CSV, index by ticket
|
||||
2. For each Trade, look up ticket in CSV
|
||||
3. Fill mfe_pips, mae_pips, duration_minutes if found
|
||||
4. Compute all derived fields for every trade (session, quality scores, etc.)
|
||||
"""
|
||||
|
||||
def __init__(self, broker_tz_offset_hours: int = 2, pip_size: float = 0.1):
|
||||
self.tz_offset = broker_tz_offset_hours # broker local = UTC + offset
|
||||
self.pip_size = pip_size # XAUUSD: 0.1 per pip
|
||||
|
||||
# ── Public ────────────────────────────────────────────────────────────────
|
||||
|
||||
def merge(
|
||||
self,
|
||||
trades: list[Trade],
|
||||
csv_path: Optional[str | Path],
|
||||
reversal_mfe_threshold_pips: float = 15.0,
|
||||
) -> list[Trade]:
|
||||
"""
|
||||
Merge TradeLogger CSV into trade list, compute all derived fields.
|
||||
If csv_path is None or unreadable, derived fields are computed without MFE/MAE.
|
||||
"""
|
||||
log_df = self._load_csv(csv_path) if csv_path else None
|
||||
|
||||
enriched = []
|
||||
for trade in trades:
|
||||
# Fill MAE/MFE from logger if available
|
||||
if log_df is not None and trade.ticket in log_df.index:
|
||||
row = log_df.loc[trade.ticket]
|
||||
trade.mfe_pips = float(row.get("mfe_pips", 0) or 0)
|
||||
trade.mae_pips = float(row.get("mae_pips", 0) or 0)
|
||||
# Override duration with logger value (tick-accurate)
|
||||
if "duration_minutes" in row:
|
||||
trade.duration_minutes = int(row["duration_minutes"] or trade.duration_minutes)
|
||||
|
||||
# Compute all derived fields
|
||||
trade = self._enrich(trade, reversal_mfe_threshold_pips)
|
||||
enriched.append(trade)
|
||||
|
||||
logger.info(
|
||||
f"Enriched {len(enriched)} trades. "
|
||||
f"MAE/MFE available: {sum(1 for t in enriched if t.mfe_pips is not None)}"
|
||||
)
|
||||
return enriched
|
||||
|
||||
# ── Internal ──────────────────────────────────────────────────────────────
|
||||
|
||||
def _load_csv(self, csv_path: str | Path) -> Optional[pd.DataFrame]:
|
||||
path = Path(csv_path)
|
||||
if not path.exists():
|
||||
logger.warning(f"TradeLogger CSV not found: {path}")
|
||||
return None
|
||||
try:
|
||||
df = pd.read_csv(path, dtype={"ticket": int})
|
||||
if "ticket" not in df.columns:
|
||||
logger.error("TradeLogger CSV missing 'ticket' column.")
|
||||
return None
|
||||
df = df.set_index("ticket")
|
||||
logger.debug(f"Loaded {len(df)} rows from TradeLogger CSV.")
|
||||
return df
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read TradeLogger CSV: {e}")
|
||||
return None
|
||||
|
||||
def _enrich(self, trade: Trade, threshold_pips: float) -> Trade:
|
||||
"""Compute all derived classification and quality fields."""
|
||||
# --- Timezone normalisation ---
|
||||
# Broker timestamps are in broker local time (UTC+offset).
|
||||
# We compute UTC hour by subtracting the offset.
|
||||
broker_hour = trade.open_time.hour
|
||||
hour_utc = (broker_hour - self.tz_offset) % 24
|
||||
trade.hour_broker = broker_hour
|
||||
trade.hour_utc = hour_utc
|
||||
trade.day_of_week = trade.open_time.weekday() # 0=Mon, 4=Fri
|
||||
trade.session = classify_session(hour_utc)
|
||||
|
||||
# --- Result class ---
|
||||
won = trade.net_money > 0
|
||||
be = abs(trade.net_money) < 0.01 # effectively breakeven
|
||||
|
||||
if be:
|
||||
trade.result_class = "be"
|
||||
elif won:
|
||||
trade.result_class = "win"
|
||||
else:
|
||||
# Check if it's a reversal: lost, but had positive MFE above threshold
|
||||
if trade.mfe_pips is not None and trade.mfe_pips >= threshold_pips:
|
||||
trade.result_class = "reversal"
|
||||
else:
|
||||
trade.result_class = "loss"
|
||||
|
||||
# --- Quality scores (only when MFE/MAE available) ---
|
||||
if trade.mfe_pips is not None and trade.mae_pips is not None:
|
||||
mfe = max(trade.mfe_pips, 0.01) # prevent division by zero
|
||||
mae = max(trade.mae_pips, 0.0)
|
||||
|
||||
# Entry quality: how far against you before move in your favour
|
||||
# High = entered well (little adverse move relative to favourable move)
|
||||
trade.entry_quality = max(0.0, min(1.0, 1.0 - (mae / (mfe + mae + 0.01))))
|
||||
|
||||
# Exit quality: what fraction of MFE did we capture
|
||||
mfe_value = mfe * self.pip_size * trade.lot_size * 100 # approx value in $
|
||||
if mfe_value > 0:
|
||||
trade.mfe_capture_ratio = max(0.0, trade.net_money / mfe_value)
|
||||
trade.exit_quality = max(0.0, min(1.0, trade.net_pips / mfe))
|
||||
else:
|
||||
trade.mfe_capture_ratio = 0.0
|
||||
trade.exit_quality = 0.0
|
||||
|
||||
return trade
|
||||
@@ -0,0 +1,338 @@
|
||||
"""
|
||||
mt5/report_parser.py
|
||||
Parses the MT5 strategy tester HTML report (production format: pure HTML tables).
|
||||
Extracts RunMetrics and paired in/out deal trades.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from lxml import html as lhtml
|
||||
from loguru import logger
|
||||
|
||||
from data.models import RunMetrics, Trade
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def _clean(s: str) -> str:
|
||||
"""Remove HTML entity remnants, spaces, currency symbols."""
|
||||
if not s:
|
||||
return ""
|
||||
# MT5 uses non-breaking spaces (0xa0) and regular spaces
|
||||
s = s.replace("\xa0", "").replace(",", "").replace(" ", "").strip()
|
||||
return s
|
||||
|
||||
|
||||
def _parse_float(s: str) -> float:
|
||||
s = _clean(s)
|
||||
# Remove everything except digits, dot, minus
|
||||
s = re.sub(r"[^\d.\-]", "", s)
|
||||
try:
|
||||
return float(s)
|
||||
except (ValueError, TypeError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _parse_int(s: str) -> int:
|
||||
s = _clean(s)
|
||||
s = re.sub(r"[^\d\-]", "", s.split("(")[0])
|
||||
try:
|
||||
return int(s)
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_dt(s: str) -> Optional[datetime]:
|
||||
s = (s or "").strip()
|
||||
for fmt in ("%Y.%m.%d %H:%M:%S", "%Y.%m.%d %H:%M", "%Y.%m.%d"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _cell_text(td) -> str:
|
||||
"""Get all inner text from an lxml element, stripping tags."""
|
||||
return "".join(td.itertext()).strip()
|
||||
|
||||
|
||||
# ── Main Parser ───────────────────────────────────────────────────────────────
|
||||
|
||||
class ReportParser:
|
||||
"""
|
||||
Parses the MT5 HTML strategy tester report.
|
||||
|
||||
Report format (confirmed from live MT5 output):
|
||||
- Summary metrics: <td>Label:</td><td><b>Value</b></td> pairs
|
||||
- Deals table: Time | Deal | Symbol | Type | Direction | Volume |
|
||||
Price | Order | Commission | Swap | Profit | Balance | Comment
|
||||
Direction='in' → position open (entry deal)
|
||||
Direction='out' → position close (exit deal, has Profit value)
|
||||
"""
|
||||
|
||||
def parse(
|
||||
self, xml_path: Optional[str], html_path: Optional[str]
|
||||
) -> tuple[Optional[RunMetrics], list[Trade]]:
|
||||
"""
|
||||
Parse the MT5 HTML report. xml_path is ignored (MT5 command-line
|
||||
runs produce .htm, not .xml). Falls back gracefully if html is missing.
|
||||
"""
|
||||
path = None
|
||||
if html_path and Path(html_path).exists():
|
||||
path = Path(html_path)
|
||||
elif xml_path and Path(xml_path).exists():
|
||||
path = Path(xml_path)
|
||||
|
||||
if path is None:
|
||||
logger.error("No report file available to parse.")
|
||||
return None, []
|
||||
|
||||
logger.debug(f"Parsing report: {path}")
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
# MT5 HTML reports are UTF-16 LE (BOM: ff fe) — detect and decode
|
||||
if raw[:2] == b'\xff\xfe':
|
||||
# Pass raw bytes; lxml's HTML parser handles UTF-16 correctly
|
||||
tree = lhtml.document_fromstring(raw)
|
||||
else:
|
||||
# Regular UTF-8 or latin-1
|
||||
try:
|
||||
content = raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
content = raw.decode("windows-1252", errors="replace")
|
||||
tree = lhtml.document_fromstring(content.encode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse HTML: {e}")
|
||||
return None, []
|
||||
|
||||
summary = self._extract_summary(tree)
|
||||
deals = self._extract_deals(tree)
|
||||
trades = self._pair_deals(deals)
|
||||
|
||||
if not summary:
|
||||
logger.warning("No summary data found in MT5 HTML report.")
|
||||
return None, trades
|
||||
|
||||
metrics = self._build_metrics(summary)
|
||||
logger.info(f"Parsed: {metrics.total_trades} trades, PF={metrics.profit_factor:.3f}")
|
||||
return metrics, trades
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────
|
||||
|
||||
def _extract_summary(self, tree) -> dict[str, str]:
|
||||
"""
|
||||
Extract label→value pairs from the stats tables.
|
||||
MT5 pattern: <td ...>Label:</td> <td ...><b>Value</b></td>
|
||||
The label and value are adjacent siblings in the same <tr>.
|
||||
"""
|
||||
result: dict[str, str] = {}
|
||||
for tr in tree.iter("tr"):
|
||||
tds = list(tr.findall(".//td"))
|
||||
if len(tds) < 2:
|
||||
continue
|
||||
for i in range(len(tds) - 1):
|
||||
label = _cell_text(tds[i]).rstrip(":")
|
||||
val = _cell_text(tds[i + 1])
|
||||
if label and val and len(label) < 60:
|
||||
result[label] = val
|
||||
logger.debug(f"Summary fields: {len(result)}")
|
||||
return result
|
||||
|
||||
# ── Deals ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _extract_deals(self, tree) -> list[dict]:
|
||||
"""
|
||||
Find the Deals table and parse every row.
|
||||
Columns: Time|Deal|Symbol|Type|Direction|Volume|Price|Order|Commission|Swap|Profit|Balance|Comment
|
||||
"""
|
||||
# Find the <th> that contains "Deals" text
|
||||
deals_header = None
|
||||
for th in tree.iter("th"):
|
||||
if "Deals" in (_cell_text(th) or ""):
|
||||
deals_header = th
|
||||
break
|
||||
|
||||
if deals_header is None:
|
||||
logger.warning("Deals table not found in report.")
|
||||
return []
|
||||
|
||||
# Walk up to find the table element
|
||||
table = deals_header
|
||||
while table is not None and table.tag != "table":
|
||||
table = table.getparent()
|
||||
if table is None:
|
||||
return []
|
||||
|
||||
rows = table.findall(".//tr")
|
||||
# Skip header rows (first 2 rows: table title + column headers)
|
||||
data_rows = []
|
||||
header_seen = 0
|
||||
for row in rows:
|
||||
ths = row.findall(".//th")
|
||||
tds = row.findall(".//td")
|
||||
if ths:
|
||||
header_seen += 1
|
||||
continue
|
||||
if not tds:
|
||||
continue
|
||||
data_rows.append(tds)
|
||||
|
||||
deals = []
|
||||
for tds in data_rows:
|
||||
texts = [_cell_text(td) for td in tds]
|
||||
if len(texts) < 13:
|
||||
continue
|
||||
# Cols: 0=Time 1=Deal 2=Symbol 3=Type 4=Direction 5=Volume
|
||||
# 6=Price 7=Order 8=Commission 9=Swap 10=Profit 11=Balance 12=Comment
|
||||
deal_type = texts[3].lower()
|
||||
if "balance" in deal_type or "credit" in deal_type:
|
||||
continue # skip balance entries at start
|
||||
|
||||
deals.append({
|
||||
"time": texts[0],
|
||||
"deal": _parse_int(texts[1]),
|
||||
"symbol": texts[2],
|
||||
"type": deal_type, # buy / sell
|
||||
"direction": texts[4].lower(), # in / out
|
||||
"volume": _parse_float(texts[5]),
|
||||
"price": _parse_float(texts[6]),
|
||||
"order": _parse_int(texts[7]),
|
||||
"commission": _parse_float(texts[8]),
|
||||
"swap": _parse_float(texts[9]),
|
||||
"profit": _parse_float(texts[10]),
|
||||
"balance": _parse_float(texts[11]),
|
||||
"comment": texts[12] if len(texts) > 12 else "",
|
||||
})
|
||||
|
||||
logger.debug(f"Raw deals extracted: {len(deals)}")
|
||||
return deals
|
||||
|
||||
# ── Pairing in→out ────────────────────────────────────────────────────────
|
||||
|
||||
def _pair_deals(self, deals: list[dict]) -> list[Trade]:
|
||||
"""
|
||||
Pair 'in' (open) and 'out' (close) deals to form complete trades.
|
||||
MT5 reports alternate: in-deal → out-deal for each closed position.
|
||||
"""
|
||||
trades: list[Trade] = []
|
||||
pending: Optional[dict] = None # the last 'in' deal
|
||||
|
||||
for d in deals:
|
||||
if d["direction"] == "in":
|
||||
pending = d
|
||||
elif d["direction"] == "out" and pending is not None:
|
||||
open_dt = _parse_dt(pending["time"])
|
||||
close_dt = _parse_dt(d["time"])
|
||||
if not open_dt or not close_dt:
|
||||
pending = None
|
||||
continue
|
||||
|
||||
direction = pending["type"] # buy / sell
|
||||
open_price = pending["price"]
|
||||
close_price = d["price"]
|
||||
net_money = d["profit"]
|
||||
lot_size = d["volume"]
|
||||
commission = d["commission"] + pending["commission"]
|
||||
swap = d["swap"] + pending["swap"]
|
||||
duration_m = max(0, int((close_dt - open_dt).total_seconds() / 60))
|
||||
|
||||
# Net pips (XAUUSD: price moves in dollars, 1 pip = 0.1)
|
||||
price_diff = (close_price - open_price) * (1 if direction == "buy" else -1)
|
||||
net_pips = round(price_diff / 0.1, 2) if price_diff != 0 else 0.0
|
||||
|
||||
trades.append(Trade(
|
||||
ticket = d["deal"],
|
||||
open_time = open_dt,
|
||||
close_time = close_dt,
|
||||
direction = direction,
|
||||
open_price = open_price,
|
||||
close_price = close_price,
|
||||
lot_size = lot_size,
|
||||
net_pips = net_pips,
|
||||
net_money = net_money,
|
||||
duration_minutes = duration_m,
|
||||
commission = commission,
|
||||
swap = swap,
|
||||
sl = 0.0, # not in deals table
|
||||
tp = 0.0,
|
||||
))
|
||||
pending = None
|
||||
# if direction is empty/"" skip it
|
||||
|
||||
logger.info(f"Paired {len(trades)} complete trades from deals.")
|
||||
return trades
|
||||
|
||||
# ── Metrics ───────────────────────────────────────────────────────────────
|
||||
|
||||
def _build_metrics(self, raw: dict[str, str]) -> RunMetrics:
|
||||
"""Build RunMetrics from the extracted label→value dictionary."""
|
||||
|
||||
def get(*keys) -> str:
|
||||
for k in keys:
|
||||
v = raw.get(k, "")
|
||||
if v:
|
||||
return v
|
||||
return "0"
|
||||
|
||||
net_profit = _parse_float(get("Total Net Profit", "Net Profit", "Balance"))
|
||||
gross_profit = _parse_float(get("Gross Profit"))
|
||||
gross_loss = _parse_float(get("Gross Loss"))
|
||||
profit_factor = _parse_float(get("Profit Factor"))
|
||||
# Total Deals = number of deal rows; Total Trades = positions
|
||||
total_trades = _parse_int(get("Total Trades", "Total Deals"))
|
||||
win_trades = _parse_int(get("Profit Trades", "Profit Trades (% of total)",
|
||||
"Profit Deals"))
|
||||
|
||||
# Drawdown: "2 160.22 (19.25%)"
|
||||
dd_str = get("Equity Drawdown Maximal", "Equity Drawdown Relative",
|
||||
"Balance Drawdown Maximal")
|
||||
max_dd_abs = _parse_float(dd_str.split("(")[0])
|
||||
pct_match = re.search(r"([\d.]+)%", dd_str)
|
||||
max_dd_pct = float(pct_match.group(1)) / 100 if pct_match else 0.0
|
||||
|
||||
initial_dep = _parse_float(get("Initial Deposit", "Deposit"))
|
||||
if initial_dep <= 0:
|
||||
initial_dep = 10_000.0
|
||||
|
||||
sharpe = _parse_float(get("Sharpe Ratio", "Sharp Ratio"))
|
||||
recovery_factor = _parse_float(get("Recovery Factor"))
|
||||
expected_payoff = _parse_float(get("Expected Payoff"))
|
||||
|
||||
# Compute max_dd_pct if only absolute was found
|
||||
if max_dd_pct == 0.0 and max_dd_abs > 0:
|
||||
total_equity = initial_dep + net_profit
|
||||
max_dd_pct = max_dd_abs / max(1, total_equity)
|
||||
|
||||
# Calmar = annualised return / max drawdown
|
||||
# Use simple ratio since we don't know exact test duration
|
||||
calmar = 0.0
|
||||
if max_dd_pct > 0:
|
||||
annual_return = net_profit / initial_dep
|
||||
calmar = round(annual_return / max_dd_pct, 4)
|
||||
|
||||
win_rate = win_trades / total_trades if total_trades > 0 else 0.0
|
||||
loss_trades = max(0, total_trades - win_trades)
|
||||
avg_win = gross_profit / win_trades if win_trades > 0 else 0.0
|
||||
avg_loss = gross_loss / loss_trades if loss_trades > 0 else 0.0
|
||||
|
||||
return RunMetrics(
|
||||
run_id = "__placeholder__",
|
||||
net_profit = net_profit,
|
||||
profit_factor = profit_factor,
|
||||
max_drawdown_abs= max_dd_abs,
|
||||
max_drawdown_pct= max_dd_pct,
|
||||
calmar_ratio = calmar,
|
||||
sharpe_ratio = sharpe,
|
||||
total_trades = total_trades,
|
||||
win_rate = win_rate,
|
||||
avg_win = avg_win,
|
||||
avg_loss = avg_loss,
|
||||
recovery_factor = recovery_factor,
|
||||
largest_loss = _parse_float(get("Largest loss trade")),
|
||||
expected_payoff = expected_payoff,
|
||||
)
|
||||
+375
@@ -0,0 +1,375 @@
|
||||
"""
|
||||
mt5/runner.py
|
||||
Robust MT5 Strategy Tester runner with:
|
||||
1. MT5 process control (kill existing, launch fresh)
|
||||
2. Pre-run environment validation
|
||||
3. Historical data readiness wait
|
||||
4. Actionable error messages
|
||||
5. Auto-retry on failure (1 retry)
|
||||
6. Report detection in MT5 native reports folder
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import psutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import yaml
|
||||
from loguru import logger
|
||||
|
||||
from data.models import RunResult
|
||||
|
||||
|
||||
# ── Custom Exceptions ─────────────────────────────────────────────────────────
|
||||
|
||||
class MT5TimeoutError(RuntimeError):
|
||||
pass
|
||||
|
||||
class MT5ValidationError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
# ── MT5Runner ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class MT5Runner:
|
||||
|
||||
POLL_INTERVAL_S = 5 # seconds between report checks
|
||||
PROCESS_SETTLE_S = 2 # seconds to wait after process exit
|
||||
MT5_INIT_WAIT_S = 8 # seconds after launch before testing begins
|
||||
KILL_WAIT_S = 3 # seconds after killing MT5 before launching fresh
|
||||
MAX_RETRIES = 1 # retry once on failure
|
||||
|
||||
MT5_EXE_NAME = "terminal64.exe"
|
||||
|
||||
def __init__(self, config_path: str | Path = "config.yaml"):
|
||||
with open(config_path) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
self.cfg = cfg
|
||||
self.terminal_exe = Path(cfg["mt5"]["terminal_exe"])
|
||||
self.timeout_s = cfg["mt5"]["tester_timeout_seconds"]
|
||||
self.appdata_path = Path(cfg["mt5"]["appdata_path"])
|
||||
# MT5 writes reports to the ROOT of the terminal data folder
|
||||
# (not a 'reports' subfolder) — confirmed by log inspection
|
||||
self.mt5_reports_dir = self.appdata_path
|
||||
self.mql5_files_dir = Path(cfg["mt5"].get(
|
||||
"mql5_files_path",
|
||||
str(self.appdata_path / "MQL5" / "Files")
|
||||
))
|
||||
self.data_wait_s = cfg["mt5"].get("data_readiness_wait_seconds", 10)
|
||||
|
||||
# ── Public entry point ────────────────────────────────────────────────────
|
||||
|
||||
def run(
|
||||
self,
|
||||
run_id: str,
|
||||
ini_path: Path,
|
||||
report_dir: Path,
|
||||
log_csv_search_dir: Optional[Path] = None,
|
||||
) -> RunResult:
|
||||
"""
|
||||
Full execution pipeline with retry:
|
||||
1. Kill existing MT5
|
||||
2. Validate environment
|
||||
3. Launch fresh MT5
|
||||
4. Wait for data readiness
|
||||
5. Poll for report
|
||||
6. Auto-retry once on failure
|
||||
"""
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
report_stem = f"Optimizer_{run_id}"
|
||||
|
||||
for attempt in range(1, self.MAX_RETRIES + 2):
|
||||
is_retry = attempt > 1
|
||||
if is_retry:
|
||||
logger.warning(f"[{run_id}] Retry attempt {attempt}...")
|
||||
|
||||
try:
|
||||
# ── Step 1: Kill any running MT5 ─────────────────────────
|
||||
self._kill_mt5(run_id)
|
||||
|
||||
# ── Step 1b: Clear stale reports from previous runs ──────
|
||||
self._clear_stale_reports(run_id)
|
||||
|
||||
# ── Step 2: Pre-run validation ───────────────────────────
|
||||
self._validate(run_id)
|
||||
|
||||
# ── Step 3: Launch fresh MT5 ─────────────────────────────
|
||||
proc = self._launch_mt5(run_id, ini_path)
|
||||
|
||||
# ── Step 4: Wait for MT5 to initialize + data ────────────
|
||||
logger.info(f"[{run_id}] Waiting {self.MT5_INIT_WAIT_S}s for MT5 to initialize...")
|
||||
time.sleep(self.MT5_INIT_WAIT_S)
|
||||
|
||||
# ── Step 5: Wait for report + TradeLog ───────────────────
|
||||
result = self._wait_for_report(run_id, proc, report_dir, report_stem)
|
||||
|
||||
# ── Step 6: Find TradeLogger CSV ─────────────────────────
|
||||
result.trade_log_csv = self._find_trade_log(run_id, log_csv_search_dir)
|
||||
|
||||
logger.success(f"[{run_id}] Run complete. Report: {result.report_xml}")
|
||||
return result
|
||||
|
||||
except MT5ValidationError as e:
|
||||
logger.error(f"[{run_id}] Validation error: {e}")
|
||||
return RunResult(run_id=run_id, success=False, error_message=str(e))
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"[{run_id}] Attempt {attempt} failed: {e}")
|
||||
if attempt > self.MAX_RETRIES:
|
||||
msg = self._diagnose_failure(str(e))
|
||||
logger.error(f"[{run_id}] All retries exhausted. {msg}")
|
||||
return RunResult(run_id=run_id, success=False, error_message=msg)
|
||||
logger.info(f"[{run_id}] Will retry after {self.KILL_WAIT_S}s...")
|
||||
time.sleep(self.KILL_WAIT_S)
|
||||
|
||||
# ── Step 1: Kill existing MT5 ─────────────────────────────────────────────
|
||||
|
||||
def _kill_mt5(self, run_id: str) -> None:
|
||||
"""Find and terminate any running MT5 processes."""
|
||||
killed = 0
|
||||
for proc in psutil.process_iter(["pid", "name", "exe"]):
|
||||
try:
|
||||
if proc.info["name"] and self.MT5_EXE_NAME.lower() in proc.info["name"].lower():
|
||||
logger.info(f"[{run_id}] Closing existing MT5 (PID {proc.pid})...")
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except psutil.TimeoutExpired:
|
||||
proc.kill()
|
||||
killed += 1
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
pass
|
||||
|
||||
if killed > 0:
|
||||
logger.info(f"[{run_id}] Closed {killed} MT5 instance(s). Waiting {self.KILL_WAIT_S}s...")
|
||||
time.sleep(self.KILL_WAIT_S)
|
||||
else:
|
||||
logger.debug(f"[{run_id}] No existing MT5 process found.")
|
||||
|
||||
# ── Step 2: Pre-run validation ────────────────────────────────────────────
|
||||
|
||||
def _validate(self, run_id: str) -> None:
|
||||
"""Validate all required files and directories exist before launch."""
|
||||
errors = []
|
||||
|
||||
# Check terminal executable
|
||||
if not self.terminal_exe.exists():
|
||||
errors.append(f"MT5 terminal not found: {self.terminal_exe}")
|
||||
|
||||
# Check EA compiled file
|
||||
ea_name = self.cfg["ea"]["file"]
|
||||
ea_candidates = [
|
||||
self.appdata_path / "MQL5" / "Experts" / f"{ea_name}.ex5",
|
||||
self.appdata_path / "MQL5" / "Experts" / f"{ea_name}",
|
||||
]
|
||||
ea_found = any(p.exists() for p in ea_candidates)
|
||||
if not ea_found:
|
||||
errors.append(
|
||||
f"EA file not found: {ea_name}.ex5 — "
|
||||
f"ensure you compiled the EA in MetaEditor before running."
|
||||
)
|
||||
|
||||
# Check appdata path
|
||||
if not self.appdata_path.exists():
|
||||
errors.append(f"MT5 appdata folder not found: {self.appdata_path}")
|
||||
|
||||
# Symbol check (basic — just ensure it's set)
|
||||
symbol = self.cfg["ea"].get("symbol", "")
|
||||
if not symbol:
|
||||
errors.append("Symbol not configured in config.yaml ea.symbol")
|
||||
|
||||
if errors:
|
||||
for e in errors:
|
||||
logger.error(f"[{run_id}] Validation: {e}")
|
||||
raise MT5ValidationError(
|
||||
"Pre-run validation failed:\n" + "\n".join(f" • {e}" for e in errors)
|
||||
)
|
||||
|
||||
logger.debug(f"[{run_id}] Pre-run validation passed.")
|
||||
|
||||
# ── Step 3: Launch MT5 ───────────────────────────────────────────────────
|
||||
|
||||
def _launch_mt5(self, run_id: str, ini_path: Path) -> subprocess.Popen:
|
||||
"""Launch a fresh MT5 instance with the given INI config."""
|
||||
cmd = [str(self.terminal_exe), f"/config:{ini_path}"]
|
||||
logger.info(f"[{run_id}] Launching MT5: {' '.join(cmd)}")
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
logger.debug(f"[{run_id}] MT5 PID: {proc.pid}")
|
||||
return proc
|
||||
|
||||
# ── Step 5: Wait for report ───────────────────────────────────────────────
|
||||
|
||||
def _wait_for_report(
|
||||
self,
|
||||
run_id: str,
|
||||
proc: subprocess.Popen,
|
||||
report_dir: Path,
|
||||
report_stem: str,
|
||||
) -> RunResult:
|
||||
"""
|
||||
Poll both our local dir and MT5's native reports folder.
|
||||
MT5 writes the report to <appdata>\\reports\\ using the name from INI Report= field.
|
||||
"""
|
||||
elapsed = 0
|
||||
search_dirs = [report_dir, self.mt5_reports_dir]
|
||||
|
||||
while elapsed < self.timeout_s:
|
||||
# Check for report in all locations
|
||||
for search in search_dirs:
|
||||
if not search.exists():
|
||||
continue
|
||||
result = self._find_report_files(search, report_stem)
|
||||
if result:
|
||||
xml_path, html_path = result
|
||||
time.sleep(self.PROCESS_SETTLE_S)
|
||||
# Archive to our local report dir
|
||||
if xml_path:
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = report_dir / Path(xml_path).name
|
||||
if not dest.exists():
|
||||
shutil.copy2(xml_path, dest)
|
||||
logger.info(f"[{run_id}] Report found in {search} after {elapsed}s")
|
||||
return RunResult(
|
||||
run_id=run_id,
|
||||
report_xml=xml_path,
|
||||
report_html=html_path,
|
||||
success=True,
|
||||
)
|
||||
|
||||
# Check if process already exited
|
||||
ret = proc.poll()
|
||||
if ret is not None:
|
||||
time.sleep(self.PROCESS_SETTLE_S)
|
||||
# Final check after process exit
|
||||
for search in search_dirs:
|
||||
if not search.exists():
|
||||
continue
|
||||
result = self._find_report_files(search, report_stem)
|
||||
if result:
|
||||
xml_path, html_path = result
|
||||
return RunResult(
|
||||
run_id=run_id,
|
||||
report_xml=xml_path,
|
||||
report_html=html_path,
|
||||
success=True,
|
||||
)
|
||||
|
||||
# Process exited but no report — diagnose
|
||||
if ret != 0:
|
||||
raise RuntimeError(
|
||||
f"MT5 exited with error code {ret}. "
|
||||
f"Possible causes: invalid INI parameters, EA not compiled, "
|
||||
f"or missing historical data."
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"MT5 exited without generating a report. "
|
||||
"Possible causes: Symbol data not downloaded, "
|
||||
"invalid date range, or EA failed to initialize."
|
||||
)
|
||||
|
||||
time.sleep(self.POLL_INTERVAL_S)
|
||||
elapsed += self.POLL_INTERVAL_S
|
||||
if elapsed % 30 == 0:
|
||||
logger.info(f"[{run_id}] Still waiting for report... {elapsed}/{self.timeout_s}s")
|
||||
|
||||
raise MT5TimeoutError(
|
||||
f"No report after {self.timeout_s}s. "
|
||||
f"MT5 may be stuck or the test is taking too long. "
|
||||
f"Consider reducing the test date range or using OHLC M1 model."
|
||||
)
|
||||
|
||||
def _clear_stale_reports(self, run_id: str) -> None:
|
||||
"""Remove stale Optimizer_* reports from MT5 appdata root to avoid false-positive detection."""
|
||||
try:
|
||||
import glob
|
||||
# Only delete files NOT matching the current run_id
|
||||
for pattern in ["*.htm", "*.xml", "*.html"]:
|
||||
for f in self.mt5_reports_dir.glob(f"Optimizer_*{pattern[-3:]}"):
|
||||
if run_id not in f.name:
|
||||
try:
|
||||
f.unlink()
|
||||
logger.debug(f"[{run_id}] Cleared stale report: {f.name}")
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.debug(f"[{run_id}] Could not clear stale reports: {e}")
|
||||
|
||||
def _find_report_files(
|
||||
self, search_dir: Path, report_stem: str
|
||||
) -> Optional[tuple[Optional[str], Optional[str]]]:
|
||||
"""Search for report XML/HTML files by stem prefix."""
|
||||
xml_list = sorted(
|
||||
list(search_dir.glob(f"{report_stem}*.xml")) +
|
||||
list(search_dir.glob(f"{report_stem}*.XML")),
|
||||
key=lambda p: p.stat().st_mtime, reverse=True
|
||||
)
|
||||
htm_list = sorted(
|
||||
list(search_dir.glob(f"{report_stem}*.htm")) +
|
||||
list(search_dir.glob(f"{report_stem}*.html")) +
|
||||
list(search_dir.glob(f"{report_stem}*.HTM")),
|
||||
key=lambda p: p.stat().st_mtime, reverse=True
|
||||
)
|
||||
|
||||
if xml_list or htm_list:
|
||||
return (
|
||||
str(xml_list[0]) if xml_list else None,
|
||||
str(htm_list[0]) if htm_list else None,
|
||||
)
|
||||
return None
|
||||
|
||||
# ── TradeLogger CSV lookup ────────────────────────────────────────────────
|
||||
|
||||
def _find_trade_log(
|
||||
self, run_id: str, search_dir: Optional[Path]
|
||||
) -> Optional[Path]:
|
||||
"""Find the TradeLogger CSV written by the EA during the backtest."""
|
||||
ea_name = self.cfg["ea"]["file"]
|
||||
symbol = self.cfg["ea"]["symbol"]
|
||||
|
||||
candidates = [
|
||||
self.mql5_files_dir / f"{ea_name}_{symbol}_TradeLog.csv",
|
||||
]
|
||||
if search_dir:
|
||||
candidates.append(search_dir / f"{ea_name}_{symbol}_TradeLog.csv")
|
||||
|
||||
for path in candidates:
|
||||
if path.exists():
|
||||
logger.debug(f"[{run_id}] TradeLog found: {path}")
|
||||
return path
|
||||
|
||||
logger.debug(f"[{run_id}] TradeLog CSV not found (fallback to report-only mode).")
|
||||
return None
|
||||
|
||||
# ── Error diagnosis ───────────────────────────────────────────────────────
|
||||
|
||||
def _diagnose_failure(self, error_msg: str) -> str:
|
||||
"""Convert technical errors to actionable user-facing messages."""
|
||||
msg = error_msg.lower()
|
||||
if "exit" in msg and "cleanly" in msg:
|
||||
return (
|
||||
"MT5 started but did not produce a report.\n"
|
||||
"✦ Check that XAUUSD historical data is downloaded in MT5\n"
|
||||
"✦ Ensure the date range (2022-2023) has data available\n"
|
||||
"✦ Verify the EA compiled successfully in MetaEditor"
|
||||
)
|
||||
if "timeout" in msg:
|
||||
return (
|
||||
f"MT5 tester timed out after {self.timeout_s}s.\n"
|
||||
"✦ Try a shorter date range in config.yaml\n"
|
||||
"✦ Switch tester_model to 4 (OHLC M1) for faster runs"
|
||||
)
|
||||
if "validation" in msg or "not found" in msg:
|
||||
return error_msg
|
||||
return (
|
||||
f"MT5 run failed: {error_msg}\n"
|
||||
"✦ Ensure MT5 is fully closed before starting the optimizer\n"
|
||||
"✦ Check config.yaml paths are correct"
|
||||
)
|
||||
Reference in New Issue
Block a user