gavindiaz/Apex_AI_MT5_EA_Optimizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(step1): Universal EA Schema Layer — replaces param_manifest.yaml

Browse Source 
NEW FILES:
  ea/__init__.py          - EA module package
  ea/schema.py            - ParameterDef + ParameterSchema (replaces YAML manifest)
  ea/set_parser.py        - Parse ANY MT5 .set file → ParameterSchema
  ea/registry.py          - EAProfile + EARegistry (ea_registry.yaml)
  ea_registry.yaml        - LEGSTECH_EA_V2 pre-registered with automation overrides

MODIFIED:
  mt5/ini_builder.py      - Accepts ParameterSchema OR legacy manifest_path
  mt5/runner.py           - Accepts optional EAProfile for EA-agnostic validation
  config.yaml             - Removed hardcoded ea: block; added ea_registry path

KEY DESIGN:
  - .set file IS the manifest: value|min|max|step auto-detects float/int/bool/enum/fixed
  - automation_overrides in EAProfile forces headless-safe values (InpShowPanel=0)
  - Phase A optimization INI: Optimization=2 (genetic), ranges from schema
  - Phase B single backtest: Optimization=0 (unchanged behavior)
  - LEGSTECH advanced mode: fully backwards compatible via legacy manifest fallback
  - runner.py still works for legacy callers (no EAProfile needed)

TESTED:
  - 49/49 LEGSTECH params parsed correctly
  - InpShowPanel=0 automation override applied
  - INI output correct for both Phase A (Optimization=2) and Phase B (Optimization=0)
  - All imports pass
This commit is contained in:
LEGSTECH Optimizer
2026-04-13 19:55:16 +00:00
parent b70a6760ae
commit d5a8973c05
10 changed files with 878 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mt5/ini_builder.py
+92 -42
View File
@@ -1,15 +1,22 @@
"""
mt5/ini_builder.py
Generates the MT5 strategy tester .ini file from a parameter dict + config.
Generates the MT5 strategy tester .ini file from a ParameterSchema + config.
Accepts either:
- ParameterSchema object (new universal path)
- manifest_path YAML string (legacy path, still supported for transition)
"""
from __future__ import annotations
import configparser
from pathlib import Path
from typing import Any
from typing import Any, Optional, TYPE_CHECKING
import yaml
from loguru import logger
if TYPE_CHECKING:
from ea.schema import ParameterSchema
# MT5 period string names (used in [Tester] Period= field)
TIMEFRAME_NAMES = {
@@ -29,22 +36,37 @@ class IniBuilder:
"""
Builds MT5 strategy tester .ini files.
Usage:
New usage (universal, recommended):
builder = IniBuilder(config_path="config.yaml", schema=my_schema)
Legacy usage (LEGSTECH transition):
builder = IniBuilder(config_path="config.yaml", manifest_path="mutation/param_manifest.yaml")
ini_path = builder.build(
run_id="run_001",
params={"InpRiskPercent": 1.5, "InpUseTrailing": True, ...},
period_start="2022.01.01",
period_end="2023.12.31",
output_dir=Path("runs/run_001"),
)
The EA identity (ex5_file, symbol, timeframe) comes from the EAProfile
passed to build(), or falls back to config["ea"] for backwards compat.
"""
def __init__(self, config_path: str | Path, manifest_path: str | Path):
def __init__(
self,
config_path: str | Path,
manifest_path: Optional[str | Path] = None,
schema: Optional["ParameterSchema"] = None,
):
with open(config_path) as f:
self.cfg = yaml.safe_load(f)
with open(manifest_path) as f:
self.manifest = yaml.safe_load(f)["parameters"]
self._schema = schema
# Legacy manifest support
self._manifest: dict = {}
if manifest_path is not None and schema is None:
with open(manifest_path) as f:
self._manifest = yaml.safe_load(f)["parameters"]
def set_schema(self, schema: "ParameterSchema") -> None:
"""Update the schema (useful when EA changes mid-session)."""
self._schema = schema
self._manifest = {}
# ── Public ────────────────────────────────────────────────────────────────
@@ -56,10 +78,19 @@ class IniBuilder:
period_end: str,
output_dir: Path,
phase: str = "explore",
ea_file: Optional[str] = None,
ea_symbol: Optional[str] = None,
ea_timeframe: Optional[str] = None,
optimize_mode: bool = False,
) -> Path:
"""
Write <run_id>.ini to output_dir and return its path.
params: dict of EA input values (partial OK — missing params use manifest defaults)
ea_file, ea_symbol, ea_timeframe: override config["ea"] values.
Pass these from EAProfile when using the universal path.
optimize_mode=True: write Phase A optimization INI (Optimization=2).
optimize_mode=False: write Phase B single backtest INI (Optimization=0).
"""
output_dir.mkdir(parents=True, exist_ok=True)
ini_path = output_dir / f"{run_id}.ini"
@@ -67,8 +98,12 @@ class IniBuilder:
report_dir.mkdir(parents=True, exist_ok=True)
full_params = self._merge_with_defaults(params)
ini_text = self._render(run_id, full_params, period_start, period_end,
report_dir, phase)
ini_text = self._render(
run_id, full_params, period_start, period_end,
report_dir, phase,
ea_file=ea_file, ea_symbol=ea_symbol, ea_timeframe=ea_timeframe,
optimize_mode=optimize_mode,
)
ini_path.write_text(ini_text, encoding="utf-8")
logger.debug(f"INI written: {ini_path}")
@@ -76,34 +111,41 @@ class IniBuilder:
def default_params(self) -> dict[str, Any]:
"""Return all EA parameters at their default values."""
if self._schema is not None:
return self._schema.defaults()
return self._merge_with_defaults({})
# ── Internal ──────────────────────────────────────────────────────────────
def _merge_with_defaults(self, overrides: dict[str, Any]) -> dict[str, Any]:
"""Merge caller-supplied overrides with manifest defaults."""
"""Merge caller-supplied overrides with defaults."""
if self._schema is not None:
return self._schema.with_overrides(overrides)
# Legacy manifest path
result: dict[str, Any] = {}
for name, spec in self.manifest.items():
if name in overrides:
result[name] = overrides[name]
else:
result[name] = spec.get("default", 0)
manifest = self._manifest
for name, spec in manifest.items():
result[name] = overrides.get(name, spec.get("default", 0))
return result
def _format_value(self, name: str, value: Any) -> str:
"""Format a parameter value for the [TesterInputs] section."""
spec = self.manifest.get(name, {})
# Schema path
if self._schema is not None:
p = self._schema.get(name)
if p is not None:
return self._schema._fmt(p, value)
return str(value)
# Legacy manifest path
spec = self._manifest.get(name, {})
ptype = spec.get("type", "float")
if ptype == "bool":
return "true" if value else "false"
if ptype == "fixed":
# Fixed params: write exact default
return str(value)
if ptype == "int" or ptype == "enum":
if ptype in ("int", "enum"):
return str(int(value))
if ptype == "float":
# Determine decimal places from step
step = spec.get("step", 0.1)
decimals = len(str(step).split(".")[-1]) if "." in str(step) else 0
return f"{float(value):.{decimals}f}"
@@ -117,30 +159,34 @@ class IniBuilder:
period_end: str,
report_dir: Path,
phase: str,
ea_file: Optional[str] = None,
ea_symbol: Optional[str] = None,
ea_timeframe: Optional[str] = None,
optimize_mode: bool = False,
) -> str:
"""Render final INI content as a string."""
ea_cfg = self.cfg["ea"]
mt5_cfg = self.cfg["mt5"]
broker_cfg = self.cfg["broker"]
# Period must be the string name (H1, M30 etc) — NOT the ENUM integer
tf_name = TIMEFRAME_NAMES.get(ea_cfg["timeframe"].upper(), "H1")
# EA identity: prefer explicit args, fall back to config["ea"] for legacy
ea_cfg = self.cfg.get("ea", {})
_ea_file = ea_file or ea_cfg.get("file", "EA")
_symbol = ea_symbol or ea_cfg.get("symbol", "XAUUSD")
_tf_str = ea_timeframe or ea_cfg.get("timeframe", "H1")
tf_name = TIMEFRAME_NAMES.get(_tf_str.upper(), "H1")
# Model: 0=Every Tick (slow), 4=OHLC M1 (fast, reliable for ini-based launch)
model_code = mt5_cfg.get("tester_model", 4)
# Report path must be RELATIVE to the MT5 terminal data folder
# MT5 appends its own base path. Use run_id as the report name.
report_name = f"Optimizer_{run_id}"
model_code = mt5_cfg.get("tester_model", 4)
report_name = f"Optimizer_{run_id}"
opt_value = 2 if optimize_mode else 0 # 2=genetic, 0=single backtest
lines = [
f"; MT5 Optimizer INI — run_id={run_id} phase={phase}",
f"",
f"[Tester]",
f"Expert={ea_cfg['file']}",
f"Symbol={ea_cfg['symbol']}",
f"Expert={_ea_file}",
f"Symbol={_symbol}",
f"Period={tf_name}",
f"Optimization=0",
f"Optimization={opt_value}",
f"Model={model_code}",
f"FromDate={period_start}",
f"ToDate={period_end}",
@@ -155,9 +201,13 @@ class IniBuilder:
f"[TesterInputs]",
]
for name, value in params.items():
formatted = self._format_value(name, value)
lines.append(f"{name}={formatted}")
# Use schema's INI renderer if available
if self._schema is not None:
lines.append(self._schema.to_ini_inputs(params, optimize_mode=optimize_mode))
else:
for name, value in params.items():
formatted = self._format_value(name, value)
lines.append(f"{name}={formatted}")
lines.append("") # trailing newline
return "\n".join(lines)
mt5/runner.py
+29 -24
View File
@@ -15,13 +15,16 @@ import subprocess
import time
import psutil
from pathlib import Path
from typing import Optional
from typing import Optional, TYPE_CHECKING
import yaml
from loguru import logger
from data.models import RunResult
if TYPE_CHECKING:
from ea.registry import EAProfile
# ── Custom Exceptions ─────────────────────────────────────────────────────────
@@ -68,15 +71,12 @@ class MT5Runner:
ini_path: Path,
report_dir: Path,
log_csv_search_dir: Optional[Path] = None,
profile: Optional["EAProfile"] = None,
) -> RunResult:
"""
Full execution pipeline with retry:
1. Kill existing MT5
2. Validate environment
3. Launch fresh MT5
4. Wait for data readiness
5. Poll for report
6. Auto-retry once on failure
Full execution pipeline with retry.
profile: EAProfile used for validation + TradeLog lookup.
Falls back to config['ea'] if not provided (legacy mode).
"""
report_dir.mkdir(parents=True, exist_ok=True)
report_stem = f"Optimizer_{run_id}"
@@ -94,7 +94,7 @@ class MT5Runner:
self._clear_stale_reports(run_id)
# ── Step 2: Pre-run validation ───────────────────────────
self._validate(run_id)
self._validate(run_id, profile=profile)
# ── Step 3: Launch fresh MT5 ─────────────────────────────
proc = self._launch_mt5(run_id, ini_path)
@@ -107,7 +107,9 @@ class MT5Runner:
result = self._wait_for_report(run_id, proc, report_dir, report_stem)
# ── Step 6: Find TradeLogger CSV ─────────────────────────
result.trade_log_csv = self._find_trade_log(run_id, log_csv_search_dir)
result.trade_log_csv = self._find_trade_log(
run_id, log_csv_search_dir, profile=profile
)
logger.success(f"[{run_id}] Run complete. Report: {result.report_xml}")
return result
@@ -151,35 +153,32 @@ class MT5Runner:
# ── Step 2: Pre-run validation ────────────────────────────────────────────
def _validate(self, run_id: str) -> None:
def _validate(self, run_id: str, profile: Optional["EAProfile"] = None) -> None:
"""Validate all required files and directories exist before launch."""
errors = []
# Check terminal executable
if not self.terminal_exe.exists():
errors.append(f"MT5 terminal not found: {self.terminal_exe}")
# Check EA compiled file
ea_name = self.cfg["ea"]["file"]
# EA file: use profile if provided, else fall back to config['ea']
ea_name = profile.ex5_file if profile else self.cfg.get("ea", {}).get("file", "")
symbol = profile.symbol if profile else self.cfg.get("ea", {}).get("symbol", "")
ea_candidates = [
self.appdata_path / "MQL5" / "Experts" / f"{ea_name}.ex5",
self.appdata_path / "MQL5" / "Experts" / f"{ea_name}",
self.appdata_path / "MQL5" / "Experts" / ea_name,
]
ea_found = any(p.exists() for p in ea_candidates)
if not ea_found:
if not any(p.exists() for p in ea_candidates):
errors.append(
f"EA file not found: {ea_name}.ex5 — "
f"ensure you compiled the EA in MetaEditor before running."
)
# Check appdata path
if not self.appdata_path.exists():
errors.append(f"MT5 appdata folder not found: {self.appdata_path}")
# Symbol check (basic — just ensure it's set)
symbol = self.cfg["ea"].get("symbol", "")
if not symbol:
errors.append("Symbol not configured in config.yaml ea.symbol")
errors.append("Symbol not configured (check EAProfile or config.yaml)")
if errors:
for e in errors:
@@ -328,11 +327,17 @@ class MT5Runner:
# ── TradeLogger CSV lookup ────────────────────────────────────────────────
def _find_trade_log(
self, run_id: str, search_dir: Optional[Path]
self, run_id: str, search_dir: Optional[Path],
profile: Optional["EAProfile"] = None,
) -> Optional[Path]:
"""Find the TradeLogger CSV written by the EA during the backtest."""
ea_name = self.cfg["ea"]["file"]
symbol = self.cfg["ea"]["symbol"]
# Use profile if available, else fall back to config['ea']
ea_cfg = self.cfg.get("ea", {})
ea_name = profile.ex5_file if profile else ea_cfg.get("file", "")
symbol = profile.symbol if profile else ea_cfg.get("symbol", "")
if not ea_name or not symbol:
return None
candidates = [
self.mql5_files_dir / f"{ea_name}_{symbol}_TradeLog.csv",