263 lines
10 KiB
Python
263 lines
10 KiB
Python
"""Phase 7 — MT5 bridge: verify a finalist against the MT5 Strategy Tester.
|
|
|
|
Pipeline (doc 07):
|
|
1. Deploy the EA (.ex5 + .mq5) to the terminal's MQL5\\Experts directory.
|
|
2. Generate a .set from the finalist's params (UTF-16-LE).
|
|
3. Generate a tester.ini (symbol, period, dates, model, shutdown).
|
|
4. Launch terminal64.exe /config:tester.ini — the tester runs headless and
|
|
closes itself when done (ShutdownTerminal=1).
|
|
5. Parse the HTML report (UTF-16-LE) the tester writes.
|
|
6. Build a Python-vs-MT5 comparison table + write auto-verification.md.
|
|
|
|
Usage:
|
|
python verify_mt5.py --trials 30 --top-n 2
|
|
|
|
Runs the optimizer first (to get finalists), then verifies each in MT5.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
PROJECT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(PROJECT))
|
|
|
|
from shared.config import get_secret, load_env
|
|
from shared.data.mt5_report import parse_mt5_report
|
|
from shared.mt5_pipeline.compare import build_comparison_table
|
|
from shared.mt5_pipeline.ini_gen import (
|
|
MODEL_OHLC,
|
|
TesterConfig,
|
|
write_tester_ini,
|
|
)
|
|
from shared.mt5_pipeline.runner import run_tester
|
|
from shared.mt5_pipeline.set_gen import write_set_file
|
|
|
|
load_env(PROJECT)
|
|
|
|
# Reuse the optimizer's assembly.
|
|
from run import find_bars_file # noqa: E402
|
|
|
|
from shared.core.engine import SizingInputs # noqa: E402
|
|
from shared.core.metrics import compute_metrics # noqa: E402
|
|
from shared.data.loaders import load_bars # noqa: E402
|
|
from shared.optimizer.objective import ( # noqa: E402
|
|
Constraints,
|
|
ObjectiveConfig,
|
|
build_objective,
|
|
)
|
|
from shared.optimizer.selector import select_diverse_topn # noqa: E402
|
|
from strategies.gold_scalper_pro.instruments import XAUUSD_REAL # noqa: E402
|
|
from strategies.gold_scalper_pro.scalper_engine import ( # noqa: E402
|
|
ScalperEngine,
|
|
engine_kwargs_from_params,
|
|
)
|
|
from strategies.gold_scalper_pro.search_space import ( # noqa: E402
|
|
FROZEN_BASELINE,
|
|
INT_PARAMS,
|
|
SEARCH_SPACE,
|
|
)
|
|
from strategies.gold_scalper_pro.set_mappings import GOLD_SCALPER_MAPPINGS # noqa: E402
|
|
from strategies.gold_scalper_pro.signals import build_signals # noqa: E402
|
|
|
|
import optuna # noqa: E402
|
|
|
|
|
|
def deploy_ea(mt5_data_path: Path) -> None:
|
|
"""Copy GoldScalperPro.ex5 + .mq5 into MQL5\\Experts so the tester finds it."""
|
|
experts_dir = mt5_data_path / "MQL5" / "Experts"
|
|
experts_dir.mkdir(parents=True, exist_ok=True)
|
|
for src_name in ("GoldScalperPro.ex5", "GoldScalperPro.mq5"):
|
|
src = PROJECT / src_name
|
|
dst = experts_dir / src_name
|
|
if src.exists():
|
|
shutil.copy2(src, dst)
|
|
print(f" deployed {src_name} → {dst.relative_to(mt5_data_path)}")
|
|
else:
|
|
raise FileNotFoundError(f"EA source missing: {src}")
|
|
|
|
|
|
def verify_finalist(
|
|
params: dict,
|
|
label: str,
|
|
*,
|
|
bars: "pd.DataFrame",
|
|
deposit: float,
|
|
mt5_install: str,
|
|
mt5_data_path: Path,
|
|
tester_profiles_dir: Path,
|
|
date_from: str,
|
|
date_to: str,
|
|
) -> int:
|
|
"""Verify one finalist in MT5 and print the comparison table."""
|
|
print(f"\n=== verifying {label} ===")
|
|
|
|
# ── 1. Python re-run (fresh snapshot — doc 04 Rule 5) ──────────────────
|
|
pack = build_signals(params, bars, XAUUSD_REAL)
|
|
engine = ScalperEngine()
|
|
result = engine.run(
|
|
bars, pack.signals_long, pack.signals_short,
|
|
pack.sl_prices, pack.tp_prices,
|
|
XAUUSD_REAL, SizingInputs(), deposit,
|
|
**engine_kwargs_from_params(params),
|
|
)
|
|
py_metrics = compute_metrics(result)
|
|
print(f" python: net={py_metrics.net_profit:+,.2f} PF={py_metrics.profit_factor:.2f} "
|
|
f"trades={py_metrics.total_trades} DD={py_metrics.max_equity_dd:.2f}")
|
|
|
|
# ── 2. Write the .set (UTF-16-LE) into the tester profiles dir ────────
|
|
set_name = f"GoldScalperPro_{label}"
|
|
set_path = tester_profiles_dir / f"{set_name}.set"
|
|
write_set_file(params, GOLD_SCALPER_MAPPINGS, set_path)
|
|
print(f" wrote .set → {set_path.name}")
|
|
|
|
# ── 3. Write tester.ini ───────────────────────────────────────────────
|
|
ini_path = tester_profiles_dir / f"{set_name}.ini"
|
|
login = int(get_secret("MT5_DEMO_LOGIN") or 0)
|
|
password = get_secret("MT5_DEMO_PASSWORD")
|
|
server = get_secret("MT5_DEMO_SERVER")
|
|
tcfg = TesterConfig(
|
|
expert=r"Experts\GoldScalperPro.ex5",
|
|
symbol="XAUUSD",
|
|
period="M5",
|
|
model=MODEL_OHLC, # 1-min OHLC for routine verification
|
|
from_date=date_from,
|
|
to_date=date_to,
|
|
deposit=deposit,
|
|
leverage=100,
|
|
report=f"report_{label}",
|
|
shutdown_terminal=True,
|
|
set_file=str(set_path),
|
|
login=login,
|
|
password=password,
|
|
server=server,
|
|
)
|
|
write_tester_ini(tcfg, ini_path)
|
|
print(f" wrote ini → {ini_path.name}")
|
|
|
|
# ── 4. Run the tester (headless; closes itself when done) ─────────────
|
|
print(f" launching MT5 tester (headless, model=OHLC) ...")
|
|
t0 = time.time()
|
|
exit_code, report_path = run_tester(
|
|
ini_path, mt5_install=mt5_install, timeout=900, poll_interval=5.0,
|
|
)
|
|
elapsed = time.time() - t0
|
|
print(f" tester finished in {elapsed:.0f}s exit={exit_code}")
|
|
if report_path is None:
|
|
print(" ✗ no report found — tester may have failed to start")
|
|
return 1
|
|
print(f" report → {report_path}")
|
|
|
|
# ── 5. Parse the report + build comparison ────────────────────────────
|
|
mt5_metrics = parse_mt5_report(report_path)
|
|
# Map MT5 report keys to our Metrics field names for the table.
|
|
mt5_mapped = {
|
|
"net_profit": mt5_metrics.get("Total Net Profit"),
|
|
"profit_factor": mt5_metrics.get("Profit Factor"),
|
|
"total_trades": mt5_metrics.get("Total Trades"),
|
|
"max_equity_dd": mt5_metrics.get("Equity Drawdown Maximal"),
|
|
"win_rate": None, # MT5 report doesn't surface this directly
|
|
"sharpe": mt5_metrics.get("Sharpe Ratio"),
|
|
}
|
|
table = build_comparison_table(py_metrics, mt5_mapped)
|
|
print("\n " + table.replace("\n", "\n "))
|
|
|
|
# ── 6. Write auto-verification.md ─────────────────────────────────────
|
|
out_md = PROJECT / "registry" / f"auto-verification_{label}.md"
|
|
out_md.parent.mkdir(parents=True, exist_ok=True)
|
|
body = (
|
|
f"# Auto-verification: {label}\n\n"
|
|
f"## Parameters\n\n```\n"
|
|
)
|
|
for k, v in params.items():
|
|
body += f" {k} = {v}\n"
|
|
body += "```\n\n## Python vs MT5\n\n" + table
|
|
body += (
|
|
"\n## Decision rule (doc 03 §7)\n"
|
|
"If the MT5 number still clears the bar after the expected fidelity "
|
|
"gap, the finalist is real. If the edge only existed in the optimistic "
|
|
"Python figure, discard it.\n"
|
|
)
|
|
out_md.write_text(body, encoding="utf-8")
|
|
print(f" wrote {out_md.relative_to(PROJECT)}")
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description="Verify GoldScalperPro finalists in MT5.")
|
|
ap.add_argument("--trials", type=int, default=30)
|
|
ap.add_argument("--top-n", type=int, default=2)
|
|
ap.add_argument("--deposit", type=float, default=10000.0)
|
|
args = ap.parse_args()
|
|
|
|
mt5_install = get_secret("MT5_TERMINAL_PATH") or r"C:\Program Files\MetaTrader 5 IC Markets Global"
|
|
mt5_install_dir = str(Path(mt5_install).parent)
|
|
mt5_data_path = Path(get_secret("MT5_DATA_PATH")
|
|
or r"C:\Users\Administrator\AppData\Roaming\MetaQuotes\Terminal"
|
|
r"\010E047102812FC0C18890992854220E")
|
|
tester_profiles_dir = mt5_data_path / "MQL5" / "Profiles" / "Tester"
|
|
tester_profiles_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
bars_path = find_bars_file()
|
|
bars = load_bars(bars_path)
|
|
date_from = bars["timestamp"].iloc[0].strftime("%Y.%m.%d")
|
|
date_to = bars["timestamp"].iloc[-1].strftime("%Y.%m.%d")
|
|
print(f"=== MT5 verification ===")
|
|
print(f"bars : {bars_path.name} ({len(bars):,} bars)")
|
|
print(f"window : {date_from} → {date_to}")
|
|
print(f"deposit: {args.deposit:,.0f} USD")
|
|
|
|
# ── Deploy EA ─────────────────────────────────────────────────────────
|
|
print(f"\ndeploying EA to {mt5_data_path.name}/MQL5/Experts/ ...")
|
|
deploy_ea(mt5_data_path)
|
|
|
|
# ── Run optimizer to get finalists ────────────────────────────────────
|
|
print(f"\noptimizing ({args.trials} trials) ...")
|
|
constraints = Constraints(min_trades=25, min_profit_factor=1.2,
|
|
max_equity_dd_pct=0.40)
|
|
obj_cfg = ObjectiveConfig(
|
|
engine=ScalperEngine(),
|
|
bars=bars,
|
|
instrument=XAUUSD_REAL,
|
|
sizing=SizingInputs(),
|
|
initial_deposit=args.deposit,
|
|
search_space=SEARCH_SPACE,
|
|
int_params=INT_PARAMS,
|
|
frozen_baseline=FROZEN_BASELINE,
|
|
constraints=constraints,
|
|
dd_weight=1.0,
|
|
build_signals=build_signals,
|
|
build_engine_kwargs=engine_kwargs_from_params,
|
|
)
|
|
objective = build_objective(obj_cfg)
|
|
optuna.logging.set_verbosity(optuna.logging.WARNING)
|
|
study = optuna.create_study(direction="maximize",
|
|
sampler=optuna.samplers.TPESampler(seed=42))
|
|
study.optimize(objective, n_trials=args.trials)
|
|
finalists = select_diverse_topn(study, args.top_n, SEARCH_SPACE)
|
|
print(f"selected {len(finalists)} finalists")
|
|
|
|
# ── Verify each finalist ───────────────────────────────────────────────
|
|
for i, t in enumerate(finalists, 1):
|
|
label = f"f{i}"
|
|
verify_finalist(
|
|
t.user_attrs["params"], label,
|
|
bars=bars, deposit=args.deposit,
|
|
mt5_install=mt5_install_dir,
|
|
mt5_data_path=mt5_data_path,
|
|
tester_profiles_dir=tester_profiles_dir,
|
|
date_from=date_from, date_to=date_to,
|
|
)
|
|
|
|
print("\n=== done ===")
|
|
print(f"verification reports in registry/auto-verification_*.md")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|