回测基本一致
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
"""Compare the dry-run MT5 report against Python on the SAME window + deposit.
|
||||
|
||||
The dry-run MT5 report shows the tester was run on 2026.04.16 - 2026.05.08
|
||||
with initial deposit 1000 USD. To make a fair Python-vs-MT5 comparison we
|
||||
re-slice the bars to that exact window and re-run the engine with the same
|
||||
deposit. Then we print the side-by-side table.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from shared.core.engine import SizingInputs
|
||||
from shared.core.metrics import compute_metrics
|
||||
from shared.data.loaders import load_bars
|
||||
from shared.data.mt5_report import parse_mt5_report
|
||||
from shared.mt5_pipeline.compare import build_comparison_table
|
||||
from strategies.gold_scalper_pro.instruments import XAUUSD_REAL
|
||||
from strategies.gold_scalper_pro.scalper_engine import (
|
||||
ScalperEngine,
|
||||
engine_kwargs_from_params,
|
||||
)
|
||||
from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE
|
||||
from strategies.gold_scalper_pro.signals import build_signals
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# ── Parse the MT5 report ──────────────────────────────────────────────
|
||||
report = PROJECT / "reports" / "ReportTester-52845377.html"
|
||||
mt5 = parse_mt5_report(report)
|
||||
print("=== MT5 report (dry run) ===")
|
||||
for k, v in mt5.items():
|
||||
if k.startswith("_"):
|
||||
continue
|
||||
print(f" {k:24s}: {v}")
|
||||
# The MT5 window + deposit (parsed from the report's _all dict).
|
||||
all_fields = mt5["_all"]
|
||||
window_label = all_fields.get("期间:", "")
|
||||
print(f" window (raw) : {window_label}")
|
||||
deposit = mt5.get("Initial Deposit") or 1000
|
||||
print(f" initial deposit: {deposit}")
|
||||
|
||||
# ── Slice Python bars to the same window ─────────────────────────────
|
||||
bars = load_bars(PROJECT / "data" / "XAUUSD_M5_2024-06-26_2026-06-26.parquet")
|
||||
# MT5 tester's ToDate is exclusive of the day (uses 00:00 of that day),
|
||||
# so the actual data ends at 2026.05.08 00:00 (not 23:59). Match it exactly.
|
||||
start = pd.Timestamp("2026-04-16 00:00:00")
|
||||
end = pd.Timestamp("2026-05-08 00:00:00")
|
||||
window = bars[(bars["timestamp"] >= start) & (bars["timestamp"] < end)].reset_index(drop=True)
|
||||
print(f"\n=== Python (matched window) ===")
|
||||
print(f" bars : {len(window):,}")
|
||||
print(f" window : {window['timestamp'].iloc[0]} → {window['timestamp'].iloc[-1]}")
|
||||
print(f" deposit : {deposit}")
|
||||
|
||||
# ── Run the engine on the matched window ─────────────────────────────
|
||||
# Override InpAtrPeriod to 15 to match the MT5 manual run (user changed
|
||||
# it from the .set's 14 to 15 in the tester UI). RSI stays at 14.
|
||||
params = dict(FROZEN_BASELINE)
|
||||
params["InpAtrPeriod"] = 15
|
||||
pack = build_signals(params, window, XAUUSD_REAL)
|
||||
# Load M1 data for tick-level exit simulation (closes the bar-level
|
||||
# optimism gap on BE/trailing — doc 03 §7).
|
||||
m1_path = PROJECT / "data" / "XAUUSD_M1_2024-06-26_2026-06-26.parquet"
|
||||
m1_bars = load_bars(m1_path) if m1_path.exists() else None
|
||||
if m1_bars is not None:
|
||||
# Slice M1 to the same window as M5 (exclusive end, matching MT5).
|
||||
m1_bars = m1_bars[
|
||||
(m1_bars["timestamp"] >= start) & (m1_bars["timestamp"] < end)
|
||||
].reset_index(drop=True)
|
||||
print(f" m1 bars : {len(m1_bars):,} (tick-level exit simulation ON)")
|
||||
engine = ScalperEngine()
|
||||
result = engine.run(
|
||||
window, pack.signals_long, pack.signals_short,
|
||||
pack.sl_prices, pack.tp_prices,
|
||||
XAUUSD_REAL, SizingInputs(), float(deposit),
|
||||
m1_bars=m1_bars,
|
||||
**engine_kwargs_from_params(params),
|
||||
)
|
||||
py = compute_metrics(result)
|
||||
print(f" trades : {py.total_trades}")
|
||||
print(f" net : {py.net_profit:+.2f}")
|
||||
print(f" PF : {py.profit_factor:.2f}")
|
||||
print(f" DD : {py.max_equity_dd:.2f} ({py.max_equity_dd_pct:.2%})")
|
||||
|
||||
# ── Build the comparison table ───────────────────────────────────────
|
||||
# Map Python Metrics → keys the compare table expects.
|
||||
py_mapped = {
|
||||
"net_profit": py.net_profit,
|
||||
"profit_factor": py.profit_factor,
|
||||
"total_trades": py.total_trades,
|
||||
"max_equity_dd": py.max_equity_dd,
|
||||
"win_rate": py.win_rate,
|
||||
"sharpe": py.sharpe,
|
||||
}
|
||||
mt5_mapped = {
|
||||
"net_profit": mt5.get("Total Net Profit"),
|
||||
"profit_factor": mt5.get("Profit Factor"),
|
||||
"total_trades": mt5.get("Total Trades"),
|
||||
"max_equity_dd": mt5.get("Equity Drawdown Maximal"),
|
||||
"win_rate": None,
|
||||
"sharpe": mt5.get("Sharpe Ratio"),
|
||||
}
|
||||
print(f"\n=== Python vs MT5 (matched window {start.date()} → {end.date()}) ===")
|
||||
print(build_comparison_table(py_mapped, mt5_mapped))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Diagnose the exit-side PnL gap between Python and MT5.
|
||||
|
||||
MT5: gross profit 185.01, gross loss -123.78, 92 trades, net 61.23.
|
||||
Python: 92 trades, net 118.79. So Python's gross profit is much higher
|
||||
OR its gross loss is much smaller. Find out which by dumping Python's
|
||||
gross profit / gross loss + per-reason breakdown.
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
import pandas as pd
|
||||
from shared.core.engine import SizingInputs
|
||||
from shared.core.metrics import compute_metrics
|
||||
from shared.data.loaders import load_bars
|
||||
from strategies.gold_scalper_pro.instruments import XAUUSD_REAL
|
||||
from strategies.gold_scalper_pro.scalper_engine import ScalperEngine, engine_kwargs_from_params
|
||||
from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE
|
||||
from strategies.gold_scalper_pro.signals import build_signals
|
||||
import collections
|
||||
|
||||
bars = load_bars(PROJECT / "data" / "XAUUSD_M5_2024-06-26_2026-06-26.parquet")
|
||||
start = pd.Timestamp("2026-04-16 00:00:00")
|
||||
end = pd.Timestamp("2026-05-08 00:00:00")
|
||||
window = bars[(bars["timestamp"] >= start) & (bars["timestamp"] < end)].reset_index(drop=True)
|
||||
|
||||
params = dict(FROZEN_BASELINE)
|
||||
params["InpAtrPeriod"] = 15
|
||||
pack = build_signals(params, window, XAUUSD_REAL)
|
||||
engine = ScalperEngine()
|
||||
result = engine.run(window, pack.signals_long, pack.signals_short,
|
||||
pack.sl_prices, pack.tp_prices, XAUUSD_REAL, SizingInputs(),
|
||||
1000.0, **engine_kwargs_from_params(params))
|
||||
|
||||
wins = [t.pnl for t in result.trades if t.pnl > 0]
|
||||
losses = [t.pnl for t in result.trades if t.pnl < 0]
|
||||
print(f"=== Python exit-side breakdown ({len(result.trades)} trades) ===")
|
||||
print(f" gross profit : {sum(wins):+.2f} ({len(wins)} trades)")
|
||||
print(f" gross loss : {sum(losses):+.2f} ({len(losses)} trades)")
|
||||
print(f" net : {sum(t.pnl for t in result.trades):+.2f}")
|
||||
print()
|
||||
print(f"=== MT5 (from report) ===")
|
||||
print(f" gross profit : +185.01")
|
||||
print(f" gross loss : -123.78")
|
||||
print(f" net : +61.23")
|
||||
print()
|
||||
|
||||
by_reason = collections.defaultdict(list)
|
||||
for t in result.trades:
|
||||
by_reason[t.exit_reason].append(t.pnl)
|
||||
print("=== Python PnL by exit reason ===")
|
||||
for reason, pnls in sorted(by_reason.items()):
|
||||
arr = __import__("numpy").array(pnls)
|
||||
print(f" {reason:14s} n={len(arr):3d} sum={arr.sum():+8.2f} "
|
||||
f"mean={arr.mean():+6.2f} min={arr.min():+7.2f} max={arr.max():+7.2f}")
|
||||
|
||||
# Distribution of win sizes — MT5's avg win = 185.01 / n_wins; need n_wins from MT5.
|
||||
# MT5 win rate unknown, but PF = GP/|GL| = 185.01/123.78 = 1.494 → matches.
|
||||
print(f"\n=== Python win/loss sizes ===")
|
||||
print(f" avg win : {sum(wins)/len(wins):+.2f} (n={len(wins)})")
|
||||
print(f" avg loss : {sum(losses)/len(losses):+.2f} (n={len(losses)})")
|
||||
print(f" Python PF: {sum(wins)/abs(sum(losses)):.2f}")
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Download XAUUSD M5 history to Parquet (doc 02 §3, doc 07 §6).
|
||||
|
||||
Pulls the full M5 window from the running MT5 terminal and saves it as a
|
||||
Parquet file in ``data/``. Re-runs of the engine then load from Parquet (fast,
|
||||
compact, no MT5 connection needed). M5 is the EA's signal timeframe — higher
|
||||
timeframes are resampled in Python from this M1/M5 base.
|
||||
|
||||
Connects to the already-running, manually-logged-in terminal (initialize()
|
||||
with no path → reuse). Keep the terminal UI open while this runs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
from shared.config import get_secret, load_env
|
||||
|
||||
load_env(PROJECT)
|
||||
|
||||
import MetaTrader5 as mt5 # type: ignore
|
||||
import pandas as pd
|
||||
|
||||
SYMBOL = "XAUUSD"
|
||||
TIMEFRAME = "M5"
|
||||
# Full window: 2 years back to today (matches the history depth probe).
|
||||
START = (datetime.now() - timedelta(days=730)).strftime("%Y-%m-%d")
|
||||
END = datetime.now().strftime("%Y-%m-%d")
|
||||
OUT = PROJECT / "data" / f"{SYMBOL}_{TIMEFRAME}_{START}_{END}.parquet"
|
||||
|
||||
|
||||
def connect(retries: int = 5, sleep_s: float = 5.0) -> bool:
|
||||
"""Connect to the running terminal (initialize() with no path → reuse)."""
|
||||
for attempt in range(1, retries + 1):
|
||||
if not mt5.initialize():
|
||||
print(f" initialize() attempt {attempt}/{retries} failed: {mt5.last_error()}")
|
||||
time.sleep(sleep_s)
|
||||
continue
|
||||
login = int(get_secret("MT5_DEMO_LOGIN") or 0)
|
||||
password = get_secret("MT5_DEMO_PASSWORD")
|
||||
server = get_secret("MT5_DEMO_SERVER")
|
||||
if not mt5.login(login, password=password, server=server):
|
||||
print(f" login() attempt {attempt}/{retries} failed: {mt5.last_error()}")
|
||||
mt5.shutdown()
|
||||
time.sleep(sleep_s)
|
||||
continue
|
||||
print(f" connected: login={login} server={server}")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not connect():
|
||||
print("could not connect — is the terminal running and logged in?")
|
||||
return 1
|
||||
try:
|
||||
tf = getattr(mt5, f"TIMEFRAME_{TIMEFRAME}")
|
||||
print(f"downloading {SYMBOL} {TIMEFRAME} {START} → {END} ...")
|
||||
rates = mt5.copy_rates_range(
|
||||
SYMBOL, tf, pd.Timestamp(START), pd.Timestamp(END)
|
||||
)
|
||||
if rates is None or len(rates) == 0:
|
||||
print(f" no bars returned: {mt5.last_error()}")
|
||||
return 2
|
||||
df = pd.DataFrame(rates)
|
||||
df["timestamp"] = pd.to_datetime(df["time"], unit="s", utc=False)
|
||||
df = df[["timestamp", "open", "high", "low", "close", "spread"]]
|
||||
df = df.sort_values("timestamp").reset_index(drop=True)
|
||||
# Deduplicate (MT5 occasionally returns overlapping bars near boundaries).
|
||||
df = df.drop_duplicates(subset=["timestamp"]).reset_index(drop=True)
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_parquet(OUT, index=False)
|
||||
print(f"saved {len(df):,} bars → {OUT.relative_to(PROJECT)}")
|
||||
print(f"range: {df['timestamp'].iloc[0]} → {df['timestamp'].iloc[-1]}")
|
||||
print(f"spread stats: min={df['spread'].min()} max={df['spread'].max()} "
|
||||
f"median={df['spread'].median():.1f}")
|
||||
return 0
|
||||
finally:
|
||||
mt5.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Download XAUUSD M1 history to Parquet for tick-level simulation.
|
||||
|
||||
M1 bars are the finest granularity MT5 exposes via copy_rates_range. We use
|
||||
each M1 bar's OHLC as 4 synthetic ticks (open→high→low→close for longs,
|
||||
open→low→high→close for shorts) to drive BE/trailing precisely inside
|
||||
each M5 bar.
|
||||
|
||||
Downloads the same 2-year window as the M5 file so the two align.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
from shared.config import get_secret, load_env
|
||||
load_env(PROJECT)
|
||||
|
||||
import MetaTrader5 as mt5 # type: ignore
|
||||
import pandas as pd
|
||||
|
||||
SYMBOL = "XAUUSD"
|
||||
TIMEFRAME = "M1"
|
||||
START = (datetime.now() - timedelta(days=730)).strftime("%Y-%m-%d")
|
||||
END = datetime.now().strftime("%Y-%m-%d")
|
||||
OUT = PROJECT / "data" / f"{SYMBOL}_{TIMEFRAME}_{START}_{END}.parquet"
|
||||
|
||||
|
||||
def connect(retries: int = 5, sleep_s: float = 5.0) -> bool:
|
||||
for attempt in range(1, retries + 1):
|
||||
if not mt5.initialize():
|
||||
print(f" initialize() attempt {attempt}/{retries} failed: {mt5.last_error()}")
|
||||
time.sleep(sleep_s)
|
||||
continue
|
||||
login = int(get_secret("MT5_DEMO_LOGIN") or 0)
|
||||
password = get_secret("MT5_DEMO_PASSWORD")
|
||||
server = get_secret("MT5_DEMO_SERVER")
|
||||
if not mt5.login(login, password=password, server=server):
|
||||
print(f" login() attempt {attempt}/{retries} failed: {mt5.last_error()}")
|
||||
mt5.shutdown()
|
||||
time.sleep(sleep_s)
|
||||
continue
|
||||
print(f" connected: login={login} server={server}")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not connect():
|
||||
print("could not connect — is the terminal running and logged in?")
|
||||
return 1
|
||||
try:
|
||||
tf = getattr(mt5, f"TIMEFRAME_{TIMEFRAME}")
|
||||
print(f"downloading {SYMBOL} {TIMEFRAME} {START} → {END} ...")
|
||||
rates = mt5.copy_rates_range(
|
||||
SYMBOL, tf, pd.Timestamp(START), pd.Timestamp(END)
|
||||
)
|
||||
if rates is None or len(rates) == 0:
|
||||
print(f" no bars returned: {mt5.last_error()}")
|
||||
return 2
|
||||
df = pd.DataFrame(rates)
|
||||
df["timestamp"] = pd.to_datetime(df["time"], unit="s", utc=False)
|
||||
df = df[["timestamp", "open", "high", "low", "close", "spread"]]
|
||||
df = df.sort_values("timestamp").reset_index(drop=True)
|
||||
df = df.drop_duplicates(subset=["timestamp"]).reset_index(drop=True)
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_parquet(OUT, index=False)
|
||||
print(f"saved {len(df):,} bars → {OUT.relative_to(PROJECT)}")
|
||||
print(f"range: {df['timestamp'].iloc[0]} → {df['timestamp'].iloc[-1]}")
|
||||
return 0
|
||||
finally:
|
||||
mt5.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Query XAUUSD symbol spec from MT5 and dump it as an InstrumentConfig sketch.
|
||||
|
||||
Run with the venv python. Uses the MetaTrader5 package (Topology A) to read
|
||||
the symbol specification (digits, point, tick value, contract size, volume
|
||||
steps) — these come from the broker, not guesses (doc 05 §1). Also downloads
|
||||
a short M5 history window for the first validation pass (doc 03 §8).
|
||||
|
||||
IPC-timeout workarounds (Stack Overflow #66492735):
|
||||
- Use forward slashes in the terminal path (backslashes trigger -10005).
|
||||
- Split initialize(path) and login() into two steps (one-shot initialize with
|
||||
login/password/server is fragile).
|
||||
- Allow reusing an already-running terminal instance (same path = same IPC).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Make the project importable when run as a script.
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
from shared.config import get_secret, load_env
|
||||
|
||||
load_env(PROJECT)
|
||||
|
||||
import MetaTrader5 as mt5 # type: ignore
|
||||
import pandas as pd
|
||||
|
||||
# Forward slashes — backslashes in this path trigger IPC timeout (-10005).
|
||||
TERMINAL = "C:/Program Files/MetaTrader 5 IC Markets Global/terminal64.exe"
|
||||
SYMBOL = "XAUUSD"
|
||||
|
||||
|
||||
def connect(retries: int = 3, sleep_s: float = 5.0) -> bool:
|
||||
"""Connect in two steps: initialize() → login(login,password,server).
|
||||
|
||||
IMPORTANT: call initialize() with NO path argument. Passing a path makes
|
||||
the package spawn a NEW headless terminal at that path, which then sits
|
||||
waiting for an interactive login it can never complete (IPC timeout -10005).
|
||||
Calling initialize() with no args CONNECTS to the already-running terminal
|
||||
(the one with the UI you logged into manually).
|
||||
"""
|
||||
for attempt in range(1, retries + 1):
|
||||
# Step 1: connect to the running terminal (no path → reuse existing).
|
||||
if not mt5.initialize():
|
||||
err = mt5.last_error()
|
||||
print(f" initialize() attempt {attempt}/{retries} failed: {err}")
|
||||
time.sleep(sleep_s)
|
||||
continue
|
||||
# Step 2: explicit login with the demo account (re-auth is safe).
|
||||
login = int(get_secret("MT5_DEMO_LOGIN") or 0)
|
||||
password = get_secret("MT5_DEMO_PASSWORD")
|
||||
server = get_secret("MT5_DEMO_SERVER")
|
||||
if not mt5.login(login, password=password, server=server):
|
||||
err = mt5.last_error()
|
||||
print(f" login() attempt {attempt}/{retries} failed: {err}")
|
||||
mt5.shutdown()
|
||||
time.sleep(sleep_s)
|
||||
continue
|
||||
print(f" connected: login={login} server={server}")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not connect(retries=5, sleep_s=10.0):
|
||||
print("could not connect to MT5 after retries")
|
||||
print("hint: confirm the demo account logs in manually in the terminal first;")
|
||||
print(" if it does, the IPC issue is path/instance related, not account.")
|
||||
return 1
|
||||
try:
|
||||
info = mt5.symbol_info(SYMBOL)
|
||||
if info is None:
|
||||
print(f"symbol_info({SYMBOL}) returned None — is the symbol in Market Watch?")
|
||||
return 2
|
||||
print(f"=== {SYMBOL} specification (IC Markets) ===")
|
||||
fields = [
|
||||
"name", "digits", "point", "trade_tick_size", "trade_tick_value",
|
||||
"trade_contract_size", "volume_min", "volume_step", "volume_max",
|
||||
"spread", "trade_stops_level", "swap_mode", "swap_long", "swap_short",
|
||||
"swap_rollover3days",
|
||||
]
|
||||
for f in fields:
|
||||
print(f" {f:24s} = {getattr(info, f, '<n/a>')}")
|
||||
# Triple-swap weekday: MT5 swap_rollover3days is 0=Mon..6=Sun.
|
||||
|
||||
print("\n=== short M5 history probe (last 5 bars) ===")
|
||||
rates = mt5.copy_rates_from_pos(SYMBOL, mt5.TIMEFRAME_M5, 0, 5)
|
||||
if rates is None or len(rates) == 0:
|
||||
print(" no rates returned")
|
||||
else:
|
||||
df = pd.DataFrame(rates)
|
||||
df["timestamp"] = pd.to_datetime(df["time"], unit="s")
|
||||
print(df[["timestamp", "open", "high", "low", "close", "spread"]].to_string(index=False))
|
||||
|
||||
print("\n=== available history depth (count bars in last 2 years) ===")
|
||||
from datetime import datetime, timedelta
|
||||
end = datetime.now()
|
||||
start = end - timedelta(days=730)
|
||||
n = mt5.copy_rates_range(SYMBOL, mt5.TIMEFRAME_M5, start, end)
|
||||
print(f" M5 bars {start.date()} → {end.date()}: {0 if n is None else len(n)}")
|
||||
return 0
|
||||
finally:
|
||||
mt5.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Smoke test: run the scalper engine end-to-end on real XAUUSD bars.
|
||||
|
||||
Uses the frozen baseline params (the saved .set config). Verifies the engine
|
||||
+ signals + metrics produce a sane result (trades, PnL, drawdown) before we
|
||||
wire the optimizer. This is the Phase 4 validation gate — not yet the MT5
|
||||
fidelity check (that's Phase 7).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from shared.core.engine import SizingInputs
|
||||
from shared.core.metrics import compute_metrics
|
||||
from shared.data.loaders import load_bars
|
||||
from strategies.gold_scalper_pro.instruments import XAUUSD_REAL
|
||||
from strategies.gold_scalper_pro.scalper_engine import ScalperConfig, ScalperEngine
|
||||
from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE
|
||||
from strategies.gold_scalper_pro.signals import build_signals
|
||||
|
||||
|
||||
def main() -> int:
|
||||
bars_path = PROJECT / "data" / "XAUUSD_M5_2024-06-26_2026-06-26.parquet"
|
||||
print(f"loading {bars_path.name} ...")
|
||||
bars = load_bars(bars_path)
|
||||
print(f" {len(bars):,} bars {bars['timestamp'].iloc[0]} → {bars['timestamp'].iloc[-1]}")
|
||||
|
||||
print("\nbuilding signals (frozen baseline params) ...")
|
||||
pack = build_signals(FROZEN_BASELINE, bars, XAUUSD_REAL)
|
||||
n_long = int(pack.signals_long.sum())
|
||||
n_short = int(pack.signals_short.sum())
|
||||
print(f" long signals : {n_long}")
|
||||
print(f" short signals: {n_short}")
|
||||
|
||||
# Build ScalperConfig from frozen baseline (mirrors EA inputs).
|
||||
cfg = ScalperConfig(
|
||||
use_break_even=FROZEN_BASELINE["InpUseBreakEven"],
|
||||
use_trailing=FROZEN_BASELINE["InpUseTrailing"],
|
||||
use_session=FROZEN_BASELINE["InpUseSession"],
|
||||
session_start_hour=FROZEN_BASELINE["InpSessionStartHour"],
|
||||
session_end_hour=FROZEN_BASELINE["InpSessionEndHour"],
|
||||
max_positions=FROZEN_BASELINE["InpMaxPositions"],
|
||||
max_trades_per_day=FROZEN_BASELINE["InpMaxTradesPerDay"],
|
||||
daily_loss_limit_pct=FROZEN_BASELINE["InpDailyLossLimit"],
|
||||
daily_profit_target_pct=FROZEN_BASELINE["InpDailyProfitTarget"],
|
||||
min_seconds_between=FROZEN_BASELINE["InpMinSecondsBetween"],
|
||||
sizing_mode=FROZEN_BASELINE["InpSizingMode"],
|
||||
fixed_lots=FROZEN_BASELINE["InpFixedLots"],
|
||||
risk_percent=FROZEN_BASELINE["InpRiskPercent"],
|
||||
break_even_points=FROZEN_BASELINE["InpBreakEvenPoints"],
|
||||
break_even_lock=FROZEN_BASELINE["InpBreakEvenLock"],
|
||||
trail_start_points=FROZEN_BASELINE["InpTrailStartPoints"],
|
||||
trail_step_points=FROZEN_BASELINE["InpTrailStepPoints"],
|
||||
)
|
||||
sizing = SizingInputs() # unused — sizing lives in ScalperConfig for this EA
|
||||
|
||||
print("\nrunning engine ...")
|
||||
engine = ScalperEngine()
|
||||
result = engine.run(
|
||||
bars, pack.signals_long, pack.signals_short,
|
||||
pack.sl_prices, pack.tp_prices,
|
||||
XAUUSD_REAL, sizing, initial_deposit=10000.0,
|
||||
scalper_cfg=cfg,
|
||||
)
|
||||
|
||||
metrics = compute_metrics(result, periods_per_year=252 * 24 * 12) # M5 → ~72/year
|
||||
print("\n=== result (frozen baseline) ===")
|
||||
print(f" trades : {metrics.total_trades}")
|
||||
print(f" net profit : {metrics.net_profit:,.2f}")
|
||||
print(f" profit factor : {metrics.profit_factor:.2f}")
|
||||
print(f" win rate : {metrics.win_rate:.2%}")
|
||||
print(f" equity DD max : {metrics.max_equity_dd:,.2f} ({metrics.max_equity_dd_pct:.2%})")
|
||||
print(f" sharpe : {metrics.sharpe:.2f}")
|
||||
if result.trades:
|
||||
reasons = {}
|
||||
for t in result.trades:
|
||||
reasons[t.exit_reason] = reasons.get(t.exit_reason, 0) + 1
|
||||
print(f" exit reasons : {reasons}")
|
||||
print(f" final balance : {result.final_balance:,.2f}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Trade-by-trade comparison to locate the PnL gap source.
|
||||
|
||||
Both sides now produce 92 trades on the same window. This script dumps the
|
||||
first ~20 trades from each side side-by-side so we can see WHERE the PnL
|
||||
diverges (entry price? exit price? lots? swap?).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from shared.core.engine import SizingInputs
|
||||
from shared.core.metrics import compute_metrics
|
||||
from shared.data.loaders import load_bars
|
||||
from shared.data.mt5_report import parse_mt5_report
|
||||
from strategies.gold_scalper_pro.instruments import XAUUSD_REAL
|
||||
from strategies.gold_scalper_pro.scalper_engine import (
|
||||
ScalperEngine,
|
||||
engine_kwargs_from_params,
|
||||
)
|
||||
from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE
|
||||
from strategies.gold_scalper_pro.signals import build_signals
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# ── Python trades ─────────────────────────────────────────────────────
|
||||
bars = load_bars(PROJECT / "data" / "XAUUSD_M5_2024-06-26_2026-06-26.parquet")
|
||||
start = pd.Timestamp("2026-04-16 00:00:00")
|
||||
end = pd.Timestamp("2026-05-08 00:00:00")
|
||||
window = bars[(bars["timestamp"] >= start) & (bars["timestamp"] < end)].reset_index(drop=True)
|
||||
pack = build_signals(FROZEN_BASELINE, window, XAUUSD_REAL)
|
||||
engine = ScalperEngine()
|
||||
result = engine.run(
|
||||
window, pack.signals_long, pack.signals_short,
|
||||
pack.sl_prices, pack.tp_prices,
|
||||
XAUUSD_REAL, SizingInputs(), 1000.0,
|
||||
**engine_kwargs_from_params(FROZEN_BASELINE),
|
||||
)
|
||||
print("=== Python first 20 trades ===")
|
||||
print(f"{'#':>3
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Phase 7 — MT5 bridge: verify a finalist against the MT5 Strategy Tester.
|
||||
|
||||
Pipeline (doc 07):
|
||||
1. Deploy the EA (.ex5 + .mq5) to the terminal's MQL5\\Experts directory.
|
||||
2. Generate a .set from the finalist's params (UTF-16-LE).
|
||||
3. Generate a tester.ini (symbol, period, dates, model, shutdown).
|
||||
4. Launch terminal64.exe /config:tester.ini — the tester runs headless and
|
||||
closes itself when done (ShutdownTerminal=1).
|
||||
5. Parse the HTML report (UTF-16-LE) the tester writes.
|
||||
6. Build a Python-vs-MT5 comparison table + write auto-verification.md.
|
||||
|
||||
Usage:
|
||||
python verify_mt5.py --trials 30 --top-n 2
|
||||
|
||||
Runs the optimizer first (to get finalists), then verifies each in MT5.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
from shared.config import get_secret, load_env
|
||||
from shared.data.mt5_report import parse_mt5_report
|
||||
from shared.mt5_pipeline.compare import build_comparison_table
|
||||
from shared.mt5_pipeline.ini_gen import (
|
||||
MODEL_OHLC,
|
||||
TesterConfig,
|
||||
write_tester_ini,
|
||||
)
|
||||
from shared.mt5_pipeline.runner import run_tester
|
||||
from shared.mt5_pipeline.set_gen import write_set_file
|
||||
|
||||
load_env(PROJECT)
|
||||
|
||||
# Reuse the optimizer's assembly.
|
||||
from run import find_bars_file # noqa: E402
|
||||
|
||||
from shared.core.engine import SizingInputs # noqa: E402
|
||||
from shared.core.metrics import compute_metrics # noqa: E402
|
||||
from shared.data.loaders import load_bars # noqa: E402
|
||||
from shared.optimizer.objective import ( # noqa: E402
|
||||
Constraints,
|
||||
ObjectiveConfig,
|
||||
build_objective,
|
||||
)
|
||||
from shared.optimizer.selector import select_diverse_topn # noqa: E402
|
||||
from strategies.gold_scalper_pro.instruments import XAUUSD_REAL # noqa: E402
|
||||
from strategies.gold_scalper_pro.scalper_engine import ( # noqa: E402
|
||||
ScalperEngine,
|
||||
engine_kwargs_from_params,
|
||||
)
|
||||
from strategies.gold_scalper_pro.search_space import ( # noqa: E402
|
||||
FROZEN_BASELINE,
|
||||
INT_PARAMS,
|
||||
SEARCH_SPACE,
|
||||
)
|
||||
from strategies.gold_scalper_pro.set_mappings import GOLD_SCALPER_MAPPINGS # noqa: E402
|
||||
from strategies.gold_scalper_pro.signals import build_signals # noqa: E402
|
||||
|
||||
import optuna # noqa: E402
|
||||
|
||||
|
||||
def deploy_ea(mt5_data_path: Path) -> None:
|
||||
"""Copy GoldScalperPro.ex5 + .mq5 into MQL5\\Experts so the tester finds it."""
|
||||
experts_dir = mt5_data_path / "MQL5" / "Experts"
|
||||
experts_dir.mkdir(parents=True, exist_ok=True)
|
||||
for src_name in ("GoldScalperPro.ex5", "GoldScalperPro.mq5"):
|
||||
src = PROJECT / src_name
|
||||
dst = experts_dir / src_name
|
||||
if src.exists():
|
||||
shutil.copy2(src, dst)
|
||||
print(f" deployed {src_name} → {dst.relative_to(mt5_data_path)}")
|
||||
else:
|
||||
raise FileNotFoundError(f"EA source missing: {src}")
|
||||
|
||||
|
||||
def verify_finalist(
|
||||
params: dict,
|
||||
label: str,
|
||||
*,
|
||||
bars: "pd.DataFrame",
|
||||
deposit: float,
|
||||
mt5_install: str,
|
||||
mt5_data_path: Path,
|
||||
tester_profiles_dir: Path,
|
||||
date_from: str,
|
||||
date_to: str,
|
||||
) -> int:
|
||||
"""Verify one finalist in MT5 and print the comparison table."""
|
||||
print(f"\n=== verifying {label} ===")
|
||||
|
||||
# ── 1. Python re-run (fresh snapshot — doc 04 Rule 5) ──────────────────
|
||||
pack = build_signals(params, bars, XAUUSD_REAL)
|
||||
engine = ScalperEngine()
|
||||
result = engine.run(
|
||||
bars, pack.signals_long, pack.signals_short,
|
||||
pack.sl_prices, pack.tp_prices,
|
||||
XAUUSD_REAL, SizingInputs(), deposit,
|
||||
**engine_kwargs_from_params(params),
|
||||
)
|
||||
py_metrics = compute_metrics(result)
|
||||
print(f" python: net={py_metrics.net_profit:+,.2f} PF={py_metrics.profit_factor:.2f} "
|
||||
f"trades={py_metrics.total_trades} DD={py_metrics.max_equity_dd:.2f}")
|
||||
|
||||
# ── 2. Write the .set (UTF-16-LE) into the tester profiles dir ────────
|
||||
set_name = f"GoldScalperPro_{label}"
|
||||
set_path = tester_profiles_dir / f"{set_name}.set"
|
||||
write_set_file(params, GOLD_SCALPER_MAPPINGS, set_path)
|
||||
print(f" wrote .set → {set_path.name}")
|
||||
|
||||
# ── 3. Write tester.ini ───────────────────────────────────────────────
|
||||
ini_path = tester_profiles_dir / f"{set_name}.ini"
|
||||
login = int(get_secret("MT5_DEMO_LOGIN") or 0)
|
||||
password = get_secret("MT5_DEMO_PASSWORD")
|
||||
server = get_secret("MT5_DEMO_SERVER")
|
||||
tcfg = TesterConfig(
|
||||
expert=r"Experts\GoldScalperPro.ex5",
|
||||
symbol="XAUUSD",
|
||||
period="M5",
|
||||
model=MODEL_OHLC, # 1-min OHLC for routine verification
|
||||
from_date=date_from,
|
||||
to_date=date_to,
|
||||
deposit=deposit,
|
||||
leverage=100,
|
||||
report=f"report_{label}",
|
||||
shutdown_terminal=True,
|
||||
set_file=str(set_path),
|
||||
login=login,
|
||||
password=password,
|
||||
server=server,
|
||||
)
|
||||
write_tester_ini(tcfg, ini_path)
|
||||
print(f" wrote ini → {ini_path.name}")
|
||||
|
||||
# ── 4. Run the tester (headless; closes itself when done) ─────────────
|
||||
print(f" launching MT5 tester (headless, model=OHLC) ...")
|
||||
t0 = time.time()
|
||||
exit_code, report_path = run_tester(
|
||||
ini_path, mt5_install=mt5_install, timeout=900, poll_interval=5.0,
|
||||
)
|
||||
elapsed = time.time() - t0
|
||||
print(f" tester finished in {elapsed:.0f}s exit={exit_code}")
|
||||
if report_path is None:
|
||||
print(" ✗ no report found — tester may have failed to start")
|
||||
return 1
|
||||
print(f" report → {report_path}")
|
||||
|
||||
# ── 5. Parse the report + build comparison ────────────────────────────
|
||||
mt5_metrics = parse_mt5_report(report_path)
|
||||
# Map MT5 report keys to our Metrics field names for the table.
|
||||
mt5_mapped = {
|
||||
"net_profit": mt5_metrics.get("Total Net Profit"),
|
||||
"profit_factor": mt5_metrics.get("Profit Factor"),
|
||||
"total_trades": mt5_metrics.get("Total Trades"),
|
||||
"max_equity_dd": mt5_metrics.get("Equity Drawdown Maximal"),
|
||||
"win_rate": None, # MT5 report doesn't surface this directly
|
||||
"sharpe": mt5_metrics.get("Sharpe Ratio"),
|
||||
}
|
||||
table = build_comparison_table(py_metrics, mt5_mapped)
|
||||
print("\n " + table.replace("\n", "\n "))
|
||||
|
||||
# ── 6. Write auto-verification.md ─────────────────────────────────────
|
||||
out_md = PROJECT / "registry" / f"auto-verification_{label}.md"
|
||||
out_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
body = (
|
||||
f"# Auto-verification: {label}\n\n"
|
||||
f"## Parameters\n\n```\n"
|
||||
)
|
||||
for k, v in params.items():
|
||||
body += f" {k} = {v}\n"
|
||||
body += "```\n\n## Python vs MT5\n\n" + table
|
||||
body += (
|
||||
"\n## Decision rule (doc 03 §7)\n"
|
||||
"If the MT5 number still clears the bar after the expected fidelity "
|
||||
"gap, the finalist is real. If the edge only existed in the optimistic "
|
||||
"Python figure, discard it.\n"
|
||||
)
|
||||
out_md.write_text(body, encoding="utf-8")
|
||||
print(f" wrote {out_md.relative_to(PROJECT)}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Verify GoldScalperPro finalists in MT5.")
|
||||
ap.add_argument("--trials", type=int, default=30)
|
||||
ap.add_argument("--top-n", type=int, default=2)
|
||||
ap.add_argument("--deposit", type=float, default=10000.0)
|
||||
args = ap.parse_args()
|
||||
|
||||
mt5_install = get_secret("MT5_TERMINAL_PATH") or r"C:\Program Files\MetaTrader 5 IC Markets Global"
|
||||
mt5_install_dir = str(Path(mt5_install).parent)
|
||||
mt5_data_path = Path(get_secret("MT5_DATA_PATH")
|
||||
or r"C:\Users\Administrator\AppData\Roaming\MetaQuotes\Terminal"
|
||||
r"\010E047102812FC0C18890992854220E")
|
||||
tester_profiles_dir = mt5_data_path / "MQL5" / "Profiles" / "Tester"
|
||||
tester_profiles_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
bars_path = find_bars_file()
|
||||
bars = load_bars(bars_path)
|
||||
date_from = bars["timestamp"].iloc[0].strftime("%Y.%m.%d")
|
||||
date_to = bars["timestamp"].iloc[-1].strftime("%Y.%m.%d")
|
||||
print(f"=== MT5 verification ===")
|
||||
print(f"bars : {bars_path.name} ({len(bars):,} bars)")
|
||||
print(f"window : {date_from} → {date_to}")
|
||||
print(f"deposit: {args.deposit:,.0f} USD")
|
||||
|
||||
# ── Deploy EA ─────────────────────────────────────────────────────────
|
||||
print(f"\ndeploying EA to {mt5_data_path.name}/MQL5/Experts/ ...")
|
||||
deploy_ea(mt5_data_path)
|
||||
|
||||
# ── Run optimizer to get finalists ────────────────────────────────────
|
||||
print(f"\noptimizing ({args.trials} trials) ...")
|
||||
constraints = Constraints(min_trades=25, min_profit_factor=1.2,
|
||||
max_equity_dd_pct=0.40)
|
||||
obj_cfg = ObjectiveConfig(
|
||||
engine=ScalperEngine(),
|
||||
bars=bars,
|
||||
instrument=XAUUSD_REAL,
|
||||
sizing=SizingInputs(),
|
||||
initial_deposit=args.deposit,
|
||||
search_space=SEARCH_SPACE,
|
||||
int_params=INT_PARAMS,
|
||||
frozen_baseline=FROZEN_BASELINE,
|
||||
constraints=constraints,
|
||||
dd_weight=1.0,
|
||||
build_signals=build_signals,
|
||||
build_engine_kwargs=engine_kwargs_from_params,
|
||||
)
|
||||
objective = build_objective(obj_cfg)
|
||||
optuna.logging.set_verbosity(optuna.logging.WARNING)
|
||||
study = optuna.create_study(direction="maximize",
|
||||
sampler=optuna.samplers.TPESampler(seed=42))
|
||||
study.optimize(objective, n_trials=args.trials)
|
||||
finalists = select_diverse_topn(study, args.top_n, SEARCH_SPACE)
|
||||
print(f"selected {len(finalists)} finalists")
|
||||
|
||||
# ── Verify each finalist ───────────────────────────────────────────────
|
||||
for i, t in enumerate(finalists, 1):
|
||||
label = f"f{i}"
|
||||
verify_finalist(
|
||||
t.user_attrs["params"], label,
|
||||
bars=bars, deposit=args.deposit,
|
||||
mt5_install=mt5_install_dir,
|
||||
mt5_data_path=mt5_data_path,
|
||||
tester_profiles_dir=tester_profiles_dir,
|
||||
date_from=date_from, date_to=date_to,
|
||||
)
|
||||
|
||||
print("\n=== done ===")
|
||||
print(f"verification reports in registry/auto-verification_*.md")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user