From ef1b66756b46e5b73d49e1be76de7b61c396ae5e Mon Sep 17 00:00:00 2001 From: FXDavid Date: Fri, 19 Jun 2026 12:36:13 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20run=5Findicator=20=E2=80=94=20backtest?= =?UTF-8?q?=20an=20indicator=20headlessly,=20no=20chart=20attach?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MT5 can't attach an indicator to a chart via API, so a generic host EA (MBT_IndicatorHost) loads any indicator with iCustom inside the Strategy Tester; the indicator's OnCalculate runs over the date range and its SignalLogger.mqh (now tester-aware: FILE_COMMON under MQL_TESTER) writes the signals to the shared Common/Files folder, which the new run_indicator tool reads back. Removes the manual "attach to a chart" step from the flow. - core/indicator_runner.py: drives the headless run (stale-guard, .set in Profiles/Tester, signal span + warm-up note, instance-clash detection) - mcp_server.py: wires the run_indicator tool - mql5/SignalLogger.mqh: tester-gated FILE_COMMON (chart behaviour unchanged) - mql5/MBT_IndicatorHost.mq5: the generic host EA (places no trades) - install.py: also copies the host EA into MQL5/Experts - README: no-chart flow documented; chart attach now optional Validated under Wine: 348 signals on XAUUSDzero H1, stale-guard + error paths confirmed; host EA + SignalLogger compile 0/0. Co-Authored-By: Claude Opus 4.8 --- README.md | 32 ++++++- core/indicator_runner.py | 189 +++++++++++++++++++++++++++++++++++++ install.py | 35 ++++++- mcp_server.py | 31 ++++++ mql5/MBT_IndicatorHost.mq5 | 102 ++++++++++++++++++++ mql5/SignalLogger.mqh | 29 +++++- 6 files changed, 407 insertions(+), 11 deletions(-) create mode 100644 core/indicator_runner.py create mode 100644 mql5/MBT_IndicatorHost.mq5 diff --git a/README.md b/README.md index 6fb62ad..46e91d9 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ That's what MBT is built on. The indicator logs what it actually did. MBT reads │ tools exposed to Claude: │ │ get_ohlcv │ │ get_signals │ + │ run_indicator (headless) │ │ backtest (+ HTML report) │ │ validate_signals │ └────────────────────────────┘ @@ -137,8 +138,16 @@ LogSignal(shift, false, entry, sl, tp, "TRENDING"); // SELL `regime` is optional — pass `""` if you don't use it. When present, the backtest report breaks results down per regime. -Compile, attach to the chart. The indicator writes `signals.csv` to -`\MQL5\Files\`. +Compile it — that's all. From here you have two ways to make the indicator log +its signals: + +- **Headless (recommended):** ask Claude to run it — `run_indicator` computes the + indicator over your chosen date range with no chart and writes the signals + automatically. Nothing to attach. +- **On a chart:** attach the compiled indicator to a chart as usual; it writes + `signals.csv` live as it runs. + +Either way you end up with the same signal file, which the `backtest` tool reads. --- @@ -146,7 +155,23 @@ Compile, attach to the chart. The indicator writes `signals.csv` to Just describe what you want in Claude Code — Claude picks the right tool automatically. -**Backtest your indicator's signals:** +**Run your indicator and backtest it — no chart needed:** + +> "run my RegimePlusePro indicator on XAUUSD H1 for the last 3 years, then backtest it" + +Claude runs the indicator headlessly (it computes over the whole range and logs +its own signals), then replays those signals on real bars and hands you the +metrics + an HTML report. You never open a chart. + +> **One-time setup for the headless run:** it needs the `tester:` block filled in +> (see `config.example.yaml`) and the bundled host EA `MBT_IndicatorHost.mq5` +> (in `mql5/`) compiled into `MQL5/Experts` — just ask Claude to "compile the MBT +> indicator host" once. The indicator must log via `SignalLogger.mqh` (a bespoke +> logger that writes only to the local Files folder won't be found under the +> tester), and it runs with its **default** inputs. + +**Or backtest signals you already have** (from a previous run, or an indicator +running live on a chart): > "backtest my signals" > "backtest signals since 2026-01-01 and give me the HTML report" @@ -230,6 +255,7 @@ a timestamp. | `validate_signals` | check SL/TP geometry of every signal | | `get_config` | show the active terminal + signal file | | `compile_ea` | compile an EA/indicator with MetaEditor → structured errors/warnings | +| `run_indicator` | run an indicator headlessly so it logs its signals — no chart attach | | `run_strategy_tester` | run MT5's real headless Strategy Tester on an EA → metrics + report | | `signal_parity` | diff two signal sets and report the first divergence | diff --git a/core/indicator_runner.py b/core/indicator_runner.py new file mode 100644 index 0000000..d106fda --- /dev/null +++ b/core/indicator_runner.py @@ -0,0 +1,189 @@ +""" +Headless indicator runner — make an indicator log its signals with no chart. + +MT5 has no API to attach an indicator to a chart, and its Strategy Tester only +runs Experts. So MBT ships a tiny generic host EA (MBT_IndicatorHost) that loads +ANY indicator via iCustom inside the tester; once loaded the indicator's +OnCalculate runs bar-by-bar over the chosen date range, and if it logs its own +signals (via SignalLogger.mqh) those signals get written automatically. + +This module drives that: it writes the host EA's .set file (which indicator to +load), clears any stale signal CSV, runs the tester headlessly, then reads back +the signals the indicator wrote to the shared Common\\Files folder. + +Requirement: the indicator must log through SignalLogger.mqh — under the tester +that logger transparently writes to Common\\Files, which is where we read. An +indicator with a bespoke logger that writes only to the local Files folder lands +in the tester's per-agent sandbox and won't be found here. +""" + +import os +import time +import subprocess + +from .connection import reports_dir +from .tester import (_tester_cfg, _data_dir, _common_files_dir, _launch_cmd, + build_tester_ini, _fmt_date) + +HOST_EA = "MBT_IndicatorHost" + + +def _profiles_tester_dir() -> str: + """MT5 reads a tester .set file from MQL5/Profiles/Tester/ — return that dir + (created if missing) under the active data dir.""" + d = os.path.join(_data_dir(), "MQL5", "Profiles", "Tester") + os.makedirs(d, exist_ok=True) + return d + + +def _write_set_file(name: str, indicator: str) -> str: + """Write a minimal .set passing InpIndicator to the host EA. Returns the bare + filename (ExpertParameters is resolved relative to Profiles/Tester).""" + fname = name + ".set" + path = os.path.join(_profiles_tester_dir(), fname) + with open(path, "w", encoding="utf-8") as f: + f.write("InpIndicator=%s\n" % indicator) + f.write("InpBuffersToPull=2\n") + return fname + + +def _signal_csv_path(signal_file: str) -> str: + """Where the hosted indicator's SignalLogger writes under the tester: the + shared Common/Files folder, by the file's bare name.""" + return os.path.join(_common_files_dir(), os.path.basename(signal_file)) + + +def _signal_span(path: str): + """Return (count, first_time, last_time) over the signal CSV's data rows + (excluding a header). Times are the raw first-column strings, or None.""" + if not os.path.isfile(path): + return 0, None, None + n = 0 + first = last = None + with open(path, encoding="cp1252", errors="replace") as f: + for line in f: + if not line.strip(): + continue + if line.lower().startswith("time,"): + continue # header (identified by content, not position) + n += 1 + stamp = line.split(",", 1)[0].strip() + if first is None: + first = stamp + last = stamp + return n, first, last + + +def run_indicator(indicator, symbol, timeframe="h1", from_date=None, to_date=None, + signal_file="signals.csv", model=None, deposit=None, + leverage=None, timeout_sec=None, host_ea=HOST_EA) -> dict: + """Run an indicator headlessly through the tester so it logs its own signals. + + indicator : name under MQL5/Indicators (e.g. 'RegimePlusePro'), .ex5 optional. + symbol : broker symbol (e.g. 'XAUUSD'). + timeframe : 1m 5m 15m 30m 1h 4h 1d 1w. + from/to : 'YYYY-MM-DD' (omit -> broker's full history). + signal_file : the indicator's SignalLogFile name; read back from Common/Files. + Uses the indicator's compiled-in default, so leave as the default + unless the indicator hard-codes a different name. + + Returns the signal CSV path and the number of signals logged. The indicator + runs with its DEFAULT inputs (iCustom uses compiled defaults) — if you tuned + its inputs on a chart, recompile the indicator with those as defaults. + """ + t = _tester_cfg() + model = model or t.get("default_model", "open_prices") + deposit = deposit or t.get("default_deposit", 10000) + leverage = leverage or t.get("default_leverage", 100) + timeout = timeout_sec or t.get("timeout_sec", 1800) + + indicator = str(indicator).strip() + if not indicator: + return {"error": "indicator name is empty."} + + host_name = host_ea if host_ea.lower().endswith(".ex5") else host_ea + ".ex5" + stamp = time.strftime("%Y%m%d_%H%M%S") + run_name = "%s_%s" % (os.path.splitext(os.path.basename(indicator))[0], stamp) + + # .set tells the host EA which indicator to load. + set_file = _write_set_file(run_name, indicator) + + ini_txt = build_tester_ini(host_name, symbol, timeframe, from_date, to_date, + model, deposit, leverage, report_path=run_name, + set_file=set_file) + ini_path = os.path.join(reports_dir(), run_name + ".ini") + with open(ini_path, "w", encoding="utf-8") as f: + f.write(ini_txt) + + # STALE GUARD: clear the signal CSV before launch, so a leftover file from a + # previous run can never be read back as this run's signals. Fail loud if it + # can't be cleared (a silent stale read is exactly what this prevents). + csv_path = _signal_csv_path(signal_file) + if os.path.isfile(csv_path): + try: + os.remove(csv_path) + except OSError: + pass + if os.path.isfile(csv_path): + return {"error": "Could not clear the stale signal file at %s " + "(file locked?). Close any running MT5 terminal and " + "retry — leaving it risks reporting the previous run's " + "signals as this run's." % csv_path} + + cmd = _launch_cmd(ini_path) + t0 = time.time() + timed_out = False + try: + subprocess.run(cmd, timeout=timeout, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.TimeoutExpired: + timed_out = True + elapsed = round(time.time() - t0, 1) + + count, first_sig, last_sig = _signal_span(csv_path) + wrote = os.path.isfile(csv_path) + + result = { + "indicator": indicator, + "host_ea": host_ea, + "symbol": symbol, + "timeframe": timeframe, + "from": _fmt_date(from_date), + "to": _fmt_date(to_date), + "model": model, + "signal_csv": csv_path if wrote else None, + "signal_count": count, + "first_signal": first_sig, + "last_signal": last_sig, + "ran_seconds": elapsed, + "timed_out": timed_out, + "command": " ".join(cmd), + "ini": ini_path, + "set_file": set_file, + } + + # The indicator computes over warm-up history the tester loads BEFORE + # from_date, so it can log signals earlier than the requested start. Surface + # that rather than let the wider span look like a bug. + if count and from_date and first_sig and first_sig[:10].replace(".", "-") < _fmt_date(from_date).replace(".", "-"): + result["note"] = ("signals start before from_date because the indicator also " + "computed over the tester's warm-up history; filter by date " + "downstream (e.g. backtest since_date) if you want only the " + "requested window.") + + if count == 0: + # An instant return with nothing written usually means another terminal + # already owns the data dir (MT5 is single-instance). + if elapsed < 8 and not wrote: + result["error"] = ("Terminal returned in %.1fs and wrote no signal file — " + "another MT5 terminal is almost certainly already " + "running on this data dir (single-instance). Close it " + "and retry." % elapsed) + else: + result["error"] = ("No signals were logged. Check that: the indicator " + "name is correct and compiled into MQL5/Indicators; " + "it logs via SignalLogger.mqh (bespoke loggers that " + "skip Common/Files land in the tester sandbox); the " + "symbol/timeframe has history for the date range; and " + "the strategy actually fires in that range.") + return result diff --git a/install.py b/install.py index b2584dc..bd2ccdb 100644 --- a/install.py +++ b/install.py @@ -8,7 +8,8 @@ It will: 1. install Python dependencies 2. create config.yaml from the example if missing 3. copy SignalLogger.mqh into your MT5 MQL5\Include folder (if it can find it) - 4. print the exact `claude mcp add` command to register the server + 4. copy MBT_IndicatorHost.mq5 into MQL5\Experts (for headless run_indicator) + 5. print the exact `claude mcp add` command to register the server """ import os @@ -66,6 +67,33 @@ def copy_include(): print(f"No Include folders found. Manually copy {src} yourself.") +def copy_host_ea(): + """Copy the headless host EA into MQL5\\Experts so `run_indicator` works. + Compile it once in MetaEditor (or ask Claude to) before first use.""" + step("MQL5 host EA (for headless run_indicator)") + src = os.path.join(ROOT, "mql5", "MBT_IndicatorHost.mq5") + base = os.path.join(os.environ.get("APPDATA", ""), "MetaQuotes", "Terminal") + if not os.path.isdir(base): + print(f"Could not auto-find the terminal folder. Manually copy {src} " + f"into your terminal's MQL5\\Experts folder and compile it.") + return + copied = 0 + for d in os.listdir(base): + exp = os.path.join(base, d, "MQL5", "Experts") + if os.path.isdir(exp): + try: + shutil.copy(src, os.path.join(exp, "MBT_IndicatorHost.mq5")) + print(f"Copied MBT_IndicatorHost.mq5 -> {exp}") + copied += 1 + except Exception as e: + print(f"Skipped {exp}: {e}") + if copied: + print("Compile it once (MetaEditor, or ask Claude to 'compile the MBT " + "indicator host') before the first headless run.") + else: + print(f"No Experts folders found. Manually copy {src} yourself.") + + def print_mcp_cmd(): step("Register the MCP server with Claude Code") server = os.path.join(ROOT, "mcp_server.py").replace("\\", "/") @@ -80,6 +108,7 @@ if __name__ == "__main__": install_deps() make_config() copy_include() + copy_host_ea() print_mcp_cmd() - print("\nDone. Edit config.yaml, attach your indicator (with SignalLogger), " - "then ask Claude to backtest.") + print("\nDone. Edit config.yaml, then ask Claude to run your indicator " + "(with SignalLogger) and backtest it — no chart needed.") diff --git a/mcp_server.py b/mcp_server.py index 13774e3..8c6443e 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -24,6 +24,7 @@ from core.report_html import render as render_html from core.tester import run_strategy_tester as _run_tester from core.compiler import compile_mql5 as _compile_mql5 from core.parity import signal_parity as _signal_parity +from core.indicator_runner import run_indicator as _run_indicator mcp = FastMCP("MBT — MT5 Backtest Toolkit") @@ -188,6 +189,36 @@ def compile_ea(source: str) -> dict: return {"error": str(e)} +@mcp.tool() +def run_indicator(indicator: str, symbol: str, timeframe: str = "1h", + from_date: str = None, to_date: str = None, + signal_file: str = "signals.csv") -> dict: + """ + Run an indicator headlessly so it logs its own signals — no chart attach. + + MT5 can't attach an indicator to a chart programmatically, so this loads the + indicator inside MT5's Strategy Tester (via a generic host EA) and lets it + compute bar-by-bar over the date range. The indicator's own SignalLogger.mqh + writes the signals, which this reads back. After it returns, run `backtest` + to evaluate those signals. + + indicator: name under MQL5/Indicators (e.g. 'RegimePlusePro'), .ex5 optional. + symbol: broker symbol (e.g. 'XAUUSD'). + timeframe: 1m 5m 15m 30m 1h 4h 1d 1w. + from_date / to_date: 'YYYY-MM-DD' (omit for the broker's full history). + signal_file: the indicator's SignalLogFile name (default 'signals.csv'). + + Requires tester.* in config.yaml, the host EA + the indicator compiled, and + the indicator to log via SignalLogger.mqh. The indicator runs with its DEFAULT + inputs. Returns the signal CSV path and how many signals were logged. + """ + try: + return _run_indicator(indicator, symbol, timeframe, from_date, to_date, + signal_file=signal_file) + except Exception as e: + return {"error": str(e)} + + @mcp.tool() def signal_parity(reference: str, candidate: str = None, price_tol: float = 0.0) -> dict: diff --git a/mql5/MBT_IndicatorHost.mq5 b/mql5/MBT_IndicatorHost.mq5 new file mode 100644 index 0000000..e54c1e2 --- /dev/null +++ b/mql5/MBT_IndicatorHost.mq5 @@ -0,0 +1,102 @@ +//+------------------------------------------------------------------+ +//| MBT_IndicatorHost.mq5 — MBT (MT5 Backtest Toolkit) | +//| | +//| A generic, do-nothing Expert Advisor whose only job is to LOAD | +//| another indicator inside MT5's Strategy Tester and make it run. | +//| | +//| Why this exists: | +//| MT5's API can't attach an indicator to a chart, and the tester | +//| only runs Experts — not indicators. But an Expert can load any | +//| indicator with iCustom(), and once loaded the indicator's | +//| OnCalculate() runs bar-by-bar over the test range. If that | +//| indicator logs its own signals (via SignalLogger.mqh), those | +//| signals get written with zero manual chart work. | +//| | +//| This EA places NO trades. It just hosts the indicator and pulls | +//| one buffer value each bar so the terminal actually computes it | +//| (a custom indicator in the tester is only calculated when its | +//| data is requested). | +//| | +//| Driven by MBT's `run_indicator` tool, which passes the indicator | +//| name through a .set file (InpIndicator). | +//+------------------------------------------------------------------+ +#property strict +#property version "1.00" +#property description "MBT host EA — runs another indicator headlessly so it logs its signals. Places no trades." + +// Indicator name relative to MQL5\Indicators (e.g. "RegimePlusePro" or +// "Subfolder\\MyIndi"), with or without the .ex5 extension. Passed by the tool. +input string InpIndicator = ""; + +// How many of the indicator's plotted buffers to request each bar. Requesting a +// buffer is what forces the tester to calculate the indicator; one is enough for +// almost every indicator, but a couple is cheap insurance. +input int InpBuffersToPull = 2; + +int g_handle = INVALID_HANDLE; +datetime g_lastBar = 0; + +//+------------------------------------------------------------------+ +int OnInit() +{ + string name = InpIndicator; + StringTrimLeft(name); + StringTrimRight(name); + if(name == "") + { + Print("MBT_IndicatorHost: InpIndicator is empty — nothing to host. ", + "Pass the indicator name via the .set file."); + return(INIT_PARAMETERS_INCORRECT); + } + // iCustom wants the name with no extension — strip a trailing .ex5/.mq5 if the + // caller passed one (the tool documents the extension as optional). + string lower = name; + StringToLower(lower); + if(StringFind(lower, ".ex5", StringLen(lower) - 4) >= 0 || + StringFind(lower, ".mq5", StringLen(lower) - 4) >= 0) + name = StringSubstr(name, 0, StringLen(name) - 4); + // iCustom resolves the name under MQL5\Indicators. + g_handle = iCustom(_Symbol, _Period, name); + if(g_handle == INVALID_HANDLE) + { + PrintFormat("MBT_IndicatorHost: could not load indicator '%s' (iCustom failed). " + "Check the name/path under MQL5\\Indicators and that it compiled.", name); + return(INIT_FAILED); + } + PrintFormat("MBT_IndicatorHost: hosting '%s' on %s %s.", + name, _Symbol, EnumToString((ENUM_TIMEFRAMES)_Period)); + return(INIT_SUCCEEDED); +} + +//+------------------------------------------------------------------+ +void OnDeinit(const int reason) +{ + if(g_handle != INVALID_HANDLE) + IndicatorRelease(g_handle); +} + +//+------------------------------------------------------------------+ +//| Once per new bar, request the hosted indicator's buffers so the | +//| tester computes it for the just-closed bar. We never act on the | +//| values — the indicator's own logging does the real work. | +//+------------------------------------------------------------------+ +void OnTick() +{ + if(g_handle == INVALID_HANDLE) + return; + + datetime barTime = iTime(_Symbol, _Period, 0); + if(barTime == g_lastBar) + return; // still the same bar — pull once per bar + g_lastBar = barTime; + + double buf[]; + int pulls = (InpBuffersToPull < 1) ? 1 : InpBuffersToPull; + for(int b = 0; b < pulls; b++) + { + // Best-effort: a buffer index the indicator doesn't have simply returns + // <=0 copied; that's fine, the request still triggers calculation. + CopyBuffer(g_handle, b, 0, 2, buf); + } +} +//+------------------------------------------------------------------+ diff --git a/mql5/SignalLogger.mqh b/mql5/SignalLogger.mqh index 02d8aa0..b5ee4b4 100644 --- a/mql5/SignalLogger.mqh +++ b/mql5/SignalLogger.mqh @@ -19,13 +19,30 @@ // Name of the CSV the toolkit reads. Must match `signal_file` in config.yaml. input string SignalLogFile = "signals.csv"; +//+------------------------------------------------------------------+ +//| Where the log is written. | +//| | +//| On a chart the file goes to the local \MQL5\Files\ . | +//| Under the Strategy Tester that folder is a per-agent sandbox the | +//| toolkit can't find, so when running in the tester we transparently| +//| switch to the shared Common\Files\ folder instead — which is | +//| exactly where MBT's headless `run_indicator` tool reads from. | +//| Every file call below ORs in this flag, so the two modes stay in | +//| lockstep (a chart write that local-deletes won't fight a tester | +//| write that common-deletes). | +//+------------------------------------------------------------------+ +int _SigCommonFlag() +{ + return MQLInfoInteger(MQL_TESTER) ? FILE_COMMON : 0; +} + //+------------------------------------------------------------------+ //| Reset the log (call once in OnInit, or on full recalculation, | //| so the file always reflects the currently drawn signals). | //+------------------------------------------------------------------+ void ResetSignalLog() { - FileDelete(SignalLogFile); + FileDelete(SignalLogFile, _SigCommonFlag()); } //+------------------------------------------------------------------+ @@ -33,8 +50,9 @@ void ResetSignalLog() //+------------------------------------------------------------------+ int _OpenSignalLog() { - bool isNew = !FileIsExist(SignalLogFile); - int h = FileOpen(SignalLogFile, FILE_READ|FILE_WRITE|FILE_TXT|FILE_ANSI); + int common = _SigCommonFlag(); + bool isNew = !FileIsExist(SignalLogFile, common); + int h = FileOpen(SignalLogFile, FILE_READ|FILE_WRITE|FILE_TXT|FILE_ANSI|common); if(h == INVALID_HANDLE) return INVALID_HANDLE; @@ -60,11 +78,12 @@ void LogSignal(int shift, bool isLong, double entry, double sl, double tp, string regime = "") { datetime barTime = iTime(_Symbol, _Period, shift); + int common = _SigCommonFlag(); // --- dedup: skip if this bar's time is already the last logged row --- - if(FileIsExist(SignalLogFile)) + if(FileIsExist(SignalLogFile, common)) { - int rh = FileOpen(SignalLogFile, FILE_READ|FILE_TXT|FILE_ANSI); + int rh = FileOpen(SignalLogFile, FILE_READ|FILE_TXT|FILE_ANSI|common); if(rh != INVALID_HANDLE) { string lastLine = "";