commit 3f763827f4042f29ad101c7426bc84fb0e2c39c6 Author: Devid HW Date: Sat Apr 18 11:41:41 2026 +0700 feat: MT5-Quant MCP server for backtesting and optimization MCP server exposing MetaTrader 5 strategy development tools to AI assistants (Claude, Cursor, etc.) on macOS (CrossOver) and Linux (Wine). Tools: - run_backtest: full pipeline — compile EA, clean cache, backtest, parse HTML/XML report, analyze deals → metrics.json + analysis.json - run_optimization: background genetic optimization with nohup/disown, UTF-16LE .set file handling, OptMode reset - compile_ea: MQL5 compilation via MetaEditor with auto-detected include/ directory sync - get_backtest_status / get_optimization_status: job polling - verify_environment: Wine/MT5 path validation Analytics: - extract.py: MT5 HTML and SpreadsheetML XML report parser - analyze.py: deal-level analysis (drawdown events, grid depth, loss sequences, monthly P&L) → analysis.json - optimize_parser.py: optimization result parser with convergence analysis Platform support: - macOS CrossOver (GUI mode, no Xvfb needed) - Linux Wine + Xvfb (headless, CI/CD compatible) - Auto-detection of Wine executable and MT5 terminal paths diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..34e3a1a --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +dist/ +build/ +.venv/ +venv/ + +# Reports (generated, not tracked) +reports/20*/ +reports/*_opt/ + +# Job metadata +.mt5mcp_jobs/ + +# Config (contains local paths) +config/mt5-quant.yaml +config/baseline.json +config/backtest_history.json + +# Claude Code (personal, not for repo) +CLAUDE.md + +# macOS +.DS_Store + +# Logs +*.log +/tmp/mt5opt_* diff --git a/README.md b/README.md new file mode 100644 index 0000000..e6110d0 --- /dev/null +++ b/README.md @@ -0,0 +1,535 @@ +# MT5-Quant + +**The MCP server for MT5 strategy development — not live trading.** + +Most MT5 MCP servers let an AI place orders. MT5-Quant lets an AI *build the strategy*: compile an Expert Advisor, run a backtest, parse the report, analyze every deal, and optimize parameters — all in one conversation, on **macOS or Linux**, without a Windows machine. + +``` +You: "Run a backtest from Jan to March and tell me what caused the drawdown spike in February" + +Claude: [compile → clean cache → backtest → parse XML report → analyze 1,847 deals] + + Worst DD event: Feb 14, BUY grid at L6. + Locking hedge placed into a trending adverse move. Locking lot 1.75× base. + Cutloss fired 17 price points later. + + Recommendation: cap locking lot multiplier to ≤1.2× to prevent cascade on adverse trend. +``` + +### How MT5-Quant is different + +| | MT5-Quant | Other MT5 MCPs¹ | QuantConnect MCP | ai-trader | +|---|---|---|---|---| +| **Backtest via MCP** | ✅ full pipeline | ❌ | ✅ cloud only | ✅ Python only | +| **EA optimization via MCP** | ✅ genetic, background | ❌ | ✅ cloud only | ❌ | +| **Deal-level analytics** | ✅ 15+ dimensions | ❌ | ❌ | ❌ | +| **MQL5 EA compilation** | ✅ | ❌ | ❌ | ❌ | +| **macOS + Linux** | ✅ | ❌ Windows only | ❌ cloud | ✅ | +| **Headless / CI-CD** | ✅ Xvfb | ❌ | ❌ | ✅ | +| **Offline / no subscription** | ✅ | ✅ | ❌ paid cloud | ✅ | +| **MT5 .set file tooling** | ✅ 8 tools | ❌ | ❌ | ❌ | +| **Backtest history ledger** | ✅ JSON archive | ❌ | ✅ | ❌ | + +¹ *ariadng/metatrader-mcp-server, Qoyyuum/mcp-metatrader5-server, Cloudmeru/MetaTrader-5-MCP-Server — all live-trading execution bridges, Windows-only.* + +### What it covers + +28 MCP tools across the full EA development loop: + +``` +list_set_files / describe_sweep / patch_set_file / set_from_optimization + ↓ prepare parameters +compile_ea + ↓ build .ex5 +run_backtest (compile → clean cache → MT5 tester → extract HTML/XML → analyze deals) + ↓ fresh results +analyze_report / compare_baseline / get_history + ↓ evaluate and record +archive_report(delete_after=true) → promote_to_baseline + ↓ clean up, lock in new production reference +run_optimization (background, nohup — takes hours) + ↓ when MT5 finishes +get_optimization_results → set_from_optimization → run_backtest (verify) +``` + +The AI drives every step. You watch and approve. + +--- + +## Quickstart + +### 1. Clone and install + +```bash +git clone https://github.com/masdevid/mt5-mcp +cd mt5-mcp +python3 -m venv .venv && source .venv/bin/activate +pip install -e . +``` + +> **Python 3.11+** required. The venv is optional but recommended. + +### 2. Install MetaTrader 5 + +MT5 runs under Wine on both macOS and Linux. Two supported paths: + +**macOS — MetaTrader 5.app (recommended, free)** + +Download from [metatrader5.com](https://www.metatrader5.com/en/download) and install to `/Applications`. Launch it once so it initializes the Wine prefix (~30 s), then quit. + +Wine binary auto-detected at: +``` +/Applications/MetaTrader 5.app/Contents/SharedSupport/wine/bin/wine64 +``` +MT5 prefix auto-detected at: +``` +~/Library/Application Support/net.metaquotes.wine.metatrader5/drive_c/Program Files/MetaTrader 5 +``` + +**macOS — CrossOver (paid, better Wine compatibility)** + +Install [CrossOver](https://www.codeweavers.com/), create a bottle named `MetaTrader5`, install MT5 inside it. + +Wine binary: +``` +/Applications/CrossOver.app/Contents/SharedSupport/CrossOver/bin/wine64 +``` +MT5 prefix (CrossOver 24+): +``` +~/Library/Application Support/MetaQuotes//drive_c/Program Files/MetaTrader 5 +``` + +> **Apple Silicon (M1/M2/M3):** `setup.sh` automatically prepends `arch -x86_64` to all Wine calls. No manual action needed. + +**Linux** + +```bash +sudo apt install wine64 xvfb # Debian/Ubuntu +# or +sudo dnf install wine xorg-x11-server-Xvfb # Fedora/RHEL + +# Install MT5 under Wine +wine64 MetaTrader5Setup.exe +``` + +MT5 prefix auto-detected at: +``` +~/.wine/drive_c/Program Files/MetaTrader 5 +``` + +### 3. Configure + +```bash +bash scripts/setup.sh # auto-detects paths and writes config +bash scripts/setup.sh --yes # non-interactive (CI / fresh machine) +``` + +`setup.sh` writes `config/mt5-quant.yaml`. To configure manually, copy the example: + +```bash +cp config/mt5-quant.example.yaml config/mt5-quant.yaml +``` + +Minimum required fields: + +```yaml +# macOS (MetaTrader 5.app) +wine_executable: "/Applications/MetaTrader 5.app/Contents/SharedSupport/wine/bin/wine64" +terminal_dir: "~/Library/Application Support/net.metaquotes.wine.metatrader5/drive_c/Program Files/MetaTrader 5" + +# Linux +# wine_executable: "/usr/bin/wine64" +# terminal_dir: "~/.wine/drive_c/Program Files/MetaTrader 5" +``` + +### 4. Register the MCP server + +```bash +# Add to Claude Code (adjust path to where you cloned the repo) +claude mcp add MT5-Quant -- python3 /path/to/mt5-quant/server/main.py + +# Or with the venv python explicitly: +claude mcp add MT5-Quant -- /path/to/mt5-quant/.venv/bin/python3 /path/to/mt5-quant/server/main.py +``` + +`setup.sh` runs this automatically. To check registration: + +```bash +claude mcp list +``` + +Expected output: +``` +MT5-Quant: python3 /path/to/mt5-quant/server/main.py +``` + +**Claude Code integration files** (CLAUDE.md template + baseline hook): +```bash +bash scripts/setup.sh --claude-code +``` + +### 5. Verify + +```bash +bash scripts/platform_detect.sh +``` + +Expected output (macOS): +``` +Wine: /Applications/MetaTrader 5.app/Contents/SharedSupport/wine/bin/wine64 +MT5 dir: ~/Library/Application Support/net.metaquotes.wine.metatrader5/.../MetaTrader 5 +Display: gui +Arch: arch -x86_64 ← Apple Silicon only +``` + +Or from Claude: *"Run verify_setup"* + +### 6. Run a backtest + +``` +Run a backtest on MyEA from 2025.01.01 to 2025.06.30 +``` + +--- + +## Headless Support + +| Platform | `display.mode` | Notes | +|----------|---------------|-------| +| macOS (MT5.app or CrossOver) | `auto` | Wine handles display internally — no Xvfb needed | +| Linux with `$DISPLAY` set | `auto` | Uses existing X11 session | +| Linux VPS / CI (no monitor) | `headless` | Auto-starts Xvfb on `:99` | + +**Linux headless setup:** + +```bash +sudo apt install xvfb +``` + +Then set in `config/mt5-quant.yaml`: +```yaml +display: + mode: headless + xvfb_display: ":99" + xvfb_screen: "1024x768x16" +``` + +`platform_detect.sh` starts Xvfb automatically before each backtest run. To test manually: + +```bash +Xvfb :99 -screen 0 1024x768x16 & +DISPLAY=:99 xdpyinfo | grep dimensions +``` + +--- + +## MCP Tools (28) + +### Core workflow + +| Tool | Description | +|------|-------------| +| `run_backtest` | Full pipeline: compile → clean → backtest → extract → analyze | +| `run_optimization` | Genetic optimization (background, returns immediately) | +| `get_optimization_results` | Parse optimization results after MT5 finishes | +| `analyze_report` | Read `analysis.json` from any report directory | +| `compare_baseline` | Compare report vs baseline, return winner/loser verdict | +| `compile_ea` | Compile MQL5 EA via MetaEditor | + +### Monitoring + +| Tool | Description | +|------|-------------| +| `verify_setup` | Check Wine/MT5 paths, Wine version, and EA/set file counts | +| `get_backtest_status` | Check live progress of a running backtest pipeline | +| `get_optimization_status` | Check live state of a background optimization job | +| `list_jobs` | All optimization jobs with compact status in one call | + +### Reports & logs + +| Tool | Description | +|------|-------------| +| `list_reports` | Compact table of all runs with key metrics — no full analysis needed | +| `tail_log` | Read last N lines of any log; `filter=errors` to see only failures | +| `prune_reports` | Delete old report directories, keep last N (skips `_opt` dirs) | + +### History & baseline + +| Tool | Description | +|------|-------------| +| `archive_report` | Convert one report dir → compact JSON entry in `backtest_history.json`, optionally delete source | +| `archive_all_reports` | Bulk-archive all report dirs then optionally delete them; keeps N newest safe | +| `get_history` | Query history with filters (EA, symbol, verdict, profit, DD) and sort options | +| `annotate_history` | Attach verdict / notes / tags to any history entry | +| `promote_to_baseline` | Write a history entry or report to `baseline.json` for compare_baseline | + +### Cache management + +| Tool | Description | +|------|-------------| +| `cache_status` | MT5 tester cache size breakdown by symbol — check before cleaning | +| `clean_cache` | Delete tester cache files; supports per-symbol and `dry_run` | + +### .set file — read / write + +| Tool | Description | +|------|-------------| +| `list_set_files` | All .set files in tester profiles dir with sweep stats and combination counts | +| `read_set_file` | Parse UTF-16LE `.set` file → structured JSON params | +| `write_set_file` | Write full params dict → UTF-16LE `.set` with `chmod 444` | +| `patch_set_file` | Update specific params in-place, return diff — replaces read→edit→write | +| `clone_set_file` | Copy `.set` to new path with optional overrides in one call | + +### .set file — analysis & generation + +| Tool | Description | +|------|-------------| +| `describe_sweep` | Swept params, value counts, and total optimization combinations | +| `diff_set_files` | Side-by-side diff of two `.set` files — only changed params returned | +| `set_from_optimization` | Generate a clean backtest `.set` from `get_optimization_results` params; optionally narrow sweep | + +Full schema: [docs/MCP_TOOLS.md](docs/MCP_TOOLS.md) + +--- + +## Architecture + +``` +AI Agent (Claude / Cursor) + │ MCP protocol (stdio) +MT5-Quant server (Python) + │ subprocess +Pipeline scripts (bash) + │ Wine/CrossOver +MetaTrader 5 (Windows/Wine) + │ +analysis.json ← AI reads this +``` + +Every backtest produces `analysis.json` — a stable, AI-readable artifact. + +The analysis engine is **strategy-agnostic**: a `PROFILES` system drives keyword matching, depth extraction, and cycle grouping so the same pipeline works for any EA type. + +| Strategy | CLI / entry point | Depth tracking | Exit keywords | Cycle grouping | +|----------|------------------|----------------|---------------|----------------| +| `grid` (default) | `mt5-analyze-grid` | `Layer #N` → L1–L8+ | locking / cutloss / zombie / timeout | magic + direction, 60 min gap | +| `scalper` | `mt5-analyze-scalper` | — | tp / sl / manual / trailing | magic, 10 min gap | +| `trend` | `mt5-analyze-trend` | — | tp / sl / trailing / breakeven / partial | magic, 4 h gap | +| `hedge` | `mt5-analyze-hedge` | — | tp / sl / net_close / partial | magic + direction, 2 h gap | +| `generic` | `mt5-analyze` | — | tp / sl (profit-sign) | magic, 60 min gap | + +**`analysis.json` fields** (strategy name controls what each field contains): + +| Field | All strategies | Grid only | +|-------|---------------|-----------| +| `strategy` | Active profile name | — | +| `summary` | KPIs + streaks + cycle win rate + dominant exit | — | +| `monthly_pnl` | P/L, trade count, green flag per month | — | +| `dd_events` | DD events; `cause` uses profile keywords | Cause = locking_cascade / cutloss / zombie_exit | +| `top_losses` | Worst closing deals | `grid_depth_at_close` populated | +| `loss_sequences` | Consecutive losing runs | — | +| `position_pairs` | Hold time, layer, P/L per closed position | — | +| `depth_histogram` | Profile-driven depth counts | L1–L8+ (empty for other strategies) | +| `cycle_stats` | Cycle win rate; grouping + gap from profile | win_rate_by_depth populated | +| `exit_reason_breakdown` | Profile-specific exit reasons | locking / cutloss / zombie / timeout | +| `direction_bias` | Buy vs sell win rate and P/L | — | +| `streak_analysis` | Max win/loss streaks, current streak | — | +| `session_breakdown` | Asian / London / London-NY / New York P/L | — | +| `weekday_pnl` | Mon–Sun P/L and win rate | — | +| `concurrent_peak` | Peak simultaneous open positions | — | +| `hourly_pnl` | Hour 0–23 (`--deep` only) | — | +| `volume_profile` | P/L by lot tier (`--deep` only) | — | + +This is what makes AI reasoning over backtest results possible — across any EA type. + +Full architecture: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) + +--- + +## Backtest History + +Every experiment can be archived into a single JSON ledger at `config/backtest_history.json` — a permanent, queryable record of every run. This lets you delete raw report directories (which can be GBs of tick data) while keeping all results as compact JSON. + +``` +[workflow] +run_backtest → archive_report(delete_after=true) # disk reclaimed + ↓ + backtest_history.json (grows forever, tiny) + ↓ + get_history(min_profit=5000, max_dd_pct=15) # query past runs + ↓ + promote_to_baseline # set new production reference +``` + +**Files** (both gitignored, live in `config/`): + +| File | Purpose | +|------|---------| +| `config/backtest_history.json` | Ledger of all archived backtest runs with full metrics, analysis summary, monthly P/L, and verdict | +| `config/baseline.json` | Current production reference — used by `compare_baseline` and the Claude Code hook | + +Each history entry captures: all `metrics.json` fields + `analysis.json` summary + monthly P/L array + worst DD event. The raw report directory can be safely deleted after archiving. + +**Typical session:** + +``` +1. run_backtest(expert, from, to) +2. archive_report(delete_after=true, verdict="winner", notes="tight SL test") +3. promote_to_baseline() ← if this is the new production config +4. get_history(ea="MyEA", verdict="winner") ← survey all winners later +``` + +--- + +## .set File Workflow + +```ini +; param=current||start||step||stop||Y (Y=sweep, N=fixed) +Min_Entry_Confidence=0.610||0.580||0.010||0.650||Y +TP_Pips=400||300||50||500||Y +Max_DD_Percent=15.0||N +``` + +MT5-Quant handles the undocumented requirements automatically: UTF-16LE encoding, `chmod 444`, OptMode reset, SpreadsheetML XML result parsing. + +**Common .set patterns:** + +``` +# Find and inspect +list_set_files(ea="MyEA") → see all variants with combination counts +describe_sweep(path=MyEA_opt.set) → verify 240 combinations before launching +diff_set_files(path_a=v1.2.set, path_b=v1.3.set) → what changed between versions + +# Edit +patch_set_file(path, patches={TP_Pips: 350}) → change one param, keep rest intact +clone_set_file(source, dest, overrides={...}) → new variant from a base in one call + +# Post-optimization +get_optimization_results(job_id=X) → results[0].params = {TP:400, Conf:0.61, ...} +set_from_optimization( → write clean backtest .set from those params + path=MyEA_v1.3.set, + params=results[0].params, + template=MyEA_base.set → fill non-swept params from existing .set +) +``` + +--- + +## Remote Agents (Linux server farm) + +Scale optimization throughput by connecting a Linux server as an MT5 agent farm. Linear speedup with agent count. + +Setup guide: [docs/REMOTE_AGENTS.md](docs/REMOTE_AGENTS.md) + +--- + +## Claude Code Integration + +`--claude-code` generates two files that make Claude aware of your trading context: + +```bash +bash scripts/setup.sh --claude-code +``` + +| File | Purpose | +|------|---------| +| `config/CLAUDE.template.md` | Copy to your EA project root as `CLAUDE.md`. Encodes MT5-Quant rules, baseline tracking policy, and symbol name reminders. | +| `.claude/hooks/user-prompt-submit.sh` | Runs before every Claude prompt. Reads `config/baseline.json` and injects your production metrics as context. | + +**Production baseline** (`config/baseline.json`, gitignored): + +```json +{ + "symbol": "XAUUSD.cent", + "period": "2024-01-01/2024-12-31", + "net_profit": 1250.50, + "profit_factor": 1.43, + "max_drawdown_pct": 18.2, + "sharpe_ratio": 0.87, + "total_trades": 342, + "notes": "Best config as of 2024-12-15" +} +``` + +Update this file whenever a new version is promoted to production — either by running `promote_to_baseline` from Claude, or by editing it manually. Every subsequent Claude prompt will automatically include these metrics so Claude can tell you whether a new backtest is actually an improvement without you having to paste the numbers. + +**Why this matters:** Without the baseline hook, you have to manually remind Claude what the production numbers are at the start of every session. With it, that context is always present. + +--- + +## Troubleshooting + +Run `verify_setup` from Claude first — it checks all paths and returns actionable hints. + +### Wine not found + +**macOS:** Confirm `/Applications/MetaTrader 5.app` exists and has been launched at least once. If using CrossOver, confirm the bottle is named correctly. + +```bash +# Check what setup.sh found: +bash scripts/platform_detect.sh +``` + +**Linux:** +```bash +sudo apt install wine64 # Debian/Ubuntu +sudo dnf install wine # Fedora/RHEL +which wine64 # confirm it's on PATH +``` + +### terminal64.exe missing + +MT5 unpacks `terminal64.exe` only after its first launch. Open MetaTrader 5.app, wait for initialization (~30 s), then quit. Re-run setup: + +```bash +bash scripts/setup.sh --yes +``` + +### MCP server not appearing in Claude + +```bash +claude mcp list # should show MT5-Quant +claude mcp remove MT5-Quant # remove stale entry if needed +claude mcp add MT5-Quant -- python3 /absolute/path/to/mt5-quant/server/main.py +``` + +Use an **absolute path** — relative paths break when Claude starts from a different working directory. + +### Report not found after backtest + +1. **Wrong symbol name** — brokers use custom names (`XAUUSDm`, `XAUUSD.cent`). Check `verify_setup` → `experts_dir`, or look in `/history/` for available symbols. +2. **No history data** — open MT5, open the symbol's chart, wait for history to download, then retry. +3. **EA crash at startup** — check `/MQL5/Logs/` for `OnInit` errors. +4. **Date range has no trades** — try a wider range or confirm the symbol was active during that period. + +### MetaEditor compile errors + +Log at `/MQL5/Logs/`. Common causes: +- Missing `#include` files — copy dependencies into `Experts/` alongside the `.mq5` +- Stale `.ex5` from a different MT5 build — delete it and recompile + +### No deals in backtest report + +- Use `model=0` (every tick) — models 1 and 2 skip intra-bar movement, producing zero deals for grid/martingale EAs +- Check the `.set` file has values appropriate for the symbol/broker +- Confirm `OnInit()` returns `INIT_SUCCEEDED` (MT5 Journal tab) + +### Optimization never finishes / no report + +```bash +# From Claude: +tail_log(job_id=X, filter=errors) +get_optimization_status(job_id=X) +``` + +If MT5 crashed mid-run, open `/terminal.ini` and remove the line containing `OptMode=-1`, then retry. + +--- + +## License + +MIT + +--- + +*Built from battle-tested production infrastructure. Every edge case in the pipeline was hit in production.* diff --git a/analytics/__init__.py b/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/analytics/analyze.py b/analytics/analyze.py new file mode 100644 index 0000000..4aea23d --- /dev/null +++ b/analytics/analyze.py @@ -0,0 +1,1000 @@ +#!/usr/bin/env python3 +""" +analyze.py — Deal-level backtest analysis engine. + +Strategy-agnostic core + strategy-specific profiles. + +Usage: + # Generic (works for any EA — no hardcoded keyword assumptions) + python3 analytics/analyze.py generic deals.csv --output-dir DIR + + # Strategy-specific presets + python3 analytics/analyze.py grid deals.csv --output-dir DIR + python3 analytics/analyze.py scalper deals.csv --output-dir DIR + python3 analytics/analyze.py trend deals.csv --output-dir DIR + python3 analytics/analyze.py hedge deals.csv --output-dir DIR + + # Legacy (no subcommand) — defaults to 'grid' for backward compatibility + python3 analytics/analyze.py deals.csv --output-dir DIR + + # Flags + --deep Add hourly_pnl and volume_profile + --stdout Print JSON to stdout instead of writing analysis.json + +Entry points (after pip install -e .): + mt5-analyze generic analysis + mt5-analyze-grid grid / martingale + mt5-analyze-scalper scalper + mt5-analyze-trend trend following + mt5-analyze-hedge hedging +""" + +import argparse +import csv +import json +import os +import re +import sys +from collections import defaultdict +from datetime import datetime +from typing import Optional + + +# ── Strategy profiles ───────────────────────────────────────────────────────── +# +# Each profile is a plain dict with: +# name – human-readable label +# depth_re – regex to extract depth number from comment (None = no depth tracking) +# exit_keywords – {reason: [keywords]} for comment-based exit classification +# dd_cause_keywords– {cause: [keywords]} for comment-based DD cause classification +# cycle_group_by – 'magic' | 'magic+direction' | None +# cycle_gap_min – minutes between opens that marks a new cycle boundary +# +# When exit_keywords is empty, classification falls back to profit-sign (tp/sl). +# When dd_cause_keywords is empty, dd_events cause = 'unknown'. + +PROFILES: dict[str, dict] = { + 'generic': { + 'name': 'Generic', + 'depth_re': None, + 'exit_keywords': {}, + 'dd_cause_keywords': {}, + 'cycle_group_by': 'magic', + 'cycle_gap_min': 60, + }, + 'grid': { + 'name': 'Grid / Martingale', + 'depth_re': r'[Ll]ayer\s*#?(\d+)', + 'exit_keywords': { + 'locking': ['locking', 'lock'], + 'cutloss': ['cutloss', 'cut loss', 'cut_loss'], + 'zombie': ['zombie'], + 'timeout': ['timeout', 'time out', 'time_out'], + }, + 'dd_cause_keywords': { + 'locking_cascade': ['locking', 'lock'], + 'cutloss': ['cutloss', 'cut'], + 'zombie_exit': ['zombie'], + 'spike_entry': ['spike'], + }, + 'cycle_group_by': 'magic+direction', + 'cycle_gap_min': 60, + }, + 'scalper': { + 'name': 'Scalper', + 'depth_re': None, + 'exit_keywords': { + 'tp': ['tp', 'take profit', 'target'], + 'sl': ['sl', 'stop loss', 'stoploss'], + 'manual': ['manual', 'close'], + 'trailing': ['trailing', 'trail'], + }, + 'dd_cause_keywords': { + 'stop_loss': ['sl', 'stop'], + 'manual_close': ['manual', 'close'], + }, + 'cycle_group_by': 'magic', + 'cycle_gap_min': 10, + }, + 'trend': { + 'name': 'Trend Following', + 'depth_re': None, + 'exit_keywords': { + # More specific patterns must come before general ones to avoid substring + # false-positives (e.g. 'stop' inside 'breakeven stop' / 'trailing stop'). + 'breakeven': ['breakeven', 'break even', 'be stop'], + 'trailing': ['trailing', 'trail'], + 'partial': ['partial', 'scale out'], + 'tp': ['tp', 'target', 'take profit'], + 'sl': ['sl', 'stop loss', 'stoploss'], + }, + 'dd_cause_keywords': { + 'whipsaw': ['stop loss', 'stoploss'], + 'trailing_stop': ['trailing'], + 'breakeven_stop': ['breakeven', 'be stop'], + }, + 'cycle_group_by': 'magic', + 'cycle_gap_min': 240, + }, + 'hedge': { + 'name': 'Hedging', + 'depth_re': None, + 'exit_keywords': { + 'tp': ['tp'], + 'sl': ['sl'], + 'net_close': ['net', 'hedge close', 'hedge_close'], + 'partial': ['partial', 'reduce'], + }, + 'dd_cause_keywords': { + 'hedge_unwind': ['net', 'hedge'], + 'correlation_break': ['sl', 'stop'], + }, + 'cycle_group_by': 'magic+direction', + 'cycle_gap_min': 120, + }, +} + + +# ── Utility ─────────────────────────────────────────────────────────────────── + +def _parse_dt(time_str: str) -> Optional[datetime]: + """Parse MT5 time string ('2025.01.10 09:30:00') to datetime.""" + if not time_str: + return None + s = time_str.strip() + for length, fmt in [(19, '%Y.%m.%d %H:%M:%S'), (19, '%Y-%m-%d %H:%M:%S'), + (10, '%Y.%m.%d'), (10, '%Y-%m-%d')]: + try: + return datetime.strptime(s[:length], fmt) + except (ValueError, IndexError): + continue + return None + + +def _extract_depth(comment: str, depth_re: Optional[str]) -> int: + """Extract a numeric depth from comment using the given regex. Returns 0 if not matched.""" + if not depth_re or not comment: + return 0 + match = re.search(depth_re, comment) + return int(match.group(1)) if match else 0 + + +def _layer_from_comment(comment: str) -> int: + """Grid-specific: extract Layer #N number. Kept for backward compatibility.""" + return _extract_depth(comment, PROFILES['grid']['depth_re']) + + +def _classify_exit(comment: str, profit: float, + profile: Optional[dict] = None) -> str: + """ + Classify exit reason from deal comment. + + If profile is provided, uses its exit_keywords dict. + If profile is None, falls back to the grid profile keywords (backward compat). + If no keywords match, returns 'tp' (profit > 0) or 'sl' (profit <= 0). + """ + kw_map = (profile or PROFILES['grid'])['exit_keywords'] + c = comment.lower() + for reason, keywords in kw_map.items(): + if any(kw in c for kw in keywords): + return reason + return 'tp' if profit > 0 else 'sl' + + +def _classify_dd_cause(comment: str, profile: dict) -> str: + """Classify DD event cause from comment using profile's dd_cause_keywords.""" + c = comment.lower() + for cause, keywords in profile.get('dd_cause_keywords', {}).items(): + if any(kw in c for kw in keywords): + return cause + return 'unknown' + + +def _lot_tier(vol: float) -> str: + if vol <= 0.01: return '0.01' + if vol <= 0.04: return '0.02-0.04' + if vol <= 0.09: return '0.05-0.09' + if vol <= 0.49: return '0.10-0.49' + if vol <= 0.99: return '0.50-0.99' + return '1.00+' + + +def _session_for_hour(hour: int) -> str: + if 13 <= hour < 17: return 'london_ny_overlap' + if 8 <= hour < 17: return 'london' + if 17 <= hour < 22: return 'new_york' + if 0 <= hour < 8: return 'asian' + return 'off_hours' + + +# ── Loaders ─────────────────────────────────────────────────────────────────── + +def load_deals(csv_path: str) -> list[dict]: + deals = [] + with open(csv_path, newline='', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + row = {k: (v or '').strip() for k, v in row.items()} + for field in ('profit', 'volume', 'price', 'balance'): + try: + row[field] = float(row.get(field, 0).replace(',', '') or 0) + except ValueError: + row[field] = 0.0 + deals.append(row) + return deals + + +def load_metrics(metrics_path: str) -> dict: + if not os.path.exists(metrics_path): + return {} + with open(metrics_path) as f: + return json.load(f) + + +# ── Generic analytics (strategy-agnostic) ──────────────────────────────────── + +def monthly_pnl(deals: list[dict]) -> list[dict]: + monthly: dict[str, dict] = defaultdict(lambda: {'pnl': 0.0, 'trades': 0}) + for deal in deals: + time_str = deal.get('time', '') + profit = deal.get('profit', 0.0) + entry = deal.get('entry', '').lower() + if 'out' not in entry and entry != '': + continue + if not time_str or profit == 0.0: + continue + try: + dt = datetime.strptime(time_str[:10].replace('.', '-'), '%Y-%m-%d') + month = dt.strftime('%Y-%m') + except (ValueError, IndexError): + continue + monthly[month]['pnl'] += profit + monthly[month]['trades'] += 1 + + return [ + {'month': m, 'pnl': round(d['pnl'], 2), 'trades': d['trades'], 'green': d['pnl'] >= 0} + for m in sorted(monthly) + for d in [monthly[m]] + ] + + +def reconstruct_dd_events(deals: list[dict], metrics: dict, + profile: Optional[dict] = None) -> list[dict]: + """Walk deals chronologically, reconstruct drawdown events. Cause classified by profile.""" + if not deals: + return [] + + _profile = profile or PROFILES['grid'] + balance_curve = [] + peak_balance = 0.0 + initial_balance = None + + for deal in deals: + balance = deal.get('balance', 0.0) + if balance > 0: + if initial_balance is None: + initial_balance = balance + peak_balance = max(peak_balance, balance) + dd_pct = (peak_balance - balance) / peak_balance * 100 if peak_balance > 0 else 0 + balance_curve.append({ + 'time': deal.get('time', ''), + 'balance': balance, + 'dd_pct': round(dd_pct, 3), + 'profit': deal.get('profit', 0.0), + 'comment': deal.get('comment', ''), + }) + + if not balance_curve: + return [] + + events, in_dd, dd_start_idx = [], False, None + threshold = 1.0 + + for i, point in enumerate(balance_curve): + if not in_dd and point['dd_pct'] > threshold: + in_dd, dd_start_idx = True, i + elif in_dd and point['dd_pct'] < threshold: + peak_idx = max(range(dd_start_idx, i + 1), + key=lambda x: balance_curve[x]['dd_pct']) + event = _build_dd_event(balance_curve, dd_start_idx, peak_idx, i, _profile) + if event['peak_dd_pct'] > 1.0: + events.append(event) + in_dd, dd_start_idx = False, None + + if in_dd and dd_start_idx is not None: + peak_idx = max(range(dd_start_idx, len(balance_curve)), + key=lambda x: balance_curve[x]['dd_pct']) + event = _build_dd_event(balance_curve, dd_start_idx, peak_idx, None, _profile) + if event['peak_dd_pct'] > 1.0: + events.append(event) + + events.sort(key=lambda e: e['peak_dd_pct'], reverse=True) + return events[:10] + + +def _build_dd_event(curve: list[dict], start_idx: int, peak_idx: int, + recovery_idx: Optional[int], profile: dict) -> dict: + start = curve[start_idx] + peak = curve[peak_idx] + + event: dict = { + 'peak_dd_pct': round(peak['dd_pct'], 2), + 'start_date': start['time'][:10].replace('.', '-') if start['time'] else '', + 'end_date': peak['time'][:10].replace('.', '-') if peak['time'] else '', + 'cause': _classify_dd_cause(peak.get('comment', ''), profile), + } + + if recovery_idx is not None: + rec = curve[recovery_idx] + event['recovery_date'] = rec['time'][:10].replace('.', '-') if rec['time'] else None + try: + s_dt = datetime.strptime(event['start_date'], '%Y-%m-%d') + r_dt = datetime.strptime(event['recovery_date'], '%Y-%m-%d') + event['recovery_days'] = (r_dt - s_dt).days + except (ValueError, TypeError): + event['recovery_days'] = None + else: + event['recovery_date'] = None + event['recovery_days'] = None + + try: + s_dt = datetime.strptime(event['start_date'], '%Y-%m-%d') + e_dt = datetime.strptime(event['end_date'], '%Y-%m-%d') + event['duration_days'] = (e_dt - s_dt).days + except ValueError: + event['duration_days'] = 0 + + return event + + +def top_losses(deals: list[dict], n: int = 10) -> list[dict]: + losses = [ + { + 'date': deal.get('time', '')[:10].replace('.', '-'), + 'loss_usd': round(deal['profit'], 2), + 'comment': deal.get('comment', ''), + 'grid_depth_at_close': _layer_from_comment(deal.get('comment', '')), + 'volume': deal.get('volume', 0.0), + } + for deal in deals + if deal.get('profit', 0.0) < 0 + ] + losses.sort(key=lambda x: x['loss_usd']) + return losses[:n] + + +def loss_sequences(deals: list[dict]) -> list[dict]: + closed = [d for d in deals + if 'out' in d.get('entry', '').lower() and d.get('profit', 0) != 0] + if not closed: + return [] + + sequences, current_seq = [], [] + for deal in closed: + if deal['profit'] < 0: + current_seq.append(deal) + else: + if len(current_seq) >= 2: + total = sum(d['profit'] for d in current_seq) + sequences.append({ + 'length': len(current_seq), + 'total_loss': round(total, 2), + 'start': current_seq[0].get('time', '')[:10].replace('.', '-'), + 'end': current_seq[-1].get('time', '')[:10].replace('.', '-'), + }) + current_seq = [] + + if len(current_seq) >= 2: + total = sum(d['profit'] for d in current_seq) + sequences.append({ + 'length': len(current_seq), + 'total_loss': round(total, 2), + 'start': current_seq[0].get('time', '')[:10].replace('.', '-'), + 'end': current_seq[-1].get('time', '')[:10].replace('.', '-'), + }) + + sequences.sort(key=lambda x: x['total_loss']) + return sequences[:5] + + +def position_pairs(deals: list[dict]) -> list[dict]: + """Match in/out deals by order ticket → hold time + depth at close.""" + open_pos: dict[str, dict] = {} + pairs = [] + + for deal in deals: + order = deal.get('order', '') + entry = deal.get('entry', '').lower() + + if 'in' in entry and 'out' not in entry: + open_pos[order] = deal + + elif 'out' in entry: + profit = deal.get('profit', 0.0) + if profit == 0.0: + continue + + in_deal = open_pos.pop(order, None) + dt_out = _parse_dt(deal.get('time', '')) + comment = deal.get('comment', '') + + hold_minutes = None + if in_deal and dt_out: + dt_in = _parse_dt(in_deal.get('time', '')) + if dt_in: + hold_minutes = round((dt_out - dt_in).total_seconds() / 60, 1) + + pairs.append({ + 'time': deal.get('time', ''), + 'type': deal.get('type', ''), + 'profit': profit, + 'volume': deal.get('volume', 0.0), + 'layer': _layer_from_comment(comment), + 'hold_minutes': hold_minutes, + 'comment': comment, + 'magic': deal.get('magic', ''), + 'order': order, + }) + + return pairs + + +def direction_bias(deals: list[dict]) -> dict: + stats: dict[str, dict] = { + 'buy': {'trades': 0, 'wins': 0, 'total_pnl': 0.0}, + 'sell': {'trades': 0, 'wins': 0, 'total_pnl': 0.0}, + } + for deal in deals: + if 'out' not in deal.get('entry', '').lower(): + continue + profit = deal.get('profit', 0.0) + if profit == 0.0: + continue + d = deal.get('type', '').lower() + if d not in stats: + continue + stats[d]['trades'] += 1 + stats[d]['total_pnl'] += profit + if profit > 0: + stats[d]['wins'] += 1 + + return { + d: { + 'trades': s['trades'], + 'win_rate': round(s['wins'] / s['trades'] * 100, 1), + 'total_pnl': round(s['total_pnl'], 2), + 'avg_pnl': round(s['total_pnl'] / s['trades'], 2), + } + for d, s in stats.items() if s['trades'] > 0 + } + + +def streak_analysis(deals: list[dict]) -> dict: + closed = [d for d in deals + if 'out' in d.get('entry', '').lower() and d.get('profit', 0.0) != 0.0] + if not closed: + return {} + + max_win_streak = max_loss_streak = cur_win = cur_loss = 0 + max_win_start = max_win_end = max_loss_start = max_loss_end = '' + win_run_start = loss_run_start = '' + + for deal in closed: + profit = deal['profit'] + t = deal.get('time', '')[:10].replace('.', '-') + if profit > 0: + if cur_win == 0: + win_run_start = t + cur_win += 1 + cur_loss = 0 + if cur_win > max_win_streak: + max_win_streak = cur_win + max_win_start = win_run_start + max_win_end = t + else: + if cur_loss == 0: + loss_run_start = t + cur_loss += 1 + cur_win = 0 + if cur_loss > max_loss_streak: + max_loss_streak = cur_loss + max_loss_start = loss_run_start + max_loss_end = t + + last = closed[-1] + return { + 'max_win_streak': max_win_streak, + 'max_win_start': max_win_start, + 'max_win_end': max_win_end, + 'max_loss_streak': max_loss_streak, + 'max_loss_start': max_loss_start, + 'max_loss_end': max_loss_end, + 'current_streak': cur_win if last['profit'] > 0 else cur_loss, + 'current_streak_type': 'win' if last['profit'] > 0 else 'loss', + } + + +def session_breakdown(deals: list[dict]) -> dict: + """P/L by trading session (UTC hour-based).""" + sessions: dict = defaultdict(lambda: {'trades': 0, 'wins': 0, 'total_pnl': 0.0}) + for deal in deals: + if 'out' not in deal.get('entry', '').lower(): + continue + profit = deal.get('profit', 0.0) + if profit == 0.0: + continue + dt = _parse_dt(deal.get('time', '')) + if not dt: + continue + s = _session_for_hour(dt.hour) + sessions[s]['trades'] += 1 + sessions[s]['total_pnl'] += profit + if profit > 0: + sessions[s]['wins'] += 1 + + return { + s: { + 'trades': d['trades'], + 'win_rate': round(d['wins'] / d['trades'] * 100, 1) if d['trades'] > 0 else 0.0, + 'total_pnl': round(d['total_pnl'], 2), + } + for s, d in sessions.items() + } + + +def weekday_pnl(deals: list[dict]) -> list[dict]: + DAY_NAMES = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] + by_day: dict = defaultdict(lambda: {'pnl': 0.0, 'trades': 0, 'wins': 0}) + for deal in deals: + if 'out' not in deal.get('entry', '').lower(): + continue + profit = deal.get('profit', 0.0) + if profit == 0.0: + continue + dt = _parse_dt(deal.get('time', '')) + if not dt: + continue + by_day[dt.weekday()]['pnl'] += profit + by_day[dt.weekday()]['trades'] += 1 + if profit > 0: + by_day[dt.weekday()]['wins'] += 1 + + return [ + { + 'day': DAY_NAMES[day], + 'pnl': round(s['pnl'], 2), + 'trades': s['trades'], + 'win_rate': round(s['wins'] / s['trades'] * 100, 1) if s['trades'] > 0 else 0.0, + } + for day in sorted(by_day) + for s in [by_day[day]] + ] + + +def hourly_pnl(deals: list[dict]) -> list[dict]: + """P/L by close hour (0–23). Intended for --deep mode.""" + by_hour: dict = defaultdict(lambda: {'pnl': 0.0, 'trades': 0, 'wins': 0}) + for deal in deals: + if 'out' not in deal.get('entry', '').lower(): + continue + profit = deal.get('profit', 0.0) + if profit == 0.0: + continue + dt = _parse_dt(deal.get('time', '')) + if not dt: + continue + by_hour[dt.hour]['pnl'] += profit + by_hour[dt.hour]['trades'] += 1 + if profit > 0: + by_hour[dt.hour]['wins'] += 1 + + return [ + { + 'hour': h, + 'pnl': round(s['pnl'], 2), + 'trades': s['trades'], + 'win_rate': round(s['wins'] / s['trades'] * 100, 1) if s['trades'] > 0 else 0.0, + } + for h in sorted(by_hour) + for s in [by_hour[h]] + ] + + +def concurrent_peak(deals: list[dict]) -> dict: + """Peak number of simultaneously open positions.""" + events = [] + for deal in deals: + entry = deal.get('entry', '').lower() + dt = _parse_dt(deal.get('time', '')) + if not dt: + continue + if 'in' in entry and 'out' not in entry: + events.append((dt, 1, deal)) + elif 'out' in entry: + events.append((dt, -1, deal)) + + events.sort(key=lambda x: x[0]) + count = peak = 0 + peak_time = '' + for dt, delta, deal in events: + count = max(0, count + delta) + if count > peak: + peak = count + peak_time = deal.get('time', '') + + return {'peak_open': peak, 'peak_time': peak_time} + + +def volume_profile(deals: list[dict]) -> list[dict]: + """P/L breakdown by lot size tier. Intended for --deep mode.""" + TIER_ORDER = ['0.01', '0.02-0.04', '0.05-0.09', '0.10-0.49', '0.50-0.99', '1.00+'] + by_tier: dict = defaultdict(lambda: {'pnl': 0.0, 'trades': 0, 'wins': 0}) + for deal in deals: + if 'out' not in deal.get('entry', '').lower(): + continue + profit = deal.get('profit', 0.0) + if profit == 0.0: + continue + tier = _lot_tier(deal.get('volume', 0.0)) + by_tier[tier]['pnl'] += profit + by_tier[tier]['trades'] += 1 + if profit > 0: + by_tier[tier]['wins'] += 1 + + return [ + { + 'lot_tier': tier, + 'pnl': round(s['pnl'], 2), + 'trades': s['trades'], + 'win_rate': round(s['wins'] / s['trades'] * 100, 1) if s['trades'] > 0 else 0.0, + } + for tier in TIER_ORDER + if tier in by_tier + for s in [by_tier[tier]] + ] + + +# ── Strategy-aware analytics ────────────────────────────────────────────────── + +def depth_histogram(deals: list[dict], profile: dict) -> dict: + """ + Count how often each depth level was reached, using the profile's depth_re. + Returns an empty dict when the profile has no depth_re (e.g. generic, scalper). + For grid profiles, keys are L1 … L8+. + """ + depth_re = profile.get('depth_re') + if not depth_re: + return {} + + hist: dict[str, int] = {'L1': 0, 'L2': 0, 'L3': 0, 'L4': 0, + 'L5': 0, 'L6': 0, 'L7': 0, 'L8+': 0} + for deal in deals: + depth = _extract_depth(deal.get('comment', ''), depth_re) + if depth: + key = f'L{depth}' if depth <= 7 else 'L8+' + hist[key] = hist.get(key, 0) + 1 + + return hist + + +def grid_depth_histogram(deals: list[dict]) -> dict: + """Backward-compatible alias: depth_histogram using the grid profile.""" + return depth_histogram(deals, PROFILES['grid']) + + +def cycle_stats(deals: list[dict], profile: Optional[dict] = None) -> dict: + """ + Group deals into cycles using profile's cycle_group_by and cycle_gap_min. + Returns overall win rate and win_rate_by_depth. + + Default profile (None) uses grid settings for backward compatibility. + """ + _profile = profile or PROFILES['grid'] + group_by = _profile.get('cycle_group_by', 'magic+direction') + gap_minutes = _profile.get('cycle_gap_min', 60) + depth_re = _profile.get('depth_re') + + # active[key] = {'in_deals': [...], 'profit': float, 'max_depth': int, 'last_open': dt} + active: dict[tuple, dict] = {} + completed: list[dict] = [] + + def _key(deal: dict) -> tuple: + magic = deal.get('magic', '') + if group_by == 'magic+direction': + return (magic, deal.get('type', '').lower()) + return (magic,) + + def _flush(k: tuple) -> None: + if k in active and active[k]['in_deals']: + completed.append(active.pop(k)) + + for deal in sorted(deals, key=lambda d: _parse_dt(d.get('time', '')) or datetime.min): + entry = deal.get('entry', '').lower() + dt = _parse_dt(deal.get('time', '')) + if not dt or not deal.get('magic', ''): + continue + + key = _key(deal) + depth = _extract_depth(deal.get('comment', ''), depth_re) + + if 'in' in entry and 'out' not in entry: + if key in active: + gap = (dt - active[key]['last_open']).total_seconds() / 60 + if gap > gap_minutes: + _flush(key) + if key not in active: + active[key] = {'in_deals': [], 'profit': 0.0, + 'max_depth': 0, 'last_open': dt} + active[key]['in_deals'].append(deal) + active[key]['last_open'] = dt + active[key]['max_depth'] = max(active[key]['max_depth'], depth) + + elif 'out' in entry: + profit = deal.get('profit', 0.0) + if profit == 0.0: + continue + if key in active: + active[key]['profit'] += profit + active[key]['last_open'] = dt + + for k in list(active.keys()): + _flush(k) + + if not completed: + return {'total_cycles': 0, 'win_rate': 0.0, 'avg_profit': 0.0, 'win_rate_by_depth': {}} + + total = len(completed) + wins = sum(1 for c in completed if c['profit'] > 0) + total_profit = sum(c['profit'] for c in completed) + + by_depth: dict[str, dict] = defaultdict(lambda: {'wins': 0, 'total': 0}) + for c in completed: + d = c['max_depth'] + label = (f'L{d}' if 0 < d <= 7 else 'L8+') if d > 0 else 'L?' + by_depth[label]['total'] += 1 + if c['profit'] > 0: + by_depth[label]['wins'] += 1 + + return { + 'total_cycles': total, + 'win_rate': round(wins / total * 100, 1), + 'avg_profit': round(total_profit / total, 2), + 'win_rate_by_depth': { + d: {'total': s['total'], 'win_rate': round(s['wins'] / s['total'] * 100, 1)} + for d, s in sorted(by_depth.items()) + }, + } + + +def exit_reason_breakdown(deals: list[dict], + profile: Optional[dict] = None) -> dict: + """ + Classify closed deals by exit reason and aggregate P/L. + + Default profile (None) uses grid keywords for backward compatibility. + Generic profile returns only 'tp' / 'sl' (profit-sign classification). + """ + _profile = profile or PROFILES['grid'] + counts: dict[str, int] = defaultdict(int) + pnl: dict[str, float] = defaultdict(float) + + for deal in deals: + if 'out' not in deal.get('entry', '').lower(): + continue + profit = deal.get('profit', 0.0) + if profit == 0.0: + continue + reason = _classify_exit(deal.get('comment', ''), profit, _profile) + counts[reason] += 1 + pnl[reason] += profit + + return { + reason: { + 'count': counts[reason], + 'total_pnl': round(pnl[reason], 2), + 'avg_pnl': round(pnl[reason] / counts[reason], 2), + } + for reason in counts + } + + +# ── Summary ─────────────────────────────────────────────────────────────────── + +def build_summary(metrics: dict, monthly: list[dict], dd_events: list[dict], + *, + streak: Optional[dict] = None, + bias: Optional[dict] = None, + exits: Optional[dict] = None, + cycles: Optional[dict] = None, + strategy: Optional[str] = None) -> dict: + green = sum(1 for m in monthly if m['green']) + worst = min(monthly, key=lambda m: m['pnl'], default={}) + + summary: dict = { + 'net_profit': metrics.get('net_profit', 0), + 'profit_factor': metrics.get('profit_factor', 0), + 'max_dd_pct': metrics.get('max_dd_pct', 0), + 'sharpe_ratio': metrics.get('sharpe_ratio', 0), + 'total_trades': metrics.get('total_trades', 0), + 'recovery_factor': metrics.get('recovery_factor', 0), + 'green_months': green, + 'total_months': len(monthly), + 'worst_month': worst.get('month', ''), + 'worst_month_pnl': worst.get('pnl', 0), + } + + if strategy: + summary['strategy'] = strategy + + if streak: + summary['max_win_streak'] = streak.get('max_win_streak', 0) + summary['max_loss_streak'] = streak.get('max_loss_streak', 0) + summary['current_streak'] = streak.get('current_streak', 0) + summary['current_streak_type']= streak.get('current_streak_type', '') + + if bias: + for direction in ('buy', 'sell'): + if direction in bias: + summary[f'{direction}_win_rate'] = bias[direction].get('win_rate', 0) + summary[f'{direction}_total_pnl'] = bias[direction].get('total_pnl', 0) + + if exits: + dominant = max(exits, key=lambda k: exits[k]['count'], default='') + summary['dominant_exit'] = dominant + + if cycles and cycles.get('total_cycles', 0): + summary['cycle_win_rate'] = cycles.get('win_rate', 0) + summary['total_cycles'] = cycles.get('total_cycles', 0) + + return summary + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def _make_parser(prog_suffix: str = '') -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog=f'mt5-analyze{prog_suffix}' if prog_suffix else 'analyze.py', + description=f'MT5 deal analysis{(" — " + PROFILES[prog_suffix.lstrip("-")]["name"]) if prog_suffix else ""}', + ) + p.add_argument('deals_csv', help='Path to deals.csv') + p.add_argument('--output-dir', default='.', help='Output directory') + p.add_argument('--deep', action='store_true', help='Add hourly_pnl and volume_profile') + p.add_argument('--stdout', action='store_true', help='Print JSON to stdout') + return p + + +def _run(deals_csv: str, output_dir: str, deep: bool, stdout: bool, + strategy_name: str) -> None: + profile = PROFILES.get(strategy_name, PROFILES['grid']) + + os.makedirs(output_dir, exist_ok=True) + deals = load_deals(deals_csv) + metrics = load_metrics(os.path.join(output_dir, 'metrics.json')) + + if not deals: + print("WARNING: No deals found in CSV", file=sys.stderr) + + # Generic analytics (always run, strategy-agnostic) + monthly = monthly_pnl(deals) + dd_ev = reconstruct_dd_events(deals, metrics, profile) + top_loss = top_losses(deals) + loss_seq = loss_sequences(deals) + pairs = position_pairs(deals) + bias = direction_bias(deals) + streak = streak_analysis(deals) + sessions = session_breakdown(deals) + wday = weekday_pnl(deals) + peak = concurrent_peak(deals) + + # Strategy-aware analytics + depth_hist = depth_histogram(deals, profile) + cycles = cycle_stats(deals, profile) + exits = exit_reason_breakdown(deals, profile) + + summary = build_summary(metrics, monthly, dd_ev, + streak=streak, bias=bias, exits=exits, + cycles=cycles, strategy=strategy_name) + + analysis: dict = { + 'strategy': strategy_name, + 'summary': summary, + 'monthly_pnl': monthly, + 'dd_events': dd_ev, + 'top_losses': top_loss, + 'loss_sequences': loss_seq, + 'position_pairs': pairs, + 'direction_bias': bias, + 'streak_analysis': streak, + 'session_breakdown': sessions, + 'weekday_pnl': wday, + 'concurrent_peak': peak, + 'depth_histogram': depth_hist, + 'cycle_stats': cycles, + 'exit_reason_breakdown': exits, + } + + # Grid backward-compat alias + if strategy_name == 'grid' and depth_hist: + analysis['grid_depth_histogram'] = depth_hist + + if deep: + analysis['hourly_pnl'] = hourly_pnl(deals) + analysis['volume_profile'] = volume_profile(deals) + + if stdout: + json.dump(analysis, sys.stdout, indent=2) + print() + return + + out_path = os.path.join(output_dir, 'analysis.json') + with open(out_path, 'w') as f: + json.dump(analysis, f, indent=2) + + print(f"Analysis complete [{PROFILES[strategy_name]['name']}]: {out_path}") + print(f" {summary.get('green_months', 0)}/{summary.get('total_months', 0)} green months") + print(f" {len(dd_ev)} DD events reconstructed") + if depth_hist: + max_layer = max( + (k for k, v in depth_hist.items() if v > 0), + key=lambda x: int(x[1:].replace('+', '9')), + default='?' + ) + print(f" Grid depth: max {max_layer}") + if cycles.get('total_cycles', 0): + print(f" {cycles['total_cycles']} cycles — {cycles['win_rate']}% win rate") + if bias: + for d in ('buy', 'sell'): + if d in bias: + b = bias[d] + print(f" {d.capitalize()}: {b['trades']} trades, " + f"{b['win_rate']}% win rate, {b['total_pnl']:+.2f}") + + +def main() -> None: + """ + Entry point — auto-detects subcommand style vs legacy style. + + analyze.py grid deals.csv [options] → grid strategy + analyze.py deals.csv [options] → grid (backward compat default) + """ + strategy_name = 'grid' + argv = sys.argv[1:] + + if argv and argv[0] in PROFILES: + strategy_name = argv.pop(0) + + parser = _make_parser() + args = parser.parse_args(argv) + _run(args.deals_csv, args.output_dir, args.deep, args.stdout, strategy_name) + + +# ── Named entry points for each strategy ───────────────────────────────────── + +def main_generic() -> None: + """Entry point: mt5-analyze (generic, strategy-agnostic).""" + _entry('generic') + +def main_grid() -> None: + """Entry point: mt5-analyze-grid""" + _entry('grid') + +def main_scalper() -> None: + """Entry point: mt5-analyze-scalper""" + _entry('scalper') + +def main_trend() -> None: + """Entry point: mt5-analyze-trend""" + _entry('trend') + +def main_hedge() -> None: + """Entry point: mt5-analyze-hedge""" + _entry('hedge') + +def _entry(strategy_name: str) -> None: + parser = _make_parser(f'-{strategy_name}') + args = parser.parse_args() + _run(args.deals_csv, args.output_dir, args.deep, args.stdout, strategy_name) + + +if __name__ == '__main__': + main() diff --git a/analytics/extract.py b/analytics/extract.py new file mode 100644 index 0000000..30fa5b4 --- /dev/null +++ b/analytics/extract.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +""" +extract.py — Single-pass MT5 report parser. + +Reads MT5 backtest report (.htm or .htm.xml / SpreadsheetML) and produces: + - metrics.json (aggregate summary) + - deals.csv (all deals, 13 columns) + - deals.json (deals as JSON array) + +Usage: + python3 analytics/extract.py report.htm --output-dir reports/20250101_123456/ +""" + +import argparse +import csv +import json +import os +import re +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Optional + + +# MT5 backtest report deals table columns (actual order from HTML): +# Time, Deal, Symbol, Type, Direction, Volume, Price, Order, Commission, Swap, Profit, Balance, Comment +DEAL_COLUMNS = [ + "time", "deal", "symbol", "type", "entry", "volume", "price", + "order", "commission", "swap", "profit", "balance", "comment" +] + + +def detect_format(path: str) -> str: + """Return 'xml' for SpreadsheetML, 'html' for legacy HTML report.""" + if path.endswith('.xml') or path.endswith('.htm.xml'): + return 'xml' + # Peek at file header + with open(path, 'rb') as f: + header = f.read(512) + if b' str: + """Read file, handling UTF-16 (MT5 default) and latin-1 fallback.""" + with open(path, 'rb') as f: + raw = f.read() + for encoding in ('utf-16', 'utf-8', 'latin-1'): + try: + return raw.decode(encoding) + except (UnicodeDecodeError, LookupError): + continue + return raw.decode('latin-1', errors='replace') + + +def strip_tags(html: str) -> str: + return re.sub(r'<[^>]+>', '', html).strip() + + +# ── HTML parser ─────────────────────────────────────────────────────────────── + +def parse_html(path: str) -> tuple[dict, list[dict]]: + text = read_text(path) + + metrics = _parse_metrics_html(text) + deals = _parse_deals_html(text) + return metrics, deals + + +def _parse_metrics_html(text: str) -> dict: + """Extract aggregate metrics from the summary table.""" + m = {} + + # MT5 report HTML format: MetricLabel:\r\nVALUE + # Helper patterns — values always wrapped in ... + _b = r'[^<]*\s*]*>\s*([-\d\s.,]+)' # plain number + _b_pct = r'[^<]*\s*]*>\s*[^(]*\(([\d.,]+)%\)' # "abs (pct%)" — capture pct + patterns = { + 'net_profit': r'Net\s+Profit' + _b, + 'profit_factor': r'Profit\s+Factor' + _b, + 'max_dd_pct': r'Equity\s+Drawdown\s+Maximal' + _b_pct, + 'sharpe_ratio': r'Sharpe\s+Ratio' + _b, + 'total_trades': r'Total\s+Trades' + _b, + 'recovery_factor': r'Recovery\s+Factor' + _b, + 'win_rate_pct': r'Profit\s+Trades\s+\(%' + _b_pct, + 'gross_profit': r'Gross\s+Profit' + _b, + 'gross_loss': r'Gross\s+Loss' + _b, + } + + for key, pattern in patterns.items(): + match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) + if match: + val = match.group(1).replace(' ', '').replace(',', '').strip() + try: + m[key] = float(val) + except ValueError: + pass + + # Trades needs int + if 'total_trades' in m: + m['total_trades'] = int(m['total_trades']) + + return m + + +def _parse_deals_html(text: str) -> list[dict]: + """Extract deal rows from the deals table.""" + # Find deals section (after "Deals" header) + deals_section = re.search( + r']*>.*?Deal.*?Time.*?Type.*?Direction.*?Volume.*?(.*)', + text, re.DOTALL | re.IGNORECASE + ) + if not deals_section: + return [] + + rows = re.findall( + r']*>(.*?)', + deals_section.group(1), + re.DOTALL | re.IGNORECASE + ) + + deals = [] + for row in rows: + cells = re.findall(r']*>(.*?)', row, re.DOTALL | re.IGNORECASE) + cells = [strip_tags(c).replace(',', '') for c in cells] + + if len(cells) < 3 or not cells[0]: + continue + # Skip balance/deposit/credit rows — 'balance' appears in the Type column (index 3) + # or sometimes in index 1; check first 5 cells + if any(c.strip().lower() in ('balance', 'credit') for c in cells[:5]): + continue + + deal = {} + for i, col in enumerate(DEAL_COLUMNS): + deal[col] = cells[i] if i < len(cells) else '' + deals.append(deal) + + return deals + + +# ── XML parser (SpreadsheetML) ───────────────────────────────────────────────── + +def parse_xml(path: str) -> tuple[dict, list[dict]]: + """Parse MT5 SpreadsheetML optimization/report XML.""" + tree = ET.parse(path) + root = tree.getroot() + + # Namespace handling — MT5 XML uses Excel namespace + ns = {} + ns_match = re.match(r'\{([^}]+)\}', root.tag) + if ns_match: + ns['ss'] = ns_match.group(1) + + def tag(name): + return f"{{{ns['ss']}}}{name}" if ns else name + + metrics = {} + deals = [] + in_deals_sheet = False + + for sheet in root.iter(tag('Worksheet')): + sheet_name = sheet.get(f"{{{ns['ss']}}}Name" if ns else 'Name', '') + + if 'result' in sheet_name.lower() or 'report' in sheet_name.lower(): + metrics = _parse_metrics_xml(sheet, tag) + elif 'deal' in sheet_name.lower() or 'trade' in sheet_name.lower(): + deals = _parse_deals_xml(sheet, tag) + elif sheet_name == '': + # Unnamed sheet — check if it has deal-like structure + rows = list(sheet.iter(tag('Row'))) + if len(rows) > 5: + # Try to parse as deals + candidate = _parse_deals_xml(sheet, tag) + if candidate: + deals = candidate + + return metrics, deals + + +def _cell_value(cell, tag) -> str: + data = cell.find(tag('Data')) + return data.text.strip() if data is not None and data.text else '' + + +def _parse_metrics_xml(sheet, tag) -> dict: + m = {} + for row in sheet.iter(tag('Row')): + cells = [_cell_value(c, tag) for c in row.iter(tag('Cell'))] + if len(cells) < 2: + continue + key = cells[0].lower() + val = cells[1].replace(',', '').strip() + try: + fval = float(val) + if 'net profit' in key or 'net_profit' in key: + m['net_profit'] = fval + elif 'profit factor' in key: + m['profit_factor'] = fval + elif 'drawdown' in key and '%' in cells[1]: + m['max_dd_pct'] = fval + elif 'sharpe' in key: + m['sharpe_ratio'] = fval + elif 'total trades' in key: + m['total_trades'] = int(fval) + except (ValueError, AttributeError): + pass + return m + + +def _parse_deals_xml(sheet, tag) -> list[dict]: + deals = [] + header_found = False + col_map = {} + + for row in sheet.iter(tag('Row')): + cells = [_cell_value(c, tag) for c in row.iter(tag('Cell'))] + + if not header_found: + # Detect header row + if any(h in str(cells).lower() for h in ('time', 'type', 'volume', 'profit')): + header_found = True + for i, h in enumerate(cells): + h_lower = h.lower().strip() + for col in DEAL_COLUMNS: + if col in h_lower or h_lower in col: + col_map[i] = col + break + continue + + if not cells or not cells[0]: + continue + + deal = {} + for i, val in enumerate(cells): + col = col_map.get(i) + if col: + deal[col] = val.replace(',', '') + + if deal: + deals.append(deal) + + return deals + + +# ── Writer ──────────────────────────────────────────────────────────────────── + +def write_outputs(metrics: dict, deals: list[dict], output_dir: str) -> dict: + os.makedirs(output_dir, exist_ok=True) + + metrics_path = os.path.join(output_dir, 'metrics.json') + deals_csv_path = os.path.join(output_dir, 'deals.csv') + deals_json_path = os.path.join(output_dir, 'deals.json') + + with open(metrics_path, 'w') as f: + json.dump(metrics, f, indent=2) + + with open(deals_json_path, 'w') as f: + json.dump(deals, f, indent=2) + + if deals: + all_keys = DEAL_COLUMNS + with open(deals_csv_path, 'w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=all_keys, extrasaction='ignore') + writer.writeheader() + writer.writerows(deals) + else: + # Write empty CSV with headers + with open(deals_csv_path, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(DEAL_COLUMNS) + + return { + 'metrics': metrics_path, + 'deals_csv': deals_csv_path, + 'deals_json': deals_json_path, + } + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser(description='Extract MT5 backtest report') + parser.add_argument('report', help='Path to report.htm or report.htm.xml') + parser.add_argument('--output-dir', default='.', help='Output directory') + parser.add_argument('--stdout', action='store_true', + help='Print metrics JSON to stdout instead of writing files') + args = parser.parse_args() + + fmt = detect_format(args.report) + + if fmt == 'xml': + metrics, deals = parse_xml(args.report) + else: + metrics, deals = parse_html(args.report) + + if not metrics: + print(f"WARNING: No aggregate metrics found in report", file=sys.stderr) + if not deals: + print(f"WARNING: No deals found in report (check date range and symbol)", file=sys.stderr) + + if args.stdout: + json.dump({'metrics': metrics, 'deals_count': len(deals)}, sys.stdout, indent=2) + print() + return + + paths = write_outputs(metrics, deals, args.output_dir) + + print(f"Extracted: {len(deals)} deals, {len(metrics)} metrics") + for name, path in paths.items(): + print(f" {name}: {path}") + + +if __name__ == '__main__': + main() diff --git a/analytics/optimize_parser.py b/analytics/optimize_parser.py new file mode 100644 index 0000000..fb28ce7 --- /dev/null +++ b/analytics/optimize_parser.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +""" +optimize_parser.py — Parse MT5 genetic optimization results. + +Handles both HTML (.htm) and SpreadsheetML XML (.htm.xml) formats. + +Usage: + python3 analytics/optimize_parser.py --job opt_20250619_143022 + python3 analytics/optimize_parser.py --file reports/opt_dir/optimization.htm + python3 analytics/optimize_parser.py --file report.htm.xml --top 30 --sort profit +""" + +import argparse +import json +import os +import re +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +ROOT_DIR = Path(__file__).parent.parent + + +def find_report(job_id: str) -> str: + """Locate optimization report from job metadata.""" + jobs_dir = ROOT_DIR / '.mt5mcp_jobs' + meta_path = jobs_dir / f'{job_id}.json' + + if not meta_path.exists(): + raise FileNotFoundError(f"Job not found: {job_id}. Check .mt5mcp_jobs/") + + with open(meta_path) as f: + meta = json.load(f) + + wine_prefix = meta.get('wine_prefix', '') + base = os.path.join(wine_prefix, 'drive_c', 'mt5mcp_opt_report') + + for ext in ('.htm', '.htm.xml', '.html'): + candidate = base + ext + if os.path.exists(candidate): + return candidate + + raise FileNotFoundError( + f"Optimization report not found. Expected: {base}.htm or {base}.htm.xml\n" + f"Is MT5 optimization still running? Check log: {meta.get('log_file', '')}" + ) + + +def detect_format(path: str) -> str: + if path.endswith('.xml') or path.endswith('.htm.xml'): + return 'xml' + with open(path, 'rb') as f: + header = f.read(512) + if b' str: + with open(path, 'rb') as f: + raw = f.read() + for enc in ('utf-16', 'utf-8', 'latin-1'): + try: + return raw.decode(enc) + except (UnicodeDecodeError, LookupError): + continue + return raw.decode('latin-1', errors='replace') + + +# ── HTML parser ─────────────────────────────────────────────────────────────── + +def parse_html(path: str) -> list[dict]: + text = read_text(path) + + rows = re.findall(r']*>(.*?)', text, re.DOTALL | re.IGNORECASE) + results = [] + headers = [] + + for row in rows: + cells = re.findall(r']*>(.*?)', row, re.DOTALL | re.IGNORECASE) + cells = [re.sub(r'<[^>]+>', '', c).strip().replace(',', '') for c in cells] + + if not cells: + continue + + # Header row detection + if not headers and cells[0].lower() in ('pass', '#', 'result', 'run'): + headers = cells + continue + + # Data row: first cell is pass number (digit) + if headers and cells[0].isdigit(): + row_data = dict(zip(headers, cells)) + results.append(row_data) + elif not headers and cells[0].isdigit() and len(cells) > 5: + # No header — use positional mapping (common MT5 layout) + results.append(_positional_row(cells)) + + return results + + +def _positional_row(cells: list[str]) -> dict: + """Map cells by position for headerless optimization tables.""" + # MT5 optimization table columns (typical order): + # Pass | Profit | Expected Payoff | Profit Factor | Recovery Factor | Sharpe | Custom | DD% | Trades | ...params + pos_names = ['pass', 'profit', 'expected_payoff', 'profit_factor', + 'recovery_factor', 'sharpe_ratio', 'custom', 'max_dd_pct', 'total_trades'] + row = {} + for i, name in enumerate(pos_names): + if i < len(cells): + row[name] = cells[i] + # Remaining are parameters + row['_params_raw'] = cells[len(pos_names):] + return row + + +# ── XML parser ──────────────────────────────────────────────────────────────── + +def parse_xml(path: str) -> list[dict]: + tree = ET.parse(path) + root = tree.getroot() + + ns = {} + ns_match = re.match(r'\{([^}]+)\}', root.tag) + if ns_match: + ns['ss'] = ns_match.group(1) + + def tag(name): + return f"{{{ns['ss']}}}{name}" if ns else name + + def cell_val(cell): + data = cell.find(tag('Data')) + return data.text.strip() if data is not None and data.text else '' + + results = [] + headers = [] + + for sheet in root.iter(tag('Worksheet')): + for row in sheet.iter(tag('Row')): + cells = [cell_val(c) for c in row.iter(tag('Cell'))] + cells = [c.replace(',', '').strip() for c in cells] + + if not cells: + continue + + if not headers: + if any(h.lower() in ('pass', 'result', 'profit') for h in cells): + headers = cells + continue + + if cells[0].isdigit(): + if headers: + row_data = {} + for i, h in enumerate(headers): + row_data[h.lower().replace(' ', '_')] = cells[i] if i < len(cells) else '' + results.append(row_data) + else: + results.append(_positional_row(cells)) + + return results + + +# ── Normalizer ──────────────────────────────────────────────────────────────── + +def normalize(raw_results: list[dict]) -> list[dict]: + """Convert raw parsed rows to typed dicts with consistent keys.""" + normalized = [] + + for r in raw_results: + def fget(keys, default=0.0): + for k in keys: + for rk, rv in r.items(): + if k in rk.lower().replace(' ', '_'): + try: + return float(rv) + except (ValueError, TypeError): + pass + return default + + def iget(keys, default=0): + v = fget(keys, default) + return int(v) + + # Extract known fields + entry = { + 'pass': iget(['pass', '#']), + 'net_profit': fget(['profit', 'net_profit']), + 'profit_factor': fget(['profit_factor']), + 'max_dd_pct': fget(['dd', 'drawdown']), + 'total_trades': iget(['trades']), + 'sharpe_ratio': fget(['sharpe']), + 'recovery_factor': fget(['recovery']), + } + + # Remaining keys are parameters + known_keys = {'pass', 'profit', 'net_profit', 'profit_factor', 'expected_payoff', + 'dd', 'drawdown', 'max_dd_pct', 'trades', 'total_trades', + 'sharpe', 'sharpe_ratio', 'recovery', 'recovery_factor', + 'custom', '#', '_params_raw'} + + params = {} + for k, v in r.items(): + if not any(kw in k.lower() for kw in known_keys): + try: + params[k] = float(v) + except (ValueError, TypeError): + params[k] = v + + entry['params'] = params + normalized.append(entry) + + return normalized + + +# ── Convergence analysis ────────────────────────────────────────────────────── + +def convergence_analysis(results: list[dict], top_n: int = 10) -> dict: + top = results[:top_n] + if not top: + return {} + + all_param_keys = set() + for r in top: + all_param_keys.update(r.get('params', {}).keys()) + + strong = {} # Same value across all top-N + uncertain = [] # Varies + + for key in all_param_keys: + values = set() + for r in top: + v = r.get('params', {}).get(key) + if v is not None: + values.add(v) + if len(values) == 1: + strong[key] = list(values)[0] + else: + uncertain.append(key) + + return { + 'top_n_agreement': strong, + 'high_variance_params': uncertain, + } + + +# ── Display ─────────────────────────────────────────────────────────────────── + +def display_results(results: list[dict], top_n: int, dd_threshold: float, conv: dict): + print(f"\nTotal passes: {len(results)}") + print(f"Showing top {min(top_n, len(results))} by profit:\n") + + print(f"{'Rank':<5} {'Profit':>10} {'PF':>6} {'DD%':>6} {'Sharpe':>7} {'Trades':>7} Params") + print("─" * 80) + + for i, r in enumerate(results[:top_n], 1): + dd = r['max_dd_pct'] + risk_flag = ' ⚠' if dd > dd_threshold else '' + params_str = ' '.join(f"{k}={v}" for k, v in list(r.get('params', {}).items())[:4]) + print( + f"#{i:<4} ${r['net_profit']:>9,.2f} " + f"{r['profit_factor']:>5.2f} " + f"{dd:>5.2f}%" + f"{risk_flag} " + f"{r['sharpe_ratio']:>6.2f} " + f"{r['total_trades']:>7} " + f"{params_str}" + ) + + if conv: + print(f"\nConvergence (top-{min(top_n, len(results))} agreement):") + if conv.get('top_n_agreement'): + print(" Stable params:", ', '.join(f"{k}={v}" for k, v in conv['top_n_agreement'].items())) + if conv.get('high_variance_params'): + print(" Uncertain params:", ', '.join(conv['high_variance_params'])) + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser(description='Parse MT5 optimization results') + parser.add_argument('--job', help='Job ID from optimize.sh output') + parser.add_argument('--file', help='Direct path to optimization.htm or .htm.xml') + parser.add_argument('--top', type=int, default=20, help='Show top N results') + parser.add_argument('--sort', choices=['profit', 'profit_factor', 'sharpe'], + default='profit', help='Sort metric') + parser.add_argument('--dd-threshold', type=float, default=20.0, + help='Flag DD above this % as high-risk') + parser.add_argument('--output', help='Save results as JSON') + args = parser.parse_args() + + # Locate report + if args.file: + report_path = args.file + elif args.job: + try: + report_path = find_report(args.job) + except FileNotFoundError as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + else: + print("ERROR: Provide --job or --file", file=sys.stderr) + sys.exit(1) + + if not os.path.exists(report_path): + print(f"ERROR: Report not found: {report_path}", file=sys.stderr) + sys.exit(1) + + # Parse + fmt = detect_format(report_path) + if fmt == 'xml': + raw = parse_xml(report_path) + else: + raw = parse_html(report_path) + + results = normalize(raw) + + if not results: + print("ERROR: No optimization passes found in report.", file=sys.stderr) + sys.exit(1) + + # Sort + sort_key = { + 'profit': 'net_profit', + 'profit_factor': 'profit_factor', + 'sharpe': 'sharpe_ratio', + }[args.sort] + results.sort(key=lambda r: r.get(sort_key, 0), reverse=True) + + # Convergence analysis + conv = convergence_analysis(results, top_n=10) + + # Display + display_results(results, args.top, args.dd_threshold, conv) + + # Optional JSON output + if args.output: + output = { + 'total_passes': len(results), + 'results': results[:args.top], + 'convergence': conv, + } + with open(args.output, 'w') as f: + json.dump(output, f, indent=2) + print(f"\nSaved: {args.output}") + + +if __name__ == '__main__': + main() diff --git a/config/example.set b/config/example.set new file mode 100644 index 0000000..60518ac --- /dev/null +++ b/config/example.set @@ -0,0 +1,27 @@ +; MT5 optimization .set file — example +; Format: param=current_value||start||step||stop||Y (Y=sweep this param) +; param=current_value||N (N=fixed, no sweep) +; +; MT5-Quant handles encoding automatically: +; - Converts to UTF-16LE (MT5 strips ||Y flags from UTF-8 files) +; - Sets read-only flag after writing +; - Resets OptMode in terminal.ini before launch +; +; Copy to your project dir and point --set to it. + +; ── Entry parameters (sweep) ───────────────────────────────────────────────── +Min_Entry_Confidence=0.610||0.580||0.010||0.650||Y +Max_Entry_Spread=30||10||5||50||Y + +; ── Take-profit and stop-loss (sweep) ──────────────────────────────────────── +TP_Pips=400||300||50||600||Y +SL_Pips=800||600||100||1200||Y + +; ── Grid / martingale settings (fixed) ─────────────────────────────────────── +MaxLayers=6||N +LotMultiplier=1.5||N +GridStep_Pips=150||N + +; ── Risk management (fixed) ────────────────────────────────────────────────── +Max_DD_Percent=15.0||N +CutLoss_Percent=25.0||N diff --git a/config/mt5-quant.example.yaml b/config/mt5-quant.example.yaml new file mode 100644 index 0000000..a51a051 --- /dev/null +++ b/config/mt5-quant.example.yaml @@ -0,0 +1,45 @@ +# mt5-quant configuration (flat key format — no YAML nesting) +# Copy this file to config/mt5-quant.yaml and fill in your paths. + +# ── MT5 / Wine paths ────────────────────────────────────────────────────────── + +# macOS (CrossOver / MetaTrader 5.app) +wine_executable: "/Applications/MetaTrader 5.app/Contents/SharedSupport/wine/bin/wine64" +terminal_dir: "$HOME/Library/Application Support/net.metaquotes.wine.metatrader5/drive_c/Program Files/MetaTrader 5" +experts_dir: "$HOME/Library/Application Support/net.metaquotes.wine.metatrader5/drive_c/Program Files/MetaTrader 5/MQL5/Experts" +tester_profiles_dir: "$HOME/Library/Application Support/net.metaquotes.wine.metatrader5/drive_c/Program Files/MetaTrader 5/MQL5/Profiles/Tester" +tester_cache_dir: "$HOME/Library/Application Support/net.metaquotes.wine.metatrader5/drive_c/Program Files/MetaTrader 5/Tester" + +# Linux (Wine) +# wine_executable: "/usr/bin/wine64" +# terminal_dir: "$HOME/.wine/drive_c/Program Files/MetaTrader 5" +# experts_dir: "$HOME/.wine/drive_c/Program Files/MetaTrader 5/MQL5/Experts" +# tester_profiles_dir: "$HOME/.wine/drive_c/Program Files/MetaTrader 5/MQL5/Profiles/Tester" +# tester_cache_dir: "$HOME/.wine/drive_c/Program Files/MetaTrader 5/Tester" + +# ── Display ─────────────────────────────────────────────────────────────────── +# auto: GUI on macOS CrossOver, Xvfb on Linux without $DISPLAY +# gui: always use display (requires active session) +# headless: always use Xvfb (requires xvfb-run) +display_mode: auto + +# ── Project ─────────────────────────────────────────────────────────────────── +# Root directory of your EA project (where .mq5 and .set files live) +project_dir: "/path/to/your/ea/project" + +# ── Backtest defaults ───────────────────────────────────────────────────────── +# These are fallbacks when not passed as MCP tool arguments +backtest_symbol: "EURUSD" +backtest_deposit: 10000 +backtest_currency: USD +backtest_leverage: 100 +backtest_model: 0 +backtest_timeframe: H1 +backtest_timeout: 900 + +# ── Optimization ────────────────────────────────────────────────────────────── +opt_log_dir: /tmp +opt_min_agents: 1 + +# ── Reports output ──────────────────────────────────────────────────────────── +reports_dir: "/path/to/your/ea/project/reports" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..9880be0 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,431 @@ +# Architecture Deep Dive + +## Design Philosophy + +**Deal-level over aggregate.** MT5's built-in HTML report gives you: profit, profit factor, max DD%, trade count. That's it. You cannot tell from those numbers whether a drawdown was caused by overleveraged locking, a bad entry during a news spike, or a grid that reached L8 in a trending market. + +MT5-Quant extracts every individual deal — entry price, exit price, P/L, comment string — and reconstructs what was happening when each loss event occurred. The `analysis.json` artifact is the result: AI-readable, stable schema, diffable between runs. + +**Pipeline idempotency.** MT5 caches aggressively. Cached `.ex5` binaries, cached `.set` files, stale `terminal.ini` flags. The pipeline exists specifically to invalidate all of these before every run. A backtest result that came from a cached EA binary is wrong in a way that's impossible to detect without the cache clear step. + +**Background isolation for optimization.** Genetic optimizations run 2-6 hours. Running them inside a parent process that can be killed (Claude task runner, SSH session, terminal) corrupts the MT5 optimization state. The only correct pattern is `nohup + disown` — fully detached from all parent process trees. + +--- + +## Component Map + +``` +MT5-Quant/ +├── server/ +│ └── main.py # MCP server (stdio transport) — 10 tool handlers +│ +├── scripts/ +│ ├── setup.sh # Auto-detect Wine/MT5, write config, register MCP +│ ├── platform_detect.sh # Sourced by all scripts — Wine path + headless detection +│ ├── backtest_pipeline.sh # 5-stage pipeline orchestrator +│ ├── optimize.sh # Genetic optimization launcher (nohup + disown) +│ └── mqlcompile.sh # MetaEditor wrapper (Wine/CrossOver) +│ +├── analytics/ +│ ├── extract.py # HTML/XML report parser → metrics.json + deals.csv +│ ├── analyze.py # Deal-level analysis engine → analysis.json +│ └── optimize_parser.py # Optimization result parser +│ +├── config/ +│ ├── MT5-Quant.example.yaml # Template config (copy to MT5-Quant.yaml) +│ ├── MT5-Quant.yaml # Live config (gitignored) +│ ├── baseline.json # Production baseline metrics (gitignored, user-maintained) +│ ├── CLAUDE.template.md # CLAUDE.md template (generated by --claude-code) +│ └── example.set # Example optimization .set file +│ +├── .claude/ +│ └── hooks/ +│ └── user-prompt-submit.sh # Injects baseline.json into every Claude prompt +│ +└── docs/ + ├── ARCHITECTURE.md # This file + ├── MCP_TOOLS.md # Full tool spec + └── REMOTE_AGENTS.md # Linux agent farm setup +``` + +--- + +## Pipeline Stages + +### Stage 1: COMPILE + +```bash +./scripts/mqlcompile.sh src/experts/MyEA.mq5 +``` + +Invokes MetaEditor via Wine with the MQL5 source file. Copies resulting `.ex5` to the MT5 Experts directory. Fails the pipeline on compile errors (stderr contains error count). + +**Why not skip this?** MT5 caches the `.ex5` binary by filename. If you edit your EA and re-run without recompiling, MT5 runs the old binary silently. Always compile. + +--- + +### Stage 2: CLEAN + +```bash +rm -f "${MT5_TESTER_DIR}/cache/*.tst" +rm -f "${MT5_PROFILES_DIR}/Tester/${EXPERT}.set" +``` + +Clears: +- Tester cache (`.tst` files): compiled test results MT5 reuses to skip re-running ticks +- Cached `.set` file: MT5 writes the current parameter values here after each run; if stale, next run picks up wrong params + +**The `.set` encoding trap:** MT5 reads parameter files as UTF-16LE with BOM. It writes them back as UTF-16LE. If you provide a UTF-8 `.set` file for optimization, MT5 reads the parameters correctly (it tries multiple encodings) but when it writes the optimization variants, it uses UTF-16LE and **strips the `||Y` optimization flags**. Every subsequent pass uses the base value. Your 500-combination optimization runs 500 identical backtests. + +Solution: always write `.set` files as UTF-16LE with BOM, mark them read-only before MT5 starts. + +```python +# Python: write UTF-16LE with BOM +content = "\n".join(lines) +with open(set_path, 'w', encoding='utf-16-le') as f: + f.write('\ufeff') # BOM + f.write(content) +os.chmod(set_path, 0o444) # read-only +``` + +--- + +### Stage 3: BACKTEST + +```bash +arch -x86_64 "${WINE}" cmd.exe /c 'C:\_backtest.bat' +``` + +The batch file sets MT5 CLI flags and launches `terminal64.exe`: + +```bat +"C:\Program Files\MetaTrader 5\terminal64.exe" /config:C:\backtest.ini +``` + +`backtest.ini` contains: +```ini +[Tester] +Expert=MyEA +Symbol=XAUUSD +Period=M5 +Deposit=10000 +Currency=USD +Leverage=500 +Model=0 +FromDate=2025.01.01 +ToDate=2025.06.30 +Report=C:\report +Optimization=0 +``` + +MT5 runs in headless mode, writes the report, and exits. + +**Process isolation note:** On macOS with CrossOver, `arch -x86_64` is required — CrossOver ships arm64 Wine wrappers that don't support the x86_64 MT5 binary correctly. Without it, MT5 appears to start but produces no output. + +--- + +### Stage 4: EXTRACT + +Single HTML/XML parse pass that produces three artifacts: + +```bash +python3 analytics/extract.py report.htm +# → metrics.json (aggregate summary) +# → deals.csv (all deals, 13 columns) +# → deals.json (same data, JSON) +``` + +**Why single-pass?** MT5 HTML reports are large (1-5MB for 14-month tests). Each regex pass over the file takes ~200ms. The old pipeline ran 5 separate grep/regex passes, one per artifact. Collapsed to one Python pass: 5× faster and no partial-read inconsistencies. + +**Format detection:** +```python +# MT5 Build 48+ saves SpreadsheetML XML, not HTML +if report_path.endswith('.xml') or report_path.endswith('.htm.xml'): + tree = ET.parse(report_path) + # parse via ElementTree +else: + with open(report_path, 'rb') as f: + raw = f.read() + try: + text = raw.decode('utf-16') + except: + text = raw.decode('latin-1', errors='replace') + # parse via regex +``` + +**Deal columns (13):** +``` +Time | Type | Direction | Volume | Price | S/L | T/P | Profit | Balance | Comment | Order | Magic | Entry +``` + +The `Comment` column is the key to grid analytics. The EA writes `"Layer #3"`, `"Locking Total"`, `"Zombie Exit"` etc. Pattern matching on comments reconstructs which position was at which layer. + +--- + +### Stage 5: ANALYZE + +```bash +# Strategy subcommand (recommended) +python3 analytics/analyze.py grid deals.csv --output-dir reports/20250101/ +python3 analytics/analyze.py scalper deals.csv --output-dir reports/20250101/ +python3 analytics/analyze.py trend deals.csv --output-dir reports/20250101/ +python3 analytics/analyze.py hedge deals.csv --output-dir reports/20250101/ +python3 analytics/analyze.py generic deals.csv --output-dir reports/20250101/ + +# Legacy positional (no subcommand) — defaults to 'grid', backward compatible +python3 analytics/analyze.py deals.csv --output-dir reports/20250101/ + +# Flags available with any strategy +python3 analytics/analyze.py grid deals.csv --output-dir DIR --deep --stdout +# → analysis.json +``` + +All functions operate on the 13-column `deals.csv` — no MT5 or Wine required. + +#### Strategy profiles + +The analysis engine is driven by a `PROFILES` dict. Each profile controls: + +| Field | Type | Controls | +|-------|------|----------| +| `depth_re` | regex or `None` | Whether/how to extract depth from comments | +| `exit_keywords` | `{reason: [kw]}` | Comment patterns for exit classification | +| `dd_cause_keywords` | `{cause: [kw]}` | Comment patterns for DD cause classification | +| `cycle_group_by` | `'magic'` or `'magic+direction'` | How deals are grouped into cycles | +| `cycle_gap_min` | int | Minutes between opens that mark a new cycle | + +Built-in profiles: + +| Profile | `depth_re` | `cycle_group_by` | `cycle_gap_min` | Exit keywords | +|---------|-----------|-----------------|----------------|---------------| +| `generic` | — | `magic` | 60 | profit-sign only (tp/sl) | +| `grid` | `Layer #N` | `magic+direction` | 60 | locking, cutloss, zombie, timeout | +| `scalper` | — | `magic` | 10 | tp, sl, manual, trailing | +| `trend` | — | `magic` | 240 | breakeven, trailing, partial, tp, sl | +| `hedge` | — | `magic+direction` | 120 | tp, sl, net_close, partial | + +#### Entry points (after `pip install -e .`) + +```bash +mt5-analyze deals.csv # generic +mt5-analyze-grid deals.csv # grid / martingale (default in pipeline) +mt5-analyze-scalper deals.csv # scalper +mt5-analyze-trend deals.csv # trend following +mt5-analyze-hedge deals.csv # hedging +``` + +#### Analytics functions + +**Core (always run, strategy-agnostic):** + +| Function | What it computes | +|----------|-----------------| +| `monthly_pnl` | P/L, trade count, green flag per calendar month | +| `reconstruct_dd_events` | Balance curve → local minima; cause from profile keywords | +| `top_losses` | Worst individual closing deals by P/L | +| `loss_sequences` | Consecutive losing closed deals (runs of length ≥ 2) | +| `position_pairs` | Match in/out by order ticket → hold time, depth at close | +| `direction_bias` | Buy vs sell win rate, total P/L, average trade | +| `streak_analysis` | Max consecutive win/loss streaks; current streak | +| `session_breakdown` | Asian (00–08h) / London (08–13h) / London-NY (13–17h) / New York (17–22h) | +| `weekday_pnl` | Mon–Sun P/L and win rate | +| `concurrent_peak` | Peak simultaneous open positions | + +**Strategy-driven (output varies by profile):** + +| Function | Generic | Grid | Scalper/Trend/Hedge | +|----------|---------|------|---------------------| +| `depth_histogram` | `{}` (empty) | L1–L8+ counts | `{}` (no `depth_re`) | +| `cycle_stats` | magic, 60-min gap | magic+direction, 60-min gap | per-profile config | +| `exit_reason_breakdown` | tp / sl | locking / cutloss / zombie / timeout | profile-specific | + +**Deep analytics (`--deep` flag):** + +| Function | What it computes | +|----------|-----------------| +| `hourly_pnl` | Hour-by-hour (0–23) P/L and win rate | +| `volume_profile` | P/L breakdown by lot size tier | + +**DD event reconstruction:** +1. Walk deals chronologically, track running balance +2. At each local minimum (DD > 1%), record timestamp, depth (%), recovery date +3. Classify `cause` using `profile['dd_cause_keywords']`; returns `"unknown"` for generic/unmatched + +**Cycle statistics:** +Deals are grouped by `cycle_group_by` key. A gap greater than `cycle_gap_min` between consecutive opens marks a new cycle boundary. Win rate is computed per cycle (not per deal), then broken down by max depth reached. + +**Exit reason classification:** +Iterates `exit_keywords` in definition order — more specific patterns must appear before general ones to avoid substring false-positives (e.g. `"stop"` inside `"breakeven stop"`). Falls back to profit-sign if no keyword matches. + +**Loss sequence detection:** +Consecutive closed deals where P/L < 0 (minimum length 2). Captures clusters of losses better than any single worst-trade metric. + +--- + +## Optimization Pipeline + +### Why `nohup + disown` is mandatory + +```bash +nohup ./scripts/optimize.sh ... > /tmp/opt.log 2>&1 & disown +``` + +MT5 optimization uses Unix signals to coordinate between `terminal64.exe` (master) and `metatester64.exe` instances (workers). When the parent process tree is killed: + +1. `SIGHUP` propagates to child processes +2. `metatester64.exe` workers receive the signal and terminate +3. The master `terminal64.exe` detects worker failure and aborts the optimization +4. `terminal.ini` is left with `OptMode=-1`, requiring manual reset before next run + +`nohup` prevents `SIGHUP` propagation. `disown` removes the process from the shell's job table so it's not killed when the shell exits. Both are required. + +--- + +### `OptMode` state machine + +`terminal.ini` contains an `OptMode` key that MT5 uses to track optimization state: + +| `OptMode` value | Meaning | +|----------------|---------| +| `0` | Normal backtest mode (ready) | +| `1` | Optimization in progress | +| `2` | Optimization complete — show results | +| `-1` | Optimization aborted / crashed | + +After any optimization run (complete or aborted), MT5 writes `-1` or `2`. On next launch with `Optimization=2` in `backtest.ini`, MT5 reads `OptMode=-1` and exits immediately without running. + +**Fix:** Before every optimization launch, force `OptMode=0` in `terminal.ini`: + +```bash +sed -i 's/OptMode=.*/OptMode=0/' "${MT5_DIR}/terminal.ini" +# Also remove LastOptimization line if present +sed -i '/LastOptimization=/d' "${MT5_DIR}/terminal.ini" +``` + +--- + +## Remote Agent Architecture + +MT5's distributed testing works via a custom TCP protocol. The master `terminal64.exe` listens on a port. Remote agents (`metatester64.exe`) connect and receive test configurations. + +``` +Mac (master) Linux server (agents) +terminal64.exe metatester64.exe × N + │ │ + └──── TCP:3000 ─────────────────────┘ +``` + +**Linux setup:** +```bash +# On Linux server (Wine required) +wine metatester64.exe /server:MAC_IP:3000 /agents:8 +``` + +MT5 shows remote agents in the agent manager as `Agent-0.0.0.0-PORT` entries when listening, and activates them when the remote `metatester64.exe` connects. + +**Throughput:** Linear scaling with agent count. 10 local + 16 remote = 26 agents. A 17,000-combination optimization that takes 3 hours locally completes in ~70 minutes. + +--- + +## Headless Operation + +MT5-Quant uses MT5's CLI mode (`terminal64.exe /config:backtest.ini`) — no user interaction, no clicking in the Strategy Tester GUI. Whether this is truly "headless" depends on platform: + +| Platform | Status | Notes | +|----------|--------|-------| +| macOS + CrossOver | Near-headless | CrossOver manages the display internally. MT5 window may flash briefly or be suppressed entirely depending on bottle settings. No monitor required in practice. | +| Linux + Wine | Requires Xvfb | Wine needs an X11 display connection. Without one, `wine64 terminal64.exe` fails with `cannot open display`. | +| Linux + Wine + Xvfb | Full headless | Virtual framebuffer satisfies Wine's X11 requirement. Use on servers with no monitor. | + +**Linux headless setup (Xvfb):** + +```bash +# Install Xvfb +sudo apt install xvfb + +# Start virtual display on :99 +Xvfb :99 -screen 0 1024x768x16 & +export DISPLAY=:99 + +# Now Wine can launch MT5 without a physical display +wine64 terminal64.exe /config:backtest.ini +``` + +**Persistent virtual display (systemd):** + +```ini +# /etc/systemd/system/xvfb.service +[Unit] +Description=Virtual Display for MT5 + +[Service] +ExecStart=/usr/bin/Xvfb :99 -screen 0 1024x768x16 +Restart=always + +[Install] +WantedBy=multi-user.target +``` + +```bash +sudo systemctl enable xvfb +sudo systemctl start xvfb +``` + +Then set `DISPLAY=:99` in MT5-Quant's environment config. + +**Note:** `metatester64.exe` (the agent worker process) is fully headless — it runs tick simulation with no display requirement. Only the master `terminal64.exe` needs a display to orchestrate the session. On Mac with CrossOver this is handled transparently. + +--- + +## Known Limitations + +**macOS-specific:** +- Requires Wine. The native MT5.app from the Mac App Store (or MetaQuotes CDN) ships bundled Wine at `MetaTrader 5.app/Contents/SharedSupport/wine/`. CrossOver is an alternative. +- `arch -x86_64` required on Apple Silicon. +- File paths must go through Wine's virtual filesystem (`C:\` = inside the Wine prefix `drive_c/`). + +**Report format dependency:** +- SpreadsheetML XML format (`.htm.xml`) has no documented schema from MetaQuotes. The parser is reverse-engineered from observed output. May break on future MT5 builds. + +**Comment-based analytics:** +- Strategy-specific analytics (depth histogram, exit reason, DD cause) depend on EA comment strings. EAs that don't write structured comments will get `generic` profile results — summary metrics, session breakdown, streaks, and direction bias all still work; only keyword-classified fields fall back to `"unknown"` or profit-sign. +- Custom comment patterns can be supported by adding a new entry to `PROFILES` in `analytics/analyze.py` — no other code changes needed. + +**Single MT5 instance:** +- MT5 is single-instance per Windows drive. Two backtests cannot run simultaneously on the same Wine prefix. Parallelism requires multiple Wine prefixes (separate installations). + +--- + +## Claude Code Integration + +`setup.sh --claude-code` generates two files that give Claude persistent context about the user's trading setup: + +### `config/CLAUDE.template.md` + +A project-level CLAUDE.md template the user copies to their EA project root. Encodes: +- MT5-Quant tool names and when to use them +- Baseline tracking policy (never call something an improvement without comparing to `baseline.json`) +- Symbol name reminder (broker-specific suffix matters — `XAUUSD.cent` ≠ `XAUUSD`) +- Backtest and optimization constraints (model 0, single instance, UTF-16LE .set files) + +### `.claude/hooks/user-prompt-submit.sh` + +A Claude Code hook that runs before every prompt submission. Reads `config/baseline.json` and outputs a JSON context block: + +```json +{"context": "## Production Baseline (config/baseline.json)\n..."} +``` + +Claude Code injects this into the system context for every conversation turn. The result: Claude always knows the current production metrics without the user having to paste them. + +**Hook execution path:** +``` +User types prompt + → user-prompt-submit.sh executes + → reads config/baseline.json + → outputs {"context": "..."} to stdout + → Claude Code prepends to system context + → Claude sees baseline in every prompt +``` + +**Graceful degradation:** If `baseline.json` doesn't exist or is malformed, the hook exits 0 silently — no prompt is blocked. The baseline section simply doesn't appear until the user creates the file. diff --git a/docs/MCP_TOOLS.md b/docs/MCP_TOOLS.md new file mode 100644 index 0000000..504cff3 --- /dev/null +++ b/docs/MCP_TOOLS.md @@ -0,0 +1,1570 @@ +# MCP Tool Specification + +Full input/output schemas for all MT5-Quant tools. + +--- + +## `run_backtest` + +Run a complete backtest pipeline: compile → clean cache → backtest → extract → analyze. + +**When to call:** Any time you need fresh backtest results. Always runs the full pipeline unless `skip_*` flags are set. + +### Input schema + +```typescript +{ + // Required + expert: string; // EA name without path or extension. e.g. "MyEA_v1.2" + + // Date range — use either preset OR from+to + preset?: "last_month" | "last_3months" | "ytd" | "last_year"; + from?: string; // "YYYY-MM-DD" + to?: string; // "YYYY-MM-DD" + + // Optional overrides + symbol?: string; // Default from config. e.g. "XAUUSD" + timeframe?: "M1" | "M5" | "M15" | "M30" | "H1" | "H4" | "D1"; // Default: M5 + deposit?: number; // Default from config. e.g. 10000 + currency?: string; // Default: "USD" + model?: 0 | 1 | 2; // 0=every tick (default), 1=1min OHLC, 2=open price + set_file?: string; // Path to .set file. If omitted, uses EA defaults. + leverage?: number; // Default: 500 + + // Pipeline flags + skip_compile?: boolean; // Skip EA compilation (use existing .ex5) + skip_clean?: boolean; // Skip cache clean (faster but risks stale cache) + skip_analyze?: boolean; // Extract only, skip deal analysis + deep_analyze?: boolean; // Add hourly_pnl and volume_profile to analysis.json + strategy?: "grid" | "scalper" | "trend" | "hedge" | "generic"; + // Analysis strategy profile (default: "grid"). + // Controls depth tracking, exit keywords, and cycle grouping. +} +``` + +### Output schema + +```typescript +{ + success: boolean; + report_dir: string; // "reports/20250619_143022_MyEA_XAUUSD_M5" + duration_seconds: number; + + // Inline summary from metrics.json (always present on success) + metrics: { + net_profit: number; + profit_factor: number; + max_dd_pct: number; + sharpe_ratio: number; + total_trades: number; + recovery_factor: number; + expected_payoff: number; + gross_profit: number; + gross_loss: number; + win_rate_pct: number; + avg_profit: number; + avg_loss: number; + }; + + // Deal analysis summary (present unless skip_analyze=true) + analysis_summary: { + green_months: number; + total_months: number; + worst_month: string; // "2025-10" + worst_month_pnl: number; + worst_dd_event_pct: number; + worst_dd_date: string; + max_grid_depth: number; // highest layer reached in any cycle + l5_plus_count: number; // cycles that reached L5+ + }; + + // File paths for direct reading + files: { + metrics_json: string; + analysis_json: string; + deals_csv: string; + deals_json: string; + }; + + error?: string; // Present on failure +} +``` + +### Example + +```json +// Input +{ + "expert": "MyEA_v1.2", + "from": "2025-01-01", + "to": "2025-06-30", + "deposit": 10000, + "model": 0 +} + +// Output +{ + "success": true, + "report_dir": "reports/20250619_143022_MyEA_XAUUSD_M5", + "duration_seconds": 287, + "metrics": { + "net_profit": 4832.10, + "profit_factor": 1.54, + "max_dd_pct": 12.3, + "sharpe_ratio": 1.18, + "total_trades": 891 + }, + "analysis_summary": { + "green_months": 5, + "total_months": 6, + "worst_month": "2025-03", + "worst_month_pnl": -412.80, + "worst_dd_event_pct": 12.3, + "worst_dd_date": "2025-03-14", + "max_grid_depth": 6, + "l5_plus_count": 8 + }, + "files": { + "metrics_json": "reports/20250619_143022_MyEA_XAUUSD_M5/metrics.json", + "analysis_json": "reports/20250619_143022_MyEA_XAUUSD_M5/analysis.json", + "deals_csv": "reports/20250619_143022_MyEA_XAUUSD_M5/deals.csv", + "deals_json": "reports/20250619_143022_MyEA_XAUUSD_M5/deals.json" + } +} +``` + +--- + +## `run_optimization` + +Launch genetic parameter optimization as a detached background process. + +**Important:** This tool returns immediately. MT5 runs for 2-6 hours. The AI agent must NOT poll for results — the user monitors MT5 and signals when done. Call `get_optimization_results` only after user confirmation. + +**Always uses model 0.** Model 1 (1-min OHLC) overfits grid/martingale EAs because intra-bar price movement is not simulated. Parameters that look optimal on model 1 fail on model 0 verification — this is a known trap. + +### Input schema + +```typescript +{ + expert: string; // EA name + set_file: string; // Path to optimization .set file (with ||Y flags) + from: string; // "YYYY-MM-DD" + to: string; // "YYYY-MM-DD" + symbol?: string; // Default from config + deposit?: number; // Default from config + currency?: string; // Default: "USD" + leverage?: number; // Default: 500 + log_file?: string; // Where to write nohup output (default: /tmp/opt_.log) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + job_id: string; // "opt_20250619_143022" + log_file: string; // "/tmp/opt_20250619_143022.log" + pid: number; // Process ID (for user monitoring if needed) + combinations: number; // Estimated from set_file analysis (product of all ||Y ranges) + message: string; // "Optimization launched. Signal me when MT5 completes." +} +``` + +### Optimization set file format + +```ini +; param=current_value||start||step||stop||Y (Y = include in sweep) +; param=value||N (N = fixed, not swept) + +Min_Entry_Confidence=0.610||0.580||0.010||0.650||Y ; 8 values +TP_Pips_Layer1=400||300||50||500||Y ; 5 values +Max_DD_Percent=15.0||N ; fixed + +; Total combinations: 8 × 5 = 40 +``` + +**MT5-Quant handles automatically:** +- UTF-16LE encoding with BOM +- `chmod 444` (read-only) before launch +- `OptMode=0` reset in `terminal.ini` +- `LastOptimization` line removal from `terminal.ini` +- `ExpertParameters` = filename only (not full path) in launch INI + +--- + +## `get_optimization_results` + +Parse completed optimization results. Handles both HTML (`.htm`) and SpreadsheetML XML (`.htm.xml`) formats transparently. + +### Input schema + +```typescript +{ + job_id?: string; // From run_optimization response. If omitted, uses latest _opt/ dir. + report_dir?: string; // Explicit path to *_opt/ directory + top_n?: number; // How many top results to return (default: 20) + dd_threshold?: number; // Flag results above this DD% as high-risk (default: 20) + sort_by?: "profit" | "profit_factor" | "sharpe"; // Default: "profit" +} +``` + +### Output schema + +```typescript +{ + success: boolean; + total_passes: number; + converged: boolean; // True if passes stopped improving in last 10% + report_format: "html" | "xml"; + + results: Array<{ + rank: number; + net_profit: number; + profit_factor: number; + max_dd_pct: number; + total_trades: number; + sharpe_ratio: number; + high_risk: boolean; // DD > dd_threshold + params: Record; // All swept parameter values + }>; + + convergence_analysis: { + top_10_agreement: Record; // Params same across top 10 = strong signal + high_variance_params: string[]; // Params that vary in top 10 = uncertain + }; + + recommendation: { + best_params: Record; + reasoning: string; + next_step: "verify_model0" | "auto_promote" | "investigate"; + }; +} +``` + +### Convergence analysis + +A parameter that appears with the same value across all top-10 results is a strong optimization signal — the genetic algorithm converged on it. A parameter that varies across top-10 means the optimizer couldn't distinguish between values — either the parameter doesn't matter much, or more passes are needed. + +--- + +## `verify_setup` + +Check all required paths, Wine version, and EA/set file inventory. Run this first if `run_backtest` or `run_optimization` fails with path errors. + +### Input schema + +```typescript +{} // No parameters required +``` + +### Output schema + +```typescript +{ + success: boolean; + wine_path: string; + wine_version: string; + mt5_dir: string; + terminal_exe: string; + experts_dir: string; + display_mode: "gui" | "headless"; + ea_count: number; // .ex5 files found in Experts/ + set_count: number; // .set files found + missing: string[]; // List of paths/tools that couldn't be found + hints: string[]; // Actionable fix hints for each missing item +} +``` + +--- + +## `get_backtest_status` + +Check the current stage and elapsed time of a running backtest pipeline by reading its `progress.log`. + +### Input schema + +```typescript +{ + report_dir: string; // Path to the report directory from run_backtest +} +``` + +### Output schema + +```typescript +{ + success: boolean; + report_dir: string; + stage: "COMPILE" | "CLEAN" | "BACKTEST" | "EXTRACT" | "ANALYZE" | "DONE"; + elapsed_seconds: number; + finished: boolean; + log_lines: string[]; // Last 5 lines of progress.log +} +``` + +--- + +## `get_optimization_status` + +Check the live state of a background optimization job (started by `run_optimization`). + +### Input schema + +```typescript +{ + job_id: string; // From run_optimization response +} +``` + +### Output schema + +```typescript +{ + success: boolean; + job_id: string; + alive: boolean; // True if the optimization process is still running + pid: number; + started_at: string; // ISO timestamp + elapsed_seconds: number; + report_found: boolean; // True if MT5 has written the result file + report_path: string | null; + log_tail: string[]; // Last 10 lines of the nohup log +} +``` + +--- + +## `prune_reports` + +Delete old report directories to reclaim disk space, keeping the most recent N runs. Optimization result directories (`*_opt/`) are always preserved. + +### Input schema + +```typescript +{ + keep_last?: number; // How many recent reports to keep (default from config, usually 10) + dry_run?: boolean; // If true, list what would be deleted without deleting (default: false) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + deleted: string[]; // Paths that were (or would be) deleted + kept: string[]; // Paths that were kept + freed_mb: number; // Approximate disk space freed +} +``` + +--- + +## `analyze_report` + +Read and summarize a completed backtest report without re-running MT5. + +### Input schema + +```typescript +{ + report_dir: string; // Path to report directory + strategy?: "grid" | "scalper" | "trend" | "hedge" | "generic"; + // Strategy profile that was used (default: "grid"). + // Only affects interpretation of analysis.json fields — + // does not re-run analysis. + include_deals?: boolean; // Include top 20 deals in output (default: false) + include_monthly?: boolean; // Include full monthly P/L table (default: true) + include_dd_events?: boolean; // Include DD event reconstruction (default: true) + deep?: boolean; // Include hourly_pnl and volume_profile (default: false) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + report_dir: string; + strategy: string; // Active profile: "grid" | "scalper" | "trend" | "hedge" | "generic" + + metrics: { /* same as run_backtest metrics */ }; + + // ── Always present (strategy-agnostic) ───────────────────────────────────── + + monthly_pnl: Array<{ + month: string; // "2025-01" + pnl: number; + trades: number; + green: boolean; + }>; + + dd_events: Array<{ + peak_dd_pct: number; + start_date: string; + end_date: string; + duration_days: number; + recovery_date: string | null; + recovery_days: number | null; + cause: string; // Profile-driven: e.g. "locking_cascade" (grid) or "whipsaw" (trend) + // Falls back to "unknown" when no keyword matched + }>; + + top_losses: Array<{ + date: string; + loss_usd: number; + grid_depth_at_close: number; // 0 for non-grid strategies + volume: number; + comment: string; + }>; + + loss_sequences: Array<{ + length: number; + total_loss: number; + start: string; + end: string; + }>; + + position_pairs: Array<{ + time: string; + type: "buy" | "sell"; + profit: number; + volume: number; + layer: number; + hold_minutes: number | null; + comment: string; + magic: string; + order: string; + }>; + + // ── Strategy-driven (content varies by profile) ──────────────────────────── + + depth_histogram: Record; + // grid: { L1: n, L2: n, …, "L8+": n } + // others: {} (empty — no depth_re in profile) + + grid_depth_histogram: Record; + // Backward-compat alias for depth_histogram (grid only) + + cycle_stats: { + total_cycles: number; + win_rate: number; // percent + avg_profit: number; + win_rate_by_depth: Record; + // win_rate_by_depth populated for grid; keys = "L?" for non-depth profiles + }; + + exit_reason_breakdown: Record< + string, // Keys depend on strategy profile exit_keywords + // grid: "locking" | "cutloss" | "zombie" | "timeout" | "tp" | "sl" + // scalper: "manual" | "trailing" | "tp" | "sl" + // trend: "breakeven" | "trailing" | "partial" | "tp" | "sl" + // generic: "tp" | "sl" + { count: number; total_pnl: number; avg_pnl: number } + >; + + direction_bias: { + buy?: { trades: number; win_rate: number; total_pnl: number; avg_pnl: number }; + sell?: { trades: number; win_rate: number; total_pnl: number; avg_pnl: number }; + }; + + streak_analysis: { + max_win_streak: number; + max_win_start: string; + max_win_end: string; + max_loss_streak: number; + max_loss_start: string; + max_loss_end: string; + current_streak: number; + current_streak_type: "win" | "loss"; + }; + + session_breakdown: Record< + "asian" | "london" | "london_ny_overlap" | "new_york" | "off_hours", + { trades: number; win_rate: number; total_pnl: number } + >; + + weekday_pnl: Array<{ + day: string; // "Monday" … "Sunday" + pnl: number; + trades: number; + win_rate: number; + }>; + + concurrent_peak: { + peak_open: number; + peak_time: string; + }; + + // ── Deep mode only (deep=true) ────────────────────────────────────────────── + + hourly_pnl?: Array<{ + hour: number; // 0–23 + pnl: number; + trades: number; + win_rate: number; + }>; + + volume_profile?: Array<{ + lot_tier: string; // "0.01" | "0.02-0.04" | "0.05-0.09" | "0.10-0.49" | … + pnl: number; + trades: number; + win_rate: number; + }>; + + // ── Optional raw deals ────────────────────────────────────────────────────── + + deals?: Array<{ /* all 13 deal columns */ }>; // Only if include_deals=true +} +``` + +--- + +## `compare_baseline` + +Compare a report against a baseline and return a structured verdict. + +### Input schema + +```typescript +{ + report_dir: string; // Report to evaluate + baseline: { + net_profit: number; + max_dd_pct: number; + total_trades?: number; + label?: string; // e.g. "v1.2 production" + }; + promote_threshold?: { + profit_gt: number; // Auto-promote if profit > this (default: baseline profit) + dd_lt: number; // AND DD < this (default: 20) + }; +} +``` + +### Output schema + +```typescript +{ + verdict: "winner" | "loser" | "marginal"; + auto_promote: boolean; + + delta: { + profit_usd: number; // positive = improvement + profit_pct: number; // relative to baseline + dd_pp: number; // positive = DD got worse + trades_delta: number; + }; + + summary: string; // Human-readable one-liner + + details: { + candidate: { net_profit: number; max_dd_pct: number; total_trades: number; }; + baseline: { net_profit: number; max_dd_pct: number; label: string; }; + }; +} +``` + +### Example + +```json +// Input +{ + "report_dir": "reports/20250619_143022_MyEA_v1.3_XAUUSD_M5", + "baseline": { + "net_profit": 8660, + "max_dd_pct": 15.66, + "label": "v1.2 production" + } +} + +// Output +{ + "verdict": "winner", + "auto_promote": true, + "delta": { + "profit_usd": 3186.32, + "profit_pct": 36.8, + "dd_pp": -7.27, + "trades_delta": -3 + }, + "summary": "+$3,186 (+37%) profit vs v1.2. DD dropped from 15.66% to 8.39%. Auto-promoting.", + "details": { + "candidate": { "net_profit": 11846.32, "max_dd_pct": 8.39, "total_trades": 1963 }, + "baseline": { "net_profit": 8660.00, "max_dd_pct": 15.66, "label": "v1.2 production" } + } +} +``` + +--- + +## `compile_ea` + +Compile an MQL5 Expert Advisor via MetaEditor (Wine/CrossOver). + +### Input schema + +```typescript +{ + expert_path: string; // e.g. "src/MyEA_v1.2.mq5" + include_dirs?: string[]; // Additional include search paths +} +``` + +### Output schema + +```typescript +{ + success: boolean; + binary_path: string; // Path where .ex5 was written + binary_size_bytes: number; + warnings: number; + errors: number; + error_list: Array<{ + file: string; + line: number; + message: string; + }>; + compile_time_ms: number; +} +``` + +--- + +## Error Handling + +All tools return `success: false` with an `error` field on failure. Pipeline failures are non-fatal by default — the tool returns partial results if any stages completed. + +```typescript +{ + success: false, + error: "COMPILE_FAILED", + error_detail: "2 errors in src/MyEA_v1.2.mq5: line 847: undeclared identifier 'Max_New_Param'", + completed_stages: ["COMPILE"], + failed_stage: "COMPILE" +} +``` + +**Error codes:** + +| Code | Stage | Cause | +|------|-------|-------| +| `COMPILE_FAILED` | COMPILE | MQL5 syntax errors | +| `WINE_NOT_FOUND` | Any | Wine/CrossOver not installed or wrong path | +| `MT5_TIMEOUT` | BACKTEST | MT5 didn't exit within timeout (default: 15min) | +| `REPORT_NOT_FOUND` | EXTRACT | MT5 produced no report (usually parameter error) | +| `EXTRACT_FAILED` | EXTRACT | Report parse error (format change?) | +| `NO_DEALS` | ANALYZE | Report has 0 trades (check date range, symbol) | +| `OPT_NOT_FINISHED` | get_opt_results | Optimization still running | + +--- + +## `list_reports` + +List all backtest report directories with compact key metrics. Use this to survey what runs exist before deciding which to analyze — much cheaper than calling `analyze_report` repeatedly. + +### Input schema + +```typescript +{ + include_opt?: boolean; // Include _opt dirs (default: false) + limit?: number; // Max reports, newest first (default: 30) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + count: number; + reports: Array<{ + name: string; // "20250619_143022_MyEA_XAUUSD_M5" + is_opt: boolean; + net_profit?: number; + max_dd_pct?: number; + total_trades?: number; + symbol?: string; + timeframe?: string; + from_date?: string; + to_date?: string; + metrics?: "missing"; // Present only if metrics.json is absent + }>; +} +``` + +--- + +## `tail_log` + +Read the last N lines of a log file. Supports `filter=errors` to return only lines containing error/fail keywords — avoids streaming full logs into context. + +### Input schema + +```typescript +{ + // Provide one of: report_dir, job_id, or log_file + report_dir?: string; // Reads progress.log from this dir (omit for latest) + job_id?: string; // Reads the nohup log for this optimization job + log_file?: string; // Absolute path to any log file + + n?: number; // Lines to return (default: 50) + filter?: "all" | "errors" | "warnings"; // Default: "all" +} +``` + +### Output schema + +```typescript +{ + success: boolean; + log_file: string; // Resolved path of the file that was read + total_lines: number; // Lines matched after filter applied + lines: string[]; // Last n of the matched lines +} +``` + +--- + +## `cache_status` + +Show the MT5 tester cache directory size broken down by symbol. Use before `clean_cache` to see what's there. + +### Input schema + +```typescript +{} // No parameters +``` + +### Output schema + +```typescript +{ + success: boolean; + cache_dir: string; + total_size_mb: number; + symbols: Array<{ + symbol: string; // Subdirectory name (broker symbol) + size_mb: number; + }>; +} +``` + +--- + +## `clean_cache` + +Delete MT5 tester cache files. Forces MT5 to regenerate tick data on the next backtest (slower first run after clean). Supports dry-run preview and per-symbol targeting. + +### Input schema + +```typescript +{ + symbol?: string; // Delete only this symbol's cache. Omit to delete all. + dry_run?: boolean; // Report what would be deleted without deleting (default: false) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + dry_run: boolean; + deleted_symbols: string[]; + freed_mb: number; + hint: string; // Reminder that next backtest will be slower +} +``` + +--- + +## `read_set_file` + +Parse an MT5 `.set` parameter file (UTF-16LE or UTF-8) into structured JSON. Handles BOM detection automatically. Use this instead of reading raw `.set` files. + +### Input schema + +```typescript +{ + path: string; // Path to .set file +} +``` + +### Output schema + +```typescript +{ + success: boolean; + path: string; + param_count: number; + comments: string[]; // Header comment lines (stripped of semicolons) + params: Record; +} +``` + +### Example + +```json +// Input +{ "path": "config/MyEA_opt.set" } + +// Output +{ + "success": true, + "path": "config/MyEA_opt.set", + "param_count": 5, + "comments": ["MyEA optimization set — XAUUSD M5"], + "params": { + "Min_Entry_Confidence": { "value": "0.610", "from": "0.580", "to": "0.650", "step": "0.010", "optimize": true }, + "TP_Pips": { "value": "400", "from": "300", "to": "500", "step": "50", "optimize": true }, + "Max_DD_Percent": { "value": "15.0" } + } +} +``` + +--- + +## `write_set_file` + +Write an MT5 `.set` parameter file with correct UTF-16LE encoding and `chmod 444`. Overwrites any existing file at the path. + +### Input schema + +```typescript +{ + path: string; // Output path for .set file + + params: Record; +} +``` + +### Output schema + +```typescript +{ + success: boolean; + path: string; + param_count: number; + encoding: "utf-16-le"; + permissions: string; // "444 (read-only, required by MT5)" +} +``` + +### Example + +```json +// Input +{ + "path": "config/MyEA_opt.set", + "params": { + "Min_Entry_Confidence": { "value": 0.61, "from": 0.58, "to": 0.65, "step": 0.01, "optimize": true }, + "TP_Pips": { "value": 400, "from": 300, "to": 500, "step": 50, "optimize": true }, + "Max_DD_Percent": 15.0 + } +} + +// Output +{ + "success": true, + "path": "config/MyEA_opt.set", + "param_count": 3, + "encoding": "utf-16-le", + "permissions": "444 (read-only, required by MT5)" +} +``` + +--- + +## `list_jobs` + +List all optimization jobs tracked in `.mt5mcp_jobs/` with compact status. Cheaper than calling `get_optimization_status` per job. + +### Input schema + +```typescript +{ + include_done?: boolean; // Include completed/failed jobs (default: true) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + count: number; + jobs: Array<{ + job_id: string; // "opt_20250619_143022" + status: "running" | "done" | "failed"; + elapsed_seconds: number | null; + expert: string; + started_at: string; // ISO timestamp + log_file: string; + }>; +} +``` + +--- + +## `patch_set_file` + +Modify specific parameters in an existing `.set` file in-place. Preserves all other params, comments, and sweep config untouched. Returns a diff of what changed. **Use instead of `read_set_file` → edit → `write_set_file`** — saves two round-trips. + +### Input schema + +```typescript +{ + path: string; // .set file to modify (must exist) + patches: Record; +} +``` + +### Output schema + +```typescript +{ + success: boolean; + path: string; + changed_count: number; + param_count: number; + changed: Array<{ name: string; old: string; new: string; }>; +} +``` + +### Example + +```json +// Input — change two params without touching the rest of the file +{ + "path": "config/MyEA_opt.set", + "patches": { + "TP_Pips": 350, + "Min_Entry_Confidence": { "value": 0.62, "from": 0.60, "to": 0.65, "optimize": true } + } +} + +// Output +{ + "success": true, + "path": "config/MyEA_opt.set", + "changed_count": 2, + "param_count": 12, + "changed": [ + { "name": "TP_Pips", "old": "400", "new": "350" }, + { "name": "Min_Entry_Confidence", "old": "0.610", "new": "0.62" } + ] +} +``` + +--- + +## `clone_set_file` + +Copy a `.set` file to a new path, applying optional param overrides. One call instead of read → modify → write. Preserves header comments. + +### Input schema + +```typescript +{ + source: string; // Source .set file + destination: string; // Output path (created if needed) + overrides?: Record; +} +``` + +### Output schema + +```typescript +{ + success: boolean; + source: string; + destination: string; + param_count: number; + overridden_count: number; + overridden: Array<{ name: string; old: string | null; new: string; }>; +} +``` + +--- + +## `set_from_optimization` + +Generate a clean backtest `.set` file directly from an optimization result's params dict. Strips all sweep flags (`||Y`) so the file is ready for `run_backtest`. Optionally fills params not in the optimization result from a template `.set`, and optionally re-adds sweep ranges to selected params for a narrowed follow-on optimization. + +**Typical call**: immediately after `get_optimization_results`, use `results[0].params` as the `params` argument. + +### Input schema + +```typescript +{ + path: string; // Output .set file path + + params: Record; + // Flat param→value dict from optimization result. + // e.g. { "TP_Pips": 400, "Min_Confidence": 0.61 } + + template?: string; // Path to existing .set. Params NOT in 'params' are + // copied from here as fixed values. + + sweep?: Record; + // Re-add sweep ranges to specific params after applying opt values. + // Used to create a narrowed follow-on optimization .set. +} +``` + +### Output schema + +```typescript +{ + success: boolean; + path: string; + param_count: number; + from_template: boolean; + opt_params_applied: number; + swept_params: number; // > 0 if sweep was provided + total_combinations: number; // 0 for pure backtest .set +} +``` + +### Example + +```json +// After get_optimization_results returned: +// results[0].params = { "TP_Pips": 400, "Min_Entry_Confidence": 0.62, "Max_DD_Percent": 15.0 } + +{ + "path": "config/MyEA_v1.3.set", + "params": { "TP_Pips": 400, "Min_Entry_Confidence": 0.62, "Max_DD_Percent": 15.0 }, + "template": "config/MyEA_base.set" +} + +// Output +{ + "success": true, + "path": "config/MyEA_v1.3.set", + "param_count": 12, + "from_template": true, + "opt_params_applied": 3, + "swept_params": 0, + "total_combinations": 0 +} +``` + +--- + +## `diff_set_files` + +Compare two `.set` files and return only the differences. Use instead of reading both files and comparing manually. + +### Input schema + +```typescript +{ + path_a: string; // Baseline / old file + path_b: string; // Candidate / new file +} +``` + +### Output schema + +```typescript +{ + success: boolean; + path_a: string; + path_b: string; + identical: boolean; + added_count: number; // Params in b but not a + removed_count: number; // Params in a but not b + changed_count: number; // Params in both but with different value or sweep flag + + added: Array<{ name: string; value: string; }>; + removed: Array<{ name: string; value: string; }>; + changed: Array<{ + name: string; + a: string; // value in path_a + b: string; // value in path_b + sweep_a?: boolean; // Present only if sweep flag differs + sweep_b?: boolean; + }>; +} +``` + +### Example + +```json +{ + "path_a": "config/MyEA_v1.2.set", + "path_b": "config/MyEA_v1.3.set" +} + +// Output +{ + "success": true, + "identical": false, + "added_count": 1, + "removed_count": 0, + "changed_count": 2, + "added": [{ "name": "Trailing_Activation", "value": "50" }], + "removed": [], + "changed": [ + { "name": "TP_Pips", "a": "400", "b": "350" }, + { "name": "Min_Entry_Confidence", "a": "0.610", "b": "0.620", "sweep_a": true, "sweep_b": false } + ] +} +``` + +--- + +## `describe_sweep` + +Show a `.set` file's sweep configuration: which params are swept, their ranges, per-param value counts, and total combinations. Use before `run_optimization` to verify scope. + +### Input schema + +```typescript +{ + path: string; +} +``` + +### Output schema + +```typescript +{ + success: boolean; + path: string; + total_params: number; + swept_count: number; + fixed_count: number; + total_combinations: number; + swept_params: Array<{ + name: string; + from: string; + to: string; + step: string; + count: number; // Number of distinct values in this param's range + }>; + hint: string; // e.g. "240 combinations. Typical range: 1–8h depending on EA tick speed." +} +``` + +### Example + +```json +// Input +{ "path": "config/MyEA_opt.set" } + +// Output +{ + "success": true, + "total_params": 12, + "swept_count": 3, + "fixed_count": 9, + "total_combinations": 240, + "swept_params": [ + { "name": "TP_Pips", "from": "300", "to": "500", "step": "50", "count": 5 }, + { "name": "Min_Entry_Confidence", "from": "0.58","to": "0.65","step": "0.01", "count": 8 }, + { "name": "Max_DD_Percent", "from": "12", "to": "20", "step": "2", "count": 5 } + ], + "hint": "240 combinations. Typical range: 1–8h depending on EA tick speed." +} +``` + +--- + +## `list_set_files` + +List all `.set` files in the MT5 tester profiles directory with param counts, swept param counts, and total combinations per file. Use to find the right `.set` without reading each one. + +### Input schema + +```typescript +{ + ea?: string; // Filter by EA name substring (case-insensitive) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + profiles_dir: string; + count: number; + files: Array<{ + name: string; // filename only + param_count: number; + swept_count: number; + total_combinations: number; // 0 for backtest-only .set files + modified: string; // "YYYY-MM-DD HH:MM" + error?: string; // Present only if file is unreadable + }>; +} +``` + +--- + +## `archive_report` + +Convert a backtest report directory into a compact JSON entry appended to `config/backtest_history.json`. Idempotent — re-archiving the same report is a no-op. Optionally deletes the source directory to reclaim disk space. + +### Input schema + +```typescript +{ + report_dir?: string; // Directory to archive. Omit for latest. + delete_after?: boolean; // Delete source dir after archiving (default: false) + verdict?: "winner" | "loser" | "marginal" | "reference"; + notes?: string; // Free-text notes for the entry + tags?: string[]; // Tags e.g. ["tight-sl", "new-filter"] +} +``` + +### Output schema + +```typescript +{ + success: boolean; + id: string; // Report dir basename used as history entry id + already_existed: boolean; + deleted_source: boolean; + history_file: string; // Absolute path to backtest_history.json + entry_summary: { + ea: string; + symbol: string; + metrics: { net_profit: number; profit_factor: number; max_dd_pct: number; sharpe_ratio: number; total_trades: number; }; + verdict: string | null; + }; +} +``` + +--- + +## `archive_all_reports` + +Bulk-archive all backtest report directories into `config/backtest_history.json`. Entries already in history are skipped. Use `delete_after=true` to reclaim disk space while preserving all results as JSON. Optimization dirs (`_opt` suffix) are never deleted. + +### Input schema + +```typescript +{ + delete_after?: boolean; // Delete source dirs after archiving (default: false) + keep_last?: number; // Protect newest N dirs from deletion even with delete_after=true (default: 5) + dry_run?: boolean; // Preview without making changes (default: false) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + dry_run: boolean; + archived_count: number; + skipped_count: number; // Already in history + deleted_count: number; + failed_count: number; // Dirs with no parseable metrics + archived: string[]; + skipped: string[]; + deleted: string[]; + failed: string[]; + history_file: string; +} +``` + +--- + +## `get_history` + +Query `config/backtest_history.json` with filters and sorting. Strips `monthly_pnl` arrays by default — set `include_monthly=true` when you need the full breakdown. + +### Input schema + +```typescript +{ + ea?: string; // Substring match on EA name + symbol?: string; // Exact match (uppercase) + verdict?: "winner" | "loser" | "marginal" | "reference"; + tag?: string; // Entry must contain this tag + min_profit?: number; // net_profit >= this + max_dd_pct?: number; // max_dd_pct <= this + sort_by?: "date" | "profit" | "dd" | "sharpe"; // Default: date, newest first + limit?: number; // Default: 20 + include_monthly?: boolean; // Include monthly_pnl arrays (default: false) +} +``` + +### Output schema + +```typescript +{ + success: boolean; + count: number; + entries: Array<{ + id: string; // Report dir basename + archived_at: string; // ISO timestamp + report_dir_deleted: boolean; + ea: string; + symbol: string; + timeframe: string; + from_date: string; + to_date: string; + metrics: { + net_profit: number; + profit_factor: number; + max_dd_pct: number; + sharpe_ratio: number; + total_trades: number; + recovery_factor: number; + win_rate_pct: number; + expected_payoff: number; + }; + summary?: { + green_months: number; + total_months: number; + worst_month: string; + worst_month_pnl: number; + dominant_exit?: string; + max_win_streak?: number; + max_loss_streak?: number; + }; + worst_dd_event?: { + peak_dd_pct: number; + start_date: string; + end_date: string; + duration_days: number; + cause: string; + }; + monthly_pnl?: Array<{ month: string; pnl: number; trades: number; green: boolean; }>; + verdict: string | null; + notes: string; + tags: string[]; + promoted_to_baseline: boolean; + }>; +} +``` + +--- + +## `promote_to_baseline` + +Write a backtest result to `config/baseline.json` — the production reference used by `compare_baseline` and the Claude Code baseline hook. Also marks the source history entry as `promoted_to_baseline: true`. + +### Input schema + +```typescript +{ + // Provide one: history_id, report_dir, or neither (uses latest report) + history_id?: string; // Entry id from get_history + report_dir?: string; // Direct path to report directory + notes?: string; // Written to baseline.json notes field +} +``` + +### Output schema + +```typescript +{ + success: boolean; + baseline_file: string; + baseline: { + ea: string; + symbol: string; + period: string; // "YYYY-MM-DD/YYYY-MM-DD" + net_profit: number; + profit_factor: number; + max_drawdown_pct: number; + sharpe_ratio: number; + total_trades: number; + recovery_factor: number; + promoted_from: string; // History entry id + promoted_at: string; // Date promoted (YYYY-MM-DD) + notes: string; + }; +} +``` + +### Example + +```json +// Input +{ "history_id": "20250619_143022_MyEA_XAUUSD_M5", "notes": "v1.3 after walk-forward validation" } + +// Output +{ + "success": true, + "baseline_file": "/path/to/config/baseline.json", + "baseline": { + "ea": "MyEA", + "symbol": "XAUUSD", + "period": "2025-01-01/2025-06-30", + "net_profit": 4832.10, + "profit_factor": 1.54, + "max_drawdown_pct": 12.3, + "sharpe_ratio": 1.18, + "total_trades": 891, + "recovery_from": "20250619_143022_MyEA_XAUUSD_M5", + "promoted_at": "2025-06-20", + "notes": "v1.3 after walk-forward validation" + } +} +``` + +--- + +## `annotate_history` + +Update the verdict, notes, or tags on an existing history entry. Use this after `compare_baseline` to record the decision, or to tag runs for later retrieval. + +### Input schema + +```typescript +{ + history_id: string; // Required — entry id to update + verdict?: "winner" | "loser" | "marginal" | "reference"; + notes?: string; // Replaces existing notes + tags?: string[]; // Replaces existing tags + add_tags?: string[]; // Appends to existing tags without overwriting +} +``` + +### Output schema + +```typescript +{ + success: boolean; + id: string; + verdict: string | null; + notes: string; + tags: string[]; +} +``` + +--- + +## Token-efficient usage patterns + +### Surveying past runs + +``` +list_reports(limit=10) → see what's there (live dirs) +get_history(ea="MyEA", limit=10) → see what's been archived +analyze_report(report_dir=X) → drill into one specific run +``` + +Never call `analyze_report` on multiple directories to find the best run — use `list_reports` or `get_history` first. + +### Checking logs without noise + +``` +tail_log(job_id=X, filter=errors) → only failures +tail_log(report_dir=X, n=20) → last 20 lines of backtest progress +``` + +### Managing disk space + +``` +archive_all_reports(dry_run=true) → preview what would be archived +archive_all_reports(delete_after=true, keep_last=3) → archive all, delete old, keep 3 newest +get_history(sort_by=profit, limit=5) → find best archived runs +``` + +### Labelling experiments + +``` +annotate_history(history_id=X, verdict="loser", notes="SL too tight, reversed at L3") +annotate_history(history_id=X, add_tags=["walk-forward-fail"]) +get_history(verdict="winner") → all winners across all sessions +``` + +### Promoting a new production config + +``` +run_backtest(...) +compare_baseline(...) → get verdict +archive_report(delete_after=true, verdict="winner") +promote_to_baseline(notes="v1.4 after WF") → update baseline.json +``` + +### Managing cache + +``` +cache_status() → see symbol breakdown and total size +clean_cache(symbol=XAUUSD, dry_run=true) → preview +clean_cache(symbol=XAUUSD) → execute +``` + +### Working with set files + +``` +# Inspect +list_set_files(ea="MyEA") → all variants, swept param counts, combinations +describe_sweep(path=MyEA_opt.set) → verify 240 combinations before launching opt +diff_set_files(a=v1.2.set, b=v1.3.set) → only changed params, not full file content + +# Edit (never read+write manually) +patch_set_file(path, {TP_Pips: 350}) → change one param, keep everything else intact +clone_set_file(src, dest, overrides) → create variant from base in one call + +# Generate after optimization +set_from_optimization( → map results[0].params → clean backtest .set + path=MyEA_v1.3.set, + params=results[0].params, + template=MyEA_base.set → fills non-swept params from existing file +) +``` + +--- + +## Autonomous Loop Pattern + +The tools are designed to support a fully autonomous experiment → evaluate → promote → optimize loop: + +``` +1. run_backtest(new_params) +2. compare_baseline(result, current_production) +3a. if winner: + - archive_report(delete_after=true, verdict="winner") + - promote_to_baseline(notes="...") + - write_set_file(new_production.set) + - run_optimization(new_production_set) + - [wait for user signal] + - get_optimization_results() + - set_from_optimization(path=verify.set, params=results[0].params, template=prod.set) + - verify top result: run_backtest(expert, set_file=verify.set, skip_compile=true) + - if still beats baseline → goto step 1 +3b. if loser: + - archive_report(delete_after=true, verdict="loser", notes="root cause") + - analyze_report(result) → find root cause + - read_set_file() → inspect current params + - propose parameter or code change → goto step 1 +``` + +No user confirmation needed between steps 1→2→3. The AI agent drives the full loop; the user monitors and signals when optimization completes (since that runs for hours). Every run is archived before the directory is deleted, so nothing is lost. diff --git a/docs/REMOTE_AGENTS.md b/docs/REMOTE_AGENTS.md new file mode 100644 index 0000000..7f4371c --- /dev/null +++ b/docs/REMOTE_AGENTS.md @@ -0,0 +1,187 @@ +# Remote Agent Setup (Linux Server) + +MT5's distributed testing lets you farm optimization work across multiple machines. A Linux server running `metatester64.exe` via Wine connects to a Mac master over a local network. + +**Throughput:** Each agent handles one pass at a time. 10 local + 16 remote = 26 agents. A 17,000-combination genetic optimization that takes 3 hours locally finishes in ~70 minutes. + +--- + +## Requirements + +**Mac (master)** +- MetaTrader 5 installed via CrossOver or Wine +- MT5-Quant configured and working for local backtests +- Port 3000 open in macOS firewall (or whichever port MT5 uses — check below) + +**Linux server (agents)** +- Wine 7.0+ (or Wine Staging for better compatibility) +- `metatester64.exe` from an MT5 installation +- Access to the same MT5 data files (tick history) as the master — or let agents download on first run + +--- + +## Step 1: Find the MT5 agent port on Mac + +After running a local optimization, check the MT5 agent directories: + +```bash +ls ~/Library/Application\ Support/MetaQuotes/*/drive_c/Program\ Files/MetaTrader\ 5/ +``` + +You'll see directories like: +``` +Agent-127.0.0.1-3000/ ← local agents (loopback) +Agent-127.0.0.1-3001/ +Agent-0.0.0.0-3000/ ← remote listener (if enabled) +``` + +If you don't see `Agent-0.0.0.0-*` directories, enable remote agents in MT5: +**Tools → Options → Expert Advisors → Allow remote agents** + +Note the port number (default: 3000). + +--- + +## Step 2: Open firewall on Mac + +```bash +# Check current firewall rules +sudo pfctl -s rules + +# Allow incoming on agent port (example: 3000) +# Add to /etc/pf.conf or use macOS Firewall in System Settings +``` + +Or use macOS System Settings → Privacy & Security → Firewall → Firewall Options → Add MetaTrader 5. + +--- + +## Step 3: Install Wine on Linux server + +```bash +# Ubuntu/Debian +sudo dpkg --add-architecture i386 +sudo apt update +sudo apt install wine64 wine32 + +# Verify +wine64 --version +``` + +--- + +## Step 4: Copy `metatester64.exe` to Linux + +From your Mac MT5 installation: +```bash +MAC_MT5="$HOME/Library/Application Support/MetaQuotes/Terminal/XXXXX/drive_c/Program Files/MetaTrader 5" + +scp "${MAC_MT5}/metatester64.exe" user@linux-server:~/mt5agents/ +scp "${MAC_MT5}/metaeditor64.exe" user@linux-server:~/mt5agents/ # not required for agents +``` + +Also copy the required DLLs (they're in the same directory): +```bash +scp "${MAC_MT5}"/*.dll user@linux-server:~/mt5agents/ +``` + +--- + +## Step 5: Launch agents on Linux + +```bash +# On the Linux server +cd ~/mt5agents/ + +# Replace MAC_IP with your Mac's local IP address +# Replace 8 with number of CPU cores - 1 +wine64 metatester64.exe /server:192.168.1.100:3000 /agents:8 +``` + +**What this does:** +- Connects to your Mac's MT5 master at `192.168.1.100:3000` +- Registers 8 worker agents +- MT5 on Mac will show 8 new agents in the agent manager + +**To run as a background service:** +```bash +nohup wine64 metatester64.exe /server:192.168.1.100:3000 /agents:8 \ + > ~/mt5agents/agents.log 2>&1 & +disown +``` + +--- + +## Step 6: Verify agents appear in MT5 + +On your Mac, open MT5: +**View → Strategy Tester → Agents tab** + +You should see entries like: +``` +Agent-192.168.1.200-3000 [Active] +Agent-192.168.1.200-3001 [Active] +... +``` + +If agents appear as `[Inactive]`, check: +1. Mac firewall is allowing incoming connections on the agent port +2. Linux server can reach Mac IP: `ping 192.168.1.100` +3. Port is open: `nc -zv 192.168.1.100 3000` + +--- + +## Step 7: Configure MT5-Quant for remote agents + +In `config/MT5-Quant.yaml`: +```yaml +optimization: + remote_agents: + enabled: true + check_agent_count: true # Verify remote agents are connected before launching + min_agents: 4 # Require at least N agents before optimizing +``` + +MT5-Quant will log the agent count before launching optimization: +``` +[optimize] Local agents: 9, Remote agents: 8, Total: 17 +[optimize] Estimated completion: ~105 minutes (17,640 passes at 17 agents) +``` + +--- + +## Tick Data Sync + +Remote agents need tick history to replay trades. On first run, MT5 automatically downloads ticks from the broker for each symbol+period combination tested. This can take 10-30 minutes per symbol. + +**Speed up first run:** Pre-populate the tick cache on the Linux server by copying from Mac: + +```bash +# On Mac — find tick data location +find ~/Library/Application\ Support/MetaQuotes -name "*.bin" | grep "XAUUSD" + +# Typical path: +# Terminal/XXXXX/drive_c/users/user/AppData/Roaming/MetaQuotes/Terminal/Common/Files/ + +scp -r "${TICK_DIR}" user@linux-server:~/mt5agents/ticks/ +``` + +--- + +## Troubleshooting + +**Agents connect then immediately disconnect** +- MT5 version mismatch between `metatester64.exe` (from Mac) and the master. Use the exact same build number. +- Check `~/mt5agents/agents.log` for Wine errors. + +**Agents show as connected but don't receive work** +- MT5 sometimes requires optimization to be started before assigning work to new agents. +- Start an optimization from the MT5 GUI first to "activate" remote agents, then cancel it and use MT5-Quant. + +**Performance is slower with remote agents** +- Network latency between Mac and Linux server. Results are sent over TCP after each pass. +- Local gigabit network: negligible. WiFi or WAN: significant overhead. Use wired connection. + +**Linux server runs out of memory** +- Each `metatester64.exe` instance uses ~200-400MB. +- 8 agents = ~2-3GB RAM. Size agent count accordingly. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5ae33be --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "mt5-quant" +version = "0.1.0" +description = "MCP server for MetaTrader 5 backtesting and optimization" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.9" +dependencies = [ + "mcp>=1.0.0", + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-asyncio>=0.21", +] + +[project.scripts] +mt5-quant = "server.main:cli" +mt5-analyze = "analytics.analyze:main_generic" +mt5-analyze-grid = "analytics.analyze:main_grid" +mt5-analyze-scalper = "analytics.analyze:main_scalper" +mt5-analyze-trend = "analytics.analyze:main_trend" +mt5-analyze-hedge = "analytics.analyze:main_hedge" + +[tool.hatch.build.targets.wheel] +packages = ["server", "analytics"] diff --git a/scripts/backtest_pipeline.sh b/scripts/backtest_pipeline.sh new file mode 100755 index 0000000..193df4a --- /dev/null +++ b/scripts/backtest_pipeline.sh @@ -0,0 +1,453 @@ +#!/usr/bin/env bash +# backtest_pipeline.sh — 5-stage MT5 backtest pipeline +# Stages: COMPILE → CLEAN → BACKTEST → EXTRACT → ANALYZE +# +# Usage: +# ./scripts/backtest_pipeline.sh [options] +# +# Options: +# --expert NAME EA name (without path or .mq5 extension) +# --symbol SYMBOL Trading symbol (default: from config) +# --from YYYY.MM.DD Start date +# --to YYYY.MM.DD End date +# --preset PRESET last_month | last_3months | ytd | last_year +# --timeframe TF M1 M5 M15 M30 H1 H4 D1 (default: M5) +# --deposit AMOUNT Initial deposit (default: from config) +# --model 0|1|2 Tick model (default: 0=every tick) +# --set FILE Path to .set parameter file +# --leverage N Leverage (default: 500) +# --skip-compile Skip compilation stage +# --skip-clean Skip cache clean stage +# --skip-analyze Skip analysis stage (extract only) +# --deep Run deep analysis (hourly + volume profile) +# --strategy NAME Analysis strategy profile: grid (default) | scalper | trend | hedge | generic +# --timeout N Backtest timeout in seconds (default: 900) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Resolve real physical path (follows symlinks) so analytics/ is found even when +# scripts/ is a symlink (e.g. ~/.config/mt5-quant/scripts -> /path/to/mt5-quant/scripts) +REAL_SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${REAL_SCRIPT_DIR}/.." && pwd)" +source "${SCRIPT_DIR}/platform_detect.sh" + +# ── Defaults from config ────────────────────────────────────────────────────── +DEFAULT_SYMBOL=$(_cfg "backtest_symbol" "XAUUSD") +DEFAULT_DEPOSIT=$(_cfg "backtest_deposit" "10000") +DEFAULT_CURRENCY=$(_cfg "backtest_currency" "USD") +DEFAULT_LEVERAGE=$(_cfg "backtest_leverage" "500") +DEFAULT_MODEL=$(_cfg "backtest_model" "0") +DEFAULT_TF=$(_cfg "backtest_timeframe" "M5") +DEFAULT_TIMEOUT=$(_cfg "backtest_timeout" "900") +REPORTS_DIR="$(_cfg "reports_dir" "${ROOT_DIR}/reports")" +# Optional: force headless terminal to a specific broker account (needed when live +# trading terminal uses a different broker than the backtest symbol requires). +DEFAULT_LOGIN=$(_cfg "backtest_login" "") +DEFAULT_SERVER=$(_cfg "backtest_server" "") + +# ── Parse arguments ─────────────────────────────────────────────────────────── +EXPERT="" +SYMBOL="$DEFAULT_SYMBOL" +FROM_DATE="" +TO_DATE="" +PRESET="" +TIMEFRAME="$DEFAULT_TF" +DEPOSIT="$DEFAULT_DEPOSIT" +CURRENCY="$DEFAULT_CURRENCY" +MODEL="$DEFAULT_MODEL" +SET_FILE="" +LEVERAGE="$DEFAULT_LEVERAGE" +SKIP_COMPILE=false +SKIP_CLEAN=false +SKIP_ANALYZE=false +DEEP_ANALYZE=false +STRATEGY="grid" +TIMEOUT="$DEFAULT_TIMEOUT" +PROJECT_DIR="$(_cfg "project_dir" "")" +GUI_MODE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --expert) EXPERT="$2"; shift 2 ;; + --project-dir) PROJECT_DIR="$2"; shift 2 ;; + --gui) GUI_MODE=true; shift ;; + --symbol) SYMBOL="$2"; shift 2 ;; + --from) FROM_DATE="$2"; shift 2 ;; + --to) TO_DATE="$2"; shift 2 ;; + --preset) PRESET="$2"; shift 2 ;; + --timeframe) TIMEFRAME="$2"; shift 2 ;; + --deposit) DEPOSIT="$2"; shift 2 ;; + --model) MODEL="$2"; shift 2 ;; + --set) SET_FILE="$2"; shift 2 ;; + --leverage) LEVERAGE="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --skip-compile) SKIP_COMPILE=true; shift ;; + --skip-clean) SKIP_CLEAN=true; shift ;; + --skip-analyze) SKIP_ANALYZE=true; shift ;; + --deep) DEEP_ANALYZE=true; shift ;; + --strategy) STRATEGY="$2"; shift 2 ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +[[ -z "$EXPERT" ]] && { echo "ERROR: --expert is required" >&2; exit 1; } + +# ── Preset date resolution ──────────────────────────────────────────────────── +if [[ -n "$PRESET" ]]; then + TODAY=$(date +%Y.%m.%d) + case "$PRESET" in + last_month) + FROM_DATE=$(date -d "1 month ago" +%Y.%m.01 2>/dev/null || \ + date -v-1m +%Y.%m.01) + TO_DATE="$TODAY" + ;; + last_3months) + FROM_DATE=$(date -d "3 months ago" +%Y.%m.01 2>/dev/null || \ + date -v-3m +%Y.%m.01) + TO_DATE="$TODAY" + ;; + ytd) + FROM_DATE=$(date +%Y.01.01) + TO_DATE="$TODAY" + ;; + last_year) + PREV_YEAR=$(( $(date +%Y) - 1 )) + FROM_DATE="${PREV_YEAR}.01.01" + TO_DATE="${PREV_YEAR}.12.31" + ;; + *) echo "ERROR: Unknown preset: $PRESET" >&2; exit 1 ;; + esac +fi + +[[ -z "$FROM_DATE" || -z "$TO_DATE" ]] && { + echo "ERROR: Provide --from/--to dates or --preset" >&2; exit 1 +} + +# ── Report directory ────────────────────────────────────────────────────────── +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +REPORT_ID="${TIMESTAMP}_${EXPERT}_${SYMBOL}_${TIMEFRAME}" +REPORT_DIR="${REPORTS_DIR}/${REPORT_ID}" +mkdir -p "$REPORT_DIR" + +PIPELINE_START=$(date +%s) +PROGRESS_LOG="${REPORT_DIR}/progress.log" +_progress() { echo "$1 $(date -u +%Y-%m-%dT%H:%M:%SZ) elapsed=$(( $(date +%s) - PIPELINE_START ))" >> "$PROGRESS_LOG"; } + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " MT5-Quant Backtest Pipeline" +echo " Expert: $EXPERT" +echo " Symbol: $SYMBOL Timeframe: $TIMEFRAME Model: $MODEL" +echo " Period: $FROM_DATE → $TO_DATE" +echo " Deposit: $CURRENCY $DEPOSIT Leverage: 1:$LEVERAGE" +[[ -n "$SET_FILE" ]] && echo " Set file: $SET_FILE" +echo " Report: $REPORT_DIR" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# ── Resolve platform ────────────────────────────────────────────────────────── +resolve_platform + +# ── Stage 1: COMPILE ────────────────────────────────────────────────────────── +if [[ "$SKIP_COMPILE" == false ]]; then + _progress "COMPILE" + echo "" + echo "[1/5] COMPILE" + + # Find source file — check project.dir first, then fall back to pipeline root + EA_SOURCE="" + search_paths=( + "${ROOT_DIR}/src/experts/${EXPERT}.mq5" + "${ROOT_DIR}/src/${EXPERT}.mq5" + "${ROOT_DIR}/${EXPERT}.mq5" + ) + if [[ -n "$PROJECT_DIR" ]]; then + search_paths=( + "${PROJECT_DIR}/src/experts/${EXPERT}.mq5" + "${PROJECT_DIR}/src/${EXPERT}.mq5" + "${PROJECT_DIR}/${EXPERT}.mq5" + "${search_paths[@]}" + ) + fi + for search_path in "${search_paths[@]}"; do + [[ -f "$search_path" ]] && { EA_SOURCE="$search_path"; break; } + done + + [[ -z "$EA_SOURCE" ]] && { + echo " ERROR: Cannot find ${EXPERT}.mq5" >&2 + exit 1 + } + + "${SCRIPT_DIR}/mqlcompile.sh" "$EA_SOURCE" +else + echo "[1/5] COMPILE skipped" +fi + +# ── Stage 2: CLEAN ──────────────────────────────────────────────────────────── +if [[ "$SKIP_CLEAN" == false ]]; then + _progress "CLEAN" + echo "" + echo "[2/5] CLEAN" + + # Clear tester cache + if [[ -d "$MT5_CACHE_DIR" ]]; then + find "$MT5_CACHE_DIR" -name "*.tst" -delete 2>/dev/null || true + echo " Cleared tester cache: $MT5_CACHE_DIR" + fi + + # Remove cached .set file for this expert + CACHED_SET="${MT5_TESTER_DIR}/${EXPERT}.set" + if [[ -f "$CACHED_SET" ]]; then + rm -f "$CACHED_SET" + echo " Removed cached .set: $CACHED_SET" + fi + + # Reset terminal.ini OptMode — after any test/optimization MT5 sets OptMode=-1 + # which causes the next headless run to exit immediately (exit 49, no report) + TERMINAL_INI="${MT5_DIR}/config/terminal.ini" + if [[ -f "$TERMINAL_INI" ]]; then + python3 -c " +import sys, re +path = sys.argv[1] +try: + with open(path, 'rb') as f: + raw = f.read() + # Detect encoding: UTF-16 with BOM, or plain text + if raw[:2] in (b'\xff\xfe', b'\xfe\xff'): + text = raw.decode('utf-16') + encoding = 'utf-16' + else: + text = raw.decode('utf-8', errors='replace') + encoding = 'utf-8' + text = re.sub(r'(?m)^OptMode=-1\s*$', 'OptMode=0', text) + text = re.sub(r'(?m)^LastOptimization=1\s*\n?', '', text) + with open(path, 'wb') as f: + f.write(text.encode(encoding)) + print(' Reset OptMode=-1 -> OptMode=0 in terminal.ini') +except Exception as e: + print(f' Warning: could not reset OptMode in terminal.ini: {e}') +" "$TERMINAL_INI" 2>/dev/null || true + echo " Reset terminal.ini OptMode" + fi +else + echo "[2/5] CLEAN skipped" +fi + +# ── Prepare .set file ───────────────────────────────────────────────────────── +if [[ -n "$SET_FILE" ]]; then + # Resolve relative paths against PROJECT_DIR (fallback: script ROOT_DIR, then CWD) + if [[ ! -f "$SET_FILE" ]]; then + for base in "$PROJECT_DIR" "$ROOT_DIR" "$(pwd)"; do + [[ -n "$base" && -f "${base}/${SET_FILE}" ]] && { SET_FILE="${base}/${SET_FILE}"; break; } + done + fi + if [[ ! -f "$SET_FILE" ]]; then + echo "ERROR: Set file not found: $SET_FILE" >&2 + exit 1 + fi + # Copy to tester profiles dir (MT5 reads from here) + mkdir -p "$MT5_TESTER_DIR" + cp "$SET_FILE" "${MT5_TESTER_DIR}/${EXPERT}.set" + SET_FILENAME="$(basename "$SET_FILE")" +fi + +# ── Stage 3: BACKTEST ───────────────────────────────────────────────────────── +_progress "BACKTEST" +echo "" +echo "[3/5] BACKTEST" + +# Guard: detect if terminal64.exe is already running (live trading mode). +# MT5 uses a single-instance lock per Wine prefix — a second headless instance +# exits in ~3s with no report. Kill the existing instance before proceeding. +if pgrep -f "wine64-preloader.*terminal64\.exe" > /dev/null 2>&1; then + echo " WARNING: MetaTrader 5 is already running — killing it to allow backtest." + echo " (Restart MT5 manually after the backtest if needed.)" + # Graceful SIGTERM first, then SIGKILL after 5s + pkill -TERM -f "wine64-preloader.*terminal64\.exe" 2>/dev/null || true + for _i in 1 2 3 4 5; do + sleep 1 + pgrep -f "wine64-preloader.*terminal64\.exe" > /dev/null 2>&1 || break + done + # Force-kill if still alive + if pgrep -f "wine64-preloader.*terminal64\.exe" > /dev/null 2>&1; then + pkill -KILL -f "wine64-preloader.*terminal64\.exe" 2>/dev/null || true + sleep 1 + fi + echo " MT5 stopped." +fi + +# Build backtest.ini +REPORT_FILENAME="${REPORT_ID}.htm" +# Relative path — MT5 resolves against its working dir (C:\Program Files\MetaTrader 5\reports\) +WINE_REPORT_PATH="reports\\${REPORT_FILENAME}" + +INI_HOST_PATH="${MT5_DIR}/backtest_config.ini" +mkdir -p "${MT5_DIR}/reports" + +# Prepend [Common] section if login/server are configured — forces the headless +# terminal to connect to the correct broker account (avoids "symbol not exist" +# when live terminal uses a different broker than the backtest symbol requires). +INI_CONTENT="" +if [[ -n "$DEFAULT_LOGIN" && -n "$DEFAULT_SERVER" ]]; then + INI_CONTENT="[Common] +Login=${DEFAULT_LOGIN} +Server=${DEFAULT_SERVER} + +" +fi + +INI_CONTENT+="[Tester] +Expert=${EXPERT}.ex5 +Symbol=${SYMBOL} +Period=${TIMEFRAME} +Optimization=0 +Model=${MODEL} +FromDate=${FROM_DATE} +ToDate=${TO_DATE} +ForwardMode=0 +Deposit=${DEPOSIT} +Currency=${CURRENCY} +ProfitInPips=1 +Leverage=${LEVERAGE} +ExecutionMode=10 +OptimizationCriterion=0 +Visual=$([[ "$GUI_MODE" == true ]] && echo 1 || echo 0) +Report=${WINE_REPORT_PATH} +ReplaceReport=1 +ShutdownTerminal=1 +" +[[ -n "$SET_FILE" ]] && INI_CONTENT+="ExpertParameters=${EXPERT}.set +" + +# MT5 requires UTF-16LE with BOM — plain UTF-8 is silently ignored +printf "%s" "$INI_CONTENT" | iconv -f UTF-8 -t UTF-16LE > "${INI_HOST_PATH}.tmp" +# Prepend BOM (FF FE) +printf '\xff\xfe' | cat - "${INI_HOST_PATH}.tmp" > "${INI_HOST_PATH}" +rm -f "${INI_HOST_PATH}.tmp" + +# Set Wine prefix — CRITICAL: without WINEPREFIX, Wine uses ~/.wine (wrong prefix) +# which causes MT5 to exit immediately (no registry, no tick data, no report) +WINE_PREFIX_DIR=$(dirname "$(dirname "$(dirname "$MT5_DIR")")") +export WINEPREFIX="$WINE_PREFIX_DIR" +export WINEDEBUG="-all" + +# Write launcher batch (start /wait works correctly once WINEPREFIX is set) +BAT_PATH="${WINE_PREFIX_DIR}/drive_c/_mt5mcp_run.bat" +cat > "$BAT_PATH" << 'BATEOF' +@echo off +cd /d "C:\Program Files\MetaTrader 5" +start /wait terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" +BATEOF + +echo " Launching MT5 (timeout: ${TIMEOUT}s) ..." +BACKTEST_START=$(date +%s) + +set +e +timeout "${TIMEOUT}" ${MT5_ARCH} "${MT5_WINE}" cmd.exe /c 'C:\_mt5mcp_run.bat' 2>/dev/null +WINE_EXIT=$? +set -e + +rm -f "$BAT_PATH" + +BACKTEST_ELAPSED=$(( $(date +%s) - BACKTEST_START )) +echo " MT5 completed in ${BACKTEST_ELAPSED}s (exit: ${WINE_EXIT})" + +# Give MT5 a moment to flush the report to disk +sleep 2 + +# ── Locate report file ──────────────────────────────────────────────────────── +MT5_REPORT="" +# Primary: expected relative path from ini +for ext in ".htm" ".htm.xml" ".html"; do + candidate="${MT5_DIR}/reports/${REPORT_ID}${ext}" + if [[ -f "$candidate" ]]; then + MT5_REPORT="$candidate" + break + fi +done +# Fallback: any HTM in MT5_DIR newer than the ini file +if [[ -z "$MT5_REPORT" ]]; then + MT5_REPORT=$(find "${MT5_DIR}" -maxdepth 3 -name "*.htm" -newer "${INI_HOST_PATH}" 2>/dev/null | head -1) +fi + +if [[ -z "$MT5_REPORT" ]]; then + echo " ERROR: MT5 produced no report." >&2 + echo " Check: symbol name, date range, EA name, and that MT5 ran to completion." >&2 + exit 1 +fi + +echo " Report: $MT5_REPORT" + +# ── Stage 4: EXTRACT ───────────────────────────────────────────────────────── +_progress "EXTRACT" +echo "" +echo "[4/5] EXTRACT" + +python3 "${ROOT_DIR}/analytics/extract.py" \ + "$MT5_REPORT" \ + --output-dir "$REPORT_DIR" \ + && echo " → metrics.json, deals.csv, deals.json" + +# ── Stage 5: ANALYZE ───────────────────────────────────────────────────────── +if [[ "$SKIP_ANALYZE" == false ]]; then + _progress "ANALYZE" + echo "" + echo "[5/5] ANALYZE" + + ANALYZE_FLAGS="$STRATEGY" + [[ "$DEEP_ANALYZE" == true ]] && ANALYZE_FLAGS="$ANALYZE_FLAGS --deep" + + python3 "${ROOT_DIR}/analytics/analyze.py" \ + $ANALYZE_FLAGS \ + "${REPORT_DIR}/deals.csv" \ + --output-dir "$REPORT_DIR" \ + && echo " → analysis.json [$STRATEGY]" +else + echo "[5/5] ANALYZE skipped" +fi + +# ── Save pipeline metadata ──────────────────────────────────────────────────── +_progress "DONE" +PIPELINE_ELAPSED=$(( $(date +%s) - PIPELINE_START )) + +python3 - << PYEOF +import json, os +meta = { + "expert": "${EXPERT}", + "symbol": "${SYMBOL}", + "timeframe": "${TIMEFRAME}", + "from_date": "${FROM_DATE}", + "to_date": "${TO_DATE}", + "deposit": ${DEPOSIT}, + "currency": "${CURRENCY}", + "model": ${MODEL}, + "leverage": ${LEVERAGE}, + "set_file": "${SET_FILE}", + "report_dir": "${REPORT_DIR}", + "duration_seconds": ${PIPELINE_ELAPSED}, + "files": { + "metrics": "${REPORT_DIR}/metrics.json", + "analysis": "${REPORT_DIR}/analysis.json", + "deals_csv": "${REPORT_DIR}/deals.csv", + "deals_json": "${REPORT_DIR}/deals.json" + } +} +with open("${REPORT_DIR}/pipeline_metadata.json", "w") as f: + json.dump(meta, f, indent=2) +PYEOF + +# ── Summary ─────────────────────────────────────────────────────────────────── +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Pipeline complete in ${PIPELINE_ELAPSED}s" +echo " Report: $REPORT_DIR" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# Print key metrics inline +if [[ -f "${REPORT_DIR}/metrics.json" ]]; then + python3 - << PYEOF +import json +with open("${REPORT_DIR}/metrics.json") as f: + m = json.load(f) +print(f" Profit: \${m.get('net_profit',0):,.2f} PF: {m.get('profit_factor',0):.2f} DD: {m.get('max_dd_pct',0):.2f}% Sharpe: {m.get('sharpe_ratio',0):.2f} Trades: {m.get('total_trades',0)}") +PYEOF +fi diff --git a/scripts/mqlcompile.sh b/scripts/mqlcompile.sh new file mode 100755 index 0000000..5207180 --- /dev/null +++ b/scripts/mqlcompile.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# mqlcompile.sh — Compile an MQL5 Expert Advisor via MetaEditor (Wine/CrossOver) +# +# Usage: +# ./scripts/mqlcompile.sh +# +# Output: +# Compiled .ex5 written to MT5_EXPERTS_DIR +# Exit 0 on success, 1 on compile errors + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/platform_detect.sh" + +# ── Args ────────────────────────────────────────────────────────────────────── +SOURCE_FILE="${1:-}" +if [[ -z "$SOURCE_FILE" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +if [[ ! -f "$SOURCE_FILE" ]]; then + echo "ERROR: Source file not found: $SOURCE_FILE" >&2 + exit 1 +fi + +SOURCE_FILE="$(realpath "$SOURCE_FILE")" +EXPERT_NAME="$(basename "$SOURCE_FILE" .mq5)" + +# ── Resolve platform ────────────────────────────────────────────────────────── +resolve_platform + +METAEDITOR="${MT5_DIR}/metaeditor64.exe" +if [[ ! -f "$METAEDITOR" ]]; then + echo "ERROR: metaeditor64.exe not found at: $METAEDITOR" >&2 + exit 1 +fi + +# ── Copy source to MT5 Experts source dir ──────────────────────────────────── +# MetaEditor requires the source file to be inside the MT5 directory tree +MT5_SRC_DIR="${MT5_DIR}/MQL5/Experts" +mkdir -p "$MT5_SRC_DIR" +cp "$SOURCE_FILE" "${MT5_SRC_DIR}/${EXPERT_NAME}.mq5" + +WINE_SRC_PATH="$(host_to_wine_path "${MT5_SRC_DIR}/${EXPERT_NAME}.mq5")" + +# ── Sync .mqh include files to MT5 Include dir ─────────────────────────────── +# Auto-detect include/ directory relative to source file. Searches up to 2 +# levels above the source file for an include/ sibling directory. +# Layout supported: +# /experts/EA.mq5 + /include//*.mqh +# /src/experts/EA.mq5 + /src/include//*.mqh +_find_include_dir() { + local source_dir="$1" + local candidate + for candidate in "$source_dir" "$(dirname "$source_dir")" "$(dirname "$(dirname "$source_dir")")"; do + if [[ -d "${candidate}/include" ]]; then + echo "${candidate}/include" + return 0 + fi + done + return 1 +} + +INCLUDE_BASE="" +INCLUDE_BASE=$(_find_include_dir "$(dirname "$SOURCE_FILE")") || true + +if [[ -n "$INCLUDE_BASE" ]]; then + synced_total=0 + # Sync each subdirectory under include/ → MQL5/Include// + while IFS= read -r -d '' subdir; do + dir_name="$(basename "$subdir")" + mt5_dest="${MT5_DIR}/MQL5/Include/${dir_name}" + rm -rf "$mt5_dest" + cp -r "$subdir" "$mt5_dest" + count=$(find "$mt5_dest" -name "*.mqh" | wc -l | tr -d ' ') + echo "[compile] Synced ${count} .mqh → Include/${dir_name}/" + synced_total=$((synced_total + count)) + done < <(find "$INCLUDE_BASE" -mindepth 1 -maxdepth 1 -type d -print0 2>/dev/null) + + # Also sync any .mqh files directly in include/ (flat layout) + while IFS= read -r -d '' mqh; do + cp "$mqh" "${MT5_DIR}/MQL5/Include/" + synced_total=$((synced_total + 1)) + done < <(find "$INCLUDE_BASE" -maxdepth 1 -name "*.mqh" -print0 2>/dev/null) + + if [[ $synced_total -eq 0 ]]; then + echo "[compile] INFO: include/ found but contains no .mqh files — skipping sync" + else + echo "[compile] Synced ${synced_total} .mqh file(s) total" + fi +else + echo "[compile] INFO: No include/ directory found — skipping .mqh sync" +fi + +# ── Set Wine prefix ─────────────────────────────────────────────────────────── +WINE_PREFIX_DIR=$(dirname "$(dirname "$(dirname "$MT5_DIR")")") +export WINEPREFIX="$WINE_PREFIX_DIR" +export WINEDEBUG="-all" + +# ── Run MetaEditor ──────────────────────────────────────────────────────────── +echo "[compile] Compiling ${EXPERT_NAME}.mq5 ..." +LOG_FILE="$(mktemp /tmp/mqlcompile_XXXXXX.log)" + +set +e +${MT5_ARCH} "${MT5_WINE}" "${METAEDITOR}" \ + /compile:"${WINE_SRC_PATH}" \ + /log:"${LOG_FILE}" \ + 2>/dev/null +WINE_EXIT=$? +set -e + +# MetaEditor always exits 0 on macOS/Wine; check log for errors +ERRORS=0 +WARNINGS=0 +if [[ -f "$LOG_FILE" ]]; then + # Log may be UTF-16LE + LOG_TEXT=$(iconv -f UTF-16LE -t UTF-8 "$LOG_FILE" 2>/dev/null || cat "$LOG_FILE") + ERRORS=$(echo "$LOG_TEXT" | grep -cE "^.*error" || true) + WARNINGS=$(echo "$LOG_TEXT" | grep -cE "^.*warning" || true) + + if [[ $ERRORS -gt 0 ]]; then + echo "[compile] FAILED: $ERRORS error(s), $WARNINGS warning(s)" + echo "$LOG_TEXT" | grep -E "error|warning" | head -20 + rm -f "$LOG_FILE" + exit 1 + fi +fi + +# ── Verify .ex5 was produced ────────────────────────────────────────────────── +EX5_PATH="${MT5_SRC_DIR}/${EXPERT_NAME}.ex5" +if [[ ! -f "$EX5_PATH" ]]; then + echo "[compile] ERROR: .ex5 not produced. MetaEditor may have failed silently." >&2 + [[ -f "$LOG_FILE" ]] && cat "$LOG_FILE" + exit 1 +fi + +BINARY_SIZE=$(stat -f%z "$EX5_PATH" 2>/dev/null || stat -c%s "$EX5_PATH") +echo "[compile] OK: ${EXPERT_NAME}.ex5 (${BINARY_SIZE} bytes, ${WARNINGS} warning(s))" + +rm -f "$LOG_FILE" +exit 0 diff --git a/scripts/optimize.sh b/scripts/optimize.sh new file mode 100755 index 0000000..888a934 --- /dev/null +++ b/scripts/optimize.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# optimize.sh — Launch MT5 genetic optimization (always background + detached) +# +# Usage: +# ./scripts/optimize.sh [options] +# +# Options: +# --expert NAME EA name +# --set FILE Optimization .set file (with ||Y flags) +# --symbol SYMBOL Trading symbol +# --from YYYY.MM.DD Start date +# --to YYYY.MM.DD End date +# --deposit AMOUNT Initial deposit +# --model 0|1|2 Tick model (ALWAYS use 0 for grid/martingale EAs) +# --log FILE Log file path (default: /tmp/mt5opt_TIMESTAMP.log) +# +# IMPORTANT: This script launches MT5 as a detached background process. +# It returns immediately. Do NOT set a timeout on this script. +# Monitor /tmp/mt5opt_*.log and wait for user signal before parsing results. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +source "${SCRIPT_DIR}/platform_detect.sh" + +# ── Defaults ────────────────────────────────────────────────────────────────── +DEFAULT_SYMBOL=$(_cfg "backtest_symbol" "XAUUSD") +DEFAULT_DEPOSIT=$(_cfg "backtest_deposit" "10000") +DEFAULT_CURRENCY=$(_cfg "backtest_currency" "USD") +DEFAULT_LEVERAGE=$(_cfg "backtest_leverage" "500") + +EXPERT="" +SET_FILE="" +SYMBOL="$DEFAULT_SYMBOL" +FROM_DATE="" +TO_DATE="" +DEPOSIT="$DEFAULT_DEPOSIT" +CURRENCY="$DEFAULT_CURRENCY" +LEVERAGE="$DEFAULT_LEVERAGE" +MODEL=0 # ALWAYS 0 for optimization — see below +LOG_FILE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --expert) EXPERT="$2"; shift 2 ;; + --set) SET_FILE="$2"; shift 2 ;; + --symbol) SYMBOL="$2"; shift 2 ;; + --from) FROM_DATE="$2"; shift 2 ;; + --to) TO_DATE="$2"; shift 2 ;; + --deposit) DEPOSIT="$2"; shift 2 ;; + --model) + # Warn if user tries to use model != 0 + if [[ "$2" != "0" ]]; then + echo "WARNING: --model $2 ignored. Optimization always uses model=0." >&2 + echo " Model 1/2 overfits martingale/grid EAs (intra-bar price not simulated)." >&2 + fi + shift 2 + ;; + --log) LOG_FILE="$2"; shift 2 ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +[[ -z "$EXPERT" ]] && { echo "ERROR: --expert is required" >&2; exit 1; } +[[ -z "$SET_FILE" ]] && { echo "ERROR: --set is required" >&2; exit 1; } +[[ -z "$FROM_DATE" ]] && { echo "ERROR: --from is required" >&2; exit 1; } +[[ -z "$TO_DATE" ]] && { echo "ERROR: --to is required" >&2; exit 1; } + +[[ ! -f "$SET_FILE" ]] && { echo "ERROR: Set file not found: $SET_FILE" >&2; exit 1; } + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_FILE="${LOG_FILE:-/tmp/mt5opt_${TIMESTAMP}.log}" +JOB_ID="opt_${TIMESTAMP}" + +# ── Resolve platform ────────────────────────────────────────────────────────── +resolve_platform + +# ── Write .set file as UTF-16LE with BOM (read-only) ───────────────────────── +# MT5 REQUIREMENT: optimization .set files must be UTF-16LE with BOM. +# If provided as UTF-8, MT5 strips the ||Y optimization flags silently — +# every pass runs with the fixed base value and optimization is useless. +python3 - << PYEOF +import sys, os, shutil + +src = "${SET_FILE}" +dst = "${MT5_TESTER_DIR}/${EXPERT}.set" +os.makedirs("${MT5_TESTER_DIR}", exist_ok=True) + +with open(src, 'r', encoding='utf-8', errors='replace') as f: + content = f.read() + +# Write UTF-16LE with BOM +with open(dst, 'w', encoding='utf-16-le') as f: + f.write('\ufeff') # BOM + f.write(content) + +# Make read-only — prevents MT5 from overwriting ||Y flags during optimization +os.chmod(dst, 0o444) +print(f" .set → {dst} (UTF-16LE, read-only)") +PYEOF + +# ── Reset OptMode in terminal.ini ───────────────────────────────────────────── +# After any optimization run (complete or aborted), MT5 writes OptMode=-1. +# On next launch, MT5 reads OptMode=-1 and exits immediately without running. +# Must reset to 0 before every optimization launch. +TERMINAL_INI="${MT5_DIR}/terminal.ini" +if [[ -f "$TERMINAL_INI" ]]; then + # Use Python for safe in-place edit (sed -i behaves differently on macOS vs Linux) + python3 - << PYEOF +import re + +ini_path = "${TERMINAL_INI}" +with open(ini_path, 'r', errors='replace') as f: + content = f.read() + +# Reset OptMode +content = re.sub(r'OptMode=.*', 'OptMode=0', content) +# Remove LastOptimization (causes MT5 to skip running) +content = re.sub(r'LastOptimization=.*\n?', '', content) + +with open(ini_path, 'w') as f: + f.write(content) +print(f" terminal.ini: OptMode reset to 0") +PYEOF +fi + +# ── Build optimization INI ──────────────────────────────────────────────────── +WINE_PREFIX_DIR=$(dirname "$(dirname "$MT5_DIR")") + +cat > "${WINE_PREFIX_DIR}/drive_c/mt5mcp_backtest.ini" << INI +[Tester] +Expert=${EXPERT} +Symbol=${SYMBOL} +Period=M5 +Deposit=${DEPOSIT} +Currency=${CURRENCY} +Leverage=${LEVERAGE} +Model=${MODEL} +FromDate=${FROM_DATE} +ToDate=${TO_DATE} +Report=C:\\mt5mcp_opt_report +Optimization=2 +ExpertParameters=${EXPERT}.set +ShutdownTerminal=1 +INI + +cat > "${WINE_PREFIX_DIR}/drive_c/mt5mcp_run.bat" << 'EOF' +@echo off +"C:\Program Files\MetaTrader 5\terminal64.exe" /config:C:\mt5mcp_backtest.ini +EOF + +# ── Count optimization combinations ────────────────────────────────────────── +COMBINATIONS=$(python3 - << PYEOF +import re, math + +with open("${SET_FILE}", 'r', errors='replace') as f: + lines = f.readlines() + +total = 1 +for line in lines: + line = line.strip() + if line.startswith(';') or '=' not in line: + continue + # Format: param=value||start||step||stop||Y + parts = line.split('||') + if len(parts) >= 5 and parts[-1].strip().upper() == 'Y': + try: + start = float(parts[1]) + step = float(parts[2]) + stop = float(parts[3]) + count = max(1, int((stop - start) / step) + 1) + total *= count + except (ValueError, ZeroDivisionError): + pass + +print(total) +PYEOF +) + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " MT5-Quant Genetic Optimization" +echo " Job ID: $JOB_ID" +echo " Expert: $EXPERT" +echo " Symbol: $SYMBOL Model: ${MODEL} (every tick)" +echo " Period: $FROM_DATE → $TO_DATE" +echo " Set file: $SET_FILE" +echo " Combos: $COMBINATIONS (genetic — converges in ~300-500 passes)" +echo " Log: $LOG_FILE" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# ── Launch detached ─────────────────────────────────────────────────────────── +# nohup: prevents SIGHUP when parent (Claude task, SSH session) exits +# disown: removes from shell job table so shell exit doesn't kill it +# Both are required for true detachment. + +nohup bash -c "${MT5_ARCH} '${MT5_WINE}' cmd.exe /c 'C:\\mt5mcp_run.bat' 2>/dev/null || true" \ + > "$LOG_FILE" 2>&1 & +OPT_PID=$! +disown $OPT_PID + +# Write job metadata +JOBS_DIR="${ROOT_DIR}/.mt5mcp_jobs" +mkdir -p "$JOBS_DIR" +cat > "${JOBS_DIR}/${JOB_ID}.json" << JEOF +{ + "job_id": "${JOB_ID}", + "pid": ${OPT_PID}, + "expert": "${EXPERT}", + "symbol": "${SYMBOL}", + "from_date": "${FROM_DATE}", + "to_date": "${TO_DATE}", + "set_file": "${SET_FILE}", + "combinations": ${COMBINATIONS}, + "log_file": "${LOG_FILE}", + "wine_prefix": "${WINE_PREFIX_DIR}", + "started_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +} +JEOF + +echo "" +echo " Launched (pid: $OPT_PID)" +echo " Optimization runs for 2-6 hours. Do NOT kill this process." +echo " Signal when MT5 shows 'Optimization complete' and use:" +echo " python3 analytics/optimize_parser.py --job $JOB_ID" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/scripts/platform_detect.sh b/scripts/platform_detect.sh new file mode 100755 index 0000000..b0bcd48 --- /dev/null +++ b/scripts/platform_detect.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# platform_detect.sh — Detect Wine path, MT5 location, and display mode +# Sourced by other scripts: source scripts/platform_detect.sh +# Sets: MT5_WINE, MT5_DIR, MT5_EXPERTS_DIR, MT5_TESTER_DIR, MT5_CACHE_DIR, DISPLAY_ENV + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Config resolution: user config (~/.config/mt5-quant/) takes precedence over repo config +_USER_CFG="${HOME}/.config/mt5-quant/config/mt5-quant.yaml" +_REPO_CFG="${SCRIPT_DIR}/../config/mt5-quant.yaml" +if [[ -f "$_USER_CFG" ]]; then + CONFIG_FILE="$_USER_CFG" +elif [[ -f "$_REPO_CFG" ]]; then + CONFIG_FILE="$_REPO_CFG" +else + CONFIG_FILE="$_REPO_CFG" # will fail gracefully in _cfg +fi + +# ── Config reader (minimal YAML parser for simple key: value) ──────────────── +_cfg() { + local key="$1" + local default="${2:-}" + if [[ -f "$CONFIG_FILE" ]]; then + local val + val=$(grep -E "^[[:space:]]*${key}[[:space:]]*:" "$CONFIG_FILE" 2>/dev/null \ + | head -1 | sed 's/.*:[[:space:]]*//' | tr -d '"' | tr -d "'" | tr -d '\r') + if [[ -n "$val" && "$val" != "null" && "$val" != '""' ]]; then + echo "$val" + return + fi + fi + echo "$default" +} + +# ── Wine / CrossOver detection ─────────────────────────────────────────────── +_detect_wine() { + local configured + configured=$(_cfg "wine_executable") + + if [[ -n "$configured" && -x "$configured" ]]; then + echo "$configured" + return 0 + fi + + # Auto-detect: native MT5.app (macOS App Store / direct download) + local mt5_app_wine="/Applications/MetaTrader 5.app/Contents/SharedSupport/wine/bin/wine64" + if [[ -x "$mt5_app_wine" ]]; then + echo "$mt5_app_wine" + return 0 + fi + + # Auto-detect: CrossOver (macOS) + local crossover_wine="/Applications/CrossOver.app/Contents/SharedSupport/CrossOver/bin/wine64" + if [[ -x "$crossover_wine" ]]; then + echo "$crossover_wine" + return 0 + fi + + # Auto-detect: Wine (Linux / Homebrew) + for candidate in wine64 wine; do + if command -v "$candidate" &>/dev/null; then + echo "$(command -v "$candidate")" + return 0 + fi + done + + echo "" + return 1 +} + +# ── MT5 terminal directory detection ───────────────────────────────────────── +_detect_mt5_dir() { + local configured + configured=$(_cfg "terminal_dir") + [[ -n "$configured" && -d "$configured" ]] && { echo "$configured"; return 0; } + + # macOS CrossOver — scan all bottles + if [[ "$(uname -s)" == "Darwin" ]]; then + # Native MT5.app sandboxed Wine prefix (most common on macOS) + local app_support="$HOME/Library/Application Support" + for prefix_pattern in \ + "net.metaquotes.wine.metatrader5" \ + "MetaTrader 5/Bottles/metatrader5"; do + local candidate="${app_support}/${prefix_pattern}/drive_c/Program Files/MetaTrader 5" + if [[ -f "${candidate}/terminal64.exe" ]]; then + echo "$candidate" + return 0 + fi + done + + # CrossOver old-style ~/.cxoffice bottles + local bottle_base="$HOME/.cxoffice" + if [[ -d "$bottle_base" ]]; then + while IFS= read -r -d '' terminal; do + echo "$(dirname "$terminal")" + return 0 + done < <(find "$bottle_base" -name "terminal64.exe" -print0 2>/dev/null | head -z -1) + fi + # CrossOver 24+ default location + local mq_base="$HOME/Library/Application Support/MetaQuotes" + if [[ -d "$mq_base" ]]; then + while IFS= read -r -d '' terminal; do + echo "$(dirname "$terminal")" + return 0 + done < <(find "$mq_base" -name "terminal64.exe" -print0 2>/dev/null | head -z -1) + fi + fi + + # Linux Wine — common default + local wine_mt5="$HOME/.wine/drive_c/Program Files/MetaTrader 5" + [[ -d "$wine_mt5" ]] && { echo "$wine_mt5"; return 0; } + + echo "" + return 1 +} + +# ── Display / headless mode ─────────────────────────────────────────────────── +_detect_display_env() { + local mode + mode=$(_cfg "display_mode" "auto") + local xvfb_display + xvfb_display=$(_cfg "xvfb_display" ":99") + local xvfb_screen + xvfb_screen=$(_cfg "xvfb_screen" "1024x768x16") + + case "$mode" in + false|gui) + # GUI mode — use whatever DISPLAY is set + echo "gui" + return 0 + ;; + true|headless) + # Force headless via Xvfb + _start_xvfb "$xvfb_display" "$xvfb_screen" + echo "headless:${xvfb_display}" + return 0 + ;; + auto|*) + # macOS: CrossOver handles display — no Xvfb needed + if [[ "$(uname -s)" == "Darwin" ]]; then + echo "gui" + return 0 + fi + # Linux: if $DISPLAY is set, use it; otherwise start Xvfb + if [[ -n "${DISPLAY:-}" ]]; then + echo "gui" + return 0 + fi + _start_xvfb "$xvfb_display" "$xvfb_screen" + echo "headless:${xvfb_display}" + return 0 + ;; + esac +} + +_start_xvfb() { + local display="$1" + local screen="$2" + + if ! command -v Xvfb &>/dev/null; then + echo "[platform_detect] ERROR: headless mode requires Xvfb. Install with:" >&2 + echo " sudo apt install xvfb # Debian/Ubuntu" >&2 + echo " sudo yum install xorg-x11-server-Xvfb # RHEL/CentOS" >&2 + exit 1 + fi + + # Check if already running + if xdpyinfo -display "$display" &>/dev/null 2>&1; then + return 0 # already up + fi + + Xvfb "$display" -screen 0 "$screen" &>/dev/null & + local xvfb_pid=$! + sleep 1 # brief wait for Xvfb to initialize + + if ! xdpyinfo -display "$display" &>/dev/null 2>&1; then + echo "[platform_detect] ERROR: Xvfb failed to start on display ${display}" >&2 + exit 1 + fi + + echo "[platform_detect] Xvfb started (pid=${xvfb_pid}, display=${display})" >&2 +} + +# ── macOS arch flag ─────────────────────────────────────────────────────────── +_arch_prefix() { + if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then + echo "arch -x86_64" + else + echo "" + fi +} + +# ── Wine path converter (host path → Windows C:\ path) ─────────────────────── +host_to_wine_path() { + local host_path="$1" + # Convert Unix absolute path to Wine C:\ equivalent + # Works for both CrossOver and standard Wine + echo "$host_path" | sed \ + -e 's|.*drive_c/|C:\\|' \ + -e 's|/|\\|g' +} + +# ── Main: resolve everything and export ─────────────────────────────────────── +resolve_platform() { + MT5_WINE=$(_detect_wine) || { + echo "[platform_detect] ERROR: Wine/CrossOver not found." >&2 + echo " Configure wine_executable in config/mt5-quant.yaml" >&2 + echo " macOS: install CrossOver from https://www.codeweavers.com/" >&2 + echo " Linux: sudo apt install wine64" >&2 + exit 1 + } + + MT5_DIR=$(_detect_mt5_dir) || { + echo "[platform_detect] ERROR: MetaTrader 5 installation not found." >&2 + echo " Configure terminal_dir in config/mt5-quant.yaml" >&2 + exit 1 + } + + # Derive sub-paths from MT5_DIR (override via config if needed) + MT5_EXPERTS_DIR="$(_cfg "experts_dir" "${MT5_DIR}/MQL5/Experts")" + MT5_TESTER_DIR="$(_cfg "tester_profiles_dir" "${MT5_DIR}/MQL5/Profiles/Tester")" + MT5_CACHE_DIR="$(_cfg "tester_cache_dir" "${MT5_DIR}/Tester")" + MT5_ARCH="$(_arch_prefix)" + + DISPLAY_MODE=$(_detect_display_env) + if [[ "$DISPLAY_MODE" == headless:* ]]; then + export DISPLAY="${DISPLAY_MODE#headless:}" + fi + + export MT5_WINE MT5_DIR MT5_EXPERTS_DIR MT5_TESTER_DIR MT5_CACHE_DIR MT5_ARCH DISPLAY_MODE +} + +# Run if executed directly (not sourced) +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + resolve_platform + echo "Wine: $MT5_WINE" + echo "MT5 dir: $MT5_DIR" + echo "Experts: $MT5_EXPERTS_DIR" + echo "Tester: $MT5_TESTER_DIR" + echo "Cache: $MT5_CACHE_DIR" + echo "Display: $DISPLAY_MODE" + echo "Arch: ${MT5_ARCH:-native}" +fi diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 0000000..7ad0057 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,832 @@ +#!/usr/bin/env bash +# setup.sh — Auto-detect MT5/Wine paths and write config/mt5-quant.yaml +# Usage: bash scripts/setup.sh [--yes] [--output /path/to/config.yaml] [--keep-last N] [--claude-code] +# +# --yes Overwrite existing config and register MCP without prompting +# --output FILE Write to a custom path instead of config/mt5-quant.yaml +# --keep-last N Keep only last N backtest reports (default: 20) +# --claude-code Generate CLAUDE.md template and .claude/hooks/user-prompt-submit.sh +# (skips main config wizard — run standalone or alongside normal setup) + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_OUT="${REPO_DIR}/config/mt5-quant.yaml" +AUTO_YES=false +KEEP_LAST=20 +CLAUDE_CODE=false + +# ── Argument parsing ────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --yes|-y) AUTO_YES=true ;; + --output) CONFIG_OUT="$2"; shift ;; + --keep-last) KEEP_LAST="$2"; shift ;; + --claude-code) CLAUDE_CODE=true ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac + shift +done + +# ── Helpers ─────────────────────────────────────────────────────────────────── +_green() { printf '\033[0;32m%s\033[0m\n' "$*"; } +_yellow() { printf '\033[0;33m%s\033[0m\n' "$*"; } +_red() { printf '\033[0;31m%s\033[0m\n' "$*"; } +_bold() { printf '\033[1m%s\033[0m\n' "$*"; } +_ok() { printf ' \033[0;32m✓\033[0m %s\n' "$*"; } +_warn() { printf ' \033[0;33m⚠\033[0m %s\n' "$*"; } +_fail() { printf ' \033[0;31m✗\033[0m %s\n' "$*"; } + +_ask() { + # _ask "Prompt text" default_value → echoes user input or default + local prompt="$1" + local default="${2:-}" + if [[ -n "$default" ]]; then + printf '%s [%s]: ' "$prompt" "$default" >&2 + else + printf '%s: ' "$prompt" >&2 + fi + local answer + read -r answer + echo "${answer:-$default}" +} + +_pick() { + # _pick "Label" item1 item2 ... → echoes chosen item + local label="$1"; shift + local items=("$@") + if [[ ${#items[@]} -eq 1 ]]; then + echo "${items[0]}" + return + fi + printf '\n%s\n' "$label" >&2 + local i + for i in "${!items[@]}"; do + printf ' [%d] %s\n' "$((i+1))" "${items[$i]}" >&2 + done + local choice + while true; do + printf ' Choose [1-%d]: ' "${#items[@]}" >&2 + read -r choice + if [[ "$choice" =~ ^[0-9]+$ ]] && (( choice >= 1 && choice <= ${#items[@]} )); then + echo "${items[$((choice-1))]}" + return + fi + printf ' Invalid choice.\n' >&2 + done +} + +# ── Wine candidate scanner ──────────────────────────────────────────────────── +_scan_wine_candidates() { + local candidates=() + local os + os="$(uname -s)" + + if [[ "$os" == "Darwin" ]]; then + # Native MT5.app (bundled Wine — most common on macOS) + local mt5_app_wine="/Applications/MetaTrader 5.app/Contents/SharedSupport/wine/bin/wine64" + [[ -x "$mt5_app_wine" ]] && candidates+=("$mt5_app_wine") + + # CrossOver (classic install) + local cxover="/Applications/CrossOver.app/Contents/SharedSupport/CrossOver/bin/wine64" + [[ -x "$cxover" ]] && candidates+=("$cxover") + + # Homebrew wine + for brew_wine in /opt/homebrew/bin/wine64 /usr/local/bin/wine64 \ + /opt/homebrew/bin/wine /usr/local/bin/wine; do + [[ -x "$brew_wine" ]] && candidates+=("$brew_wine") + done + fi + + # Linux / fallback + for sys_wine in wine64 wine; do + local found + found="$(command -v "$sys_wine" 2>/dev/null || true)" + [[ -n "$found" && -x "$found" ]] && candidates+=("$found") + done + + # Deduplicate + local seen=() + local c + for c in "${candidates[@]}"; do + local dup=false + local s + for s in "${seen[@]:-}"; do [[ "$s" == "$c" ]] && dup=true && break; done + $dup || seen+=("$c") + done + + printf '%s\n' "${seen[@]:-}" +} + +# ── MT5 prefix scanner ──────────────────────────────────────────────────────── +_scan_mt5_prefixes() { + # Returns list of terminal_dir paths (each containing terminal64.exe) + local results=() + local os + os="$(uname -s)" + + if [[ "$os" == "Darwin" ]]; then + local search_roots=( + # Native MT5.app sandboxed prefix + "$HOME/Library/Application Support" + # CrossOver old + "$HOME/.cxoffice" + # CrossOver 24+ + "$HOME/Library/Application Support/MetaQuotes" + ) + local root + for root in "${search_roots[@]}"; do + [[ -d "$root" ]] || continue + while IFS= read -r -d '' terminal_exe; do + results+=("$(dirname "$terminal_exe")") + done < <(find "$root" -maxdepth 8 -name "terminal64.exe" -print0 2>/dev/null) + done + fi + + # Linux Wine prefixes + local linux_roots=( + "$HOME/.wine" + "$HOME/.wine64" + ) + local root + for root in "${linux_roots[@]}"; do + [[ -d "$root" ]] || continue + local mt5_dir="${root}/drive_c/Program Files/MetaTrader 5" + [[ -f "${mt5_dir}/terminal64.exe" ]] && results+=("$mt5_dir") + done + + # Deduplicate + local seen=() + local c + for c in "${results[@]:-}"; do + local dup=false + local s + for s in "${seen[@]:-}"; do [[ "$s" == "$c" ]] && dup=true && break; done + $dup || seen+=("$c") + done + + printf '%s\n' "${seen[@]:-}" +} + +# Score an MT5 dir by activity: count .ex5 + .set files (higher = more active) +_score_mt5_dir() { + local dir="$1" + local count=0 + count=$(find "$dir" -name "*.ex5" -o -name "*.set" 2>/dev/null | wc -l | tr -d ' ') + echo "$count" +} + +# Pick the most active MT5 dir from a list +_best_mt5_dir() { + local best="" best_score=-1 + local d + while IFS= read -r d; do + [[ -z "$d" ]] && continue + local score + score=$(_score_mt5_dir "$d") + if (( score > best_score )); then + best="$d" + best_score=$score + fi + done + echo "$best" +} + +# ── MT5 auto-installer ─────────────────────────────────────────────────────── +# +# MetaTrader 5 provides official installers for both platforms: +# macOS — DMG from MetaQuotes CDN (includes bundled Wine, no CrossOver needed) +# Linux — Official bash installer from MetaQuotes (handles Wine + MT5 prefix) +# +# After install, MT5 must be launched once to unpack terminal64.exe into the +# Wine prefix. _install_mt5() handles this automatically on Linux (headless via +# Xvfb). On macOS the user must launch the app once manually (GUI required). + +MT5_DMG_URL="https://download.mql5.com/cdn/web/metaquotes.software.corp/mt5/MetaTrader5.dmg" +MT5_LINUX_URL="https://download.mql5.com/cdn/web/metaquotes.software.corp/mt5/mt5ubuntu.sh" +MT5_EXE_URL="https://download.mql5.com/cdn/web/metaquotes.software.corp/mt5/mt5setup.exe" + +_install_mt5() { + local os + os="$(uname -s)" + + if [[ "$os" == "Darwin" ]]; then + _install_mt5_macos + else + _install_mt5_linux + fi +} + +_install_mt5_macos() { + echo "" + _bold "Installing MetaTrader 5 for macOS..." + + if [[ -d "/Applications/MetaTrader 5.app" ]]; then + _ok "MetaTrader 5.app already present in /Applications" + return 0 + fi + + # Prefer mas (Mac App Store CLI) — avoids Gatekeeper issues + if command -v mas &>/dev/null; then + _ok "Using mas (Mac App Store CLI)..." + # MT5 App Store ID: 413698442 + mas install 413698442 && { + _ok "Installed via Mac App Store" + return 0 + } || _warn "mas install failed — falling back to direct download" + fi + + # Direct download from MetaQuotes CDN + local dmg="/tmp/MetaTrader5.dmg" + _ok "Downloading MetaTrader5.dmg from MetaQuotes CDN..." + if ! curl -L --progress-bar --connect-timeout 30 "$MT5_DMG_URL" -o "$dmg"; then + _fail "Download failed. Check your internet connection." + echo "" + echo " Manual install: open https://www.metatrader5.com/en/terminal/help/start_advanced/install_mac" + return 1 + fi + + _ok "Mounting DMG..." + local mnt="/tmp/mt5_mount" + if ! hdiutil attach "$dmg" -mountpoint "$mnt" -quiet -nobrowse; then + _fail "Could not mount DMG: $dmg" + return 1 + fi + + _ok "Copying MetaTrader 5.app to /Applications..." + cp -R "${mnt}/MetaTrader 5.app" /Applications/ 2>/dev/null || { + _fail "Copy failed — try: sudo cp -R \"${mnt}/MetaTrader 5.app\" /Applications/" + hdiutil detach "$mnt" -quiet 2>/dev/null + return 1 + } + + hdiutil detach "$mnt" -quiet 2>/dev/null + rm -f "$dmg" + _ok "MetaTrader 5.app installed to /Applications" + + echo "" + _yellow "ACTION REQUIRED: Launch MetaTrader 5.app once to complete initialization." + echo " It will create the Wine prefix and download terminal64.exe." + echo " After it loads (shows the login screen), you can close it." + echo "" + if ! $AUTO_YES; then + _ask "Press Enter when MT5 has been launched and closed..." "" + fi +} + +_install_mt5_linux() { + echo "" + _bold "Installing MetaTrader 5 for Linux..." + + # Check for existing Wine terminal64.exe anywhere + local existing + existing=$(find "$HOME/.wine" "$HOME/.wine64" \ + "$HOME/Library/Application Support" 2>/dev/null \ + -name "terminal64.exe" -print -quit 2>/dev/null || true) + if [[ -n "$existing" ]]; then + _ok "terminal64.exe already found at: $existing" + return 0 + fi + + # MetaQuotes provides an official Ubuntu installer that handles Wine + MT5 + local has_wget has_curl + has_wget=$(command -v wget &>/dev/null && echo yes || echo no) + has_curl=$(command -v curl &>/dev/null && echo yes || echo no) + + local script="/tmp/mt5ubuntu.sh" + _ok "Downloading MetaQuotes Linux installer..." + if [[ "$has_wget" == "yes" ]]; then + wget -q --show-progress "$MT5_LINUX_URL" -O "$script" || { + _fail "Download failed. Check: $MT5_LINUX_URL" + return 1 + } + elif [[ "$has_curl" == "yes" ]]; then + curl -L --progress-bar "$MT5_LINUX_URL" -o "$script" || { + _fail "Download failed. Check: $MT5_LINUX_URL" + return 1 + } + else + _fail "Neither wget nor curl found. Install one and retry." + return 1 + fi + + chmod +x "$script" + _ok "Running MetaQuotes Linux installer (may prompt for sudo)..." + bash "$script" || { + _fail "Installer failed. Check output above." + rm -f "$script" + return 1 + } + rm -f "$script" + + # The MetaQuotes installer creates the Wine prefix and launches MT5 briefly. + # On headless systems, we use Xvfb to allow MT5 to initialize. + if [[ -z "${DISPLAY:-}" ]]; then + _ok "Headless system — running MT5 init via Xvfb..." + if ! command -v Xvfb &>/dev/null; then + _warn "Xvfb not found. Install with: sudo apt install xvfb" + _warn "Then launch MT5 manually: DISPLAY=:99 metatrader5" + return 0 + fi + Xvfb :99 -screen 0 1024x768x16 &>/dev/null & + local xvfb_pid=$! + sleep 2 + DISPLAY=:99 metatrader5 &>/dev/null & + local mt5_pid=$! + _ok "MT5 initializing (30s)..." + sleep 30 + kill "$mt5_pid" 2>/dev/null || true + kill "$xvfb_pid" 2>/dev/null || true + else + _ok "Launching MT5 to initialize Wine prefix (closes in 20s)..." + metatrader5 &>/dev/null & + local mt5_pid=$! + sleep 20 + kill "$mt5_pid" 2>/dev/null || true + fi + + _ok "MT5 initialization complete" +} + +# ── Display mode detection ──────────────────────────────────────────────────── +_detect_display_mode() { + if [[ "$(uname -s)" == "Darwin" ]]; then + echo "auto" # macOS always GUI + elif [[ -n "${DISPLAY:-}" ]]; then + echo "auto" # Linux with display + else + echo "auto" # headless Linux — auto will pick Xvfb + fi +} + +# ── YAML writer ─────────────────────────────────────────────────────────────── +_write_yaml() { + local wine="$1" + local terminal_dir="$2" + local display_mode="$3" + local keep_last="${4:-20}" + + # Quote paths that contain spaces for YAML (wrap in double quotes) + local wine_q="\"${wine}\"" + local terminal_dir_q="\"${terminal_dir}\"" + local experts_q="\"${terminal_dir}/MQL5/Experts\"" + local tester_q="\"${terminal_dir}/MQL5/Profiles/Tester\"" + local cache_q="\"${terminal_dir}/Tester\"" + + cat > "$CONFIG_OUT" </dev/null | wc -l | tr -d ' ') + _ok "experts_dir: ${ea_count} .ex5 file(s) found" + else + _warn "experts_dir not found yet (will be created by MT5 on first run)" + fi + + local tester_dir="${terminal_dir}/MQL5/Profiles/Tester" + if [[ -d "$tester_dir" ]]; then + local set_count + set_count=$(find "$tester_dir" -name "*.set" 2>/dev/null | wc -l | tr -d ' ') + _ok "tester_profiles_dir: ${set_count} .set file(s) found" + else + _warn "tester_profiles_dir not found yet (will be created by MT5 on first run)" + fi + + $ok +} + +# ── Claude Code generation ──────────────────────────────────────────────────── +# +# Writes two files to help users integrate Claude Code with their backtesting workflow: +# +# config/CLAUDE.template.md — copy to your EA project root as CLAUDE.md +# .claude/hooks/user-prompt-submit.sh — injects production baseline into every prompt +# +# The hook reads config/baseline.json (gitignored, user-maintained). +# baseline.json schema: +# { +# "symbol": "XAUUSD.cent", +# "period": "2024-01-01/2024-12-31", +# "net_profit": 1250.50, +# "profit_factor": 1.43, +# "max_drawdown_pct": 18.2, +# "sharpe_ratio": 0.87, +# "total_trades": 342, +# "notes": "Best config as of 2024-12-15" +# } + +_generate_claude_code() { + echo "" + _bold "Generating Claude Code integration files..." + + # ── CLAUDE.md template ──────────────────────────────────────────────────── + local template_out="${REPO_DIR}/config/CLAUDE.template.md" + if [[ -f "$template_out" ]] && ! $AUTO_YES; then + local ans + ans=$(_ask "config/CLAUDE.template.md already exists. Overwrite?" "no") + if [[ ! "$ans" =~ ^[Yy] ]]; then + _warn "Skipping CLAUDE.md template" + else + _write_claude_template "$template_out" + fi + else + _write_claude_template "$template_out" + fi + + # ── .claude/hooks/user-prompt-submit.sh ─────────────────────────────────── + local hooks_dir="${REPO_DIR}/.claude/hooks" + mkdir -p "$hooks_dir" + local hook_out="${hooks_dir}/user-prompt-submit.sh" + if [[ -f "$hook_out" ]] && ! $AUTO_YES; then + local ans + ans=$(_ask ".claude/hooks/user-prompt-submit.sh already exists. Overwrite?" "no") + if [[ ! "$ans" =~ ^[Yy] ]]; then + _warn "Skipping hook generation" + return + fi + fi + _write_baseline_hook "$hook_out" + chmod +x "$hook_out" + _ok "Written: $hook_out" + + echo "" + _green "Claude Code files ready." + echo "" + echo " Next steps:" + echo " 1. Copy config/CLAUDE.template.md to your EA project root as CLAUDE.md" + echo " 2. Create config/baseline.json with your production backtest metrics" + echo " (see comments in .claude/hooks/user-prompt-submit.sh for the schema)" + echo " 3. The hook auto-injects your baseline into every Claude prompt" + echo "" +} + +_write_claude_template() { + local out="$1" + cat > "$out" <<'CLAUDE_MD' +# CLAUDE.md — [Your EA Project Name] + +## MT5 MCP Integration + +This project uses [mt5-quant](https://github.com/masdevid/mt5-quant) for backtesting and +optimization via Claude Code MCP tools. + +Available MCP tools: `run_backtest`, `run_optimization`, `get_results`, +`list_reports`, `get_report`, `get_analysis` + +## Baseline Tracking + +Production baseline is in `config/baseline.json` (gitignored, user-maintained). + +- Always compare new backtest results against the baseline before calling a result + an improvement. A result only counts as better if it beats the baseline on the + primary metric without significantly degrading secondary metrics. +- Update `baseline.json` only after confirming improvement in live/demo testing. +- Key metrics: `net_profit`, `profit_factor`, `max_drawdown_pct`, `sharpe_ratio`, + `total_trades`. + +## Symbol Name Rules + +- Always use the **exact** symbol name from the broker + (e.g., `"XAUUSD.cent"` not `"XAUUSD"` — they are different instruments). +- The symbol is set in `config/mt5-quant.yaml` under `backtest.symbol`. +- Never hardcode a symbol name in tool calls — read it from config. + +## Backtest Rules + +- Model 0 (every tick) is required for martingale/grid EAs. +- Never run two backtests in parallel — MT5 uses a single Wine prefix. +- After optimization, reset `OptMode` in `terminal.ini` before the next backtest. + +## Optimization Rules + +- Optimizations run in the background (`nohup+disown`) — never add a timeout. +- Use Model 0 only — Model 1 overfits martingale/grid strategies. +- `.set` files must be UTF-16LE; UTF-8 causes MT5 to strip `||Y` parameter flags. +CLAUDE_MD + _ok "Written: $out (copy to your EA project root as CLAUDE.md)" +} + +_write_baseline_hook() { + local out="$1" + cat > "$out" <<'HOOK_SH' +#!/usr/bin/env bash +# .claude/hooks/user-prompt-submit.sh +# Injects the current production baseline into every Claude Code prompt. +# +# Reads: config/baseline.json (gitignored — create and maintain manually) +# +# baseline.json schema: +# { +# "symbol": "XAUUSD.cent", +# "period": "2024-01-01/2024-12-31", +# "net_profit": 1250.50, +# "profit_factor": 1.43, +# "max_drawdown_pct": 18.2, +# "sharpe_ratio": 0.87, +# "total_trades": 342, +# "notes": "Best config as of 2024-12-15" +# } + +HOOK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "${HOOK_DIR}/../.." && pwd)" +BASELINE="${REPO_DIR}/config/baseline.json" + +[[ -f "$BASELINE" ]] || exit 0 + +python3 - "$BASELINE" <<'PY' +import json, sys + +path = sys.argv[1] +try: + with open(path) as f: + baseline = json.load(f) +except Exception as e: + sys.exit(0) # Malformed baseline — don't block the prompt + +context = ( + "## Production Baseline (config/baseline.json)\n" + "Compare all backtest results against these metrics. " + "A result is an improvement only if it beats the baseline on the primary " + "metric without significantly degrading secondary metrics.\n" + "```json\n" + + json.dumps(baseline, indent=2) + + "\n```" +) +print(json.dumps({"context": context})) +PY +HOOK_SH +} + +# ── Main ────────────────────────────────────────────────────────────────────── +main() { + # --claude-code: skip the config wizard — only generate Claude Code integration files + if $CLAUDE_CODE; then + _generate_claude_code + exit 0 + fi + + echo "" + _bold "MT5-Quant setup — auto-detecting Wine and MT5 paths" + echo "────────────────────────────────────────────────────" + + # ── Check if config already exists ─────────────────────────────────────── + if [[ -f "$CONFIG_OUT" ]] && ! $AUTO_YES; then + _yellow "Config already exists: $CONFIG_OUT" + local ans + ans=$(_ask "Overwrite?" "no") + if [[ ! "$ans" =~ ^[Yy] ]]; then + echo "Aborted — existing config unchanged." + exit 0 + fi + fi + + # ── Detect Wine ─────────────────────────────────────────────────────────── + echo "" + _bold "Scanning for Wine..." + local wine_candidates=() + while IFS= read -r line; do + [[ -n "$line" ]] && wine_candidates+=("$line") + done < <(_scan_wine_candidates) + + local wine="" + if [[ ${#wine_candidates[@]} -eq 0 ]]; then + _fail "No Wine installation found." + local install_ans="yes" + if ! $AUTO_YES; then + install_ans=$(_ask "Download and install MetaTrader 5 automatically?" "yes") + fi + if [[ "$install_ans" =~ ^[Yy] ]]; then + _install_mt5 || { _red "Auto-install failed. Install MT5 manually and re-run setup.sh."; exit 1; } + # Re-scan after install + while IFS= read -r line; do + [[ -n "$line" ]] && wine_candidates+=("$line") + done < <(_scan_wine_candidates) + fi + if [[ ${#wine_candidates[@]} -eq 0 ]]; then + if ! $AUTO_YES; then + wine=$(_ask "Enter wine64 path manually (or press Enter to abort)") + fi + [[ -z "$wine" ]] && { _red "Cannot continue without Wine."; exit 1; } + fi + fi + if [[ -z "$wine" ]]; then + if [[ ${#wine_candidates[@]} -eq 1 ]]; then + wine="${wine_candidates[0]}" + _ok "Found: $wine" + else + _ok "Found ${#wine_candidates[@]} Wine installations" + if $AUTO_YES; then + wine="${wine_candidates[0]}" + _ok "Auto-selected: $wine" + else + wine=$(_pick "Select Wine executable:" "${wine_candidates[@]}") + fi + fi + fi + + # ── Detect MT5 ──────────────────────────────────────────────────────────── + echo "" + _bold "Scanning for MetaTrader 5 installations..." + local mt5_candidates=() + while IFS= read -r line; do + [[ -n "$line" ]] && mt5_candidates+=("$line") + done < <(_scan_mt5_prefixes) + + local terminal_dir="" + if [[ ${#mt5_candidates[@]} -eq 0 ]]; then + _fail "No MetaTrader 5 installation found." + # Wine was found but MT5 prefix doesn't exist yet — offer to run installer + local install_ans2="yes" + if ! $AUTO_YES; then + install_ans2=$(_ask "Install MetaTrader 5 and initialize Wine prefix?" "yes") + fi + if [[ "$install_ans2" =~ ^[Yy] ]]; then + _install_mt5 || true + # Re-scan + while IFS= read -r line; do + [[ -n "$line" ]] && mt5_candidates+=("$line") + done < <(_scan_mt5_prefixes) + fi + if [[ ${#mt5_candidates[@]} -eq 0 ]]; then + if ! $AUTO_YES; then + terminal_dir=$(_ask "Enter terminal_dir path manually (or press Enter to abort)") + fi + [[ -z "$terminal_dir" ]] && { _red "Cannot continue without terminal_dir."; exit 1; } + fi + fi + if [[ -z "$terminal_dir" ]]; then + if [[ ${#mt5_candidates[@]} -eq 1 ]]; then + terminal_dir="${mt5_candidates[0]}" + _ok "Found: $terminal_dir" + else + _ok "Found ${#mt5_candidates[@]} MT5 installations" + # Auto-pick: highest activity score + local best + best=$(printf '%s\n' "${mt5_candidates[@]}" | _best_mt5_dir) + if $AUTO_YES; then + terminal_dir="$best" + _ok "Auto-selected (most active): $terminal_dir" + else + # Show scores for each candidate + echo "" + local i + for i in "${!mt5_candidates[@]}"; do + local d="${mt5_candidates[$i]}" + local score + score=$(_score_mt5_dir "$d") + printf ' [%d] %s (%d files)\n' "$((i+1))" "$d" "$score" >&2 + [[ "$d" == "$best" ]] && printf ' ^ most active\n' >&2 + done + terminal_dir=$(_pick "Select MT5 installation:" "${mt5_candidates[@]}") + fi + fi + fi + + # ── Display mode ────────────────────────────────────────────────────────── + local display_mode + display_mode=$(_detect_display_mode) + + # ── Validate ────────────────────────────────────────────────────────────── + if ! _validate "$wine" "$terminal_dir"; then + _red "Validation failed. Config NOT written." + if ! $AUTO_YES; then + local ans + ans=$(_ask "Write config anyway?" "no") + [[ ! "$ans" =~ ^[Yy] ]] && exit 1 + else + exit 1 + fi + fi + + # ── Write YAML ──────────────────────────────────────────────────────────── + echo "" + _bold "Writing config..." + _write_yaml "$wine" "$terminal_dir" "$display_mode" "$KEEP_LAST" + _ok "Written: $CONFIG_OUT" + echo " Tip: see config/example.set for optimization .set file format" + + # ── Register with Claude Code ────────────────────────────────────────────── + _offer_mcp_register + + echo "" + _green "Setup complete!" + echo "" +} + +# ── MCP registration ────────────────────────────────────────────────────────── +_offer_mcp_register() { + echo "" + _bold "Registering with Claude Code..." + + if ! command -v claude &>/dev/null; then + _warn "claude CLI not found — register manually:" + echo "" + echo " claude mcp add mt5-quant -- python3 \"${REPO_DIR}/server/main.py\"" + echo "" + return + fi + + local register=true + if ! $AUTO_YES; then + local ans + ans=$(_ask "Register mt5-quant with Claude Code now?" "yes") + [[ ! "$ans" =~ ^[Yy] ]] && register=false + fi + + if $register; then + local out + out=$(claude mcp add mt5-quant -- python3 "${REPO_DIR}/server/main.py" 2>&1) || true + if echo "$out" | grep -qi "already\|exists"; then + _ok "Already registered (no change needed)" + elif echo "$out" | grep -qi "error\|failed"; then + _warn "Registration failed: $out" + echo " Run manually: claude mcp add mt5-quant -- python3 \"${REPO_DIR}/server/main.py\"" + else + _ok "Registered: claude mcp add mt5-quant" + fi + else + echo " Skipped. Run manually:" + echo " claude mcp add mt5-quant -- python3 \"${REPO_DIR}/server/main.py\"" + fi +} + +main diff --git a/server/__init__.py b/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/main.py b/server/main.py new file mode 100644 index 0000000..3ad66fc --- /dev/null +++ b/server/main.py @@ -0,0 +1,2448 @@ +#!/usr/bin/env python3 +""" +MT5-Quant MCP Server + +Exposes MT5 backtest and optimization tools via the Model Context Protocol. +Run with: python3 server/main.py + +Add to Claude Code: + claude mcp add MT5-Quant -- python3 /path/to/mt5-quant/server/main.py +""" + +import asyncio +import difflib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +try: + import mcp.server.stdio + import mcp.types as types + from mcp.server import Server +except ImportError: + print("ERROR: mcp package not installed. Run: pip install mcp", file=sys.stderr) + sys.exit(1) + +# Config/ROOT_DIR resolution (priority order): +# 1. MT5_MCP_HOME env var (explicit override) +# 2. ~/.config/mt5-quant/ (installed-package default) +# 3. parent of this file (development / run-from-repo) +def _resolve_root() -> Path: + env_home = os.environ.get('MT5_MCP_HOME') + if env_home: + return Path(env_home).expanduser().resolve() + user_cfg = Path.home() / '.config' / 'mt5-quant' + if (user_cfg / 'config' / 'mt5-quant.yaml').exists(): + return user_cfg + return Path(__file__).parent.parent + +ROOT_DIR = _resolve_root() + +# SCRIPTS_DIR always points to the package's scripts/ (adjacent to server/main.py), +# not the config dir. Scripts are never copied to ~/.config/mt5-quant. +SCRIPTS_DIR = Path(__file__).parent.parent / 'scripts' +if not SCRIPTS_DIR.exists(): + # Installed via pip — scripts landed in the package root via pyproject.toml include + SCRIPTS_DIR = ROOT_DIR / 'scripts' + +sys.path.insert(0, str(ROOT_DIR)) +# Also ensure analytics imports resolve from the package dir +_pkg_dir = str(Path(__file__).parent.parent) +if _pkg_dir not in sys.path: + sys.path.insert(0, _pkg_dir) + +from analytics.extract import detect_format, parse_html, parse_xml, write_outputs +from analytics.analyze import ( + load_deals, load_metrics, monthly_pnl, reconstruct_dd_events, + grid_depth_histogram, top_losses, loss_sequences, build_summary +) +from analytics.optimize_parser import ( + detect_format as opt_detect_format, + parse_html as opt_parse_html, + parse_xml as opt_parse_xml, + normalize, convergence_analysis +) + +app = Server("MT5-Quant") + +# ── Config ──────────────────────────────────────────────────────────────────── + +def load_config() -> dict: + config_path = ROOT_DIR / 'config' / 'mt5-quant.yaml' + if not config_path.exists(): + return {} + config = {} + with open(config_path) as f: + # Simple YAML key: value parser (no nested support needed for basic config) + for line in f: + line = line.strip() + if line.startswith('#') or ':' not in line: + continue + key, _, val = line.partition(':') + val = val.strip().strip('"').strip("'") + if val and val not in ('null', '~', ''): + config[key.strip()] = val + return config + + +CONFIG = load_config() + + +def cfg(key: str, default: str = '') -> str: + return CONFIG.get(key, default) or default + + +REPORTS_DIR = ROOT_DIR / cfg('reports_dir', 'reports') +HISTORY_FILE = ROOT_DIR / 'config' / 'backtest_history.json' +BASELINE_FILE = ROOT_DIR / 'config' / 'baseline.json' + + +def _validate_environment() -> dict | None: + """Fast pre-flight check — returns error dict if environment is broken, None if OK.""" + config_path = ROOT_DIR / 'config' / 'mt5-quant.yaml' + missing = [] + if not config_path.exists(): + missing.append('config/mt5-quant.yaml not found') + wine = cfg('wine_executable') + if not wine: + missing.append('wine_executable not set in config') + elif not os.access(wine, os.X_OK): + missing.append(f'wine_executable not found or not executable: {wine}') + terminal_dir = cfg('terminal_dir') + if not terminal_dir: + missing.append('terminal_dir not set in config') + elif not Path(terminal_dir).is_dir(): + missing.append(f'terminal_dir not found: {terminal_dir}') + if missing: + return { + 'success': False, + 'error': 'SETUP_REQUIRED', + 'missing': missing, + 'hint': 'Run: bash scripts/setup.sh', + } + return None + + +def _check_symbol(symbol: str) -> tuple[str | None, list[str]]: + """Check if symbol exists in MT5 history dir. Returns (warning, suggestions).""" + terminal_dir = cfg('terminal_dir') + if not terminal_dir: + return None, [] + history_dir = Path(terminal_dir) / 'history' + if not history_dir.is_dir(): + return None, [] + known = [d.name for d in history_dir.iterdir() if d.is_dir()] + if not known or symbol in known: + return None, [] + suggestions = difflib.get_close_matches(symbol, known, n=3, cutoff=0.6) + sample = ', '.join(known[:5]) + ('...' if len(known) > 5 else '') + warning = f"Symbol '{symbol}' not found in MT5 history. Available: {sample}" + return warning, suggestions + + +# ── History helpers ─────────────────────────────────────────────────────────── + +def load_history() -> list[dict]: + if not HISTORY_FILE.exists(): + return [] + with open(HISTORY_FILE) as f: + return json.load(f) + + +def save_history(entries: list[dict]) -> None: + HISTORY_FILE.parent.mkdir(exist_ok=True) + with open(HISTORY_FILE, 'w') as f: + json.dump(entries, f, indent=2) + + +def _build_history_entry(report_dir: str) -> dict | None: + """Build a compact, self-contained history entry from a report directory.""" + from datetime import datetime, timezone + d = Path(report_dir) + metrics = read_json(str(d / 'metrics.json')) + analysis = read_json(str(d / 'analysis.json')) + + if not metrics and not analysis: + return None + + entry: dict = { + 'id': d.name, + 'report_dir': str(d), + 'report_dir_deleted': False, + 'archived_at': datetime.now(timezone.utc).isoformat(), + 'ea': metrics.get('expert') or metrics.get('ea') or '', + 'symbol': metrics.get('symbol', ''), + 'timeframe': metrics.get('timeframe', ''), + 'from_date': metrics.get('from_date') or metrics.get('testing_from', ''), + 'to_date': metrics.get('to_date') or metrics.get('testing_to', ''), + 'metrics': { + 'net_profit': metrics.get('net_profit'), + 'profit_factor': metrics.get('profit_factor'), + 'max_dd_pct': metrics.get('max_dd_pct'), + 'sharpe_ratio': metrics.get('sharpe_ratio'), + 'total_trades': metrics.get('total_trades'), + 'recovery_factor': metrics.get('recovery_factor'), + 'win_rate_pct': metrics.get('win_rate_pct'), + 'expected_payoff': metrics.get('expected_payoff'), + }, + 'verdict': None, + 'notes': '', + 'tags': [], + 'promoted_to_baseline': False, + } + + if analysis: + summary = analysis.get('summary', {}) + entry['summary'] = {k: summary.get(k) for k in ( + 'green_months', 'total_months', 'worst_month', 'worst_month_pnl', + 'worst_dd_event_pct', 'max_grid_depth', 'l5_plus_count', + 'dominant_exit', 'max_win_streak', 'max_loss_streak', + 'current_streak', 'current_streak_type', + ) if summary.get(k) is not None} + monthly = analysis.get('monthly_pnl', []) + if monthly: + entry['monthly_pnl'] = monthly + dd_events = analysis.get('dd_events', []) + if dd_events: + entry['worst_dd_event'] = dd_events[0] + + return entry + + +# ── Tool helpers ────────────────────────────────────────────────────────────── + +def run_script(cmd: list[str], timeout: int = 900) -> tuple[bool, str]: + """Run a shell script synchronously and return (success, output).""" + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + cwd=str(ROOT_DIR), + ) + output = result.stdout + result.stderr + return result.returncode == 0, output + except subprocess.TimeoutExpired: + return False, f"Timeout after {timeout}s" + except Exception as e: + return False, str(e) + + +def latest_report_dir() -> str | None: + """Find most recently created report directory.""" + REPORTS_DIR.mkdir(exist_ok=True) + dirs = sorted(REPORTS_DIR.iterdir(), reverse=True) + for d in dirs: + if d.is_dir() and not d.name.endswith('_opt'): + return str(d) + return None + + +def read_json(path: str) -> dict: + if not os.path.exists(path): + return {} + with open(path) as f: + return json.load(f) + + +def format_result(data: dict) -> str: + return json.dumps(data, indent=2) + + +# ── Tool definitions ────────────────────────────────────────────────────────── + +@app.list_tools() +async def list_tools() -> list[types.Tool]: + return [ + types.Tool( + name="run_backtest", + description=( + "Run a complete MT5 backtest pipeline: compile → clean cache → " + "backtest → extract → analyze. Returns profit, DD%, Sharpe, monthly P/L, " + "and drawdown event reconstruction. Always compiles and clears cache unless " + "skip flags are set." + ), + inputSchema={ + "type": "object", + "required": ["expert"], + "properties": { + "expert": { + "type": "string", + "description": "EA name without path or extension. e.g. 'MyEA_v1.2'" + }, + "symbol": { + "type": "string", + "description": "Trading symbol. Use your broker's exact name. e.g. 'XAUUSD'" + }, + "from_date": { + "type": "string", + "description": "Start date in YYYY.MM.DD format" + }, + "to_date": { + "type": "string", + "description": "End date in YYYY.MM.DD format" + }, + "preset": { + "type": "string", + "enum": ["last_month", "last_3months", "ytd", "last_year"], + "description": "Date preset (alternative to from/to)" + }, + "timeframe": { + "type": "string", + "enum": ["M1", "M5", "M15", "M30", "H1", "H4", "D1"], + "description": "Chart timeframe (default: M5)" + }, + "deposit": { + "type": "number", + "description": "Initial deposit (default: from config)" + }, + "model": { + "type": "integer", + "enum": [0, 1, 2], + "description": "0=every tick (default), 1=1min OHLC, 2=open price" + }, + "set_file": { + "type": "string", + "description": "Path to .set parameter file" + }, + "skip_compile": { + "type": "boolean", + "description": "Skip compilation (use existing .ex5)" + }, + "skip_clean": { + "type": "boolean", + "description": "Skip cache clean (faster, risks stale results)" + }, + "skip_analyze": { + "type": "boolean", + "description": "Extract only, skip deal analysis" + }, + "deep": { + "type": "boolean", + "description": "Run deep analysis (grid + regime breakdown)" + }, + "gui": { + "type": "boolean", + "description": "Show MT5 visual mode (chart animation). Default: false (headless). Use for debugging or demo." + }, + "timeout": { + "type": "integer", + "description": "Backtest timeout in seconds (default: 900)" + }, + }, + }, + ), + types.Tool( + name="run_optimization", + description=( + "Launch MT5 genetic parameter optimization as a detached background process. " + "Returns immediately — MT5 runs for 2-6 hours. " + "Always uses model=0 (every tick). Call get_optimization_results only after " + "user confirms MT5 has finished." + ), + inputSchema={ + "type": "object", + "required": ["expert", "set_file", "from_date", "to_date"], + "properties": { + "expert": {"type": "string"}, + "set_file": { + "type": "string", + "description": "Path to optimization .set file with ||Y sweep flags" + }, + "from_date": {"type": "string"}, + "to_date": {"type": "string"}, + "symbol": {"type": "string"}, + "deposit": {"type": "number"}, + }, + }, + ), + types.Tool( + name="get_optimization_results", + description=( + "Parse completed MT5 optimization results. Call only after user signals " + "that MT5 optimization has finished. Returns top passes sorted by profit " + "with convergence analysis." + ), + inputSchema={ + "type": "object", + "properties": { + "job_id": { + "type": "string", + "description": "Job ID from run_optimization response" + }, + "report_file": { + "type": "string", + "description": "Direct path to optimization.htm or .htm.xml" + }, + "top_n": { + "type": "integer", + "description": "Number of top results to return (default: 20)" + }, + "dd_threshold": { + "type": "number", + "description": "Flag results above this DD% as high-risk (default: 20)" + }, + }, + }, + ), + types.Tool( + name="analyze_report", + description=( + "Read and summarize a completed backtest report. Does not re-run MT5. " + "Returns monthly P/L, drawdown events, grid depth histogram, and top losses." + ), + inputSchema={ + "type": "object", + "properties": { + "report_dir": { + "type": "string", + "description": "Path to report directory. If omitted, uses latest." + }, + }, + }, + ), + types.Tool( + name="compare_baseline", + description=( + "Compare a backtest report against a baseline. Returns winner/loser verdict " + "and delta metrics. Baseline must include net_profit and max_dd_pct." + ), + inputSchema={ + "type": "object", + "required": ["baseline"], + "properties": { + "report_dir": { + "type": "string", + "description": "Report to evaluate. If omitted, uses latest." + }, + "baseline": { + "type": "object", + "required": ["net_profit", "max_dd_pct"], + "properties": { + "net_profit": {"type": "number"}, + "max_dd_pct": {"type": "number"}, + "total_trades": {"type": "integer"}, + "label": {"type": "string"}, + }, + }, + "promote_dd_limit": { + "type": "number", + "description": "Auto-promote only if DD < this % (default: 20)" + }, + }, + }, + ), + types.Tool( + name="compile_ea", + description="Compile an MQL5 Expert Advisor via MetaEditor (Wine/CrossOver).", + inputSchema={ + "type": "object", + "required": ["expert_path"], + "properties": { + "expert_path": { + "type": "string", + "description": "Path to .mq5 source file" + }, + }, + }, + ), + types.Tool( + name="verify_setup", + description=( + "Verify the MT5-Quant environment without launching MT5. " + "Checks Wine executable, MT5 installation paths, and config file. " + "Run this first if other tools return SETUP_REQUIRED errors." + ), + inputSchema={"type": "object", "properties": {}}, + ), + types.Tool( + name="get_backtest_status", + description=( + "Check progress of a running or recently completed backtest pipeline. " + "Returns current stage (COMPILE/CLEAN/BACKTEST/EXTRACT/ANALYZE/DONE), " + "elapsed time, and whether the pipeline has finished." + ), + inputSchema={ + "type": "object", + "properties": { + "report_dir": { + "type": "string", + "description": "Report directory path. If omitted, uses latest.", + }, + }, + }, + ), + types.Tool( + name="get_optimization_status", + description=( + "Check whether a background optimization job is still running. " + "Returns process alive status, elapsed time, last 20 log lines, " + "and whether the report file has appeared (definitive completion signal)." + ), + inputSchema={ + "type": "object", + "required": ["job_id"], + "properties": { + "job_id": { + "type": "string", + "description": "Job ID returned by run_optimization.", + }, + }, + }, + ), + types.Tool( + name="prune_reports", + description=( + "Delete old backtest report directories, keeping only the N most recent. " + "Optimization result directories (_opt suffix) are never deleted." + ), + inputSchema={ + "type": "object", + "properties": { + "keep_last": { + "type": "integer", + "description": "Number of most recent reports to keep (default: 20).", + }, + }, + }, + ), + types.Tool( + name="list_reports", + description=( + "List all backtest report directories with compact key metrics " + "(profit, DD%, trades, date). Much cheaper than calling analyze_report " + "repeatedly. Use this to survey what runs exist before drilling in." + ), + inputSchema={ + "type": "object", + "properties": { + "include_opt": { + "type": "boolean", + "description": "Include optimization result dirs (_opt suffix). Default: false.", + }, + "limit": { + "type": "integer", + "description": "Max reports to return, newest first (default: 30).", + }, + }, + }, + ), + types.Tool( + name="tail_log", + description=( + "Read the last N lines of a backtest progress log or optimization log. " + "Use filter='errors' to get only ERROR/WARN lines. Cheaper than " + "get_optimization_status when you only need log content." + ), + inputSchema={ + "type": "object", + "properties": { + "report_dir": { + "type": "string", + "description": "Backtest report dir (reads progress.log). Omit for latest.", + }, + "job_id": { + "type": "string", + "description": "Optimization job ID (reads its log file).", + }, + "log_file": { + "type": "string", + "description": "Absolute path to any log file.", + }, + "n": { + "type": "integer", + "description": "Number of lines to return (default: 50).", + }, + "filter": { + "type": "string", + "enum": ["all", "errors", "warnings"], + "description": "Line filter (default: all).", + }, + }, + }, + ), + types.Tool( + name="cache_status", + description=( + "Show MT5 tester cache size breakdown by symbol/timeframe directory. " + "Call before clean_cache to understand what will be deleted." + ), + inputSchema={"type": "object", "properties": {}}, + ), + types.Tool( + name="clean_cache", + description=( + "Delete MT5 tester cache files to force fresh price data on next backtest. " + "Optionally target a specific symbol. Returns bytes freed." + ), + inputSchema={ + "type": "object", + "properties": { + "symbol": { + "type": "string", + "description": "Delete only cache for this symbol. Omit to delete all.", + }, + "dry_run": { + "type": "boolean", + "description": "Report what would be deleted without deleting. Default: false.", + }, + }, + }, + ), + types.Tool( + name="read_set_file", + description=( + "Parse an MT5 .set parameter file (UTF-16LE or UTF-8) into structured JSON. " + "Returns each parameter with its value and optimization sweep config. " + "Use this instead of reading raw .set files." + ), + inputSchema={ + "type": "object", + "required": ["path"], + "properties": { + "path": { + "type": "string", + "description": "Path to .set file.", + }, + }, + }, + ), + types.Tool( + name="write_set_file", + description=( + "Write an MT5 .set parameter file in UTF-16LE encoding (required by MT5). " + "Accepts a dict of params. For optimization sweeps include from/to/step keys. " + "Existing file is overwritten and chmod 444 is applied." + ), + inputSchema={ + "type": "object", + "required": ["path", "params"], + "properties": { + "path": { + "type": "string", + "description": "Output path for .set file.", + }, + "params": { + "type": "object", + "description": ( + "Dict of param_name → value or dict with keys: " + "value, from, to, step, optimize (bool)." + ), + }, + }, + }, + ), + types.Tool( + name="patch_set_file", + description=( + "Modify specific parameters in an existing .set file in-place. " + "Preserves all other params, comments, and sweep config. " + "Returns a diff of what changed. " + "Use instead of read_set_file → edit → write_set_file (saves 2 round-trips)." + ), + inputSchema={ + "type": "object", + "required": ["path", "patches"], + "properties": { + "path": { + "type": "string", + "description": "Path to the .set file to modify.", + }, + "patches": { + "type": "object", + "description": ( + "Params to update. Each key is a param name. " + "Value can be a scalar (just updates value) or a dict " + "with keys: value, from, to, step, optimize." + ), + }, + }, + }, + ), + types.Tool( + name="clone_set_file", + description=( + "Copy a .set file to a new path, applying optional overrides. " + "One call instead of read → modify → write. " + "Useful for creating variant .set files from a base config." + ), + inputSchema={ + "type": "object", + "required": ["source", "destination"], + "properties": { + "source": { + "type": "string", + "description": "Path to source .set file.", + }, + "destination": { + "type": "string", + "description": "Output path for the cloned .set file.", + }, + "overrides": { + "type": "object", + "description": ( + "Optional param overrides to apply in the clone. " + "Same format as patch_set_file patches." + ), + }, + }, + }, + ), + types.Tool( + name="set_from_optimization", + description=( + "Generate a .set file directly from an optimization result's params dict. " + "Strips all sweep flags (||Y) to produce a clean backtest .set. " + "Optionally uses a template .set for params not in the optimization result. " + "Optionally re-adds sweep ranges to selected params for follow-on optimization. " + "Use immediately after get_optimization_results — params dict comes from results[0].params." + ), + inputSchema={ + "type": "object", + "required": ["path", "params"], + "properties": { + "path": { + "type": "string", + "description": "Output path for the generated .set file.", + }, + "params": { + "type": "object", + "description": ( + "Flat dict of param_name → value from optimization result. " + "e.g. {'TP_Pips': 400, 'Min_Confidence': 0.61}" + ), + }, + "template": { + "type": "string", + "description": ( + "Optional path to an existing .set file. " + "Params not in 'params' are filled from the template as fixed values." + ), + }, + "sweep": { + "type": "object", + "description": ( + "Optional: re-add sweep ranges to specific params after applying opt values. " + "Dict of param_name → {from, to, step, optimize: true}. " + "Use to create a narrowed follow-on optimization .set." + ), + }, + }, + }, + ), + types.Tool( + name="diff_set_files", + description=( + "Compare two .set files and return only the differences: " + "params added, removed, or changed (value or sweep flag). " + "Use instead of reading both files and comparing manually." + ), + inputSchema={ + "type": "object", + "required": ["path_a", "path_b"], + "properties": { + "path_a": {"type": "string", "description": "First .set file (baseline/old)."}, + "path_b": {"type": "string", "description": "Second .set file (candidate/new)."}, + }, + }, + ), + types.Tool( + name="describe_sweep", + description=( + "Show a .set file's sweep configuration: which params are swept, " + "their ranges, value counts, and total optimization combinations. " + "Use before run_optimization to verify scope." + ), + inputSchema={ + "type": "object", + "required": ["path"], + "properties": { + "path": {"type": "string", "description": "Path to .set file."}, + }, + }, + ), + types.Tool( + name="list_set_files", + description=( + "List all .set files in the MT5 tester profiles directory with " + "param counts, swept param counts, and total optimization combinations. " + "Use instead of reading each file individually to find the right .set." + ), + inputSchema={ + "type": "object", + "properties": { + "ea": { + "type": "string", + "description": "Filter by EA name substring (case-insensitive).", + }, + }, + }, + ), + types.Tool( + name="list_jobs", + description=( + "List all optimization jobs with compact status (alive/done/failed, elapsed). " + "Cheaper than calling get_optimization_status for each job individually." + ), + inputSchema={ + "type": "object", + "properties": { + "include_done": { + "type": "boolean", + "description": "Include completed jobs (default: true).", + }, + }, + }, + ), + types.Tool( + name="archive_report", + description=( + "Convert a backtest report directory into a compact JSON entry appended to " + "config/backtest_history.json. Captures all metrics, analysis summary, monthly P/L, " + "and worst DD event. Optionally deletes the source directory to reclaim disk space. " + "Skips if the report is already in history (idempotent)." + ), + inputSchema={ + "type": "object", + "properties": { + "report_dir": { + "type": "string", + "description": "Report directory to archive. If omitted, uses latest.", + }, + "delete_after": { + "type": "boolean", + "description": "Delete source directory after archiving (default: false).", + }, + "verdict": { + "type": "string", + "enum": ["winner", "loser", "marginal", "reference"], + "description": "Optional verdict to attach to the entry.", + }, + "notes": { + "type": "string", + "description": "Free-text notes to attach to the entry.", + }, + "tags": { + "type": "array", + "items": {"type": "string"}, + "description": "Tags to attach (e.g. ['tight-sl', 'new-entry-filter']).", + }, + }, + }, + ), + types.Tool( + name="archive_all_reports", + description=( + "Bulk-archive all backtest report directories into config/backtest_history.json, " + "then optionally delete the source directories. Skips dirs already in history. " + "Optimization dirs (_opt suffix) are never deleted. " + "Use this to clean up disk space while preserving all results as JSON." + ), + inputSchema={ + "type": "object", + "properties": { + "delete_after": { + "type": "boolean", + "description": "Delete source directories after archiving (default: false).", + }, + "keep_last": { + "type": "integer", + "description": "Keep this many newest dirs even if delete_after=true (default: 5).", + }, + "dry_run": { + "type": "boolean", + "description": "Report what would happen without making changes (default: false).", + }, + }, + }, + ), + types.Tool( + name="get_history", + description=( + "Query config/backtest_history.json with filters. Returns compact entries sorted " + "newest-first by default. Use this to compare past runs, find regressions, or " + "pick a candidate to promote to baseline." + ), + inputSchema={ + "type": "object", + "properties": { + "ea": { + "type": "string", + "description": "Filter by EA name (substring match).", + }, + "symbol": { + "type": "string", + "description": "Filter by symbol (exact match).", + }, + "verdict": { + "type": "string", + "enum": ["winner", "loser", "marginal", "reference"], + "description": "Filter by verdict.", + }, + "tag": { + "type": "string", + "description": "Filter entries that contain this tag.", + }, + "min_profit": { + "type": "number", + "description": "Filter entries with net_profit >= this value.", + }, + "max_dd_pct": { + "type": "number", + "description": "Filter entries with max_dd_pct <= this value.", + }, + "sort_by": { + "type": "string", + "enum": ["date", "profit", "dd", "sharpe"], + "description": "Sort order (default: date, newest first).", + }, + "limit": { + "type": "integer", + "description": "Max entries to return (default: 20).", + }, + "include_monthly": { + "type": "boolean", + "description": "Include monthly_pnl arrays (default: false, saves tokens).", + }, + }, + }, + ), + types.Tool( + name="promote_to_baseline", + description=( + "Promote a backtest result to config/baseline.json — the reference used by " + "compare_baseline and the Claude Code baseline hook. " + "Accepts a history entry id, a report_dir, or defaults to the latest report. " + "Also marks the history entry as promoted." + ), + inputSchema={ + "type": "object", + "properties": { + "history_id": { + "type": "string", + "description": "Entry id from get_history (report dir basename).", + }, + "report_dir": { + "type": "string", + "description": "Direct path to report directory (alternative to history_id).", + }, + "notes": { + "type": "string", + "description": "Notes written to baseline.json (e.g. 'v1.3 promoted after 3-month walk-forward').", + }, + }, + }, + ), + types.Tool( + name="annotate_history", + description=( + "Add or update notes, verdict, or tags on a history entry in " + "config/backtest_history.json. Use this after compare_baseline to record " + "the verdict, or to tag runs for later retrieval." + ), + inputSchema={ + "type": "object", + "required": ["history_id"], + "properties": { + "history_id": { + "type": "string", + "description": "Entry id (report dir basename) to update.", + }, + "verdict": { + "type": "string", + "enum": ["winner", "loser", "marginal", "reference"], + }, + "notes": { + "type": "string", + "description": "Free-text notes (replaces existing notes).", + }, + "tags": { + "type": "array", + "items": {"type": "string"}, + "description": "Tags to set (replaces existing tags).", + }, + "add_tags": { + "type": "array", + "items": {"type": "string"}, + "description": "Tags to append without replacing existing ones.", + }, + }, + }, + ), + ] + + +# ── Tool handlers ───────────────────────────────────────────────────────────── + +@app.call_tool() +async def call_tool(name: str, arguments: dict[str, Any]) -> list[types.TextContent]: + try: + if name == "run_backtest": + result = await handle_run_backtest(arguments) + elif name == "run_optimization": + result = await handle_run_optimization(arguments) + elif name == "get_optimization_results": + result = await handle_get_optimization_results(arguments) + elif name == "analyze_report": + result = await handle_analyze_report(arguments) + elif name == "compare_baseline": + result = await handle_compare_baseline(arguments) + elif name == "compile_ea": + result = await handle_compile_ea(arguments) + elif name == "verify_setup": + result = await handle_verify_setup(arguments) + elif name == "get_backtest_status": + result = await handle_get_backtest_status(arguments) + elif name == "get_optimization_status": + result = await handle_get_optimization_status(arguments) + elif name == "prune_reports": + result = await handle_prune_reports(arguments) + elif name == "list_reports": + result = await handle_list_reports(arguments) + elif name == "tail_log": + result = await handle_tail_log(arguments) + elif name == "cache_status": + result = await handle_cache_status(arguments) + elif name == "clean_cache": + result = await handle_clean_cache(arguments) + elif name == "read_set_file": + result = await handle_read_set_file(arguments) + elif name == "write_set_file": + result = await handle_write_set_file(arguments) + elif name == "patch_set_file": + result = await handle_patch_set_file(arguments) + elif name == "clone_set_file": + result = await handle_clone_set_file(arguments) + elif name == "set_from_optimization": + result = await handle_set_from_optimization(arguments) + elif name == "diff_set_files": + result = await handle_diff_set_files(arguments) + elif name == "describe_sweep": + result = await handle_describe_sweep(arguments) + elif name == "list_set_files": + result = await handle_list_set_files(arguments) + elif name == "list_jobs": + result = await handle_list_jobs(arguments) + elif name == "archive_report": + result = await handle_archive_report(arguments) + elif name == "archive_all_reports": + result = await handle_archive_all_reports(arguments) + elif name == "get_history": + result = await handle_get_history(arguments) + elif name == "promote_to_baseline": + result = await handle_promote_to_baseline(arguments) + elif name == "annotate_history": + result = await handle_annotate_history(arguments) + else: + result = {"error": f"Unknown tool: {name}"} + except Exception as e: + result = {"error": str(e), "success": False} + + return [types.TextContent(type="text", text=format_result(result))] + + +async def handle_run_backtest(args: dict) -> dict: + env_error = _validate_environment() + if env_error: + return env_error + + symbol = args.get('symbol') or cfg('backtest_symbol', 'XAUUSD') + symbol_warning, symbol_suggestions = _check_symbol(symbol) + + cmd = [str(SCRIPTS_DIR / 'backtest_pipeline.sh')] + cmd += ['--expert', args['expert']] + project_dir = cfg('project_dir', '') + if project_dir: + cmd += ['--project-dir', project_dir] + + if 'symbol' in args: + cmd += ['--symbol', args['symbol']] + if 'preset' in args: + cmd += ['--preset', args['preset']] + if 'from_date' in args: + cmd += ['--from', args['from_date']] + if 'to_date' in args: + cmd += ['--to', args['to_date']] + if 'timeframe' in args: + cmd += ['--timeframe', args['timeframe']] + if 'deposit' in args: + cmd += ['--deposit', str(args['deposit'])] + if 'model' in args: + cmd += ['--model', str(args['model'])] + if 'set_file' in args: + set_file = args['set_file'] + # Resolve relative paths against project_dir (where the EA repo lives) + if project_dir and not os.path.isabs(set_file): + set_file = os.path.join(project_dir, set_file) + cmd += ['--set', set_file] + if args.get('skip_compile'): + cmd.append('--skip-compile') + if args.get('skip_clean'): + cmd.append('--skip-clean') + if args.get('skip_analyze'): + cmd.append('--skip-analyze') + if args.get('deep'): + cmd.append('--deep') + if args.get('gui'): + cmd.append('--gui') + + timeout = args.get('timeout', 900) + success, output = run_script(cmd, timeout=timeout) + + if not success: + return {'success': False, 'error': output[-2000:]} # last 2k chars + + # Parse report dir from pipeline output (reliable, avoids stale REPORTS_DIR at startup) + report_dir = None + for line in output.splitlines(): + if line.strip().startswith('Report:') or ' Report: ' in line: + parts = line.split('Report:', 1) + if len(parts) == 2: + candidate = parts[1].strip() + if os.path.isdir(candidate): + report_dir = candidate + break + if not report_dir: + report_dir = latest_report_dir() + if not report_dir: + return {'success': False, 'error': 'Pipeline completed but no report directory found'} + + metrics = read_json(os.path.join(report_dir, 'metrics.json')) + analysis = read_json(os.path.join(report_dir, 'analysis.json')) + + result = { + 'success': True, + 'report_dir': report_dir, + 'metrics': metrics, + 'analysis_summary': analysis.get('summary', {}), + 'worst_dd_event': analysis.get('dd_events', [{}])[0] if analysis.get('dd_events') else None, + 'monthly_pnl': analysis.get('monthly_pnl', []), + 'grid_depth_histogram': analysis.get('grid_depth_histogram', {}), + 'output': output[-1000:], + } + if symbol_warning: + result['symbol_warning'] = symbol_warning + result['symbol_suggestions'] = symbol_suggestions + return result + + +async def handle_run_optimization(args: dict) -> dict: + env_error = _validate_environment() + if env_error: + return env_error + + cmd = [str(SCRIPTS_DIR / 'optimize.sh')] + cmd += ['--expert', args['expert']] + cmd += ['--set', args['set_file']] + cmd += ['--from', args['from_date']] + cmd += ['--to', args['to_date']] + + if 'symbol' in args: + cmd += ['--symbol', args['symbol']] + if 'deposit' in args: + cmd += ['--deposit', str(args['deposit'])] + + success, output = run_script(cmd, timeout=60) # script returns quickly (nohup) + + # Extract job ID from output + import re + job_match = re.search(r'opt_\d{8}_\d{6}', output) + job_id = job_match.group(0) if job_match else None + + return { + 'success': success, + 'job_id': job_id, + 'message': 'Optimization launched in background. Do NOT poll. Signal me when MT5 completes.', + 'output': output[-500:], + } + + +async def handle_get_optimization_results(args: dict) -> dict: + from analytics.optimize_parser import find_report as find_opt_report + + # Locate report + report_path = None + if 'report_file' in args: + report_path = args['report_file'] + elif 'job_id' in args: + try: + report_path = find_opt_report(args['job_id']) + except FileNotFoundError as e: + return {'success': False, 'error': str(e)} + + if not report_path or not os.path.exists(report_path): + return {'success': False, 'error': 'Report not found. Is optimization still running?'} + + fmt = opt_detect_format(report_path) + if fmt == 'xml': + raw = opt_parse_xml(report_path) + else: + raw = opt_parse_html(report_path) + + results = normalize(raw) + results.sort(key=lambda r: r.get('net_profit', 0), reverse=True) + + top_n = args.get('top_n', 20) + dd_threshold = args.get('dd_threshold', 20.0) + conv = convergence_analysis(results, top_n=10) + + # Flag high-risk results + for r in results: + r['high_risk'] = r.get('max_dd_pct', 0) > dd_threshold + + return { + 'success': True, + 'total_passes': len(results), + 'results': results[:top_n], + 'convergence': conv, + 'recommendation': _opt_recommendation(results, dd_threshold), + } + + +def _opt_recommendation(results: list[dict], dd_threshold: float) -> dict: + safe = [r for r in results if r.get('max_dd_pct', 999) < dd_threshold] + if not safe: + return { + 'verdict': 'all_high_risk', + 'message': f'All top results exceed DD threshold ({dd_threshold}%). Widen parameter ranges or increase DD threshold.', + } + best = safe[0] + return { + 'verdict': 'verify_model0' if best.get('model', 0) != 0 else 'promote_candidate', + 'best_params': best.get('params', {}), + 'net_profit': best.get('net_profit', 0), + 'max_dd_pct': best.get('max_dd_pct', 0), + 'message': f"Run verification backtest with these params before promoting.", + } + + +async def handle_analyze_report(args: dict) -> dict: + report_dir = args.get('report_dir') or latest_report_dir() + if not report_dir: + return {'success': False, 'error': 'No report directory found'} + + metrics = read_json(os.path.join(report_dir, 'metrics.json')) + analysis = read_json(os.path.join(report_dir, 'analysis.json')) + + if not metrics and not analysis: + # Try re-running analysis on deals.csv + deals_csv = os.path.join(report_dir, 'deals.csv') + if os.path.exists(deals_csv): + deals = load_deals(deals_csv) + monthly = monthly_pnl(deals) + dd_events = reconstruct_dd_events(deals, metrics) + analysis = { + 'summary': build_summary(metrics, monthly, dd_events), + 'monthly_pnl': monthly, + 'dd_events': dd_events, + 'grid_depth_histogram': grid_depth_histogram(deals), + 'top_losses': top_losses(deals), + 'loss_sequences': loss_sequences(deals), + } + else: + return {'success': False, 'error': f'No data found in {report_dir}'} + + return { + 'success': True, + 'report_dir': report_dir, + 'metrics': metrics, + **analysis, + } + + +async def handle_compare_baseline(args: dict) -> dict: + report_dir = args.get('report_dir') or latest_report_dir() + if not report_dir: + return {'success': False, 'error': 'No report directory found'} + + metrics = read_json(os.path.join(report_dir, 'metrics.json')) + baseline = args['baseline'] + dd_limit = args.get('promote_dd_limit', 20.0) + + candidate_profit = metrics.get('net_profit', 0) + candidate_dd = metrics.get('max_dd_pct', 999) + baseline_profit = baseline['net_profit'] + baseline_dd = baseline['max_dd_pct'] + + profit_delta = candidate_profit - baseline_profit + dd_delta = candidate_dd - baseline_dd + profit_pct = (profit_delta / baseline_profit * 100) if baseline_profit else 0 + + is_winner = candidate_profit > baseline_profit and candidate_dd < dd_limit + + if is_winner: + verdict = 'winner' + elif candidate_profit > baseline_profit: + verdict = 'marginal' # Better profit but DD too high + else: + verdict = 'loser' + + sign = '+' if profit_delta >= 0 else '' + dd_sign = '+' if dd_delta >= 0 else '' + + return { + 'success': True, + 'verdict': verdict, + 'auto_promote': is_winner, + 'delta': { + 'profit_usd': round(profit_delta, 2), + 'profit_pct': round(profit_pct, 1), + 'dd_pp': round(dd_delta, 2), + }, + 'summary': ( + f"{sign}${profit_delta:,.2f} ({sign}{profit_pct:.1f}%) profit vs {baseline.get('label', 'baseline')}. " + f"DD: {candidate_dd:.2f}% vs {baseline_dd:.2f}% ({dd_sign}{dd_delta:.2f}pp). " + f"{'Auto-promoting.' if is_winner else 'Not promoting (DD too high).' if verdict == 'marginal' else 'Regression.'}" + ), + 'candidate': { + 'net_profit': candidate_profit, + 'max_dd_pct': candidate_dd, + 'total_trades': metrics.get('total_trades', 0), + }, + 'baseline': baseline, + } + + +async def handle_compile_ea(args: dict) -> dict: + env_error = _validate_environment() + if env_error: + return env_error + + expert_path = args['expert_path'] + cmd = [str(SCRIPTS_DIR / 'mqlcompile.sh'), expert_path] + success, output = run_script(cmd, timeout=120) + + return { + 'success': success, + 'output': output, + 'expert_path': expert_path, + } + + +async def handle_verify_setup(args: dict) -> dict: + checks: dict = {} + all_ok = True + + # Config file + config_path = ROOT_DIR / 'config' / 'mt5-quant.yaml' + checks['config_file'] = { + 'ok': config_path.exists(), + 'detail': str(config_path) if config_path.exists() else 'Not found — run: bash scripts/setup.sh', + } + if not config_path.exists(): + all_ok = False + + # Wine executable + wine = cfg('wine_executable') + if not wine: + checks['wine_executable'] = {'ok': False, 'detail': 'Not configured in mt5-quant.yaml'} + all_ok = False + else: + executable = os.access(wine, os.X_OK) + version = '' + if executable: + try: + r = subprocess.run([wine, '--version'], capture_output=True, text=True, timeout=5) + version = ((r.stdout or '') + (r.stderr or '')).strip().splitlines()[0] + except Exception as e: + version = f'error: {e}' + checks['wine_executable'] = { + 'ok': executable, + 'version': version, + 'detail': wine if executable else f'Not executable: {wine}', + } + if not executable: + all_ok = False + + # terminal_dir and derived paths + terminal_dir = cfg('terminal_dir') + if not terminal_dir: + checks['terminal_dir'] = {'ok': False, 'detail': 'Not configured in mt5-quant.yaml'} + all_ok = False + else: + td_ok = Path(terminal_dir).is_dir() + checks['terminal_dir'] = { + 'ok': td_ok, + 'detail': terminal_dir if td_ok else f'Directory not found: {terminal_dir}', + } + if not td_ok: + all_ok = False + + terminal_exe = Path(terminal_dir) / 'terminal64.exe' + checks['terminal64_exe'] = { + 'ok': terminal_exe.exists(), + 'detail': str(terminal_exe) if terminal_exe.exists() + else 'Not found — launch MT5 once to unpack it', + } + + experts_dir = Path(cfg('experts_dir') or os.path.join(terminal_dir, 'MQL5', 'Experts')) + ea_count = len(list(experts_dir.glob('*.ex5'))) if experts_dir.is_dir() else 0 + checks['experts_dir'] = { + 'ok': experts_dir.is_dir(), + 'detail': f'{ea_count} .ex5 file(s)' if experts_dir.is_dir() + else f'Not found (will be created on first EA compile): {experts_dir}', + } + + tester_dir = Path(cfg('tester_profiles_dir') or os.path.join(terminal_dir, 'MQL5', 'Profiles', 'Tester')) + set_count = len(list(tester_dir.glob('*.set'))) if tester_dir.is_dir() else 0 + checks['tester_profiles_dir'] = { + 'ok': tester_dir.is_dir(), + 'detail': f'{set_count} .set file(s)' if tester_dir.is_dir() + else f'Not found (will be created on first backtest): {tester_dir}', + } + + cache_dir = Path(cfg('tester_cache_dir') or os.path.join(terminal_dir, 'Tester')) + checks['tester_cache_dir'] = { + 'ok': cache_dir.is_dir(), + 'detail': str(cache_dir) if cache_dir.is_dir() else f'Not found: {cache_dir}', + } + + return { + 'all_ok': all_ok, + 'checks': checks, + 'hint': 'Run: bash scripts/setup.sh' if not all_ok else 'Environment looks good.', + } + + +async def handle_get_backtest_status(args: dict) -> dict: + report_dir = args.get('report_dir') or latest_report_dir() + if not report_dir: + return {'success': False, 'error': 'No report directory found'} + + progress_log = Path(report_dir) / 'progress.log' + pipeline_meta = Path(report_dir) / 'pipeline_metadata.json' + + stages = [] + if progress_log.exists(): + for line in progress_log.read_text().splitlines(): + parts = line.split() + if len(parts) >= 3: + stages.append({'stage': parts[0], 'timestamp': parts[1], 'elapsed': parts[2]}) + + current_stage = stages[-1]['stage'] if stages else 'UNKNOWN' + finished = pipeline_meta.exists() or current_stage == 'DONE' + elapsed = None + if stages: + try: + elapsed = int(stages[-1]['elapsed'].replace('elapsed=', '').rstrip('s')) + except (ValueError, AttributeError): + pass + + return { + 'success': True, + 'report_dir': report_dir, + 'current_stage': current_stage, + 'elapsed_seconds': elapsed, + 'finished': finished, + 'stages': stages, + } + + +async def handle_get_optimization_status(args: dict) -> dict: + job_id = args['job_id'] + meta_path = ROOT_DIR / '.mt5mcp_jobs' / f'{job_id}.json' + + if not meta_path.exists(): + return {'success': False, 'error': f'Job not found: {job_id}. Check .mt5mcp_jobs/'} + + with open(meta_path) as f: + meta = json.load(f) + + pid = meta.get('pid') + log_file = meta.get('log_file', '') + wine_prefix = meta.get('wine_prefix', '') + started_at = meta.get('started_at', '') + + # Check process alive via kill -0 + alive = False + if pid: + try: + os.kill(int(pid), 0) + alive = True + except (OSError, ProcessLookupError): + alive = False + + # Report file existence = definitive completion signal + report_found = False + report_path = None + if wine_prefix: + base = os.path.join(wine_prefix, 'drive_c', 'mt5mcp_opt_report') + for ext in ('.htm', '.htm.xml', '.html'): + candidate = base + ext + if os.path.exists(candidate): + report_found = True + report_path = candidate + break + + # Tail log + log_tail: list[str] = [] + if log_file and os.path.exists(log_file): + try: + log_tail = Path(log_file).read_text(errors='replace').splitlines()[-20:] + except Exception: + pass + + # Elapsed time + elapsed_seconds = None + if started_at: + try: + from datetime import datetime, timezone + start_dt = datetime.fromisoformat(started_at.replace('Z', '+00:00')) + elapsed_seconds = int((datetime.now(timezone.utc) - start_dt).total_seconds()) + except Exception: + pass + + if report_found: + hint = f'Optimization complete. Call get_optimization_results with job_id="{job_id}".' + elif alive: + hint = f'Still running. Monitor: tail -f {log_file}' + else: + hint = f'Process not running and no report found. Check log: {log_file}' + + return { + 'success': True, + 'job_id': job_id, + 'alive': alive, + 'finished': report_found, + 'elapsed_seconds': elapsed_seconds, + 'report_found': report_found, + 'report_path': report_path, + 'log_file': log_file, + 'log_tail': log_tail, + 'hint': hint, + } + + +async def handle_prune_reports(args: dict) -> dict: + keep_last = int(args.get('keep_last') or cfg('keep_last', '20') or 20) + REPORTS_DIR.mkdir(exist_ok=True) + + all_dirs = sorted( + [d for d in REPORTS_DIR.iterdir() if d.is_dir() and not d.name.endswith('_opt')], + key=lambda d: d.stat().st_mtime, + ) + to_delete = all_dirs[:-keep_last] if len(all_dirs) > keep_last else [] + kept = all_dirs[-keep_last:] if len(all_dirs) > keep_last else all_dirs + + deleted_names = [] + for d in to_delete: + try: + shutil.rmtree(str(d)) + deleted_names.append(d.name) + except Exception: + pass + + return { + 'success': True, + 'deleted_count': len(deleted_names), + 'kept_count': len(kept), + 'deleted_dirs': deleted_names, + 'kept_dirs': [d.name for d in kept], + } + + +async def handle_list_reports(args: dict) -> dict: + REPORTS_DIR.mkdir(exist_ok=True) + include_opt = args.get('include_opt', False) + limit = int(args.get('limit') or 30) + + dirs = sorted( + [d for d in REPORTS_DIR.iterdir() if d.is_dir()], + key=lambda d: d.stat().st_mtime, + reverse=True, + ) + if not include_opt: + dirs = [d for d in dirs if not d.name.endswith('_opt')] + dirs = dirs[:limit] + + rows = [] + for d in dirs: + m = read_json(str(d / 'metrics.json')) + row: dict = {'name': d.name, 'is_opt': d.name.endswith('_opt')} + if m: + row['net_profit'] = m.get('net_profit') + row['max_dd_pct'] = m.get('max_dd_pct') + row['total_trades'] = m.get('total_trades') + row['symbol'] = m.get('symbol') + row['timeframe'] = m.get('timeframe') + row['from_date'] = m.get('from_date') or m.get('testing_from') + row['to_date'] = m.get('to_date') or m.get('testing_to') + else: + row['metrics'] = 'missing' + rows.append(row) + + return {'success': True, 'count': len(rows), 'reports': rows} + + +async def handle_tail_log(args: dict) -> dict: + n = int(args.get('n') or 50) + filt = args.get('filter', 'all') + + log_path: str | None = args.get('log_file') + + if not log_path and 'job_id' in args: + job_id = args['job_id'] + meta_path = ROOT_DIR / '.mt5mcp_jobs' / f'{job_id}.json' + if not meta_path.exists(): + return {'success': False, 'error': f'Job not found: {job_id}'} + with open(meta_path) as f: + meta = json.load(f) + log_path = meta.get('log_file', '') + + if not log_path: + report_dir = args.get('report_dir') or latest_report_dir() + if report_dir: + log_path = str(Path(report_dir) / 'progress.log') + + if not log_path or not os.path.exists(log_path): + return {'success': False, 'error': f'Log file not found: {log_path}'} + + try: + lines = Path(log_path).read_text(errors='replace').splitlines() + except Exception as e: + return {'success': False, 'error': str(e)} + + if filt == 'errors': + lines = [l for l in lines if 'error' in l.lower() or 'fail' in l.lower() or 'err:' in l.lower()] + elif filt == 'warnings': + lines = [l for l in lines if 'warn' in l.lower() or 'error' in l.lower()] + + return { + 'success': True, + 'log_file': log_path, + 'total_lines': len(lines), + 'lines': lines[-n:], + } + + +def _dir_size(path: Path) -> int: + return sum(f.stat().st_size for f in path.rglob('*') if f.is_file()) + + +async def handle_cache_status(args: dict) -> dict: + terminal_dir = cfg('terminal_dir') + if not terminal_dir: + return {'success': False, 'error': 'terminal_dir not configured'} + + cache_dir = Path(cfg('tester_cache_dir') or os.path.join(terminal_dir, 'Tester')) + if not cache_dir.is_dir(): + return {'success': False, 'error': f'Cache dir not found: {cache_dir}'} + + total_bytes = 0 + breakdown: list[dict] = [] + + for item in sorted(cache_dir.iterdir()): + if item.is_dir(): + sz = _dir_size(item) + total_bytes += sz + breakdown.append({'symbol': item.name, 'size_mb': round(sz / 1024 / 1024, 2)}) + elif item.is_file(): + sz = item.stat().st_size + total_bytes += sz + + return { + 'success': True, + 'cache_dir': str(cache_dir), + 'total_size_mb': round(total_bytes / 1024 / 1024, 2), + 'symbols': breakdown, + } + + +async def handle_clean_cache(args: dict) -> dict: + terminal_dir = cfg('terminal_dir') + if not terminal_dir: + return {'success': False, 'error': 'terminal_dir not configured'} + + cache_dir = Path(cfg('tester_cache_dir') or os.path.join(terminal_dir, 'Tester')) + if not cache_dir.is_dir(): + return {'success': False, 'error': f'Cache dir not found: {cache_dir}'} + + symbol = args.get('symbol', '').strip() + dry_run = bool(args.get('dry_run', False)) + + targets: list[Path] = [] + if symbol: + target = cache_dir / symbol + if target.is_dir(): + targets.append(target) + else: + return {'success': False, 'error': f'No cache found for symbol: {symbol}'} + else: + targets = [d for d in cache_dir.iterdir() if d.is_dir()] + + freed_bytes = sum(_dir_size(t) for t in targets) + names = [t.name for t in targets] + + if not dry_run: + for t in targets: + shutil.rmtree(str(t)) + + return { + 'success': True, + 'dry_run': dry_run, + 'deleted_symbols': names, + 'freed_mb': round(freed_bytes / 1024 / 1024, 2), + 'hint': 'Next backtest will regenerate tick data (slower first run).', + } + + +# ── .set file helpers ───────────────────────────────────────────────────────── + +def _parse_set_line(line: str) -> tuple[str, dict] | None: + """Parse one .set file line → (name, param_dict) or None.""" + line = line.strip() + if not line or line.startswith(';') or '=' not in line: + return None + name, _, raw = line.partition('=') + name = name.strip() + parts = raw.split('||') + value = parts[0].strip() + param: dict = {'value': value} + if len(parts) >= 4: + param['from'] = parts[1].strip() + param['to'] = parts[2].strip() + param['step'] = parts[3].strip() if len(parts) > 3 else '' + param['optimize'] = parts[4].strip() == 'Y' if len(parts) > 4 else False + return name, param + + +def _decode_set(path: str) -> tuple[dict, list[str]]: + """Load a .set file → (params, comments). Raises ValueError on decode failure.""" + content = None + raw = Path(path).read_bytes() + for enc in ('utf-16-le', 'utf-16', 'utf-8-sig', 'utf-8'): + try: + if enc in ('utf-16-le', 'utf-16') and raw[:2] in (b'\xff\xfe', b'\xfe\xff'): + content = raw.decode('utf-16') + else: + content = raw.decode(enc) + break + except (UnicodeDecodeError, LookupError): + continue + if content is None: + raise ValueError(f'Cannot decode {path} — unknown encoding') + + params: dict = {} + comments: list[str] = [] + for line in content.splitlines(): + if line.strip().startswith(';'): + comments.append(line.strip().lstrip(';').strip()) + continue + result = _parse_set_line(line) + if result: + name, param = result + params[name] = param + return params, comments + + +def _encode_set(params: dict, comments: list[str] | None = None) -> bytes: + """Serialize params (and optional header comments) to UTF-16LE bytes.""" + lines: list[str] = [] + if comments: + for c in comments: + lines.append(f'; {c}') + for name, spec in params.items(): + if isinstance(spec, dict): + value = str(spec.get('value', '')) + if 'from' in spec: + flag = 'Y' if spec.get('optimize', False) else 'N' + lines.append(f"{name}={value}||{spec['from']}||{spec.get('to', value)}||{spec.get('step', '1')}||{flag}") + else: + lines.append(f"{name}={value}") + else: + lines.append(f"{name}={spec}") + return ('\r\n'.join(lines) + '\r\n').encode('utf-16-le') + + +def _write_set(path: str, data: bytes) -> None: + """Write bytes to path and apply chmod 444 (required by MT5).""" + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + # chmod 644 first in case file already exists as 444 + if p.exists(): + os.chmod(path, 0o644) + p.write_bytes(data) + os.chmod(path, 0o444) + + +def _sweep_combinations(params: dict) -> tuple[list[dict], int]: + """Return (swept_param_details, total_combinations) for a parsed params dict.""" + import math + swept = [] + total = 1 + for name, spec in params.items(): + if not isinstance(spec, dict) or not spec.get('optimize'): + continue + try: + f = float(spec['from']) + t = float(spec['to']) + s = float(spec['step']) + count = max(1, math.floor(abs(t - f) / s) + 1) if s else 1 + except (KeyError, ValueError, ZeroDivisionError): + count = 1 + swept.append({ + 'name': name, + 'from': spec.get('from'), + 'to': spec.get('to'), + 'step': spec.get('step'), + 'count': count, + }) + total *= count + return swept, total + + +async def handle_read_set_file(args: dict) -> dict: + path = args['path'] + if not os.path.exists(path): + return {'success': False, 'error': f'File not found: {path}'} + try: + params, comments = _decode_set(path) + except Exception as e: + return {'success': False, 'error': str(e)} + return { + 'success': True, + 'path': path, + 'param_count': len(params), + 'comments': comments, + 'params': params, + } + + +async def handle_write_set_file(args: dict) -> dict: + path = args['path'] + params: dict = args['params'] + try: + _write_set(path, _encode_set(params)) + except Exception as e: + return {'success': False, 'error': str(e)} + return { + 'success': True, + 'path': path, + 'param_count': len(params), + 'encoding': 'utf-16-le', + 'permissions': '444 (read-only, required by MT5)', + } + + +async def handle_patch_set_file(args: dict) -> dict: + path = args['path'] + patches: dict = args['patches'] + if not os.path.exists(path): + return {'success': False, 'error': f'File not found: {path}'} + try: + params, comments = _decode_set(path) + except Exception as e: + return {'success': False, 'error': str(e)} + + changed: list[dict] = [] + for name, new_spec in patches.items(): + old = params.get(name, {}) + old_value = old.get('value') if isinstance(old, dict) else str(old) + if isinstance(new_spec, dict): + # Merge: keep existing sweep config unless overridden + merged = dict(old) if isinstance(old, dict) else {'value': old_value} + merged.update(new_spec) + params[name] = merged + new_value = str(merged.get('value', '')) + else: + new_value = str(new_spec) + if isinstance(params.get(name), dict): + params[name] = dict(params[name]) + params[name]['value'] = new_value + else: + params[name] = {'value': new_value} + if old_value != new_value: + changed.append({'name': name, 'old': old_value, 'new': new_value}) + + try: + _write_set(path, _encode_set(params, comments)) + except Exception as e: + return {'success': False, 'error': str(e)} + + return { + 'success': True, + 'path': path, + 'changed_count': len(changed), + 'changed': changed, + 'param_count': len(params), + } + + +async def handle_clone_set_file(args: dict) -> dict: + source = args['source'] + destination = args['destination'] + overrides: dict = args.get('overrides', {}) + if not os.path.exists(source): + return {'success': False, 'error': f'Source not found: {source}'} + try: + params, comments = _decode_set(source) + except Exception as e: + return {'success': False, 'error': str(e)} + + changed: list[dict] = [] + for name, new_spec in overrides.items(): + old = params.get(name, {}) + old_value = old.get('value') if isinstance(old, dict) else str(old) if old else None + if isinstance(new_spec, dict): + merged = dict(old) if isinstance(old, dict) else {} + merged.update(new_spec) + params[name] = merged + new_value = str(merged.get('value', '')) + else: + new_value = str(new_spec) + if isinstance(params.get(name), dict): + params[name] = dict(params[name]) + params[name]['value'] = new_value + else: + params[name] = {'value': new_value} + if old_value != new_value: + changed.append({'name': name, 'old': old_value, 'new': new_value}) + + try: + _write_set(destination, _encode_set(params, comments)) + except Exception as e: + return {'success': False, 'error': str(e)} + + return { + 'success': True, + 'source': source, + 'destination': destination, + 'param_count': len(params), + 'overridden_count': len(changed), + 'overridden': changed, + } + + +async def handle_set_from_optimization(args: dict) -> dict: + path = args['path'] + opt_params: dict = args['params'] # {name: value} from optimization result + template: str | None = args.get('template') + sweep: dict = args.get('sweep', {}) # {name: {from, to, step}} to add sweep flags + + base_params: dict = {} + base_comments: list[str] = [] + + if template: + if not os.path.exists(template): + return {'success': False, 'error': f'Template not found: {template}'} + try: + base_params, base_comments = _decode_set(template) + except Exception as e: + return {'success': False, 'error': str(e)} + + # Start from template (or empty), apply opt values, strip all sweep flags + merged: dict = {} + for name, spec in base_params.items(): + # Copy as fixed value (no sweep) + value = spec.get('value') if isinstance(spec, dict) else str(spec) + merged[name] = {'value': value} + + # Apply optimization result values (overwrite template values, add new params) + for name, value in opt_params.items(): + merged[name] = {'value': str(value)} + + # Optionally re-add sweep ranges for a subset of params + for name, sweep_spec in sweep.items(): + if name in merged: + merged[name].update({ + 'from': str(sweep_spec.get('from', '')), + 'to': str(sweep_spec.get('to', '')), + 'step': str(sweep_spec.get('step', '1')), + 'optimize': bool(sweep_spec.get('optimize', True)), + }) + + try: + _write_set(path, _encode_set(merged, base_comments)) + except Exception as e: + return {'success': False, 'error': str(e)} + + swept, total = _sweep_combinations(merged) + return { + 'success': True, + 'path': path, + 'param_count': len(merged), + 'from_template': bool(template), + 'opt_params_applied': len(opt_params), + 'swept_params': len(swept), + 'total_combinations': total if swept else 0, + } + + +async def handle_diff_set_files(args: dict) -> dict: + path_a = args['path_a'] + path_b = args['path_b'] + + for p in (path_a, path_b): + if not os.path.exists(p): + return {'success': False, 'error': f'File not found: {p}'} + try: + params_a, _ = _decode_set(path_a) + params_b, _ = _decode_set(path_b) + except Exception as e: + return {'success': False, 'error': str(e)} + + keys_a = set(params_a) + keys_b = set(params_b) + + added = [] + for k in sorted(keys_b - keys_a): + spec = params_b[k] + added.append({'name': k, 'value': spec.get('value') if isinstance(spec, dict) else str(spec)}) + + removed = [] + for k in sorted(keys_a - keys_b): + spec = params_a[k] + removed.append({'name': k, 'value': spec.get('value') if isinstance(spec, dict) else str(spec)}) + + changed = [] + for k in sorted(keys_a & keys_b): + sa = params_a[k] + sb = params_b[k] + va = sa.get('value') if isinstance(sa, dict) else str(sa) + vb = sb.get('value') if isinstance(sb, dict) else str(sb) + opt_a = sa.get('optimize', False) if isinstance(sa, dict) else False + opt_b = sb.get('optimize', False) if isinstance(sb, dict) else False + if va != vb or opt_a != opt_b: + entry: dict = {'name': k, 'a': va, 'b': vb} + if opt_a != opt_b: + entry['sweep_a'] = opt_a + entry['sweep_b'] = opt_b + changed.append(entry) + + identical = not added and not removed and not changed + return { + 'success': True, + 'path_a': path_a, + 'path_b': path_b, + 'identical': identical, + 'added_count': len(added), + 'removed_count': len(removed), + 'changed_count': len(changed), + 'added': added, + 'removed': removed, + 'changed': changed, + } + + +async def handle_describe_sweep(args: dict) -> dict: + path = args['path'] + if not os.path.exists(path): + return {'success': False, 'error': f'File not found: {path}'} + try: + params, comments = _decode_set(path) + except Exception as e: + return {'success': False, 'error': str(e)} + + swept, total = _sweep_combinations(params) + fixed_count = len(params) - len(swept) + + return { + 'success': True, + 'path': path, + 'total_params': len(params), + 'swept_count': len(swept), + 'fixed_count': fixed_count, + 'total_combinations': total, + 'swept_params': swept, + 'hint': ( + 'No swept params — this is a backtest .set, not an optimization .set.' + if not swept else + f'{total:,} combinations. Typical range: 1–8h depending on EA tick speed.' + ), + } + + +async def handle_list_set_files(args: dict) -> dict: + terminal_dir = cfg('terminal_dir') + if not terminal_dir: + return {'success': False, 'error': 'terminal_dir not configured'} + + profiles_dir = Path(cfg('tester_profiles_dir') or + os.path.join(terminal_dir, 'MQL5', 'Profiles', 'Tester')) + if not profiles_dir.is_dir(): + return {'success': False, 'error': f'Tester profiles dir not found: {profiles_dir}'} + + ea_filter = args.get('ea', '').lower() + rows: list[dict] = [] + + for f in sorted(profiles_dir.glob('*.set'), key=lambda x: x.stat().st_mtime, reverse=True): + if ea_filter and ea_filter not in f.stem.lower(): + continue + try: + params, _ = _decode_set(str(f)) + swept, total = _sweep_combinations(params) + rows.append({ + 'name': f.name, + 'param_count': len(params), + 'swept_count': len(swept), + 'total_combinations': total if swept else 0, + 'modified': f.stat().st_mtime, + }) + except Exception: + rows.append({'name': f.name, 'error': 'unreadable'}) + + # Convert mtime to ISO for readability + from datetime import datetime + for r in rows: + if 'modified' in r: + r['modified'] = datetime.fromtimestamp(r['modified']).strftime('%Y-%m-%d %H:%M') + + return { + 'success': True, + 'profiles_dir': str(profiles_dir), + 'count': len(rows), + 'files': rows, + } + + +async def handle_archive_report(args: dict) -> dict: + report_dir = args.get('report_dir') or latest_report_dir() + if not report_dir: + return {'success': False, 'error': 'No report directory found'} + + entry = _build_history_entry(report_dir) + if not entry: + return {'success': False, 'error': f'No metrics.json or analysis.json in {report_dir}'} + + if args.get('verdict'): + entry['verdict'] = args['verdict'] + if args.get('notes'): + entry['notes'] = args['notes'] + if args.get('tags'): + entry['tags'] = args['tags'] + + history = load_history() + existing_ids = {e['id'] for e in history} + already_exists = entry['id'] in existing_ids + + if not already_exists: + history.append(entry) + save_history(history) + + deleted = False + if args.get('delete_after') and not already_exists: + try: + shutil.rmtree(report_dir) + deleted = True + # Update the entry in history to reflect deletion + for e in history: + if e['id'] == entry['id']: + e['report_dir_deleted'] = True + break + save_history(history) + except Exception as exc: + return {'success': False, 'error': f'Archive succeeded but delete failed: {exc}'} + + return { + 'success': True, + 'id': entry['id'], + 'already_existed': already_exists, + 'deleted_source': deleted, + 'history_file': str(HISTORY_FILE), + 'entry_summary': { + 'ea': entry['ea'], + 'symbol': entry['symbol'], + 'metrics': entry['metrics'], + 'verdict': entry['verdict'], + }, + } + + +async def handle_archive_all_reports(args: dict) -> dict: + REPORTS_DIR.mkdir(exist_ok=True) + delete_after = bool(args.get('delete_after', False)) + keep_last = int(args.get('keep_last', 5)) + dry_run = bool(args.get('dry_run', False)) + + all_dirs = sorted( + [d for d in REPORTS_DIR.iterdir() if d.is_dir() and not d.name.endswith('_opt')], + key=lambda d: d.stat().st_mtime, + ) + + history = load_history() + existing_ids = {e['id'] for e in history} + + # Dirs protected from deletion regardless of keep_last + protected = {d.name for d in all_dirs[-keep_last:]} if keep_last > 0 else set() + + results = {'archived': [], 'skipped': [], 'deleted': [], 'failed': []} + + for d in all_dirs: + if d.name in existing_ids: + results['skipped'].append(d.name) + continue + + entry = _build_history_entry(str(d)) + if not entry: + results['failed'].append(d.name) + continue + + if not dry_run: + history.append(entry) + results['archived'].append(d.name) + + should_delete = delete_after and d.name not in protected + if should_delete and not dry_run: + try: + shutil.rmtree(str(d)) + entry['report_dir_deleted'] = True + results['deleted'].append(d.name) + except Exception: + results['failed'].append(d.name) + + if not dry_run and results['archived']: + save_history(history) + + return { + 'success': True, + 'dry_run': dry_run, + 'archived_count': len(results['archived']), + 'skipped_count': len(results['skipped']), + 'deleted_count': len(results['deleted']), + 'failed_count': len(results['failed']), + 'history_file': str(HISTORY_FILE), + **results, + } + + +async def handle_get_history(args: dict) -> dict: + history = load_history() + if not history: + return {'success': True, 'count': 0, 'entries': []} + + ea_filter = args.get('ea', '').lower() + symbol_filter = args.get('symbol', '').upper() + verdict_filter = args.get('verdict') + tag_filter = args.get('tag', '') + min_profit = args.get('min_profit') + max_dd = args.get('max_dd_pct') + sort_by = args.get('sort_by', 'date') + limit = int(args.get('limit') or 20) + include_monthly = bool(args.get('include_monthly', False)) + + filtered = [] + for e in history: + if ea_filter and ea_filter not in e.get('ea', '').lower(): + continue + if symbol_filter and e.get('symbol', '').upper() != symbol_filter: + continue + if verdict_filter and e.get('verdict') != verdict_filter: + continue + if tag_filter and tag_filter not in e.get('tags', []): + continue + m = e.get('metrics', {}) + if min_profit is not None and (m.get('net_profit') or 0) < min_profit: + continue + if max_dd is not None and (m.get('max_dd_pct') or 999) > max_dd: + continue + filtered.append(e) + + key_map = { + 'date': lambda e: e.get('archived_at', ''), + 'profit': lambda e: (e.get('metrics') or {}).get('net_profit') or 0, + 'dd': lambda e: (e.get('metrics') or {}).get('max_dd_pct') or 999, + 'sharpe': lambda e: (e.get('metrics') or {}).get('sharpe_ratio') or 0, + } + reverse = sort_by != 'dd' + filtered.sort(key=key_map.get(sort_by, key_map['date']), reverse=reverse) + filtered = filtered[:limit] + + if not include_monthly: + for e in filtered: + e.pop('monthly_pnl', None) + + return {'success': True, 'count': len(filtered), 'entries': filtered} + + +async def handle_promote_to_baseline(args: dict) -> dict: + from datetime import datetime, timezone + + # Resolve source: history entry, explicit report_dir, or latest + entry: dict | None = None + report_dir: str | None = None + + if 'history_id' in args: + history = load_history() + matches = [e for e in history if e['id'] == args['history_id']] + if not matches: + return {'success': False, 'error': f"History entry not found: {args['history_id']}"} + entry = matches[0] + report_dir = entry.get('report_dir') if not entry.get('report_dir_deleted') else None + else: + report_dir = args.get('report_dir') or latest_report_dir() + if not report_dir: + return {'success': False, 'error': 'No report directory found'} + + # Load metrics — prefer live report dir, fall back to history entry + if report_dir and Path(report_dir).is_dir(): + metrics = read_json(os.path.join(report_dir, 'metrics.json')) + elif entry: + metrics = entry.get('metrics', {}) + else: + return {'success': False, 'error': 'Source not found (report dir missing and no history entry)'} + + if not metrics: + return {'success': False, 'error': 'No metrics found in source'} + + now = datetime.now(timezone.utc).strftime('%Y-%m-%d') + ea = (entry or {}).get('ea') or metrics.get('expert') or metrics.get('ea') or '' + symbol = (entry or {}).get('symbol') or metrics.get('symbol') or '' + from_date = (entry or {}).get('from_date') or '' + to_date = (entry or {}).get('to_date') or '' + period = f"{from_date}/{to_date}" if from_date and to_date else '' + + baseline = { + 'ea': ea, + 'symbol': symbol, + 'period': period, + 'net_profit': metrics.get('net_profit'), + 'profit_factor': metrics.get('profit_factor'), + 'max_drawdown_pct': metrics.get('max_dd_pct'), + 'sharpe_ratio': metrics.get('sharpe_ratio'), + 'total_trades': metrics.get('total_trades'), + 'recovery_factor': metrics.get('recovery_factor'), + 'promoted_from': (entry or {}).get('id') or Path(report_dir or '').name, + 'promoted_at': now, + 'notes': args.get('notes', f'Promoted {now}'), + } + + BASELINE_FILE.parent.mkdir(exist_ok=True) + with open(BASELINE_FILE, 'w') as f: + json.dump(baseline, f, indent=2) + + # Mark in history + if entry: + history = load_history() + for e in history: + if e['id'] == entry['id']: + e['promoted_to_baseline'] = True + e['verdict'] = e.get('verdict') or 'reference' + break + save_history(history) + + return { + 'success': True, + 'baseline_file': str(BASELINE_FILE), + 'baseline': baseline, + } + + +async def handle_annotate_history(args: dict) -> dict: + history_id = args['history_id'] + history = load_history() + + target = next((e for e in history if e['id'] == history_id), None) + if not target: + return {'success': False, 'error': f'Entry not found: {history_id}'} + + if 'verdict' in args: + target['verdict'] = args['verdict'] + if 'notes' in args: + target['notes'] = args['notes'] + if 'tags' in args: + target['tags'] = args['tags'] + if 'add_tags' in args: + existing = target.get('tags') or [] + for t in args['add_tags']: + if t not in existing: + existing.append(t) + target['tags'] = existing + + save_history(history) + + return { + 'success': True, + 'id': history_id, + 'verdict': target.get('verdict'), + 'notes': target.get('notes'), + 'tags': target.get('tags'), + } + + +async def handle_list_jobs(args: dict) -> dict: + jobs_dir = ROOT_DIR / '.mt5mcp_jobs' + if not jobs_dir.is_dir(): + return {'success': True, 'jobs': [], 'count': 0} + + include_done = args.get('include_done', True) + rows: list[dict] = [] + + from datetime import datetime, timezone + + for meta_file in sorted(jobs_dir.glob('*.json'), reverse=True): + try: + with open(meta_file) as f: + meta = json.load(f) + except Exception: + continue + + job_id = meta_file.stem + pid = meta.get('pid') + started_at = meta.get('started_at', '') + log_file = meta.get('log_file', '') + wine_prefix = meta.get('wine_prefix', '') + + alive = False + if pid: + try: + os.kill(int(pid), 0) + alive = True + except (OSError, ProcessLookupError): + pass + + report_found = False + if wine_prefix: + base = os.path.join(wine_prefix, 'drive_c', 'mt5mcp_opt_report') + for ext in ('.htm', '.htm.xml', '.html'): + if os.path.exists(base + ext): + report_found = True + break + + status = 'running' if alive else ('done' if report_found else 'failed') + + elapsed_seconds = None + if started_at: + try: + start_dt = datetime.fromisoformat(started_at.replace('Z', '+00:00')) + elapsed_seconds = int((datetime.now(timezone.utc) - start_dt).total_seconds()) + except Exception: + pass + + if not include_done and status != 'running': + continue + + rows.append({ + 'job_id': job_id, + 'status': status, + 'elapsed_seconds': elapsed_seconds, + 'expert': meta.get('expert', ''), + 'started_at': started_at, + 'log_file': log_file, + }) + + return {'success': True, 'count': len(rows), 'jobs': rows} + + +# ── Entry point ─────────────────────────────────────────────────────────────── + +async def main(): + async with mcp.server.stdio.stdio_server() as (read_stream, write_stream): + await app.run( + read_stream, + write_stream, + app.create_initialization_options(), + ) + + +def cli(): + """Sync entry point for [project.scripts] — pyproject.toml requires a sync callable.""" + asyncio.run(main()) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/sample_deals.csv b/tests/fixtures/sample_deals.csv new file mode 100644 index 0000000..5581eb8 --- /dev/null +++ b/tests/fixtures/sample_deals.csv @@ -0,0 +1,24 @@ +time,type,direction,volume,price,sl,tp,profit,balance,comment,order,magic,entry +2025.01.10 09:00:00,buy,in,0.01,1900.00,1880.00,1920.00,0.0,10000.0,Layer #1,1001,12345,in +2025.01.10 09:30:00,buy,out,0.01,1915.00,0.0,0.0,15.00,10015.0,Layer #1,1001,12345,out +2025.01.15 10:00:00,sell,in,0.01,1910.00,1930.00,1890.00,0.0,10015.0,Layer #1,1002,12345,in +2025.01.15 10:45:00,sell,out,0.01,1920.00,0.0,0.0,-10.00,10005.0,Layer #1,1002,12345,out +2025.01.20 14:00:00,buy,in,0.02,1905.00,1880.00,1935.00,0.0,10005.0,Layer #2,1003,12345,in +2025.01.21 09:00:00,buy,in,0.04,1895.00,1870.00,1925.00,0.0,10005.0,Layer #3,1004,12345,in +2025.01.21 15:30:00,buy,out,0.06,1925.00,0.0,0.0,30.00,10035.0,Layer #3,1003,12345,out +2025.02.05 10:00:00,sell,in,0.01,1930.00,1950.00,1910.00,0.0,10035.0,Layer #1,1005,12345,in +2025.02.05 14:00:00,sell,out,0.01,1945.00,0.0,0.0,-15.00,10020.0,Layer #1,1005,12345,out +2025.02.12 11:00:00,buy,in,0.01,1920.00,1900.00,1940.00,0.0,10020.0,Layer #1,1006,12345,in +2025.02.12 16:00:00,buy,out,0.01,1938.00,0.0,0.0,18.00,10038.0,Layer #1,1006,12345,out +2025.01.06 02:30:00,buy,in,0.01,1885.00,1865.00,1905.00,0.0,10038.0,Layer #1,2001,23456,in +2025.01.06 03:15:00,buy,out,0.01,1898.00,0.0,0.0,12.00,10050.0,Layer #1 TP,2001,23456,out +2025.01.07 20:00:00,sell,in,0.01,1910.00,1930.00,1890.00,0.0,10050.0,Layer #1,2002,23456,in +2025.01.07 21:30:00,sell,out,0.01,1918.00,0.0,0.0,-8.00,10042.0,Layer #1 cutloss,2002,23456,out +2025.01.08 09:30:00,buy,in,0.01,1892.00,1872.00,1912.00,0.0,10042.0,Layer #1,2003,23456,in +2025.01.08 10:20:00,buy,out,0.01,1905.00,0.0,0.0,20.00,10062.0,Layer #1,2003,23456,out +2025.01.13 08:00:00,sell,in,0.01,1925.00,1945.00,1905.00,0.0,10062.0,Layer #1,2004,23456,in +2025.01.13 08:01:00,sell,in,0.02,1928.00,1948.00,1908.00,0.0,10062.0,Layer #2,2005,23456,in +2025.01.13 08:02:00,sell,in,0.04,1931.00,1951.00,1911.00,0.0,10062.0,Layer #3,2006,23456,in +2025.01.13 09:30:00,sell,out,0.07,1905.00,0.0,0.0,-45.00,10017.0,Layer #3 cutloss,2004,23456,out +2025.01.16 15:00:00,buy,in,0.01,1900.00,1880.00,1920.00,0.0,10017.0,Layer #1,2007,23456,in +2025.01.16 17:30:00,buy,out,0.01,1915.00,0.0,0.0,22.00,10039.0,Layer #1,2007,23456,out diff --git a/tests/fixtures/sample_report.htm b/tests/fixtures/sample_report.htm new file mode 100644 index 0000000..74f3964 --- /dev/null +++ b/tests/fixtures/sample_report.htm @@ -0,0 +1,25 @@ + +Strategy Tester Report + + + + + + + + + + + +
Net profit1234.56
Profit factor1.25
Maximal drawdown500.00 (5.00%)
Sharpe Ratio0.75
Total trades150
Recovery factor2.50
Profit trades (% of total)90 (60.00%)
Gross profit2000.00
Gross loss-765.44
+ + + + + + + + +
Deal TimeTypeDirectionVolumePriceS/LT/PProfitBalanceCommentOrderMagicEntry
2025.01.10 09:30:00buyout0.011915.000015.0010015.00Layer #1100112345out
2025.01.15 10:45:00sellout0.011920.0000-10.0010005.00Layer #1100212345out
2025.01.21 15:30:00buyout0.061925.000030.0010035.00Layer #3100312345out
2025.02.05 14:00:00sellout0.011945.0000-15.0010020.00Layer #1100512345out
2025.02.12 16:00:00buyout0.011938.000018.0010038.00Layer #1100612345out
+ + diff --git a/tests/fixtures/sample_report.htm.xml b/tests/fixtures/sample_report.htm.xml new file mode 100644 index 0000000..2bdf44d --- /dev/null +++ b/tests/fixtures/sample_report.htm.xml @@ -0,0 +1,88 @@ + + + + + + Net profit + 1234.56 + + + Profit factor + 1.25 + + + Sharpe Ratio + 0.75 + + + Total trades + 150 + +
+ + + + + time + type + direction + volume + price + sl + tp + profit + balance + comment + order + magic + entry + + + 2025.01.10 09:30:00 + buy + out + 0.01 + 1915.00 + 0 + 0 + 15.00 + 10015.00 + Layer #1 + 1001 + 12345 + out + + + 2025.01.15 10:45:00 + sell + out + 0.01 + 1920.00 + 0 + 0 + -10.00 + 10005.00 + Layer #1 + 1002 + 12345 + out + + + 2025.02.05 14:00:00 + sell + out + 0.01 + 1945.00 + 0 + 0 + -15.00 + 10020.00 + Layer #1 + 1005 + 12345 + out + +
+ + diff --git a/tests/test_analyze.py b/tests/test_analyze.py new file mode 100644 index 0000000..b1799ad --- /dev/null +++ b/tests/test_analyze.py @@ -0,0 +1,673 @@ +"""Tests for analytics/analyze.py — runs without MT5 or Wine.""" + +import sys +from pathlib import Path + +import pytest + +FIXTURES = Path(__file__).parent / 'fixtures' +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from analytics.analyze import ( + PROFILES, + load_deals, monthly_pnl, reconstruct_dd_events, + grid_depth_histogram, depth_histogram, top_losses, loss_sequences, build_summary, + position_pairs, cycle_stats, exit_reason_breakdown, + direction_bias, streak_analysis, session_breakdown, + weekday_pnl, hourly_pnl, concurrent_peak, volume_profile, + _parse_dt, _classify_exit, _extract_depth, _classify_dd_cause, + _lot_tier, _session_for_hour, +) + + +@pytest.fixture +def deals(): + return load_deals(str(FIXTURES / 'sample_deals.csv')) + + +def test_load_deals_count(deals): + assert len(deals) > 0 + + +def test_load_deals_numeric_fields(deals): + for deal in deals: + assert isinstance(deal['profit'], float) + assert isinstance(deal['balance'], float) + assert isinstance(deal['volume'], float) + + +def test_monthly_pnl_groups_correctly(deals): + result = monthly_pnl(deals) + assert isinstance(result, list) + assert len(result) >= 1 + for entry in result: + assert 'month' in entry + assert 'pnl' in entry + assert 'trades' in entry + assert 'green' in entry + assert isinstance(entry['green'], bool) + + +def test_monthly_pnl_only_out_entries(deals): + """Only 'out' entries should be counted.""" + result = monthly_pnl(deals) + # All trades in fixture are closed, so at least one month should have trades + total_trades = sum(m['trades'] for m in result) + assert total_trades > 0 + + +def test_monthly_pnl_has_jan_and_feb(deals): + result = monthly_pnl(deals) + months = [m['month'] for m in result] + assert '2025-01' in months + assert '2025-02' in months + + +def test_reconstruct_dd_events_returns_list(deals): + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} + result = reconstruct_dd_events(deals, metrics) + assert isinstance(result, list) + + +def test_reconstruct_dd_events_empty_on_no_deals(): + result = reconstruct_dd_events([], {}) + assert result == [] + + +def test_grid_depth_histogram_keys(deals): + hist = grid_depth_histogram(deals) + assert isinstance(hist, dict) + assert 'L1' in hist + assert 'L2' in hist + assert 'L3' in hist + assert 'L8+' in hist + + +def test_grid_depth_histogram_counts_layers(deals): + hist = grid_depth_histogram(deals) + # Fixture has Layer #1, #2, #3 comments + assert hist['L1'] > 0 + assert hist['L3'] > 0 + + +def test_top_losses_are_negative(deals): + losses = top_losses(deals) + assert isinstance(losses, list) + for loss in losses: + assert loss['loss_usd'] < 0 + + +def test_top_losses_sorted_ascending(deals): + losses = top_losses(deals) + if len(losses) >= 2: + assert losses[0]['loss_usd'] <= losses[1]['loss_usd'] + + +def test_loss_sequences_structure(deals): + seqs = loss_sequences(deals) + assert isinstance(seqs, list) + for seq in seqs: + assert 'length' in seq + assert 'total_loss' in seq + assert seq['total_loss'] < 0 + + +def test_build_summary_keys(deals): + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5, 'total_trades': 11, + 'profit_factor': 1.2, 'sharpe_ratio': 0.5, 'recovery_factor': 2.0} + monthly = monthly_pnl(deals) + dd = reconstruct_dd_events(deals, metrics) + summary = build_summary(metrics, monthly, dd) + + expected_keys = ['net_profit', 'profit_factor', 'max_dd_pct', 'sharpe_ratio', + 'total_trades', 'green_months', 'total_months', + 'worst_month', 'worst_month_pnl'] + for k in expected_keys: + assert k in summary, f"Missing key: {k}" + + +def test_build_summary_green_months(deals): + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} + monthly = monthly_pnl(deals) + dd = reconstruct_dd_events(deals, metrics) + summary = build_summary(metrics, monthly, dd) + assert summary['green_months'] >= 0 + assert summary['total_months'] >= summary['green_months'] + + +# ── Utility helpers ──────────────────────────────────────────────────────────── + +def test_parse_dt_standard_format(): + dt = _parse_dt('2025.01.10 09:30:00') + assert dt is not None + assert dt.year == 2025 + assert dt.month == 1 + assert dt.day == 10 + assert dt.hour == 9 + + +def test_parse_dt_iso_format(): + dt = _parse_dt('2025-02-05 14:00:00') + assert dt is not None + assert dt.month == 2 + + +def test_parse_dt_invalid_returns_none(): + assert _parse_dt('') is None + assert _parse_dt('not-a-date') is None + + +def test_classify_exit_locking(): + assert _classify_exit('locking hedge', -50.0) == 'locking' + + +def test_classify_exit_cutloss(): + assert _classify_exit('cutloss fired', -20.0) == 'cutloss' + assert _classify_exit('cut loss', -20.0) == 'cutloss' + + +def test_classify_exit_tp_sl_by_profit(): + assert _classify_exit('Layer #1', 15.0) == 'tp' + assert _classify_exit('Layer #1', -10.0) == 'sl' + + +def test_lot_tier(): + assert _lot_tier(0.01) == '0.01' + assert _lot_tier(0.02) == '0.02-0.04' + assert _lot_tier(0.04) == '0.02-0.04' + assert _lot_tier(0.06) == '0.05-0.09' + assert _lot_tier(0.10) == '0.10-0.49' + assert _lot_tier(1.0) == '1.00+' + + +def test_session_for_hour(): + assert _session_for_hour(3) == 'asian' + assert _session_for_hour(9) == 'london' + assert _session_for_hour(14) == 'london_ny_overlap' + assert _session_for_hour(18) == 'new_york' + assert _session_for_hour(23) == 'off_hours' + + +# ── Position pairs ───────────────────────────────────────────────────────────── + +def test_position_pairs_count(deals): + pairs = position_pairs(deals) + assert isinstance(pairs, list) + assert len(pairs) > 0 + + +def test_position_pairs_hold_minutes(deals): + pairs = position_pairs(deals) + for p in pairs: + if p['hold_minutes'] is not None: + assert p['hold_minutes'] > 0 + + +def test_position_pairs_has_layer(deals): + pairs = position_pairs(deals) + layers = [p['layer'] for p in pairs if p['layer'] > 0] + assert len(layers) > 0 + + +def test_position_pairs_profit_nonzero(deals): + pairs = position_pairs(deals) + for p in pairs: + assert p['profit'] != 0.0 + + +# ── Cycle stats ──────────────────────────────────────────────────────────────── + +def test_cycle_stats_structure(deals): + result = cycle_stats(deals) + assert 'total_cycles' in result + assert 'win_rate' in result + assert 'avg_profit' in result + assert 'win_rate_by_depth' in result + + +def test_cycle_stats_total_cycles(deals): + result = cycle_stats(deals) + assert result['total_cycles'] > 0 + + +def test_cycle_stats_win_rate_range(deals): + result = cycle_stats(deals) + assert 0.0 <= result['win_rate'] <= 100.0 + + +def test_cycle_stats_empty(): + result = cycle_stats([]) + assert result['total_cycles'] == 0 + + +# ── Exit reason breakdown ────────────────────────────────────────────────────── + +def test_exit_reason_breakdown_structure(deals): + result = exit_reason_breakdown(deals) + assert isinstance(result, dict) + for reason, data in result.items(): + assert 'count' in data + assert 'total_pnl' in data + assert 'avg_pnl' in data + assert data['count'] > 0 + + +def test_exit_reason_breakdown_has_cutloss(deals): + result = exit_reason_breakdown(deals) + # fixture has 'cutloss' in comment for some deals + assert 'cutloss' in result + + +def test_exit_reason_breakdown_counts_match(deals): + result = exit_reason_breakdown(deals) + total_counted = sum(r['count'] for r in result.values()) + closed_with_pnl = [d for d in deals + if 'out' in d.get('entry', '').lower() and d.get('profit', 0.0) != 0.0] + assert total_counted == len(closed_with_pnl) + + +# ── Direction bias ───────────────────────────────────────────────────────────── + +def test_direction_bias_keys(deals): + result = direction_bias(deals) + assert isinstance(result, dict) + # fixture has both buy and sell + assert 'buy' in result + assert 'sell' in result + + +def test_direction_bias_win_rate_range(deals): + result = direction_bias(deals) + for d, s in result.items(): + assert 0.0 <= s['win_rate'] <= 100.0 + assert s['trades'] > 0 + + +def test_direction_bias_buy_profitable(deals): + result = direction_bias(deals) + # fixture: buy deals net positive + assert result['buy']['total_pnl'] > 0 + + +# ── Streak analysis ──────────────────────────────────────────────────────────── + +def test_streak_analysis_structure(deals): + result = streak_analysis(deals) + assert isinstance(result, dict) + for key in ('max_win_streak', 'max_loss_streak', 'current_streak', 'current_streak_type'): + assert key in result + + +def test_streak_analysis_nonnegative(deals): + result = streak_analysis(deals) + assert result['max_win_streak'] >= 0 + assert result['max_loss_streak'] >= 0 + assert result['current_streak'] >= 1 + + +def test_streak_analysis_type_valid(deals): + result = streak_analysis(deals) + assert result['current_streak_type'] in ('win', 'loss') + + +def test_streak_analysis_empty(): + assert streak_analysis([]) == {} + + +# ── Session breakdown ────────────────────────────────────────────────────────── + +def test_session_breakdown_structure(deals): + result = session_breakdown(deals) + assert isinstance(result, dict) + for session, data in result.items(): + assert 'trades' in data + assert 'win_rate' in data + assert 'total_pnl' in data + + +def test_session_breakdown_has_sessions(deals): + result = session_breakdown(deals) + # fixture has deals at 09:00, 10:00, 14:00, 15:00, 16:00 (London + London/NY) + # and 02:30, 03:15 (Asian), 20:00-21:30 (NY) + known_sessions = {'london', 'london_ny_overlap', 'asian', 'new_york'} + assert len(set(result.keys()) & known_sessions) >= 2 + + +def test_session_breakdown_win_rate_range(deals): + result = session_breakdown(deals) + for session, data in result.items(): + assert 0.0 <= data['win_rate'] <= 100.0 + + +# ── Weekday P/L ──────────────────────────────────────────────────────────────── + +def test_weekday_pnl_structure(deals): + result = weekday_pnl(deals) + assert isinstance(result, list) + for entry in result: + assert 'day' in entry + assert 'pnl' in entry + assert 'trades' in entry + assert 'win_rate' in entry + + +def test_weekday_pnl_day_names(deals): + result = weekday_pnl(deals) + valid_days = {'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'} + for entry in result: + assert entry['day'] in valid_days + + +def test_weekday_pnl_has_results(deals): + result = weekday_pnl(deals) + assert len(result) >= 1 + + +# ── Hourly P/L ───────────────────────────────────────────────────────────────── + +def test_hourly_pnl_structure(deals): + result = hourly_pnl(deals) + assert isinstance(result, list) + for entry in result: + assert 'hour' in entry + assert 0 <= entry['hour'] <= 23 + assert 'pnl' in entry + assert 'trades' in entry + + +def test_hourly_pnl_has_results(deals): + result = hourly_pnl(deals) + assert len(result) >= 1 + + +# ── Concurrent peak ──────────────────────────────────────────────────────────── + +def test_concurrent_peak_structure(deals): + result = concurrent_peak(deals) + assert 'peak_open' in result + assert 'peak_time' in result + + +def test_concurrent_peak_at_least_one(deals): + result = concurrent_peak(deals) + assert result['peak_open'] >= 1 + + +def test_concurrent_peak_multi_layer(deals): + # fixture has a cycle where L2 and L3 open before close → peak >= 2 + result = concurrent_peak(deals) + assert result['peak_open'] >= 2 + + +# ── Volume profile ───────────────────────────────────────────────────────────── + +def test_volume_profile_structure(deals): + result = volume_profile(deals) + assert isinstance(result, list) + for entry in result: + assert 'lot_tier' in entry + assert 'pnl' in entry + assert 'trades' in entry + assert 'win_rate' in entry + + +def test_volume_profile_has_micro_lots(deals): + result = volume_profile(deals) + tiers = [e['lot_tier'] for e in result] + assert '0.01' in tiers + + +def test_volume_profile_win_rate_range(deals): + result = volume_profile(deals) + for entry in result: + assert 0.0 <= entry['win_rate'] <= 100.0 + + +# ── build_summary with new stats ─────────────────────────────────────────────── + +def test_build_summary_with_streak(deals): + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} + monthly = monthly_pnl(deals) + dd = reconstruct_dd_events(deals, metrics) + streak = streak_analysis(deals) + summary = build_summary(metrics, monthly, dd, streak=streak) + assert 'max_win_streak' in summary + assert 'max_loss_streak' in summary + assert 'current_streak_type' in summary + + +def test_build_summary_with_bias(deals): + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} + monthly = monthly_pnl(deals) + dd = reconstruct_dd_events(deals, metrics) + bias = direction_bias(deals) + summary = build_summary(metrics, monthly, dd, bias=bias) + assert 'buy_win_rate' in summary or 'sell_win_rate' in summary + + +def test_build_summary_with_cycles(deals): + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} + monthly = monthly_pnl(deals) + dd = reconstruct_dd_events(deals, metrics) + cycles = cycle_stats(deals) + summary = build_summary(metrics, monthly, dd, cycles=cycles) + assert 'cycle_win_rate' in summary + assert 'total_cycles' in summary + + +# ── Strategy profiles ────────────────────────────────────────────────────────── + +def test_profiles_registry(): + """All expected strategy names are registered.""" + for name in ('generic', 'grid', 'scalper', 'trend', 'hedge'): + assert name in PROFILES + p = PROFILES[name] + assert 'name' in p + assert 'exit_keywords' in p + assert 'dd_cause_keywords' in p + assert 'cycle_group_by' in p + assert 'cycle_gap_min' in p + + +def test_profiles_depth_re(): + assert PROFILES['grid']['depth_re'] is not None + assert PROFILES['generic']['depth_re'] is None + assert PROFILES['scalper']['depth_re'] is None + assert PROFILES['trend']['depth_re'] is None + assert PROFILES['hedge']['depth_re'] is None + + +# ── _extract_depth ───────────────────────────────────────────────────────────── + +def test_extract_depth_grid_pattern(): + depth_re = PROFILES['grid']['depth_re'] + assert _extract_depth('Layer #3', depth_re) == 3 + assert _extract_depth('Layer #1', depth_re) == 1 + assert _extract_depth('layer 7', depth_re) == 7 + + +def test_extract_depth_no_pattern(): + assert _extract_depth('Layer #3', None) == 0 + assert _extract_depth('', None) == 0 + + +def test_extract_depth_no_match(): + assert _extract_depth('TP hit', PROFILES['grid']['depth_re']) == 0 + + +# ── _classify_exit with profiles ────────────────────────────────────────────── + +def test_classify_exit_grid_locking(): + assert _classify_exit('locking hedge', -50.0, PROFILES['grid']) == 'locking' + + +def test_classify_exit_grid_cutloss(): + assert _classify_exit('cutloss fired', -20.0, PROFILES['grid']) == 'cutloss' + + +def test_classify_exit_scalper_manual(): + assert _classify_exit('manual close', -5.0, PROFILES['scalper']) == 'manual' + + +def test_classify_exit_scalper_trailing(): + assert _classify_exit('trailing stop', 10.0, PROFILES['scalper']) == 'trailing' + + +def test_classify_exit_trend_breakeven(): + assert _classify_exit('breakeven stop', 0.5, PROFILES['trend']) == 'breakeven' + + +def test_classify_exit_trend_partial(): + assert _classify_exit('partial scale out', 15.0, PROFILES['trend']) == 'partial' + + +def test_classify_exit_hedge_net_close(): + assert _classify_exit('net close', -30.0, PROFILES['hedge']) == 'net_close' + + +def test_classify_exit_generic_fallback(): + """Generic profile has no keywords — falls back to profit sign.""" + assert _classify_exit('Layer #3 locking', -50.0, PROFILES['generic']) == 'sl' + assert _classify_exit('Layer #1', 15.0, PROFILES['generic']) == 'tp' + + +# ── _classify_dd_cause ──────────────────────────────────────────────────────── + +def test_classify_dd_cause_grid(): + assert _classify_dd_cause('locking total', PROFILES['grid']) == 'locking_cascade' + assert _classify_dd_cause('cutloss fired', PROFILES['grid']) == 'cutloss' + assert _classify_dd_cause('zombie exit', PROFILES['grid']) == 'zombie_exit' + + +def test_classify_dd_cause_generic_unknown(): + assert _classify_dd_cause('locking total', PROFILES['generic']) == 'unknown' + assert _classify_dd_cause('', PROFILES['generic']) == 'unknown' + + +def test_classify_dd_cause_scalper_stop(): + assert _classify_dd_cause('sl hit', PROFILES['scalper']) == 'stop_loss' + + +def test_classify_dd_cause_trend_whipsaw(): + assert _classify_dd_cause('stop loss', PROFILES['trend']) == 'whipsaw' + + +# ── depth_histogram with profiles ───────────────────────────────────────────── + +def test_depth_histogram_grid_returns_layers(deals): + result = depth_histogram(deals, PROFILES['grid']) + assert isinstance(result, dict) + assert 'L1' in result + assert result['L1'] > 0 + + +def test_depth_histogram_generic_returns_empty(deals): + """Generic profile has no depth_re → empty dict.""" + result = depth_histogram(deals, PROFILES['generic']) + assert result == {} + + +def test_depth_histogram_scalper_returns_empty(deals): + result = depth_histogram(deals, PROFILES['scalper']) + assert result == {} + + +def test_grid_depth_histogram_is_alias(deals): + """grid_depth_histogram must equal depth_histogram with grid profile.""" + assert grid_depth_histogram(deals) == depth_histogram(deals, PROFILES['grid']) + + +# ── cycle_stats with profiles ───────────────────────────────────────────────── + +def test_cycle_stats_grid_profile(deals): + result = cycle_stats(deals, PROFILES['grid']) + assert result['total_cycles'] > 0 + assert 0.0 <= result['win_rate'] <= 100.0 + + +def test_cycle_stats_scalper_profile(deals): + """Scalper uses magic-only grouping and 10-min gap.""" + result = cycle_stats(deals, PROFILES['scalper']) + assert 'total_cycles' in result + assert result['total_cycles'] > 0 + + +def test_cycle_stats_generic_profile(deals): + result = cycle_stats(deals, PROFILES['generic']) + assert 'total_cycles' in result + + +def test_cycle_stats_scalper_vs_grid_differ(deals): + """Different grouping rules can produce different cycle counts.""" + grid_result = cycle_stats(deals, PROFILES['grid']) + scalper_result = cycle_stats(deals, PROFILES['scalper']) + # Both must be valid; counts may differ due to grouping + assert grid_result['total_cycles'] >= 0 + assert scalper_result['total_cycles'] >= 0 + + +# ── exit_reason_breakdown with profiles ─────────────────────────────────────── + +def test_exit_reason_breakdown_grid(deals): + result = exit_reason_breakdown(deals, PROFILES['grid']) + assert 'cutloss' in result # fixture has "cutloss" in comments + + +def test_exit_reason_breakdown_generic_only_tp_sl(deals): + """Generic profile has no keywords → only 'tp' and 'sl' keys.""" + result = exit_reason_breakdown(deals, PROFILES['generic']) + for reason in result: + assert reason in ('tp', 'sl'), f"Unexpected reason '{reason}' from generic profile" + + +def test_exit_reason_breakdown_scalper_keywords(deals): + """Scalper profile recognises 'cutloss' comment as 'manual' (not 'cutloss').""" + result = exit_reason_breakdown(deals, PROFILES['scalper']) + # 'cutloss' is not a scalper keyword → falls back to profit-sign → 'sl' + assert 'cutloss' not in result + + +def test_exit_reason_breakdown_counts_sum(deals): + """Total count must equal number of non-zero closed deals, regardless of profile.""" + closed = [d for d in deals + if 'out' in d.get('entry', '').lower() and d.get('profit', 0.0) != 0.0] + for profile in PROFILES.values(): + result = exit_reason_breakdown(deals, profile) + assert sum(r['count'] for r in result.values()) == len(closed) + + +# ── reconstruct_dd_events with profiles ─────────────────────────────────────── + +def test_dd_events_cause_generic_unknown(deals): + """Generic profile → all causes must be 'unknown'.""" + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} + events = reconstruct_dd_events(deals, metrics, PROFILES['generic']) + for ev in events: + assert ev['cause'] == 'unknown' + + +def test_dd_events_cause_grid_classified(deals): + """Grid profile → cause is classified from comment keywords.""" + metrics = {'net_profit': 38.0, 'max_dd_pct': 5.0} + events = reconstruct_dd_events(deals, metrics, PROFILES['grid']) + valid = {'locking_cascade', 'cutloss', 'zombie_exit', 'spike_entry', 'unknown'} + for ev in events: + assert ev['cause'] in valid + + +# ── build_summary strategy field ────────────────────────────────────────────── + +def test_build_summary_strategy_field(deals): + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} + monthly = monthly_pnl(deals) + dd = reconstruct_dd_events(deals, metrics) + summary = build_summary(metrics, monthly, dd, strategy='scalper') + assert summary['strategy'] == 'scalper' + + +def test_build_summary_no_strategy_field(deals): + metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} + monthly = monthly_pnl(deals) + dd = reconstruct_dd_events(deals, metrics) + summary = build_summary(metrics, monthly, dd) + assert 'strategy' not in summary diff --git a/tests/test_extract.py b/tests/test_extract.py new file mode 100644 index 0000000..dd7375b --- /dev/null +++ b/tests/test_extract.py @@ -0,0 +1,146 @@ +"""Tests for analytics/extract.py — runs without MT5 or Wine.""" + +import json +import os +import sys +import tempfile +from pathlib import Path + +import pytest + +FIXTURES = Path(__file__).parent / 'fixtures' +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from analytics.extract import ( + detect_format, parse_html, parse_xml, write_outputs, + _parse_metrics_html, _parse_deals_html, +) + + +def test_detect_format_html(): + assert detect_format(str(FIXTURES / 'sample_report.htm')) == 'html' + + +def test_detect_format_xml(): + assert detect_format(str(FIXTURES / 'sample_report.htm.xml')) == 'xml' + + +# HTML parsing is tested via internal functions to avoid the UTF-16 decode dance +# (read_text tries UTF-16 first, which silently garbles plain UTF-8/ASCII files). +# The fixture is used for format-detection only. + +HTML_TEXT = """ + + + + + + + + + + +
Net profit1234.56
Profit factor1.25
Maximal drawdown500.00 (5.00%)
Sharpe Ratio0.75
Total trades150
Recovery factor2.50
Profit trades (% of total)90 (60.00%)
Gross profit2000.00
Gross loss-765.44
+ + + + +
Deal TimeTypeDirectionVolumePriceS/LT/PProfitBalanceCommentOrderMagicEntry
2025.01.10 09:30:00buyout0.011915.000015.0010015.00Layer #1100112345out
2025.02.05 14:00:00sellout0.011945.0000-15.0010020.00Layer #1100512345out
+""" + + +@pytest.fixture +def html_report_path(tmp_path): + """Write HTML fixture as UTF-16 LE with BOM so read_text() decodes it correctly.""" + p = tmp_path / 'report.htm' + p.write_bytes(b'\xff\xfe' + HTML_TEXT.encode('utf-16-le')) + return str(p) + + +def test_parse_html_returns_metrics(html_report_path): + metrics, _ = parse_html(html_report_path) + assert isinstance(metrics, dict) + assert 'net_profit' in metrics + assert metrics['net_profit'] == pytest.approx(1234.56) + assert metrics['total_trades'] == 150 + + +def test_parse_html_returns_deals(html_report_path): + _, deals = parse_html(html_report_path) + assert isinstance(deals, list) + assert len(deals) >= 1 + deal = deals[0] + assert 'profit' in deal + assert 'balance' in deal + + +def test_parse_metrics_html_directly(): + """Test HTML metric extraction without encoding layer.""" + metrics = _parse_metrics_html(HTML_TEXT) + assert metrics['net_profit'] == pytest.approx(1234.56) + assert metrics['profit_factor'] == pytest.approx(1.25) + assert metrics['max_dd_pct'] == pytest.approx(5.00) + assert metrics['total_trades'] == 150 + + +def test_parse_deals_html_directly(): + """Test HTML deal extraction without encoding layer.""" + deals = _parse_deals_html(HTML_TEXT) + assert len(deals) == 2 + assert float(deals[0]['profit']) == pytest.approx(15.00) + assert float(deals[1]['profit']) == pytest.approx(-15.00) + + +def test_parse_xml_returns_metrics(): + metrics, deals = parse_xml(str(FIXTURES / 'sample_report.htm.xml')) + assert isinstance(metrics, dict) + assert 'net_profit' in metrics + assert metrics['net_profit'] == pytest.approx(1234.56) + assert metrics['total_trades'] == 150 + + +def test_parse_xml_returns_deals(): + metrics, deals = parse_xml(str(FIXTURES / 'sample_report.htm.xml')) + assert isinstance(deals, list) + assert len(deals) >= 1 + deal = deals[0] + assert deal.get('profit') is not None + + +def test_write_outputs_creates_files(): + metrics = {'net_profit': 100.0, 'total_trades': 5} + deals = [ + {'time': '2025.01.10', 'type': 'buy', 'direction': 'out', 'volume': '0.01', + 'price': '1900', 'sl': '0', 'tp': '0', 'profit': '10.00', + 'balance': '10010', 'comment': '', 'order': '1', 'magic': '1', 'entry': 'out'}, + ] + with tempfile.TemporaryDirectory() as tmp: + paths = write_outputs(metrics, deals, tmp) + assert Path(paths['metrics']).exists() + assert Path(paths['deals_csv']).exists() + assert Path(paths['deals_json']).exists() + # Verify metrics.json content + with open(paths['metrics']) as f: + saved = json.load(f) + assert saved['net_profit'] == 100.0 + + +def test_parse_html_skips_balance_rows(): + """Rows with type='balance' should be filtered out.""" + html = """ + + + + +
Deal TimeTypeDirectionVolumePriceS/LT/PProfitBalanceCommentOrderMagicEntry
2025.01.10 09:30:00balance000001000000
2025.01.10 10:00:00buyout0.011910005.0010005Layer #111out
+ """ + import tempfile, os + with tempfile.NamedTemporaryFile(mode='w', suffix='.htm', delete=False) as f: + f.write(html) + path = f.name + try: + _, deals = parse_html(path) + types = [d.get('type', '').lower() for d in deals] + assert 'balance' not in types + finally: + os.unlink(path)