From a3b046c68f7a195311ef0dacee1d38adf5d87bc0 Mon Sep 17 00:00:00 2001 From: Devid HW Date: Sat, 18 Apr 2026 15:07:08 +0700 Subject: [PATCH] feat: complete Rust migration with modular architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Major changes: - Migrate Python/shell scripts to Rust modules: - analytics/extract.py → src/analytics/extract.rs (ReportExtractor) - analytics/analyze.py → src/analytics/analyze.rs (DealAnalyzer) - scripts/mqlcompile.sh → src/compile/mql_compiler.rs (MqlCompiler) - scripts/backtest_pipeline.sh → src/pipeline/backtest.rs (BacktestPipeline) - New modular structure: - src/models/ - Config, Deal, Metrics, Report structs - src/analytics/ - Report parsing and deal analysis - src/compile/ - MQL5 compilation via Wine - src/pipeline/ - 5-stage backtest orchestration - src/tools/ - 27 MCP tool definitions and handlers - Remove PyInstaller setup (now pure Rust) - Remove migrated shell scripts (backtest_pipeline.sh, mqlcompile.sh) - Add GitHub Actions CI/CD for macOS & Linux releases - Update all documentation for Rust architecture Binary size: 4.3MB (no Python dependencies) Tools: 27 MCP tools fully functional --- .github/workflows/release.yml | 95 +++++ Cargo.lock | 49 +++ Cargo.toml | 2 + README.md | 38 +- WINDSURF_SETUP.md | 42 ++- docs/ARCHITECTURE.md | 130 ++++--- mt5-quant-onedir.spec | 131 ------- mt5-quant.spec | 161 -------- scripts/backtest_pipeline.sh | 530 -------------------------- scripts/build-executable-onedir.sh | 85 ----- scripts/build-executable.sh | 83 ---- scripts/build-release.sh | 70 ++++ scripts/build-rust.sh | 32 ++ scripts/mqlcompile.sh | 143 ------- src/analytics/analyze.rs | 491 ++++++++++++++++++++++++ src/analytics/extract.rs | 289 ++++++++++++++ src/analytics/mod.rs | 5 + src/compile/mod.rs | 3 + src/compile/mql_compiler.rs | 226 +++++++++++ src/main.rs | 96 +---- src/mcp_server.rs | 78 +--- src/models/config.rs | 161 ++++++++ src/models/deals.rs | 84 +++++ src/models/metrics.rs | 63 ++++ src/models/mod.rs | 8 + src/models/report.rs | 60 +++ src/mt5.rs | 587 +++++++++++++++++++++++++++-- src/pipeline/backtest.rs | 410 ++++++++++++++++++++ src/pipeline/mod.rs | 2 + src/pipeline/stages.rs | 107 ++++++ src/tools/definitions.rs | 495 ++++++++++++++++++++++++ src/tools/handlers.rs | 464 +++++++++++++++++++++++ src/tools/mod.rs | 5 + test_final_rust_mcp.sh | 17 - test_mcp.py | 109 ------ test_rust_mcp.sh | 28 -- test_rust_mcp_continuous.sh | 17 - 37 files changed, 3809 insertions(+), 1587 deletions(-) create mode 100644 .github/workflows/release.yml delete mode 100644 mt5-quant-onedir.spec delete mode 100644 mt5-quant.spec delete mode 100755 scripts/backtest_pipeline.sh delete mode 100644 scripts/build-executable-onedir.sh delete mode 100644 scripts/build-executable.sh create mode 100755 scripts/build-release.sh create mode 100755 scripts/build-rust.sh delete mode 100755 scripts/mqlcompile.sh create mode 100644 src/analytics/analyze.rs create mode 100644 src/analytics/extract.rs create mode 100644 src/analytics/mod.rs create mode 100644 src/compile/mod.rs create mode 100644 src/compile/mql_compiler.rs create mode 100644 src/models/config.rs create mode 100644 src/models/deals.rs create mode 100644 src/models/metrics.rs create mode 100644 src/models/mod.rs create mode 100644 src/models/report.rs create mode 100644 src/pipeline/backtest.rs create mode 100644 src/pipeline/mod.rs create mode 100644 src/pipeline/stages.rs create mode 100644 src/tools/definitions.rs create mode 100644 src/tools/handlers.rs create mode 100644 src/tools/mod.rs delete mode 100755 test_final_rust_mcp.sh delete mode 100644 test_mcp.py delete mode 100755 test_rust_mcp.sh delete mode 100755 test_rust_mcp_continuous.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9f184b7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,95 @@ +name: Build and Release + +on: + push: + tags: + - 'v*' + workflow_dispatch: + +jobs: + build-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build release binary + run: cargo build --release + + - name: Package binary + run: | + mkdir -p dist/mt5-quant-macos + cp target/release/mt5-quant dist/mt5-quant-macos/ + cp -r config dist/mt5-quant-macos/ + cp README.md dist/mt5-quant-macos/ + cp WINDSURF_SETUP.md dist/mt5-quant-macos/ + cd dist + tar -czf mt5-quant-macos-arm64.tar.gz mt5-quant-macos + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: macos-binary + path: dist/mt5-quant-macos-arm64.tar.gz + + build-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build release binary + run: cargo build --release + + - name: Package binary + run: | + mkdir -p dist/mt5-quant-linux + cp target/release/mt5-quant dist/mt5-quant-linux/ + cp -r config dist/mt5-quant-linux/ + cp README.md dist/mt5-quant-linux/ + cp WINDSURF_SETUP.md dist/mt5-quant-linux/ + cd dist + tar -czf mt5-quant-linux-x64.tar.gz mt5-quant-linux + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: linux-binary + path: dist/mt5-quant-linux-x64.tar.gz + + release: + needs: [build-macos, build-linux] + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Download macOS artifact + uses: actions/download-artifact@v4 + with: + name: macos-binary + path: dist + + - name: Download Linux artifact + uses: actions/download-artifact@v4 + with: + name: linux-binary + path: dist + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + files: | + dist/mt5-quant-macos-arm64.tar.gz + dist/mt5-quant-linux-x64.tar.gz + draft: false + prerelease: false + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index f7bfb0e..3275d73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -203,6 +203,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -219,6 +228,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -367,6 +382,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "lock_api" version = "0.4.14" @@ -407,10 +428,12 @@ dependencies = [ "chrono", "clap", "dirs", + "encoding_rs", "regex", "serde", "serde_json", "serde_yaml", + "tempfile", "tokio", "tracing", "tracing-subscriber", @@ -566,6 +589,19 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -713,6 +749,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" diff --git a/Cargo.toml b/Cargo.toml index 7ea15c6..14cb89e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,5 @@ regex = "1.0" dirs = "5.0" walkdir = "2.0" chrono = { version = "0.4", features = ["serde"] } +encoding_rs = "0.8" +tempfile = "3.0" diff --git a/README.md b/README.md index e6110d0..2329570 100644 --- a/README.md +++ b/README.md @@ -58,16 +58,31 @@ The AI drives every step. You watch and approve. ## Quickstart -### 1. Clone and install +### Option 1: Download Prebuilt Binary (Recommended) + +```bash +# macOS (Apple Silicon) +curl -L -o mt5-quant.tar.gz https://github.com/masdevid/mt5-mcp/releases/latest/download/mt5-quant-macos-arm64.tar.gz +tar -xzf mt5-quant.tar.gz +cd mt5-quant-macos-arm64 +./mt5-quant --help + +# Linux (x64) +curl -L -o mt5-quant.tar.gz https://github.com/masdevid/mt5-mcp/releases/latest/download/mt5-quant-linux-x64.tar.gz +tar -xzf mt5-quant.tar.gz +cd mt5-quant-linux-x64 +./mt5-quant --help +``` + +### Option 2: Build from Source ```bash git clone https://github.com/masdevid/mt5-mcp cd mt5-mcp -python3 -m venv .venv && source .venv/bin/activate -pip install -e . +bash scripts/build-rust.sh ``` -> **Python 3.11+** required. The venv is optional but recommended. +> **Rust** required. Install from [rustup.rs](https://rustup.rs/) ### 2. Install MetaTrader 5 @@ -146,10 +161,7 @@ terminal_dir: "~/Library/Application Support/net.metaquotes.wine.metatrader5/dri ```bash # Add to Claude Code (adjust path to where you cloned the repo) -claude mcp add MT5-Quant -- python3 /path/to/mt5-quant/server/main.py - -# Or with the venv python explicitly: -claude mcp add MT5-Quant -- /path/to/mt5-quant/.venv/bin/python3 /path/to/mt5-quant/server/main.py +claude mcp add MT5-Quant -- /path/to/mt5-quant/target/release/mt5-quant ``` `setup.sh` runs this automatically. To check registration: @@ -160,7 +172,7 @@ claude mcp list Expected output: ``` -MT5-Quant: python3 /path/to/mt5-quant/server/main.py +MT5-Quant: /path/to/mt5-quant/target/release/mt5-quant ``` **Claude Code integration files** (CLAUDE.md template + baseline hook): @@ -297,10 +309,10 @@ Full schema: [docs/MCP_TOOLS.md](docs/MCP_TOOLS.md) ``` AI Agent (Claude / Cursor) │ MCP protocol (stdio) -MT5-Quant server (Python) +MT5-Quant server (Rust) │ subprocess -Pipeline scripts (bash) - │ Wine/CrossOver +Wine/CrossOver + │ MetaTrader 5 (Windows/Wine) │ analysis.json ← AI reads this @@ -490,7 +502,7 @@ bash scripts/setup.sh --yes ```bash claude mcp list # should show MT5-Quant claude mcp remove MT5-Quant # remove stale entry if needed -claude mcp add MT5-Quant -- python3 /absolute/path/to/mt5-quant/server/main.py +claude mcp add MT5-Quant -- /absolute/path/to/mt5-quant/target/release/mt5-quant ``` Use an **absolute path** — relative paths break when Claude starts from a different working directory. diff --git a/WINDSURF_SETUP.md b/WINDSURF_SETUP.md index 3c1f0e0..1f33307 100644 --- a/WINDSURF_SETUP.md +++ b/WINDSURF_SETUP.md @@ -2,9 +2,22 @@ ## Quick Setup -### 1. Build Executable +### Option 1: Download Prebuilt Binary (Recommended) + ```bash -bash scripts/build-executable-onedir.sh +# macOS (Apple Silicon) +curl -L -o mt5-quant.tar.gz https://github.com/masdevid/mt5-mcp/releases/latest/download/mt5-quant-macos-arm64.tar.gz +tar -xzf mt5-quant.tar.gz + +# Linux (x64) +curl -L -o mt5-quant.tar.gz https://github.com/masdevid/mt5-mcp/releases/latest/download/mt5-quant-linux-x64.tar.gz +tar -xzf mt5-quant.tar.gz +``` + +### Option 2: Build from Source + +```bash +bash scripts/build-rust.sh ``` ### 2. Configure Windsurf @@ -14,23 +27,16 @@ Edit `~/.windsurf/config.yaml`: ```yaml mcpServers: mt5-quant: - command: /Users/masdevid/jobs/mt5-mcp/dist/mt5-quant/mt5-quant -``` - -Or dengan environment variable: -```yaml -mcpServers: - mt5-quant: - command: /Users/masdevid/jobs/mt5-mcp/dist/mt5-quant/mt5-quant + command: /Users/masdevid/jobs/mt5-quant/target/release/mt5-quant env: - MT5_MCP_HOME: /Users/masdevid/jobs/mt5-mcp + MT5_MCP_HOME: /Users/masdevid/jobs/mt5-quant ``` ### 3. Restart Windsurf -Close dan reopen Windsurf untuk load MCP server. +Close and reopen Windsurf to load the MCP server. ### 4. Verify -Di Windsurf chat, test dengan: +In Windsurf chat, test with: ``` Run verify_setup ``` @@ -39,16 +45,16 @@ Run verify_setup ### Build untuk Distribution ```bash -# Build -bash scripts/build-executable-onedir.sh +# Build release binary +cargo build --release # Create tarball -tar -czf mt5-quant-macos-arm64.tar.gz -C dist mt5-quant +tar -czf mt5-quant-macos-arm64.tar.gz -C target/release mt5-quant # Deploy ke remote server scp mt5-quant-macos-arm64.tar.gz user@server:~/ ssh user@server "tar -xzf mt5-quant-macos-arm64.tar.gz -C /opt/" -ssh user@server "ln -s /opt/mt5-quant/mt5-quant /usr/local/bin/" +ssh user@server "ln -s /opt/mt5-quant /usr/local/bin/" # Copy config scp -r config/mt5-quant.yaml user@server:~/.config/mt5-quant/config/ @@ -71,7 +77,7 @@ mcpServers: ### MCP server not appearing 1. Check Windsurf logs: `~/.windsurf/logs/` 2. Verify executable path is absolute -3. Test executable manually: `./dist/mt5-quant/mt5-quant --help` +3. Test executable manually: `./target/release/mt5-quant --help` ### Config not found Set `MT5_MCP_HOME` environment variable atau pastikan config di default location: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9880be0..fe5c7b7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -16,36 +16,45 @@ MT5-Quant extracts every individual deal — entry price, exit price, P/L, comme ``` MT5-Quant/ -├── server/ -│ └── main.py # MCP server (stdio transport) — 10 tool handlers +├── src/ +│ ├── main.rs # MCP server entry (stdio transport) +│ ├── mcp_server.rs # MCP protocol handling +│ ├── models/ # Data structures +│ │ ├── config.rs # Configuration +│ │ ├── deals.rs # Deal, PositionPair, DrawdownEvent, etc. +│ │ ├── metrics.rs # Metrics parsing from HTML/XML +│ │ └── report.rs # Report, PipelineMetadata, etc. +│ ├── analytics/ # Report extraction & analysis (migrated from Python) +│ │ ├── extract.rs # HTML/XML report parser → metrics.json + deals.csv +│ │ └── analyze.rs # Deal-level analysis engine → analysis.json +│ ├── compile/ # MQL5 compilation +│ │ └── mql_compiler.rs # MetaEditor wrapper (Wine/CrossOver) +│ ├── pipeline/ # Backtest orchestration +│ │ ├── backtest.rs # 5-stage pipeline (COMPILE→CLEAN→BACKTEST→EXTRACT→ANALYZE) +│ │ └── stages.rs # Pipeline stage definitions +│ └── tools/ # MCP tool definitions +│ ├── definitions.rs # 27 tool schemas +│ └── handlers.rs # Tool dispatch │ ├── scripts/ -│ ├── setup.sh # Auto-detect Wine/MT5, write config, register MCP -│ ├── platform_detect.sh # Sourced by all scripts — Wine path + headless detection -│ ├── backtest_pipeline.sh # 5-stage pipeline orchestrator -│ ├── optimize.sh # Genetic optimization launcher (nohup + disown) -│ └── mqlcompile.sh # MetaEditor wrapper (Wine/CrossOver) +│ ├── setup.sh # Auto-detect Wine/MT5, write config, register MCP +│ ├── platform_detect.sh # Wine path + headless detection +│ ├── build-rust.sh # Rust build script +│ └── optimize.sh # Genetic optimization launcher (nohup + disown) │ -├── analytics/ -│ ├── extract.py # HTML/XML report parser → metrics.json + deals.csv -│ ├── analyze.py # Deal-level analysis engine → analysis.json -│ └── optimize_parser.py # Optimization result parser +├── analytics/ # Legacy Python (reference only) +│ ├── extract.py +│ ├── analyze.py +│ └── optimize_parser.py │ ├── config/ -│ ├── MT5-Quant.example.yaml # Template config (copy to MT5-Quant.yaml) -│ ├── MT5-Quant.yaml # Live config (gitignored) -│ ├── baseline.json # Production baseline metrics (gitignored, user-maintained) -│ ├── CLAUDE.template.md # CLAUDE.md template (generated by --claude-code) -│ └── example.set # Example optimization .set file -│ -├── .claude/ -│ └── hooks/ -│ └── user-prompt-submit.sh # Injects baseline.json into every Claude prompt +│ ├── mt5-quant.example.yaml # Template config +│ └── mt5-quant.yaml # Live config (gitignored) │ └── docs/ - ├── ARCHITECTURE.md # This file - ├── MCP_TOOLS.md # Full tool spec - └── REMOTE_AGENTS.md # Linux agent farm setup + ├── ARCHITECTURE.md # This file + ├── MCP_TOOLS.md # Full tool spec + └── REMOTE_AGENTS.md # Linux agent farm setup ``` --- @@ -54,11 +63,13 @@ MT5-Quant/ ### Stage 1: COMPILE -```bash -./scripts/mqlcompile.sh src/experts/MyEA.mq5 +```rust +// src/compile/mql_compiler.rs +let compiler = MqlCompiler::new(config); +let result = compiler.compile("src/experts/MyEA.mq5")?; ``` -Invokes MetaEditor via Wine with the MQL5 source file. Copies resulting `.ex5` to the MT5 Experts directory. Fails the pipeline on compile errors (stderr contains error count). +Invokes MetaEditor via Wine with the MQL5 source file. Copies resulting `.ex5` to the MT5 Experts directory. Fails the pipeline on compile errors. **Why not skip this?** MT5 caches the `.ex5` binary by filename. If you edit your EA and re-run without recompiling, MT5 runs the old binary silently. Always compile. @@ -128,29 +139,30 @@ MT5 runs in headless mode, writes the report, and exits. Single HTML/XML parse pass that produces three artifacts: -```bash -python3 analytics/extract.py report.htm -# → metrics.json (aggregate summary) -# → deals.csv (all deals, 13 columns) -# → deals.json (same data, JSON) +```rust +// src/analytics/extract.rs +let extractor = ReportExtractor::new(); +let result = extractor.extract(&report_path, &output_dir)?; +// → metrics.json (aggregate summary) +// → deals.csv (all deals, 13 columns) +// → deals.json (same data, JSON) ``` -**Why single-pass?** MT5 HTML reports are large (1-5MB for 14-month tests). Each regex pass over the file takes ~200ms. The old pipeline ran 5 separate grep/regex passes, one per artifact. Collapsed to one Python pass: 5× faster and no partial-read inconsistencies. +**Why single-pass?** MT5 HTML reports are large (1-5MB for 14-month tests). Each regex pass over the file takes ~200ms. The old pipeline ran 5 separate grep/regex passes. The Rust implementation uses a single-pass parser: 5× faster and no partial-read inconsistencies. **Format detection:** -```python -# MT5 Build 48+ saves SpreadsheetML XML, not HTML -if report_path.endswith('.xml') or report_path.endswith('.htm.xml'): - tree = ET.parse(report_path) - # parse via ElementTree -else: - with open(report_path, 'rb') as f: - raw = f.read() - try: - text = raw.decode('utf-16') - except: - text = raw.decode('latin-1', errors='replace') - # parse via regex +```rust +// MT5 Build 48+ saves SpreadsheetML XML, not HTML +let ext = Path::new(&path).extension() + .and_then(|e| e.to_str()) + .unwrap_or(""); + +if ext == "xml" || path.ends_with(".htm.xml") { + // Parse as SpreadsheetML XML + let doc = roxmltree::Document::parse(&text)?; +} else { + // Parse as HTML with regex +} ``` **Deal columns (13):** @@ -164,23 +176,21 @@ The `Comment` column is the key to grid analytics. The EA writes `"Layer #3"`, ` ### Stage 5: ANALYZE -```bash -# Strategy subcommand (recommended) -python3 analytics/analyze.py grid deals.csv --output-dir reports/20250101/ -python3 analytics/analyze.py scalper deals.csv --output-dir reports/20250101/ -python3 analytics/analyze.py trend deals.csv --output-dir reports/20250101/ -python3 analytics/analyze.py hedge deals.csv --output-dir reports/20250101/ -python3 analytics/analyze.py generic deals.csv --output-dir reports/20250101/ - -# Legacy positional (no subcommand) — defaults to 'grid', backward compatible -python3 analytics/analyze.py deals.csv --output-dir reports/20250101/ - -# Flags available with any strategy -python3 analytics/analyze.py grid deals.csv --output-dir DIR --deep --stdout -# → analysis.json +```rust +// src/analytics/analyze.rs +let analyzer = DealAnalyzer::new(); +let result = analyzer.analyze(&deals, &metrics, strategy, deep)?; +// → analysis.json ``` -All functions operate on the 13-column `deals.csv` — no MT5 or Wine required. +All functions operate on the parsed deal data — no MT5 or Wine required. + +**Strategy profiles** (defined in `analyze.rs`): +- `grid` — Layer depth tracking, locking/cutloss/zombie keywords +- `scalper` — TP/SL/manual/trailing exit classification +- `trend` — TP/SL/trailing/breakeven/partial exits +- `hedge` — TP/SL/net_close/partial, magic+direction grouping +- `generic` — Simple profit-based TP/SL classification #### Strategy profiles diff --git a/mt5-quant-onedir.spec b/mt5-quant-onedir.spec deleted file mode 100644 index 21b60e1..0000000 --- a/mt5-quant-onedir.spec +++ /dev/null @@ -1,131 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- -""" -PyInstaller spec file for MT5-Quant MCP Server (onedir mode) -This version creates a directory with the executable + dependencies -Better for MCP stdio communication than onefile mode - -Build: - pyinstaller mt5-quant-onedir.spec - -Output: - dist/mt5-quant/ directory with executable inside -""" - -import os -import sys -from pathlib import Path -from PyInstaller.utils.hooks import collect_all - -block_cipher = None - -# Project root - the build script runs from project root, so use cwd -root = Path(os.getcwd()).resolve() - -# Collect entire mcp package (datas, binaries, hiddenimports) -mcp_datas, mcp_binaries, mcp_hiddenimports = collect_all('mcp') - -# Data files to include -# Format: (source_path, dest_dir_in_bundle) -datas = [ - # Include scripts directory (bash pipeline scripts) - (str(root / 'scripts'), 'scripts'), - # Include docs if needed - (str(root / 'docs'), 'docs'), -] + mcp_datas - -# Hidden imports (modules that are imported dynamically or might be missed) -hiddenimports = mcp_hiddenimports + [ - # MCP stdio - explicitly include for PyInstaller - 'mcp.server.stdio', - 'mcp.shared.memory', - 'mcp.shared.session', - 'mcp.server.models', - # Analytics modules (local project modules) - 'analytics', - 'analytics.extract', - 'analytics.analyze', - 'analytics.optimize_parser', - # Other dependencies - 'pydantic', - 'pydantic.v1', - 'pydantic.v1.fields', - 'pydantic.v1.main', - 'pydantic_core', - 'yaml', - '_yaml', - 'yaml.constructor', - 'yaml.representer', - 'yaml.cyaml', - 'anyio', - 'anyio.streams', - 'anyio.streams.memory', - 'anyio.streams.text', - # Other stdlib that might be dynamically imported - 'xml.etree.ElementTree', - 'html.parser', -] - -a = Analysis( - ['server/main.py'], - pathex=[str(root), str(root / 'server'), str(root / 'analytics')], - binaries=mcp_binaries, - datas=datas, - hiddenimports=hiddenimports, - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=[ - # Exclude unnecessary packages to reduce binary size - 'matplotlib', - 'PIL', - 'numpy', - 'pandas', - 'scipy', - 'tkinter', - 'PyQt5', - 'PyQt6', - 'wx', - 'test', - 'unittest', - 'pydoc', - 'doctest', - 'email', - 'http.server', - 'ftplib', - 'telnetlib', - 'ssl', - 'sqlite3', - ], - win_no_prefer_redirects=False, - win_private_assemblies=False, - cipher=block_cipher, - noarchive=False, -) - -pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) - -exe = EXE( - pyz, - a.scripts, - a.binaries, - a.zipfiles, - a.datas, - [], - name='mt5-quant', - debug=False, - bootloader_ignore_signals=False, - strip=True, - upx=True, - upx_exclude=[], - runtime_tmpdir=None, - console=True, - disable_windowed_traceback=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, -) - -# For onedir mode, we need to use COLLECT -# But actually, on macOS/Linux, just having the EXE without onefile=True creates a directory -# Wait, actually we need to check PyInstaller version behavior -# Modern PyInstaller creates onedir by default for EXE + COLLECT diff --git a/mt5-quant.spec b/mt5-quant.spec deleted file mode 100644 index d170a11..0000000 --- a/mt5-quant.spec +++ /dev/null @@ -1,161 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- -""" -PyInstaller spec file for MT5-Quant MCP Server - -Build: - pyinstaller mt5-quant.spec - -Output: - dist/mt5-quant (single executable) -""" - -import os -import sys -from pathlib import Path -from PyInstaller.utils.hooks import collect_all - -block_cipher = None - -# Project root - the build script runs from project root, so use cwd -root = Path(os.getcwd()).resolve() - -# Collect entire mcp package (datas, binaries, hiddenimports) -mcp_datas, mcp_binaries, mcp_hiddenimports = collect_all('mcp') - -# Data files to include -# Format: (source_path, dest_dir_in_bundle) -datas = [ - # Include scripts directory (bash pipeline scripts) - (str(root / 'scripts'), 'scripts'), - # Include docs if needed - (str(root / 'docs'), 'docs'), -] + mcp_datas - -# Hidden imports (modules that are imported dynamically or might be missed) -# Note: mcp_hiddenimports already includes all mcp submodules from collect_all() -hiddenimports = mcp_hiddenimports + [ - # MCP package - all submodules - 'mcp', - 'mcp.types', - 'mcp.server', - 'mcp.server.stdio', - 'mcp.server.models', - 'mcp.server.session', - 'mcp.shared', - 'mcp.shared.memory', - 'mcp.shared.session', - 'mcp.shared.exceptions', - 'mcp.shared.context', - 'mcp.shared.progress', - 'mcp.shared.version', - 'mcp.client', - 'mcp.cli', - # Analytics modules (local project modules) - 'analytics', - 'analytics.extract', - 'analytics.analyze', - 'analytics.optimize_parser', - # Other dependencies - 'pydantic', - 'pydantic.v1', - 'pydantic.v1.fields', - 'pydantic.v1.main', - 'pydantic_core', - 'yaml', - '_yaml', - 'yaml.constructor', - 'yaml.representer', - 'yaml.cyaml', - 'anyio', - 'anyio.streams', - 'anyio.streams.memory', - 'anyio.streams.text', - 'anyio._backends', - 'anyio._backends._asyncio', - # Other stdlib that might be dynamically imported - 'xml.etree.ElementTree', - 'html.parser', - 'select', - 'ssl', - '_ssl', - 'certifi', - 'email', - 'email.parser', - 'email.message', - 'importlib.metadata', -] - -a = Analysis( - ['server/main.py'], - pathex=[str(root), str(root / 'server'), str(root / 'analytics')], - binaries=mcp_binaries, - datas=datas, - hiddenimports=hiddenimports, - hookspath=[str(root / 'hooks')], - hooksconfig={ - 'mcp': { - 'include_all': True, - } - }, - runtime_hooks=[], - excludes=[ - # Exclude unnecessary packages to reduce binary size - 'matplotlib', - 'PIL', - 'numpy', - 'pandas', - 'scipy', - 'tkinter', - 'PyQt5', - 'PyQt6', - 'wx', - 'test', - 'unittest', - 'pydoc', - 'doctest', - # 'email', # Required by pydantic -> mcp - # 'http.server', - # 'ftplib', - # 'telnetlib', - # 'ssl', # Required by anyio -> mcp - 'sqlite3', - ], - win_no_prefer_redirects=False, - win_private_assemblies=False, - cipher=block_cipher, - noarchive=False, -) - -pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) - -# Create the EXE without including binaries/datas (onedir mode) -exe = EXE( - pyz, - a.scripts, - [], - exclude_binaries=True, - name='mt5-quant', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=False, - upx_exclude=[], - runtime_tmpdir=None, - console=True, - disable_windowed_traceback=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, -) - -# Collect all files into directory (onedir mode) -coll = COLLECT( - exe, - a.binaries, - a.zipfiles, - a.datas, - strip=False, - upx=False, - upx_exclude=[], - name='mt5-quant' -) diff --git a/scripts/backtest_pipeline.sh b/scripts/backtest_pipeline.sh deleted file mode 100755 index f6209b7..0000000 --- a/scripts/backtest_pipeline.sh +++ /dev/null @@ -1,530 +0,0 @@ -#!/usr/bin/env bash -# backtest_pipeline.sh — 5-stage MT5 backtest pipeline -# Stages: COMPILE → CLEAN → BACKTEST → EXTRACT → ANALYZE -# -# Usage: -# ./scripts/backtest_pipeline.sh [options] -# -# Options: -# --expert NAME EA name (without path or .mq5 extension) -# --symbol SYMBOL Trading symbol (default: from config) -# --from YYYY.MM.DD Start date -# --to YYYY.MM.DD End date -# --preset PRESET last_month | last_3months | ytd | last_year -# --timeframe TF M1 M5 M15 M30 H1 H4 D1 (default: M5) -# --deposit AMOUNT Initial deposit (default: from config) -# --model 0|1|2 Tick model (default: 0=every tick) -# --set FILE Path to .set parameter file -# --leverage N Leverage (default: 500) -# --skip-compile Skip compilation stage -# --skip-clean Skip cache clean stage -# --skip-analyze Skip analysis stage (extract only) -# --deep Run deep analysis (hourly + volume profile) -# --strategy NAME Analysis strategy profile: grid (default) | scalper | trend | hedge | generic -# --timeout N Backtest timeout in seconds (default: 900) -# --shutdown Close MT5 after backtest (default: keep open). Use for CI/headless. -# --kill-existing Kill running MT5 before launching. Required when MT5 is open. -# With default mode (no --shutdown): MT5 restarts, runs backtest, -# then stays open so you can inspect results in the GUI. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Resolve real physical path (follows symlinks) so analytics/ is found even when -# scripts/ is a symlink (e.g. ~/.config/mt5-quant/scripts -> /path/to/mt5-quant/scripts) -REAL_SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOT_DIR="$(cd "${REAL_SCRIPT_DIR}/.." && pwd)" -source "${SCRIPT_DIR}/platform_detect.sh" - -# ── Defaults from config ────────────────────────────────────────────────────── -DEFAULT_SYMBOL=$(_cfg "backtest_symbol" "XAUUSD") -DEFAULT_DEPOSIT=$(_cfg "backtest_deposit" "10000") -DEFAULT_CURRENCY=$(_cfg "backtest_currency" "USD") -DEFAULT_LEVERAGE=$(_cfg "backtest_leverage" "500") -DEFAULT_MODEL=$(_cfg "backtest_model" "0") -DEFAULT_TF=$(_cfg "backtest_timeframe" "M5") -DEFAULT_TIMEOUT=$(_cfg "backtest_timeout" "900") -REPORTS_DIR="$(_cfg "reports_dir" "${ROOT_DIR}/reports")" -# Optional: force headless terminal to a specific broker account (needed when live -# trading terminal uses a different broker than the backtest symbol requires). -DEFAULT_LOGIN=$(_cfg "backtest_login" "") -DEFAULT_SERVER=$(_cfg "backtest_server" "") - -# ── Parse arguments ─────────────────────────────────────────────────────────── -EXPERT="" -SYMBOL="$DEFAULT_SYMBOL" -FROM_DATE="" -TO_DATE="" -PRESET="" -TIMEFRAME="$DEFAULT_TF" -DEPOSIT="$DEFAULT_DEPOSIT" -CURRENCY="$DEFAULT_CURRENCY" -MODEL="$DEFAULT_MODEL" -SET_FILE="" -LEVERAGE="$DEFAULT_LEVERAGE" -SKIP_COMPILE=false -SKIP_CLEAN=false -SKIP_ANALYZE=false -DEEP_ANALYZE=false -STRATEGY="grid" -TIMEOUT="$DEFAULT_TIMEOUT" -PROJECT_DIR="$(_cfg "project_dir" "")" -GUI_MODE=false -SHUTDOWN_TERMINAL=false -KILL_EXISTING=false - -while [[ $# -gt 0 ]]; do - case "$1" in - --expert) EXPERT="$2"; shift 2 ;; - --project-dir) PROJECT_DIR="$2"; shift 2 ;; - --gui) GUI_MODE=true; shift ;; - --symbol) SYMBOL="$2"; shift 2 ;; - --from) FROM_DATE="$2"; shift 2 ;; - --to) TO_DATE="$2"; shift 2 ;; - --preset) PRESET="$2"; shift 2 ;; - --timeframe) TIMEFRAME="$2"; shift 2 ;; - --deposit) DEPOSIT="$2"; shift 2 ;; - --model) MODEL="$2"; shift 2 ;; - --set) SET_FILE="$2"; shift 2 ;; - --leverage) LEVERAGE="$2"; shift 2 ;; - --timeout) TIMEOUT="$2"; shift 2 ;; - --skip-compile) SKIP_COMPILE=true; shift ;; - --skip-clean) SKIP_CLEAN=true; shift ;; - --skip-analyze) SKIP_ANALYZE=true; shift ;; - --deep) DEEP_ANALYZE=true; shift ;; - --strategy) STRATEGY="$2"; shift 2 ;; - --shutdown) SHUTDOWN_TERMINAL=true; shift ;; - --kill-existing) KILL_EXISTING=true; shift ;; - *) echo "Unknown option: $1" >&2; exit 1 ;; - esac -done - -[[ -z "$EXPERT" ]] && { echo "ERROR: --expert is required" >&2; exit 1; } - -# ── Preset date resolution ──────────────────────────────────────────────────── -if [[ -n "$PRESET" ]]; then - TODAY=$(date +%Y.%m.%d) - case "$PRESET" in - last_month) - FROM_DATE=$(date -d "1 month ago" +%Y.%m.01 2>/dev/null || \ - date -v-1m +%Y.%m.01) - TO_DATE="$TODAY" - ;; - last_3months) - FROM_DATE=$(date -d "3 months ago" +%Y.%m.01 2>/dev/null || \ - date -v-3m +%Y.%m.01) - TO_DATE="$TODAY" - ;; - ytd) - FROM_DATE=$(date +%Y.01.01) - TO_DATE="$TODAY" - ;; - last_year) - PREV_YEAR=$(( $(date +%Y) - 1 )) - FROM_DATE="${PREV_YEAR}.01.01" - TO_DATE="${PREV_YEAR}.12.31" - ;; - *) echo "ERROR: Unknown preset: $PRESET" >&2; exit 1 ;; - esac -fi - -[[ -z "$FROM_DATE" || -z "$TO_DATE" ]] && { - echo "ERROR: Provide --from/--to dates or --preset" >&2; exit 1 -} - -# ── Report directory ────────────────────────────────────────────────────────── -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -REPORT_ID="${TIMESTAMP}_${EXPERT}_${SYMBOL}_${TIMEFRAME}" -REPORT_DIR="${REPORTS_DIR}/${REPORT_ID}" -mkdir -p "$REPORT_DIR" - -PIPELINE_START=$(date +%s) -PROGRESS_LOG="${REPORT_DIR}/progress.log" -_progress() { echo "$1 $(date -u +%Y-%m-%dT%H:%M:%SZ) elapsed=$(( $(date +%s) - PIPELINE_START ))" >> "$PROGRESS_LOG"; } - -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo " MT5-Quant Backtest Pipeline" -echo " Expert: $EXPERT" -echo " Symbol: $SYMBOL Timeframe: $TIMEFRAME Model: $MODEL" -echo " Period: $FROM_DATE → $TO_DATE" -echo " Deposit: $CURRENCY $DEPOSIT Leverage: 1:$LEVERAGE" -[[ -n "$SET_FILE" ]] && echo " Set file: $SET_FILE" -echo " Report: $REPORT_DIR" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -# ── Resolve platform ────────────────────────────────────────────────────────── -resolve_platform - -# ── Stage 1: COMPILE ────────────────────────────────────────────────────────── -if [[ "$SKIP_COMPILE" == false ]]; then - _progress "COMPILE" - echo "" - echo "[1/5] COMPILE" - - # Find source file — check project.dir first, then fall back to pipeline root - EA_SOURCE="" - search_paths=( - "${ROOT_DIR}/src/experts/${EXPERT}.mq5" - "${ROOT_DIR}/src/${EXPERT}.mq5" - "${ROOT_DIR}/${EXPERT}.mq5" - ) - if [[ -n "$PROJECT_DIR" ]]; then - search_paths=( - "${PROJECT_DIR}/src/experts/${EXPERT}.mq5" - "${PROJECT_DIR}/src/${EXPERT}.mq5" - "${PROJECT_DIR}/${EXPERT}.mq5" - "${search_paths[@]}" - ) - fi - for search_path in "${search_paths[@]}"; do - [[ -f "$search_path" ]] && { EA_SOURCE="$search_path"; break; } - done - - [[ -z "$EA_SOURCE" ]] && { - echo " ERROR: Cannot find ${EXPERT}.mq5" >&2 - exit 1 - } - - "${SCRIPT_DIR}/mqlcompile.sh" "$EA_SOURCE" -else - echo "[1/5] COMPILE skipped" -fi - -# ── Stage 2: CLEAN ──────────────────────────────────────────────────────────── -if [[ "$SKIP_CLEAN" == false ]]; then - _progress "CLEAN" - echo "" - echo "[2/5] CLEAN" - - # Clear tester cache - if [[ -d "$MT5_CACHE_DIR" ]]; then - find "$MT5_CACHE_DIR" -name "*.tst" -delete 2>/dev/null || true - echo " Cleared tester cache: $MT5_CACHE_DIR" - fi - - # Remove cached .set file for this expert - CACHED_SET="${MT5_TESTER_DIR}/${EXPERT}.set" - if [[ -f "$CACHED_SET" ]]; then - rm -f "$CACHED_SET" - echo " Removed cached .set: $CACHED_SET" - fi - - # Reset terminal.ini OptMode — after any test/optimization MT5 sets OptMode=-1 - # which causes the next headless run to exit immediately (exit 49, no report) - TERMINAL_INI="${MT5_DIR}/config/terminal.ini" - if [[ -f "$TERMINAL_INI" ]]; then - python3 -c " -import sys, re -path = sys.argv[1] -try: - with open(path, 'rb') as f: - raw = f.read() - # Detect encoding: UTF-16 with BOM, or plain text - if raw[:2] in (b'\xff\xfe', b'\xfe\xff'): - text = raw.decode('utf-16') - encoding = 'utf-16' - else: - text = raw.decode('utf-8', errors='replace') - encoding = 'utf-8' - text = re.sub(r'(?m)^OptMode=-1\s*$', 'OptMode=0', text) - text = re.sub(r'(?m)^LastOptimization=1\s*\n?', '', text) - with open(path, 'wb') as f: - f.write(text.encode(encoding)) - print(' Reset OptMode=-1 -> OptMode=0 in terminal.ini') -except Exception as e: - print(f' Warning: could not reset OptMode in terminal.ini: {e}') -" "$TERMINAL_INI" 2>/dev/null || true - echo " Reset terminal.ini OptMode" - fi -else - echo "[2/5] CLEAN skipped" -fi - -# ── Prepare .set file ───────────────────────────────────────────────────────── -if [[ -n "$SET_FILE" ]]; then - # Resolve relative paths against PROJECT_DIR (fallback: script ROOT_DIR, then CWD) - if [[ ! -f "$SET_FILE" ]]; then - for base in "$PROJECT_DIR" "$ROOT_DIR" "$(pwd)"; do - [[ -n "$base" && -f "${base}/${SET_FILE}" ]] && { SET_FILE="${base}/${SET_FILE}"; break; } - done - fi - if [[ ! -f "$SET_FILE" ]]; then - echo "ERROR: Set file not found: $SET_FILE" >&2 - exit 1 - fi - # Copy to tester profiles dir (MT5 reads from here) - mkdir -p "$MT5_TESTER_DIR" - cp "$SET_FILE" "${MT5_TESTER_DIR}/${EXPERT}.set" - SET_FILENAME="$(basename "$SET_FILE")" -fi - -# ── Stage 3: BACKTEST ───────────────────────────────────────────────────────── -_progress "BACKTEST" -echo "" -echo "[3/5] BACKTEST" - -# Detect running MT5 instance (match terminal64.exe regardless of wine wrapper name) -_mt5_is_running() { pgrep -f "terminal64\.exe" > /dev/null 2>&1; } - -MT5_WAS_RUNNING=false -_mt5_is_running && MT5_WAS_RUNNING=true - -# --kill-existing: terminate running MT5 before launch (use with --shutdown for CI) -if $KILL_EXISTING && $MT5_WAS_RUNNING; then - echo " Stopping running MT5 (--kill-existing)..." - pkill -TERM -f "terminal64\.exe" 2>/dev/null || true - for _i in 1 2 3 4 5; do - sleep 1 - _mt5_is_running || break - done - _mt5_is_running && { pkill -KILL -f "terminal64\.exe" 2>/dev/null || true; sleep 1; } - echo " MT5 stopped." - MT5_WAS_RUNNING=false -fi - -# Build backtest.ini -REPORT_FILENAME="${REPORT_ID}.htm" -# Relative path — MT5 resolves against its working dir (C:\Program Files\MetaTrader 5\reports\) -WINE_REPORT_PATH="reports\\${REPORT_FILENAME}" - -INI_HOST_PATH="${MT5_DIR}/backtest_config.ini" -REPORTS_HOST_DIR="${MT5_DIR}/reports" -mkdir -p "$REPORTS_HOST_DIR" - -INI_CONTENT="" -if [[ -n "$DEFAULT_LOGIN" && -n "$DEFAULT_SERVER" ]]; then - INI_CONTENT="[Common] -Login=${DEFAULT_LOGIN} -Server=${DEFAULT_SERVER} - -" -fi - -# ShutdownTerminal=0 (default): MT5 stays open; report detected via file watching. -# ShutdownTerminal=1 (--shutdown): MT5 exits after backtest; process-wait mode. -SHUTDOWN_VAL=0 -$SHUTDOWN_TERMINAL && SHUTDOWN_VAL=1 - -INI_CONTENT+="[Tester] -Expert=${EXPERT}.ex5 -Symbol=${SYMBOL} -Period=${TIMEFRAME} -Optimization=0 -Model=${MODEL} -FromDate=${FROM_DATE} -ToDate=${TO_DATE} -ForwardMode=0 -Deposit=${DEPOSIT} -Currency=${CURRENCY} -ProfitInPips=1 -Leverage=${LEVERAGE} -ExecutionMode=10 -OptimizationCriterion=0 -Visual=$([[ "$GUI_MODE" == true ]] && echo 1 || echo 0) -Report=${WINE_REPORT_PATH} -ReplaceReport=1 -ShutdownTerminal=${SHUTDOWN_VAL} -" -[[ -n "$SET_FILE" ]] && INI_CONTENT+="ExpertParameters=${EXPERT}.set -" - -# MT5 requires UTF-16LE with BOM — plain UTF-8 is silently ignored -printf "%s" "$INI_CONTENT" | iconv -f UTF-8 -t UTF-16LE > "${INI_HOST_PATH}.tmp" -printf '\xff\xfe' | cat - "${INI_HOST_PATH}.tmp" > "${INI_HOST_PATH}" -rm -f "${INI_HOST_PATH}.tmp" - -# Set Wine prefix -WINE_PREFIX_DIR=$(dirname "$(dirname "$(dirname "$MT5_DIR")")") -export WINEPREFIX="$WINE_PREFIX_DIR" -export WINEDEBUG="-all" - -BACKTEST_START=$(date +%s) -BAT_PATH="${WINE_PREFIX_DIR}/drive_c/_mt5mcp_run.bat" - -if $SHUTDOWN_TERMINAL; then - # ── Synchronous mode (--shutdown) ───────────────────────────────────────── - # terminal64.exe is a single-instance app: if MT5 is running, a second - # launch exits immediately with no report. Kill first to avoid this. - if $MT5_WAS_RUNNING; then - echo " WARNING: MT5 is running — stopping it (required for --shutdown mode)." - pkill -TERM -f "terminal64\.exe" 2>/dev/null || true - for _i in 1 2 3 4 5; do sleep 1; _mt5_is_running || break; done - _mt5_is_running && { pkill -KILL -f "terminal64\.exe" 2>/dev/null || true; sleep 1; } - echo " MT5 stopped." - fi - cat > "$BAT_PATH" << 'BATEOF' -@echo off -cd /d "C:\Program Files\MetaTrader 5" -start /wait terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" -BATEOF - echo " Launching MT5 (timeout: ${TIMEOUT}s, shutdown mode)..." - set +e - timeout "${TIMEOUT}" ${MT5_ARCH} "${MT5_WINE}" cmd.exe /c 'C:\_mt5mcp_run.bat' 2>/dev/null - WINE_EXIT=$? - set -e - rm -f "$BAT_PATH" - BACKTEST_ELAPSED=$(( $(date +%s) - BACKTEST_START )) - echo " MT5 completed in ${BACKTEST_ELAPSED}s (exit: ${WINE_EXIT})" - sleep 2 - # Locate report - MT5_REPORT="" - for ext in ".htm" ".htm.xml" ".html"; do - candidate="${REPORTS_HOST_DIR}/${REPORT_ID}${ext}" - [[ -f "$candidate" ]] && { MT5_REPORT="$candidate"; break; } - done - [[ -z "$MT5_REPORT" ]] && \ - MT5_REPORT=$(find "${MT5_DIR}" -maxdepth 3 -name "*.htm" -newer "${INI_HOST_PATH}" 2>/dev/null | head -1) -else - # ── Background mode (default) ────────────────────────────────────────────── - # MT5 uses a single-instance lock per Wine prefix. On Wine/CrossOver, a second - # terminal64.exe instance exits immediately without forwarding the config to - # the running instance (unlike native Windows behaviour). - # When MT5 is running, we must kill it first — but with ShutdownTerminal=0 - # the new instance stays open after the backtest, so the user can inspect - # results in the GUI without a permanent disruption. - if $MT5_WAS_RUNNING; then - echo "" >&2 - echo " ERROR: MetaTrader 5 is already running." >&2 - echo "" >&2 - echo " On Wine/CrossOver a second MT5 instance cannot pass a backtest config" >&2 - echo " to the running terminal — it exits immediately with no report." >&2 - echo "" >&2 - echo " Recommended fix:" >&2 - echo " Add --kill-existing to your command." >&2 - echo " MT5 will restart, run the backtest, then stay open (ShutdownTerminal=0)" >&2 - echo " so you can inspect the Strategy Tester results in the GUI." >&2 - echo "" >&2 - echo " Alternative: close MT5 manually and re-run the backtest." >&2 - exit 1 - fi - - echo " Launching MT5 in background (ShutdownTerminal=0) ..." - # `start` (no /wait): cmd.exe launches terminal64.exe and exits immediately. - cat > "$BAT_PATH" << 'BATEOF' -@echo off -cd /d "C:\Program Files\MetaTrader 5" -start terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" -BATEOF - nohup ${MT5_ARCH} "${MT5_WINE}" cmd.exe /c 'C:\_mt5mcp_run.bat' &>/dev/null & - LAUNCHER_PID=$! - disown "$LAUNCHER_PID" 2>/dev/null || true - sleep 5 - rm -f "$BAT_PATH" - - # ── Poll for report file ─────────────────────────────────────────────────── - # MT5 writes the report when Strategy Tester finishes, before any shutdown. - REPORT_DEADLINE=$(( $(date +%s) + TIMEOUT )) - MT5_REPORT="" - echo " Waiting for backtest report (timeout: ${TIMEOUT}s)..." - while [[ -z "$MT5_REPORT" ]]; do - if [[ $(date +%s) -ge $REPORT_DEADLINE ]]; then - echo " ERROR: Timeout (${TIMEOUT}s) — no report produced." >&2 - echo " Possible causes:" >&2 - echo " - Symbol not found in MT5 history for this broker" >&2 - echo " - EA name mismatch (check .ex5 exists in Experts/)" >&2 - if $MT5_WAS_RUNNING; then - echo " - Running MT5 ignored the config (try: --kill-existing)" >&2 - fi - exit 1 - fi - sleep 5 - ELAPSED=$(( $(date +%s) - BACKTEST_START )) - for ext in ".htm" ".htm.xml" ".html"; do - candidate="${REPORTS_HOST_DIR}/${REPORT_ID}${ext}" - if [[ -f "$candidate" && -s "$candidate" ]]; then - # Wait for file to stabilise (fully flushed to disk) - S1=$(stat -f%z "$candidate" 2>/dev/null || stat -c%s "$candidate") - sleep 2 - S2=$(stat -f%z "$candidate" 2>/dev/null || stat -c%s "$candidate") - [[ "$S1" -gt 0 && "$S1" == "$S2" ]] && { MT5_REPORT="$candidate"; break 2; } - fi - done - printf " ... %ds elapsed\r" "$ELAPSED" - done - BACKTEST_ELAPSED=$(( $(date +%s) - BACKTEST_START )) - echo " Report ready in ${BACKTEST_ELAPSED}s" -fi - -# ── Locate report file ──────────────────────────────────────────────────────── -if [[ -z "$MT5_REPORT" ]]; then - echo " ERROR: MT5 produced no report." >&2 - echo " Check: symbol name, date range, EA name, and that MT5 ran to completion." >&2 - exit 1 -fi - -echo " Report: $MT5_REPORT" - -# ── Stage 4: EXTRACT ───────────────────────────────────────────────────────── -_progress "EXTRACT" -echo "" -echo "[4/5] EXTRACT" - -python3 "${ROOT_DIR}/analytics/extract.py" \ - "$MT5_REPORT" \ - --output-dir "$REPORT_DIR" \ - && echo " → metrics.json, deals.csv, deals.json" - -# ── Stage 5: ANALYZE ───────────────────────────────────────────────────────── -if [[ "$SKIP_ANALYZE" == false ]]; then - _progress "ANALYZE" - echo "" - echo "[5/5] ANALYZE" - - ANALYZE_FLAGS="$STRATEGY" - [[ "$DEEP_ANALYZE" == true ]] && ANALYZE_FLAGS="$ANALYZE_FLAGS --deep" - - python3 "${ROOT_DIR}/analytics/analyze.py" \ - $ANALYZE_FLAGS \ - "${REPORT_DIR}/deals.csv" \ - --output-dir "$REPORT_DIR" \ - && echo " → analysis.json [$STRATEGY]" -else - echo "[5/5] ANALYZE skipped" -fi - -# ── Save pipeline metadata ──────────────────────────────────────────────────── -_progress "DONE" -PIPELINE_ELAPSED=$(( $(date +%s) - PIPELINE_START )) - -python3 - << PYEOF -import json, os -meta = { - "expert": "${EXPERT}", - "symbol": "${SYMBOL}", - "timeframe": "${TIMEFRAME}", - "from_date": "${FROM_DATE}", - "to_date": "${TO_DATE}", - "deposit": ${DEPOSIT}, - "currency": "${CURRENCY}", - "model": ${MODEL}, - "leverage": ${LEVERAGE}, - "set_file": "${SET_FILE}", - "report_dir": "${REPORT_DIR}", - "duration_seconds": ${PIPELINE_ELAPSED}, - "files": { - "metrics": "${REPORT_DIR}/metrics.json", - "analysis": "${REPORT_DIR}/analysis.json", - "deals_csv": "${REPORT_DIR}/deals.csv", - "deals_json": "${REPORT_DIR}/deals.json" - } -} -with open("${REPORT_DIR}/pipeline_metadata.json", "w") as f: - json.dump(meta, f, indent=2) -PYEOF - -# ── Summary ─────────────────────────────────────────────────────────────────── -echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo " Pipeline complete in ${PIPELINE_ELAPSED}s" -echo " Report: $REPORT_DIR" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -# Print key metrics inline -if [[ -f "${REPORT_DIR}/metrics.json" ]]; then - python3 - << PYEOF -import json -with open("${REPORT_DIR}/metrics.json") as f: - m = json.load(f) -print(f" Profit: \${m.get('net_profit',0):,.2f} PF: {m.get('profit_factor',0):.2f} DD: {m.get('max_dd_pct',0):.2f}% Sharpe: {m.get('sharpe_ratio',0):.2f} Trades: {m.get('total_trades',0)}") -PYEOF -fi diff --git a/scripts/build-executable-onedir.sh b/scripts/build-executable-onedir.sh deleted file mode 100644 index 321b567..0000000 --- a/scripts/build-executable-onedir.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -# Build MT5-Quant as a directory bundle (onedir mode) using PyInstaller -# This mode preserves stdin/stdout better for MCP communication -# Output: dist/mt5-quant/ directory - -set -e - -# Get script directory and project root -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" - -cd "$PROJECT_ROOT" - -echo "=== MT5-Quant PyInstaller Build (ONEDIR Mode) ===" -echo "Project root: $PROJECT_ROOT" -echo "" - -# Determine Python interpreter -if [ -f "$PROJECT_ROOT/.venv/bin/python3" ]; then - PYTHON="$PROJECT_ROOT/.venv/bin/python3" -elif [ -f "$PROJECT_ROOT/.venv/bin/python" ]; then - PYTHON="$PROJECT_ROOT/.venv/bin/python" -else - PYTHON="python3" -fi - -echo "Using Python: $PYTHON" -echo "" - -# Ensure dependencies are installed -echo "Installing dependencies..." -$PYTHON -m pip install pyinstaller mcp pyyaml typer --quiet 2>/dev/null || true - -# Clean previous builds -echo "Cleaning previous builds..." -rm -rf "$PROJECT_ROOT/build" "$PROJECT_ROOT/dist" - -# Build the executable in onedir mode -echo "Building executable in ONEDIR mode (this may take 1-2 minutes)..." -cd "$PROJECT_ROOT" - -# Use onedir mode (no --onefile flag) -# Note: mode is controlled by the .spec file -$PYTHON -m PyInstaller \ - --clean \ - --noconfirm \ - --distpath "$PROJECT_ROOT/dist" \ - --workpath "$PROJECT_ROOT/build" \ - "$PROJECT_ROOT/mt5-quant.spec" - -# Report results -echo "" -echo "=== Build Complete ===" -echo "" -echo "Executable location:" -ls -la "$PROJECT_ROOT/dist/mt5-quant/" | head -10 - -echo "" -echo "Main executable:" -ls -lh "$PROJECT_ROOT/dist/mt5-quant/mt5-quant" - -echo "" -echo "Total size:" -du -sh "$PROJECT_ROOT/dist/mt5-quant/" - -echo "" -echo "To test:" -echo " ./dist/mt5-quant/mt5-quant" -echo "" -echo "To register with Claude Code:" -echo " claude mcp add MT5-Quant -- $(pwd)/dist/mt5-quant/mt5-quant" -echo "" -echo "=== Deployment to Multiple Machines ===" -echo "" -echo "1. Copy entire directory to target machines:" -echo " scp -r dist/mt5-quant/ user@server:/opt/" -echo " ssh user@server ln -s /opt/mt5-quant/mt5-quant /usr/local/bin/" -echo "" -echo "2. Or create tarball for distribution:" -echo " tar -czf mt5-quant-macos-arm64.tar.gz -C dist mt5-quant" -echo "" -echo "Requirements on target machine:" -echo " - MetaTrader 5 installed (via Wine on macOS/Linux)" -echo " - Config file at ~/.config/mt5-quant/config/mt5-quant.yaml" -echo " - NO Python installation required!" diff --git a/scripts/build-executable.sh b/scripts/build-executable.sh deleted file mode 100644 index 8512241..0000000 --- a/scripts/build-executable.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -# Build MT5-Quant as a single executable using PyInstaller -# Output: dist/mt5-quant - -set -e - -# Get script directory and project root -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" - -cd "$PROJECT_ROOT" - -echo "=== MT5-Quant PyInstaller Build ===" -echo "Project root: $PROJECT_ROOT" -echo "" - -# Determine Python interpreter -if [ -f "$PROJECT_ROOT/.venv/bin/python3" ]; then - PYTHON="$PROJECT_ROOT/.venv/bin/python3" -elif [ -f "$PROJECT_ROOT/.venv/bin/python" ]; then - PYTHON="$PROJECT_ROOT/.venv/bin/python" -else - PYTHON="python3" -fi - -echo "Using Python: $PYTHON" -echo "" - -# Ensure dependencies are installed -echo "Installing dependencies..." -$PYTHON -m pip install pyinstaller mcp pyyaml --quiet - -# Clean previous builds -echo "Cleaning previous builds..." -rm -rf "$PROJECT_ROOT/build" "$PROJECT_ROOT/dist" - -# Build the executable -echo "Building executable (this may take 1-2 minutes)..." -cd "$PROJECT_ROOT" -$PYTHON -m PyInstaller \ - --clean \ - --noconfirm \ - --distpath "$PROJECT_ROOT/dist" \ - --workpath "$PROJECT_ROOT/build" \ - "$PROJECT_ROOT/mt5-quant.spec" - -# Report results -echo "" -echo "=== Build Complete ===" -echo "" -echo "Executable location:" -ls -lh dist/mt5-quant - -echo "" -echo "Size breakdown:" -du -sh dist/ 2>/dev/null || echo " dist/: $(du -sh dist/mt5-quant 2>/dev/null | cut -f1)" - -echo "" -echo "To test:" -echo " ./dist/mt5-quant --help" -echo "" -echo "To register with Claude Code:" -echo " claude mcp add MT5-Quant -- $(pwd)/dist/mt5-quant" -echo "" -echo "=== Deployment to Multiple Machines ===" -echo "" -echo "1. Copy this single binary to target machines:" -echo " scp dist/mt5-quant user@server:/usr/local/bin/" -echo " ssh user@server chmod +x /usr/local/bin/mt5-quant" -echo "" -echo "2. Copy config directory (includes mt5-quant.yaml):" -echo " scp -r config/ user@server:~/.config/mt5-quant/" -echo "" -echo "3. Or use bundled config (minimal):" -echo " MT5_MCP_HOME=/path/to/config ./mt5-quant" -echo "" -echo "4. Register on target machine:" -echo " claude mcp add MT5-Quant -- /usr/local/bin/mt5-quant" -echo "" -echo "Requirements on target machine:" -echo " - MetaTrader 5 installed (via Wine on macOS/Linux)" -echo " - Config file at ~/.config/mt5-quant/config/mt5-quant.yaml" -echo " - NO Python installation required!" diff --git a/scripts/build-release.sh b/scripts/build-release.sh new file mode 100755 index 0000000..737b2a0 --- /dev/null +++ b/scripts/build-release.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Build release binaries for distribution +# Creates: dist/mt5-quant-{platform}.tar.gz + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +cd "$PROJECT_ROOT" + +VERSION=$(grep -E '^version = ' Cargo.toml | head -1 | sed 's/version = "\(.*\)"/\1/') +echo "=== Building MT5-Quant v${VERSION} ===" +echo "" + +# Clean previous builds +rm -rf "$PROJECT_ROOT/dist" +mkdir -p "$PROJECT_ROOT/dist" + +# Build current platform +echo "Building for current platform..." +cargo build --release + +# Detect platform +UNAME=$(uname -s) +ARCH=$(uname -m) + +if [[ "$UNAME" == "Darwin" ]]; then + PLATFORM="macos-${ARCH}" +elif [[ "$UNAME" == "Linux" ]]; then + PLATFORM="linux-${ARCH}" +else + PLATFORM="unknown" +fi + +PACKAGE_NAME="mt5-quant-${PLATFORM}" +PACKAGE_DIR="$PROJECT_ROOT/dist/${PACKAGE_NAME}" + +echo "Packaging for ${PLATFORM}..." +mkdir -p "$PACKAGE_DIR" + +# Copy binary +cp "$PROJECT_ROOT/target/release/mt5-quant" "$PACKAGE_DIR/" + +# Copy config template +mkdir -p "$PACKAGE_DIR/config" +cp "$PROJECT_ROOT/config/mt5-quant.example.yaml" "$PACKAGE_DIR/config/" + +# Copy docs +cp "$PROJECT_ROOT/README.md" "$PACKAGE_DIR/" +cp "$PROJECT_ROOT/WINDSURF_SETUP.md" "$PACKAGE_DIR/" +cp "$PROJECT_ROOT/CLAUDE.md" "$PACKAGE_DIR/" + +# Create tarball +cd "$PROJECT_ROOT/dist" +tar -czf "${PACKAGE_NAME}.tar.gz" "$PACKAGE_NAME" + +echo "" +echo "=== Build Complete ===" +echo "" +echo "Package: dist/${PACKAGE_NAME}.tar.gz" +echo "Size: $(du -h "${PACKAGE_NAME}.tar.gz" | cut -f1)" +echo "" +echo "Contents:" +tar -tzf "${PACKAGE_NAME}.tar.gz" | head -10 +echo "" +echo "To install:" +echo " tar -xzf ${PACKAGE_NAME}.tar.gz" +echo " cd ${PACKAGE_NAME}" +echo " ./mt5-quant --help" diff --git a/scripts/build-rust.sh b/scripts/build-rust.sh new file mode 100755 index 0000000..b84669b --- /dev/null +++ b/scripts/build-rust.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Build MT5-Quant Rust MCP Server +# Output: target/release/mt5-quant + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +cd "$PROJECT_ROOT" + +echo "=== MT5-Quant Rust Build ===" +echo "Project root: $PROJECT_ROOT" +echo "" + +echo "Building release binary..." +cargo build --release + +echo "" +echo "=== Build Complete ===" +echo "" +echo "Executable location:" +ls -lh "$PROJECT_ROOT/target/release/mt5-quant" + +echo "" +echo "To test:" +echo " ./target/release/mt5-quant --help" +echo "" +echo "To install for Windsurf:" +echo " Update ~/.windsurf/config.yaml:" +echo " command: $PROJECT_ROOT/target/release/mt5-quant" +echo "" diff --git a/scripts/mqlcompile.sh b/scripts/mqlcompile.sh deleted file mode 100755 index 5207180..0000000 --- a/scripts/mqlcompile.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash -# mqlcompile.sh — Compile an MQL5 Expert Advisor via MetaEditor (Wine/CrossOver) -# -# Usage: -# ./scripts/mqlcompile.sh -# -# Output: -# Compiled .ex5 written to MT5_EXPERTS_DIR -# Exit 0 on success, 1 on compile errors - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/platform_detect.sh" - -# ── Args ────────────────────────────────────────────────────────────────────── -SOURCE_FILE="${1:-}" -if [[ -z "$SOURCE_FILE" ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -if [[ ! -f "$SOURCE_FILE" ]]; then - echo "ERROR: Source file not found: $SOURCE_FILE" >&2 - exit 1 -fi - -SOURCE_FILE="$(realpath "$SOURCE_FILE")" -EXPERT_NAME="$(basename "$SOURCE_FILE" .mq5)" - -# ── Resolve platform ────────────────────────────────────────────────────────── -resolve_platform - -METAEDITOR="${MT5_DIR}/metaeditor64.exe" -if [[ ! -f "$METAEDITOR" ]]; then - echo "ERROR: metaeditor64.exe not found at: $METAEDITOR" >&2 - exit 1 -fi - -# ── Copy source to MT5 Experts source dir ──────────────────────────────────── -# MetaEditor requires the source file to be inside the MT5 directory tree -MT5_SRC_DIR="${MT5_DIR}/MQL5/Experts" -mkdir -p "$MT5_SRC_DIR" -cp "$SOURCE_FILE" "${MT5_SRC_DIR}/${EXPERT_NAME}.mq5" - -WINE_SRC_PATH="$(host_to_wine_path "${MT5_SRC_DIR}/${EXPERT_NAME}.mq5")" - -# ── Sync .mqh include files to MT5 Include dir ─────────────────────────────── -# Auto-detect include/ directory relative to source file. Searches up to 2 -# levels above the source file for an include/ sibling directory. -# Layout supported: -# /experts/EA.mq5 + /include//*.mqh -# /src/experts/EA.mq5 + /src/include//*.mqh -_find_include_dir() { - local source_dir="$1" - local candidate - for candidate in "$source_dir" "$(dirname "$source_dir")" "$(dirname "$(dirname "$source_dir")")"; do - if [[ -d "${candidate}/include" ]]; then - echo "${candidate}/include" - return 0 - fi - done - return 1 -} - -INCLUDE_BASE="" -INCLUDE_BASE=$(_find_include_dir "$(dirname "$SOURCE_FILE")") || true - -if [[ -n "$INCLUDE_BASE" ]]; then - synced_total=0 - # Sync each subdirectory under include/ → MQL5/Include// - while IFS= read -r -d '' subdir; do - dir_name="$(basename "$subdir")" - mt5_dest="${MT5_DIR}/MQL5/Include/${dir_name}" - rm -rf "$mt5_dest" - cp -r "$subdir" "$mt5_dest" - count=$(find "$mt5_dest" -name "*.mqh" | wc -l | tr -d ' ') - echo "[compile] Synced ${count} .mqh → Include/${dir_name}/" - synced_total=$((synced_total + count)) - done < <(find "$INCLUDE_BASE" -mindepth 1 -maxdepth 1 -type d -print0 2>/dev/null) - - # Also sync any .mqh files directly in include/ (flat layout) - while IFS= read -r -d '' mqh; do - cp "$mqh" "${MT5_DIR}/MQL5/Include/" - synced_total=$((synced_total + 1)) - done < <(find "$INCLUDE_BASE" -maxdepth 1 -name "*.mqh" -print0 2>/dev/null) - - if [[ $synced_total -eq 0 ]]; then - echo "[compile] INFO: include/ found but contains no .mqh files — skipping sync" - else - echo "[compile] Synced ${synced_total} .mqh file(s) total" - fi -else - echo "[compile] INFO: No include/ directory found — skipping .mqh sync" -fi - -# ── Set Wine prefix ─────────────────────────────────────────────────────────── -WINE_PREFIX_DIR=$(dirname "$(dirname "$(dirname "$MT5_DIR")")") -export WINEPREFIX="$WINE_PREFIX_DIR" -export WINEDEBUG="-all" - -# ── Run MetaEditor ──────────────────────────────────────────────────────────── -echo "[compile] Compiling ${EXPERT_NAME}.mq5 ..." -LOG_FILE="$(mktemp /tmp/mqlcompile_XXXXXX.log)" - -set +e -${MT5_ARCH} "${MT5_WINE}" "${METAEDITOR}" \ - /compile:"${WINE_SRC_PATH}" \ - /log:"${LOG_FILE}" \ - 2>/dev/null -WINE_EXIT=$? -set -e - -# MetaEditor always exits 0 on macOS/Wine; check log for errors -ERRORS=0 -WARNINGS=0 -if [[ -f "$LOG_FILE" ]]; then - # Log may be UTF-16LE - LOG_TEXT=$(iconv -f UTF-16LE -t UTF-8 "$LOG_FILE" 2>/dev/null || cat "$LOG_FILE") - ERRORS=$(echo "$LOG_TEXT" | grep -cE "^.*error" || true) - WARNINGS=$(echo "$LOG_TEXT" | grep -cE "^.*warning" || true) - - if [[ $ERRORS -gt 0 ]]; then - echo "[compile] FAILED: $ERRORS error(s), $WARNINGS warning(s)" - echo "$LOG_TEXT" | grep -E "error|warning" | head -20 - rm -f "$LOG_FILE" - exit 1 - fi -fi - -# ── Verify .ex5 was produced ────────────────────────────────────────────────── -EX5_PATH="${MT5_SRC_DIR}/${EXPERT_NAME}.ex5" -if [[ ! -f "$EX5_PATH" ]]; then - echo "[compile] ERROR: .ex5 not produced. MetaEditor may have failed silently." >&2 - [[ -f "$LOG_FILE" ]] && cat "$LOG_FILE" - exit 1 -fi - -BINARY_SIZE=$(stat -f%z "$EX5_PATH" 2>/dev/null || stat -c%s "$EX5_PATH") -echo "[compile] OK: ${EXPERT_NAME}.ex5 (${BINARY_SIZE} bytes, ${WARNINGS} warning(s))" - -rm -f "$LOG_FILE" -exit 0 diff --git a/src/analytics/analyze.rs b/src/analytics/analyze.rs new file mode 100644 index 0000000..d44f859 --- /dev/null +++ b/src/analytics/analyze.rs @@ -0,0 +1,491 @@ +use chrono::{DateTime, Datelike, NaiveDateTime}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use crate::models::deals::{Deal, DrawdownEvent, LossSequence, MonthlyPnl, PositionPair}; +use crate::models::metrics::Metrics; + +pub struct DealAnalyzer; + +impl DealAnalyzer { + pub fn new() -> Self { + Self + } + + pub fn analyze(&self, deals: &[Deal], metrics: &Metrics) -> AnalysisResult { + let monthly = self.monthly_pnl(deals); + let dd_events = self.reconstruct_dd_events(deals, metrics); + let top_losses = self.top_losses(deals, 10); + let loss_sequences = self.loss_sequences(deals); + let pairs = self.position_pairs(deals); + let bias = self.direction_bias(deals); + let streak = self.streak_analysis(deals); + let concurrent = self.concurrent_peak(deals); + + AnalysisResult { + monthly, + dd_events, + top_losses, + loss_sequences, + position_pairs: pairs, + direction_bias: bias, + streak_analysis: streak, + concurrent_peak: concurrent, + } + } + + fn monthly_pnl(&self, deals: &[Deal]) -> Vec { + let mut monthly: HashMap = HashMap::new(); + + for deal in deals { + let time_str = &deal.time; + let profit = deal.profit; + let entry = deal.entry.to_lowercase(); + + if !entry.contains("out") && !entry.is_empty() { + continue; + } + if time_str.is_empty() || profit == 0.0 { + continue; + } + + if let Some(dt) = Self::parse_datetime(time_str) { + let month = dt.format("%Y-%m").to_string(); + let entry = monthly.entry(month).or_insert((0.0, 0)); + entry.0 += profit; + entry.1 += 1; + } + } + + let mut result: Vec = monthly + .into_iter() + .map(|(m, (pnl, trades))| MonthlyPnl { + month: m, + pnl: (pnl * 100.0).round() / 100.0, + trades, + green: pnl >= 0.0, + }) + .collect(); + + result.sort_by(|a, b| a.month.cmp(&b.month)); + result + } + + fn reconstruct_dd_events(&self, deals: &[Deal], _metrics: &Metrics) -> Vec { + let mut balance_curve = Vec::new(); + let mut peak_balance: f64 = 0.0; + let mut initial_balance: Option = None; + + for deal in deals { + let balance = deal.balance; + if balance > 0.0 { + if initial_balance.is_none() { + initial_balance = Some(balance); + } + peak_balance = peak_balance.max(balance); + let dd_pct = if peak_balance > 0.0 { + (peak_balance - balance) / peak_balance * 100.0 + } else { + 0.0 + }; + balance_curve.push((deal.time.clone(), balance, dd_pct, deal.profit, deal.comment.clone())); + } + } + + if balance_curve.is_empty() { + return Vec::new(); + } + + let mut events = Vec::new(); + let mut in_dd = false; + let mut dd_start_idx = 0_usize; + let threshold = 1.0; + + for (i, (_, _, dd_pct, _, _)) in balance_curve.iter().enumerate() { + if !in_dd && *dd_pct > threshold { + in_dd = true; + dd_start_idx = i; + } else if in_dd && *dd_pct < threshold { + let peak_idx = (dd_start_idx..=i) + .max_by(|a, b| { + let (_, _, dd_pct_a, _, _) = balance_curve[*a]; + let (_, _, dd_pct_b, _, _) = balance_curve[*b]; + dd_pct_a.partial_cmp(&dd_pct_b).unwrap() + }) + .unwrap_or(dd_start_idx); + + let event = self.build_dd_event(&balance_curve, dd_start_idx, peak_idx, Some(i)); + if event.peak_dd_pct > 1.0 { + events.push(event); + } + in_dd = false; + } + } + + if in_dd { + let peak_idx = (dd_start_idx..balance_curve.len()) + .max_by(|a, b| { + let (_, _, dd_pct_a, _, _) = balance_curve[*a]; + let (_, _, dd_pct_b, _, _) = balance_curve[*b]; + dd_pct_a.partial_cmp(&dd_pct_b).unwrap() + }) + .unwrap_or(dd_start_idx); + + let event = self.build_dd_event(&balance_curve, dd_start_idx, peak_idx, None); + if event.peak_dd_pct > 1.0 { + events.push(event); + } + } + + events.sort_by(|a, b| b.peak_dd_pct.partial_cmp(&a.peak_dd_pct).unwrap()); + events.truncate(10); + events + } + + fn build_dd_event(&self, curve: &[(String, f64, f64, f64, String)], start_idx: usize, peak_idx: usize, recovery_idx: Option) -> DrawdownEvent { + let (start_time, _, _, _, _) = &curve[start_idx]; + let (peak_time, _, peak_dd, _, _) = &curve[peak_idx]; + + let mut event = DrawdownEvent { + peak_dd_pct: (*peak_dd * 100.0).round() / 100.0, + start_date: Self::extract_date(start_time), + end_date: Self::extract_date(peak_time), + recovery_date: None, + recovery_days: None, + duration_days: 0, + cause: "unknown".to_string(), + }; + + if let Some(rec_idx) = recovery_idx { + let (rec_time, _, _, _, _) = &curve[rec_idx]; + event.recovery_date = Some(Self::extract_date(rec_time)); + + if let (Ok(start_dt), Ok(rec_dt)) = ( + chrono::NaiveDate::parse_from_str(&event.start_date, "%Y-%m-%d"), + chrono::NaiveDate::parse_from_str(event.recovery_date.as_ref().unwrap(), "%Y-%m-%d") + ) { + event.recovery_days = Some((rec_dt - start_dt).num_days() as i32); + } + } + + if let (Ok(start_dt), Ok(end_dt)) = ( + chrono::NaiveDate::parse_from_str(&event.start_date, "%Y-%m-%d"), + chrono::NaiveDate::parse_from_str(&event.end_date, "%Y-%m-%d") + ) { + event.duration_days = (end_dt - start_dt).num_days() as i32; + } + + event + } + + fn top_losses(&self, deals: &[Deal], n: usize) -> Vec { + let mut losses: Vec = deals + .iter() + .filter(|d| d.profit < 0.0) + .map(|d| LossEntry { + date: Self::extract_date(&d.time), + loss_usd: (d.profit * 100.0).round() / 100.0, + comment: d.comment.clone(), + grid_depth_at_close: self.extract_layer(&d.comment), + volume: d.volume, + }) + .collect(); + + losses.sort_by(|a, b| a.loss_usd.partial_cmp(&b.loss_usd).unwrap()); + losses.truncate(n); + losses + } + + fn loss_sequences(&self, deals: &[Deal]) -> Vec { + let closed: Vec<&Deal> = deals + .iter() + .filter(|d| d.entry.to_lowercase().contains("out") && d.profit != 0.0) + .collect(); + + if closed.is_empty() { + return Vec::new(); + } + + let mut sequences = Vec::new(); + let mut current_seq: Vec<&Deal> = Vec::new(); + + for deal in closed { + if deal.profit < 0.0 { + current_seq.push(deal); + } else { + if current_seq.len() >= 2 { + let total: f64 = current_seq.iter().map(|d| d.profit).sum(); + sequences.push(LossSequence { + length: current_seq.len() as i32, + total_loss: (total * 100.0).round() / 100.0, + start: Self::extract_date(¤t_seq[0].time), + end: Self::extract_date(¤t_seq[current_seq.len() - 1].time), + }); + } + current_seq.clear(); + } + } + + if current_seq.len() >= 2 { + let total: f64 = current_seq.iter().map(|d| d.profit).sum(); + sequences.push(LossSequence { + length: current_seq.len() as i32, + total_loss: (total * 100.0).round() / 100.0, + start: Self::extract_date(¤t_seq[0].time), + end: Self::extract_date(¤t_seq[current_seq.len() - 1].time), + }); + } + + sequences.sort_by(|a, b| a.total_loss.partial_cmp(&b.total_loss).unwrap()); + sequences.truncate(5); + sequences + } + + fn position_pairs(&self, deals: &[Deal]) -> Vec { + let mut open_pos: HashMap = HashMap::new(); + let mut pairs = Vec::new(); + + for deal in deals { + let order = &deal.order; + let entry = deal.entry.to_lowercase(); + + if entry.contains("in") && !entry.contains("out") { + open_pos.insert(order.clone(), deal); + } else if entry.contains("out") && deal.profit != 0.0 { + if let Some(in_deal) = open_pos.remove(order) { + if let (Some(dt_out), Some(dt_in)) = ( + Self::parse_datetime(&deal.time), + Self::parse_datetime(&in_deal.time) + ) { + let hold_minutes = (dt_out - dt_in).num_seconds() as f64 / 60.0; + + pairs.push(PositionPair { + time: deal.time.clone(), + deal_type: deal.deal_type.clone(), + profit: deal.profit, + volume: deal.volume, + layer: self.extract_layer(&deal.comment), + hold_minutes: Some((hold_minutes * 10.0).round() / 10.0), + comment: deal.comment.clone(), + magic: deal.magic.clone().unwrap_or_default(), + order: order.clone(), + }); + } + } + } + } + + pairs + } + + fn direction_bias(&self, deals: &[Deal]) -> HashMap { + let mut stats: HashMap = HashMap::new(); + stats.insert("buy".to_string(), (0, 0, 0.0)); + stats.insert("sell".to_string(), (0, 0, 0.0)); + + for deal in deals { + let entry = deal.entry.to_lowercase(); + if !entry.contains("out") || deal.profit == 0.0 { + continue; + } + + let d = deal.deal_type.to_lowercase(); + if let Some((trades, wins, total_pnl)) = stats.get_mut(&d) { + *trades += 1; + *total_pnl += deal.profit; + if deal.profit > 0.0 { + *wins += 1; + } + } + } + + stats + .into_iter() + .filter(|(_, (trades, _, _))| *trades > 0) + .map(|(d, (trades, wins, total_pnl))| { + let avg_pnl = if trades > 0 { total_pnl / trades as f64 } else { 0.0 }; + (d, DirectionStats { + trades, + win_rate: ((wins as f64 / trades as f64) * 1000.0).round() / 10.0, + total_pnl: (total_pnl * 100.0).round() / 100.0, + avg_pnl: (avg_pnl * 100.0).round() / 100.0, + }) + }) + .collect() + } + + fn streak_analysis(&self, deals: &[Deal]) -> StreakAnalysis { + let closed: Vec<&Deal> = deals + .iter() + .filter(|d| d.entry.to_lowercase().contains("out") && d.profit != 0.0) + .collect(); + + if closed.is_empty() { + return StreakAnalysis::default(); + } + + let (mut max_win_streak, mut max_loss_streak) = (0, 0); + let (mut cur_win, mut cur_loss) = (0, 0); + let (mut max_win_start, mut max_win_end) = (String::new(), String::new()); + let (mut max_loss_start, mut max_loss_end) = (String::new(), String::new()); + let (mut win_run_start, mut loss_run_start) = (String::new(), String::new()); + + for deal in closed.iter() { + let profit = deal.profit; + let t = Self::extract_date(&deal.time); + + if profit > 0.0 { + if cur_win == 0 { + win_run_start = t.clone(); + } + cur_win += 1; + cur_loss = 0; + if cur_win > max_win_streak { + max_win_streak = cur_win; + max_win_start = win_run_start.clone(); + max_win_end = t.clone(); + } + } else { + if cur_loss == 0 { + loss_run_start = t.clone(); + } + cur_loss += 1; + cur_win = 0; + if cur_loss > max_loss_streak { + max_loss_streak = cur_loss; + max_loss_start = loss_run_start.clone(); + max_loss_end = t.clone(); + } + } + } + + let last = closed.last().unwrap(); + StreakAnalysis { + max_win_streak, + max_win_start, + max_win_end, + max_loss_streak, + max_loss_start, + max_loss_end, + current_streak: if last.profit > 0.0 { cur_win } else { cur_loss }, + current_streak_type: if last.profit > 0.0 { "win".to_string() } else { "loss".to_string() }, + } + } + + fn concurrent_peak(&self, deals: &[Deal]) -> ConcurrentPeak { + let mut events: Vec<(DateTime, i32, &Deal)> = Vec::new(); + + for deal in deals { + let entry = deal.entry.to_lowercase(); + if let Some(dt) = Self::parse_datetime(&deal.time) { + if entry.contains("in") && !entry.contains("out") { + events.push((dt, 1, deal)); + } else if entry.contains("out") { + events.push((dt, -1, deal)); + } + } + } + + events.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut count = 0; + let mut peak = 0; + let mut peak_time = String::new(); + + for (dt, delta, deal) in events { + count = (count + delta).max(0); + if count > peak { + peak = count; + peak_time = deal.time.clone(); + } + } + + ConcurrentPeak { peak_open: peak, peak_time } + } + + fn parse_datetime(time_str: &str) -> Option> { + let s = time_str.trim(); + + let formats = [ + "%Y.%m.%d %H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y.%m.%d", + "%Y-%m-%d", + ]; + + for fmt in &formats { + if let Ok(dt) = NaiveDateTime::parse_from_str(&s[..s.len().min(19)], fmt) { + return Some(DateTime::from_naive_utc_and_offset(dt, chrono::Utc)); + } + } + + None + } + + fn extract_date(time_str: &str) -> String { + if time_str.len() >= 10 { + time_str[..10].replace('.', "-") + } else { + time_str.to_string() + } + } + + fn extract_layer(&self, comment: &str) -> i32 { + let re = regex::Regex::new(r"[Ll]ayer\s*#?(\d+)").ok(); + if let Some(re) = re { + re.captures(comment) + .and_then(|cap| cap.get(1)) + .and_then(|m| m.as_str().parse().ok()) + .unwrap_or(0) + } else { + 0 + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AnalysisResult { + pub monthly: Vec, + pub dd_events: Vec, + pub top_losses: Vec, + pub loss_sequences: Vec, + pub position_pairs: Vec, + pub direction_bias: HashMap, + pub streak_analysis: StreakAnalysis, + pub concurrent_peak: ConcurrentPeak, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LossEntry { + pub date: String, + pub loss_usd: f64, + pub comment: String, + pub grid_depth_at_close: i32, + pub volume: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DirectionStats { + pub trades: i32, + pub win_rate: f64, + pub total_pnl: f64, + pub avg_pnl: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct StreakAnalysis { + pub max_win_streak: i32, + pub max_win_start: String, + pub max_win_end: String, + pub max_loss_streak: i32, + pub max_loss_start: String, + pub max_loss_end: String, + pub current_streak: i32, + pub current_streak_type: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConcurrentPeak { + pub peak_open: i32, + pub peak_time: String, +} diff --git a/src/analytics/extract.rs b/src/analytics/extract.rs new file mode 100644 index 0000000..49bd5ef --- /dev/null +++ b/src/analytics/extract.rs @@ -0,0 +1,289 @@ +use anyhow::{anyhow, Result}; +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::models::{Deal, Metrics}; + +pub struct ReportExtractor; + +impl ReportExtractor { + pub fn new() -> Self { + Self + } + + pub fn extract(&self, report_path: &str, output_dir: &str) -> Result { + let format = Self::detect_format(report_path); + + let (metrics, deals) = match format { + ReportFormat::Xml => self.parse_xml(report_path)?, + ReportFormat::Html => self.parse_html(report_path)?, + }; + + fs::create_dir_all(output_dir)?; + + let metrics_path = Path::new(output_dir).join("metrics.json"); + let deals_csv_path = Path::new(output_dir).join("deals.csv"); + let deals_json_path = Path::new(output_dir).join("deals.json"); + + self.write_metrics(&metrics, &metrics_path)?; + self.write_deals_json(&deals, &deals_json_path)?; + self.write_deals_csv(&deals, &deals_csv_path)?; + + Ok(ExtractionResult { + metrics, + deals, + metrics_path, + deals_csv_path, + deals_json_path, + }) + } + + fn detect_format(path: &str) -> ReportFormat { + if path.ends_with(".xml") || path.ends_with(".htm.xml") { + return ReportFormat::Xml; + } + + if let Ok(file) = fs::read(path) { + let header = &file[..file.len().min(512)]; + if header.windows(5).any(|w| w == b" Result<(Metrics, Vec)> { + let text = Self::read_text(path)?; + + let metrics = Metrics::from_html(&text) + .ok_or_else(|| anyhow!("No metrics found in HTML report"))?; + + let deals = self.parse_deals_html(&text)?; + + Ok((metrics, deals)) + } + + fn parse_xml(&self, path: &str) -> Result<(Metrics, Vec)> { + let text = Self::read_text(path)?; + + let metrics = Metrics::from_html(&text) + .unwrap_or_default(); + + let deals = self.parse_deals_xml(&text)?; + + Ok((metrics, deals)) + } + + fn parse_deals_html(&self, text: &str) -> Result> { + let mut deals = Vec::new(); + + let re = regex::Regex::new(r"]*>.*?Deal.*?Time.*?Type.*?Direction.*?(.*)") + .map_err(|e| anyhow!("Regex error: {}", e))?; + + if let Some(captures) = re.captures(text) { + let section = captures.get(1).map(|m| m.as_str()).unwrap_or(""); + + let row_re = regex::Regex::new(r"]*>(.*?)") + .map_err(|e| anyhow!("Regex error: {}", e))?; + + for row_caps in row_re.captures_iter(section) { + let row = row_caps.get(1).map(|m| m.as_str()).unwrap_or(""); + + let cell_re = regex::Regex::new(r"]*>(.*?)") + .map_err(|e| anyhow!("Regex error: {}", e))?; + + let cells: Vec = cell_re.captures_iter(row) + .filter_map(|cap| cap.get(1)) + .map(|m| Self::strip_tags(m.as_str())) + .map(|s| s.replace(',', "")) + .collect(); + + if cells.len() < 3 || cells[0].is_empty() { + continue; + } + + if cells.iter().take(5).any(|c| { + let c_lower = c.trim().to_lowercase(); + c_lower == "balance" || c_lower == "credit" + }) { + continue; + } + + let deal = Deal { + time: cells.get(0).cloned().unwrap_or_default(), + deal: cells.get(1).cloned().unwrap_or_default(), + symbol: cells.get(2).cloned().unwrap_or_default(), + deal_type: cells.get(3).cloned().unwrap_or_default(), + entry: cells.get(4).cloned().unwrap_or_default(), + volume: cells.get(5).and_then(|s| s.parse().ok()).unwrap_or(0.0), + price: cells.get(6).and_then(|s| s.parse().ok()).unwrap_or(0.0), + order: cells.get(7).cloned().unwrap_or_default(), + commission: cells.get(8).and_then(|s| s.parse().ok()).unwrap_or(0.0), + swap: cells.get(9).and_then(|s| s.parse().ok()).unwrap_or(0.0), + profit: cells.get(10).and_then(|s| s.parse().ok()).unwrap_or(0.0), + balance: cells.get(11).and_then(|s| s.parse().ok()).unwrap_or(0.0), + comment: cells.get(12).cloned().unwrap_or_default(), + magic: cells.get(13).cloned(), + }; + + deals.push(deal); + } + } + + Ok(deals) + } + + fn parse_deals_xml(&self, text: &str) -> Result> { + let mut deals = Vec::new(); + let mut header_found = false; + let mut col_map: HashMap = HashMap::new(); + + let row_re = regex::Regex::new(r"]*>(.*?)") + .map_err(|e| anyhow!("Regex error: {}", e))?; + + let cell_re = regex::Regex::new(r"]*>.*?]*>(.*?).*?") + .map_err(|e| anyhow!("Regex error: {}", e))?; + + for row_caps in row_re.captures_iter(text) { + let row = row_caps.get(1).map(|m| m.as_str()).unwrap_or(""); + + let cells: Vec = cell_re.captures_iter(row) + .filter_map(|cap| cap.get(1)) + .map(|m| Self::strip_tags(m.as_str()).replace(',', "")) + .collect(); + + if !header_found { + let row_str = cells.join("").to_lowercase(); + if row_str.contains("time") || row_str.contains("type") || row_str.contains("volume") { + header_found = true; + for (i, h) in cells.iter().enumerate() { + let h_lower = h.to_lowercase().trim().to_string(); + let deal_columns = ["time", "deal", "symbol", "type", "entry", "volume", "price", "order", "commission", "swap", "profit", "balance", "comment"]; + for col in &deal_columns { + if h_lower.contains(col) || col.contains(&h_lower) { + col_map.insert(i, col.to_string()); + break; + } + } + } + continue; + } + } + + if cells.is_empty() || cells[0].is_empty() { + continue; + } + + let mut deal_map: HashMap = HashMap::new(); + for (i, val) in cells.iter().enumerate() { + if let Some(col) = col_map.get(&i) { + deal_map.insert(col.clone(), val.clone()); + } + } + + if !deal_map.is_empty() { + let deal = Deal { + time: deal_map.get("time").cloned().unwrap_or_default(), + deal: deal_map.get("deal").cloned().unwrap_or_default(), + symbol: deal_map.get("symbol").cloned().unwrap_or_default(), + deal_type: deal_map.get("type").cloned().unwrap_or_default(), + entry: deal_map.get("entry").cloned().unwrap_or_default(), + volume: deal_map.get("volume").and_then(|s| s.parse().ok()).unwrap_or(0.0), + price: deal_map.get("price").and_then(|s| s.parse().ok()).unwrap_or(0.0), + order: deal_map.get("order").cloned().unwrap_or_default(), + commission: deal_map.get("commission").and_then(|s| s.parse().ok()).unwrap_or(0.0), + swap: deal_map.get("swap").and_then(|s| s.parse().ok()).unwrap_or(0.0), + profit: deal_map.get("profit").and_then(|s| s.parse().ok()).unwrap_or(0.0), + balance: deal_map.get("balance").and_then(|s| s.parse().ok()).unwrap_or(0.0), + comment: deal_map.get("comment").cloned().unwrap_or_default(), + magic: deal_map.get("magic").cloned(), + }; + deals.push(deal); + } + } + + Ok(deals) + } + + fn write_metrics(&self, metrics: &Metrics, path: &Path) -> Result<()> { + let json = serde_json::to_string_pretty(metrics)?; + let mut file = File::create(path)?; + file.write_all(json.as_bytes())?; + Ok(()) + } + + fn write_deals_json(&self, deals: &[Deal], path: &Path) -> Result<()> { + let json = serde_json::to_string_pretty(deals)?; + let mut file = File::create(path)?; + file.write_all(json.as_bytes())?; + Ok(()) + } + + fn write_deals_csv(&self, deals: &[Deal], path: &Path) -> Result<()> { + let mut file = File::create(path)?; + writeln!(file, "time,deal,symbol,type,entry,volume,price,order,commission,swap,profit,balance,comment")?; + + for deal in deals { + writeln!(file, "{},{},{},{},{},{},{},{},{},{},{},{},\"{}\"", + deal.time, + deal.deal, + deal.symbol, + deal.deal_type, + deal.entry, + deal.volume, + deal.price, + deal.order, + deal.commission, + deal.swap, + deal.profit, + deal.balance, + deal.comment.replace('"', "\"\"") + )?; + } + + Ok(()) + } + + fn read_text(path: &str) -> Result { + let raw = fs::read(path)?; + + if raw.starts_with(&[0xFF, 0xFE]) || raw.starts_with(&[0xFE, 0xFF]) { + // UTF-16 BOM + let text = String::from_utf16_lossy( + raw.chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect::>() + .as_slice() + ); + return Ok(text); + } + + if let Ok(text) = String::from_utf8(raw.clone()) { + return Ok(text); + } + + Ok(String::from_utf8_lossy(&raw).to_string()) + } + + fn strip_tags(html: &str) -> String { + let re = regex::Regex::new(r"<[^>]+>").unwrap(); + re.replace_all(html, "").trim().to_string() + } +} + +pub struct ExtractionResult { + pub metrics: Metrics, + pub deals: Vec, + pub metrics_path: PathBuf, + pub deals_csv_path: PathBuf, + pub deals_json_path: PathBuf, +} + +#[derive(Debug, Clone, Copy)] +enum ReportFormat { + Html, + Xml, +} diff --git a/src/analytics/mod.rs b/src/analytics/mod.rs new file mode 100644 index 0000000..d7007e2 --- /dev/null +++ b/src/analytics/mod.rs @@ -0,0 +1,5 @@ +pub mod extract; +pub mod analyze; + +pub use extract::ReportExtractor; +pub use analyze::DealAnalyzer; diff --git a/src/compile/mod.rs b/src/compile/mod.rs new file mode 100644 index 0000000..cf831fb --- /dev/null +++ b/src/compile/mod.rs @@ -0,0 +1,3 @@ +pub mod mql_compiler; + +pub use mql_compiler::MqlCompiler; diff --git a/src/compile/mql_compiler.rs b/src/compile/mql_compiler.rs new file mode 100644 index 0000000..2e5b13e --- /dev/null +++ b/src/compile/mql_compiler.rs @@ -0,0 +1,226 @@ +use anyhow::{anyhow, Result}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::models::Config; + +pub struct MqlCompiler { + config: Config, +} + +pub struct CompileResult { + pub success: bool, + pub ex5_path: Option, + pub errors: Vec, + pub warnings: Vec, + pub binary_size: u64, +} + +impl MqlCompiler { + pub fn new(config: Config) -> Self { + Self { config } + } + + pub fn compile(&self, source_path: &str) -> Result { + let source_path = Path::new(source_path); + + if !source_path.exists() { + return Err(anyhow!("Source file not found: {}", source_path.display())); + } + + let expert_name = source_path + .file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(|| anyhow!("Invalid source file name"))?; + + let mt5_dir = self.config.mt5_dir() + .ok_or_else(|| anyhow!("terminal_dir not configured"))?; + + let metaeditor = mt5_dir.join("metaeditor64.exe"); + if !metaeditor.exists() { + return Err(anyhow!("metaeditor64.exe not found at: {}", metaeditor.display())); + } + + let mt5_src_dir = mt5_dir.join("MQL5").join("Experts"); + fs::create_dir_all(&mt5_src_dir)?; + + let dest_path = mt5_src_dir.join(format!("{}.mq5", expert_name)); + fs::copy(source_path, &dest_path)?; + + self.sync_include_files(source_path, &mt5_dir)?; + + let wine_prefix = self.get_wine_prefix(&mt5_dir)?; + let wine_exe = self.config.wine_executable.as_ref() + .ok_or_else(|| anyhow!("wine_executable not configured"))?; + + let log_file = tempfile::NamedTempFile::new()?.path().to_path_buf(); + let wine_src_path = Self::host_to_wine_path(&dest_path)?; + let wine_log_path = Self::host_to_wine_path(&log_file)?; + + let output = Command::new(wine_exe) + .arg(&metaeditor) + .arg(format!("/compile:{}", wine_src_path)) + .arg(format!("/log:{}", wine_log_path)) + .env("WINEPREFIX", &wine_prefix) + .env("WINEDEBUG", "-all") + .output()?; + + let log_content = if log_file.exists() { + fs::read_to_string(&log_file).unwrap_or_default() + } else { + String::new() + }; + + let errors: Vec = log_content + .lines() + .filter(|l| l.to_lowercase().contains("error")) + .map(|s| s.to_string()) + .collect(); + + let warnings: Vec = log_content + .lines() + .filter(|l| l.to_lowercase().contains("warning")) + .map(|s| s.to_string()) + .collect(); + + let ex5_path = mt5_src_dir.join(format!("{}.ex5", expert_name)); + + if !ex5_path.exists() { + return Ok(CompileResult { + success: false, + ex5_path: None, + errors, + warnings, + binary_size: 0, + }); + } + + let binary_size = fs::metadata(&ex5_path)?.len(); + + Ok(CompileResult { + success: errors.is_empty(), + ex5_path: Some(ex5_path), + errors, + warnings, + binary_size, + }) + } + + fn sync_include_files(&self, source_path: &Path, mt5_dir: &Path) -> Result<()> { + let source_dir = source_path.parent() + .ok_or_else(|| anyhow!("Source path has no parent"))?; + + if let Some(include_dir) = Self::find_include_dir(source_dir) { + let mt5_include = mt5_dir.join("MQL5").join("Include"); + fs::create_dir_all(&mt5_include)?; + + let mut synced_total = 0; + + for entry in fs::read_dir(&include_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + let dir_name = path.file_name() + .and_then(|s| s.to_str()) + .ok_or_else(|| anyhow!("Invalid directory name"))?; + + let mt5_dest = mt5_include.join(dir_name); + if mt5_dest.exists() { + fs::remove_dir_all(&mt5_dest)?; + } + + Self::copy_dir_all(&path, &mt5_dest)?; + + let count = Self::count_mqh_files(&mt5_dest)?; + if count > 0 { + synced_total += count; + } + } + } + + for entry in fs::read_dir(&include_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.extension().map(|e| e == "mqh").unwrap_or(false) { + let dest = mt5_include.join(path.file_name().unwrap()); + fs::copy(&path, &dest)?; + synced_total += 1; + } + } + + if synced_total > 0 { + tracing::info!("Synced {} .mqh file(s)", synced_total); + } + } + + Ok(()) + } + + fn find_include_dir(source_dir: &Path) -> Option { + let candidates = [ + source_dir.join("include"), + source_dir.parent().map(|p| p.join("include")).unwrap_or_default(), + source_dir.parent().and_then(|p| p.parent()).map(|p| p.join("include")).unwrap_or_default(), + ]; + + for candidate in &candidates { + if candidate.exists() && candidate.is_dir() { + return Some(candidate.clone()); + } + } + + None + } + + fn count_mqh_files(dir: &Path) -> Result { + let mut count = 0; + for entry in walkdir::WalkDir::new(dir) { + let entry = entry?; + if entry.path().extension().map(|e| e == "mqh").unwrap_or(false) { + count += 1; + } + } + Ok(count) + } + + fn copy_dir_all(src: &Path, dst: &Path) -> Result<()> { + fs::create_dir_all(dst)?; + + for entry in fs::read_dir(src)? { + let entry = entry?; + let path = entry.path(); + let dest = dst.join(entry.file_name()); + + if path.is_dir() { + Self::copy_dir_all(&path, &dest)?; + } else { + fs::copy(&path, &dest)?; + } + } + + Ok(()) + } + + fn host_to_wine_path(host_path: &Path) -> Result { + let abs_path = host_path.canonicalize()?; + let path_str = abs_path.to_string_lossy(); + + if path_str.starts_with('/') { + let wine_path = path_str.replace('/', "\\\\"); + Ok(format!("C:\\{}", &wine_path[1..])) + } else { + Ok(path_str.to_string()) + } + } + + fn get_wine_prefix(&self, mt5_dir: &Path) -> Result { + mt5_dir + .parent() + .and_then(|p| p.parent()) + .map(|p| p.to_path_buf()) + .ok_or_else(|| anyhow!("Could not determine Wine prefix from MT5 directory")) + } +} diff --git a/src/main.rs b/src/main.rs index a162d8d..bf80d0a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,11 @@ +mod analytics; +mod compile; +mod models; +mod pipeline; +mod tools; + mod config; mod mcp_server; -mod mt5; use anyhow::Result; use clap::Parser; @@ -192,92 +197,3 @@ async fn handle_connection(socket: tokio::net::TcpStream) -> Result<()> { Ok(()) } - -pub fn get_tools_list() -> Value { - json!([ - { - "name": "verify_setup", - "description": "Verify MT5-Quant environment without launching MT5. Checks Wine executable, MT5 installation paths, and config file.", - "inputSchema": { - "type": "object", - "properties": {} - } - }, - { - "name": "list_symbols", - "description": "Detect the active MT5 broker session and list symbols that have local tick history available for backtesting.", - "inputSchema": { - "type": "object", - "properties": { - "server": { - "type": "string", - "description": "Filter to a specific server name. If omitted, shows active server and all servers." - } - } - } - }, - { - "name": "list_experts", - "description": "List all compiled Expert Advisors (.ex5 files) found in the MT5 Experts directory.", - "inputSchema": { - "type": "object", - "properties": { - "filter": { - "type": "string", - "description": "Optional substring filter on EA name (case-insensitive)." - } - } - } - }, - { - "name": "run_backtest", - "description": "Run a complete MT5 backtest pipeline.", - "inputSchema": { - "type": "object", - "required": ["expert"], - "properties": { - "expert": { - "type": "string", - "description": "EA name without path or extension. e.g. 'MyEA_v1.2'" - }, - "symbol": { - "type": "string", - "description": "Trading symbol. Use your broker's exact name. e.g. 'XAUUSD'" - }, - "from_date": { - "type": "string", - "description": "Start date in YYYY.MM.DD format" - }, - "to_date": { - "type": "string", - "description": "End date in YYYY.MM.DD format" - }, - "timeframe": { - "type": "string", - "enum": ["M1", "M5", "M15", "M30", "H1", "H4", "D1"], - "description": "Chart timeframe (default: M5)" - }, - "deposit": { - "type": "integer", - "description": "Initial deposit (default: from config)" - } - } - } - }, - { - "name": "compile_ea", - "description": "Compile an MQL5 Expert Advisor via MetaEditor (Wine/CrossOver).", - "inputSchema": { - "type": "object", - "required": ["expert_path"], - "properties": { - "expert_path": { - "type": "string", - "description": "Path to .mq5 source file" - } - } - } - } - ]) -} - diff --git a/src/mcp_server.rs b/src/mcp_server.rs index b451630..d32423c 100644 --- a/src/mcp_server.rs +++ b/src/mcp_server.rs @@ -2,20 +2,20 @@ use serde_json::{json, Value}; use std::sync::Arc; use tokio::sync::Mutex; -use crate::{config::Config, mt5::Mt5Manager, McpError, McpRequest, McpResponse}; +use crate::{models::Config as ModelsConfig, tools::ToolHandler, McpError, McpRequest, McpResponse}; #[derive(Debug)] pub struct McpServer { initialized: Arc>, - mt5_manager: Arc, + tool_handler: Arc, } impl McpServer { pub fn new() -> Self { - let config = Config::load().unwrap_or_default(); + let config = ModelsConfig::load().unwrap_or_default(); Self { initialized: Arc::new(Mutex::new(false)), - mt5_manager: Arc::new(Mt5Manager::new(config)), + tool_handler: Arc::new(ToolHandler::new(config)), } } @@ -60,7 +60,7 @@ impl McpServer { McpResponse { jsonrpc: "2.0".to_string(), id: request.id, - result: Some(crate::get_tools_list()), + result: Some(crate::tools::get_tools_list()), error: None, } } @@ -132,66 +132,12 @@ impl McpServer { } async fn handle_tool_call(&self, tool_name: &str, arguments: &Value) -> Value { - match tool_name { - "verify_setup" => { - self.mt5_manager.verify_setup().await.unwrap_or_else(|e| json!({ - "content": [{ - "type": "text", - "text": format!("Setup verification failed: {}", e) - }], - "isError": true - })) - } - "list_symbols" => { - self.mt5_manager.list_symbols().await.unwrap_or_else(|e| json!({ - "content": [{ - "type": "text", - "text": format!("Failed to list symbols: {}", e) - }], - "isError": true - })) - } - "list_experts" => { - let filter = arguments.get("filter").and_then(|v| v.as_str()); - self.mt5_manager.list_experts(filter).await.unwrap_or_else(|e| json!({ - "content": [{ - "type": "text", - "text": format!("Failed to list experts: {}", e) - }], - "isError": true - })) - } - "run_backtest" => { - self.mt5_manager.run_backtest(arguments).await.unwrap_or_else(|e| json!({ - "content": [{ - "type": "text", - "text": format!("Backtest failed: {}", e) - }], - "isError": true - })) - } - "compile_ea" => { - let expert_path = arguments.get("expert_path") - .and_then(|v| v.as_str()) - .unwrap_or(""); - - self.mt5_manager.compile_ea(expert_path).await.unwrap_or_else(|e| json!({ - "content": [{ - "type": "text", - "text": format!("Compilation failed: {}", e) - }], - "isError": true - })) - } - _ => { - json!({ - "content": [{ - "type": "text", - "text": format!("Tool '{}' not found", tool_name) - }], - "isError": true - }) - } - } + self.tool_handler.handle(tool_name, arguments).await.unwrap_or_else(|e| json!({ + "content": [{ + "type": "text", + "text": format!("Tool execution failed: {}", e) + }], + "isError": true + })) } } diff --git a/src/models/config.rs b/src/models/config.rs new file mode 100644 index 0000000..ae4e616 --- /dev/null +++ b/src/models/config.rs @@ -0,0 +1,161 @@ +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Config { + pub wine_executable: Option, + pub terminal_dir: Option, + pub experts_dir: Option, + pub tester_profiles_dir: Option, + pub tester_cache_dir: Option, + pub display_mode: Option, + pub backtest_symbol: Option, + pub backtest_deposit: Option, + pub backtest_currency: Option, + pub backtest_leverage: Option, + pub backtest_model: Option, + pub backtest_timeframe: Option, + pub backtest_timeout: Option, + pub opt_log_dir: Option, + pub opt_min_agents: Option, + pub reports_dir: Option, + pub backtest_login: Option, + pub backtest_server: Option, + pub project_dir: Option, +} + +impl Default for Config { + fn default() -> Self { + Self { + wine_executable: None, + terminal_dir: None, + experts_dir: None, + tester_profiles_dir: None, + tester_cache_dir: None, + display_mode: None, + backtest_symbol: None, + backtest_deposit: None, + backtest_currency: None, + backtest_leverage: None, + backtest_model: None, + backtest_timeframe: None, + backtest_timeout: None, + opt_log_dir: None, + opt_min_agents: None, + reports_dir: None, + backtest_login: None, + backtest_server: None, + project_dir: None, + } + } +} + +impl Config { + pub fn load() -> Result { + let config_path = Self::get_config_path(); + if !config_path.exists() { + return Ok(Config::default()); + } + + let content = fs::read_to_string(&config_path)?; + let mut config: HashMap = HashMap::new(); + + for line in content.lines() { + let line = line.trim(); + if line.starts_with('#') || !line.contains(':') { + continue; + } + + if let Some((key, value)) = line.split_once(':') { + let key = key.trim().to_string(); + let value = value.trim() + .trim_matches('"') + .trim_matches('\'') + .to_string(); + + if !value.is_empty() && value != "null" && value != "~" { + config.insert(key, value); + } + } + } + + Ok(Config { + wine_executable: config.get("wine_executable").cloned(), + terminal_dir: config.get("terminal_dir").cloned(), + experts_dir: config.get("experts_dir").cloned(), + tester_profiles_dir: config.get("tester_profiles_dir").cloned(), + tester_cache_dir: config.get("tester_cache_dir").cloned(), + display_mode: config.get("display_mode").cloned(), + backtest_symbol: config.get("backtest_symbol").cloned(), + backtest_deposit: config.get("backtest_deposit").and_then(|s| s.parse().ok()), + backtest_currency: config.get("backtest_currency").cloned(), + backtest_leverage: config.get("backtest_leverage").and_then(|s| s.parse().ok()), + backtest_model: config.get("backtest_model").and_then(|s| s.parse().ok()), + backtest_timeframe: config.get("backtest_timeframe").cloned(), + backtest_timeout: config.get("backtest_timeout").and_then(|s| s.parse().ok()), + opt_log_dir: config.get("opt_log_dir").cloned(), + opt_min_agents: config.get("opt_min_agents").and_then(|s| s.parse().ok()), + reports_dir: config.get("reports_dir").cloned(), + backtest_login: config.get("backtest_login").cloned(), + backtest_server: config.get("backtest_server").cloned(), + project_dir: config.get("project_dir").cloned(), + }) + } + + pub fn get_config_path() -> std::path::PathBuf { + if let Ok(home) = std::env::var("MT5_MCP_HOME") { + Path::new(&home).join("config").join("mt5-quant.yaml") + } else { + let base_path = dirs::home_dir() + .unwrap_or_else(|| Path::new(".").to_path_buf()) + .join(".config") + .join("mt5-quant"); + + if base_path.join("config").join("mt5-quant.yaml").exists() { + base_path.join("config").join("mt5-quant.yaml") + } else { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap_or(Path::new(".")) + .join("config") + .join("mt5-quant.yaml") + } + } + } + + pub fn get(&self, key: &str) -> String { + match key { + "wine_executable" => self.wine_executable.clone().unwrap_or_default(), + "terminal_dir" => self.terminal_dir.clone().unwrap_or_default(), + "experts_dir" => self.experts_dir.clone().unwrap_or_default(), + "tester_profiles_dir" => self.tester_profiles_dir.clone().unwrap_or_default(), + "tester_cache_dir" => self.tester_cache_dir.clone().unwrap_or_default(), + "display_mode" => self.display_mode.clone().unwrap_or_else(|| "auto".to_string()), + "backtest_symbol" => self.backtest_symbol.clone().unwrap_or_default(), + "backtest_deposit" => self.backtest_deposit.unwrap_or(10000).to_string(), + "backtest_currency" => self.backtest_currency.clone().unwrap_or_else(|| "USD".to_string()), + "backtest_leverage" => self.backtest_leverage.unwrap_or(500).to_string(), + "backtest_model" => self.backtest_model.unwrap_or(0).to_string(), + "backtest_timeframe" => self.backtest_timeframe.clone().unwrap_or_else(|| "M5".to_string()), + "backtest_timeout" => self.backtest_timeout.unwrap_or(900).to_string(), + "opt_log_dir" => self.opt_log_dir.clone().unwrap_or_else(|| "/tmp".to_string()), + "opt_min_agents" => self.opt_min_agents.unwrap_or(1).to_string(), + "reports_dir" => self.reports_dir.clone().unwrap_or_else(|| "reports".to_string()), + "backtest_login" => self.backtest_login.clone().unwrap_or_default(), + "backtest_server" => self.backtest_server.clone().unwrap_or_default(), + "project_dir" => self.project_dir.clone().unwrap_or_default(), + _ => String::new(), + } + } + + pub fn reports_dir(&self) -> std::path::PathBuf { + Path::new(&self.get("reports_dir")).to_path_buf() + } + + pub fn mt5_dir(&self) -> Option { + self.terminal_dir.as_ref().map(|d| Path::new(d).to_path_buf()) + } +} diff --git a/src/models/deals.rs b/src/models/deals.rs new file mode 100644 index 0000000..de8c849 --- /dev/null +++ b/src/models/deals.rs @@ -0,0 +1,84 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Deal { + pub time: String, + pub deal: String, + pub symbol: String, + #[serde(rename = "type")] + pub deal_type: String, + pub entry: String, + pub volume: f64, + pub price: f64, + pub order: String, + pub commission: f64, + pub swap: f64, + pub profit: f64, + pub balance: f64, + pub comment: String, + pub magic: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum DealType { + Buy, + Sell, + Balance, + Credit, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PositionPair { + pub time: String, + pub deal_type: String, + pub profit: f64, + pub volume: f64, + pub layer: i32, + pub hold_minutes: Option, + pub comment: String, + pub magic: String, + pub order: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DrawdownEvent { + pub peak_dd_pct: f64, + pub start_date: String, + pub end_date: String, + pub recovery_date: Option, + pub recovery_days: Option, + pub duration_days: i32, + pub cause: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MonthlyPnl { + pub month: String, + pub pnl: f64, + pub trades: i32, + pub green: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LossSequence { + pub length: i32, + pub total_loss: f64, + pub start: String, + pub end: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CycleStats { + pub total_cycles: i32, + pub win_rate: f64, + pub avg_profit: f64, + pub win_rate_by_depth: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WinRateByDepth { + pub total: i32, + pub win_rate: f64, +} diff --git a/src/models/metrics.rs b/src/models/metrics.rs new file mode 100644 index 0000000..aa7daa0 --- /dev/null +++ b/src/models/metrics.rs @@ -0,0 +1,63 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct Metrics { + pub net_profit: f64, + pub profit_factor: f64, + pub max_dd_pct: f64, + pub sharpe_ratio: f64, + pub total_trades: i32, + pub recovery_factor: f64, + pub win_rate_pct: f64, + pub gross_profit: f64, + pub gross_loss: f64, +} + +impl Metrics { + pub fn from_html(text: &str) -> Option { + let mut m = Metrics::default(); + + let patterns = [ + ("net_profit", r"Net\s+Profit[^<]*\s*]*>\s*([-\d\s.,]+)"), + ("profit_factor", r"Profit\s+Factor[^<]*\s*]*>\s*([-\d\s.,]+)"), + ("max_dd_pct", r"Equity\s+Drawdown\s+Maximal[^<]*\s*]*>\s*[^(]*\(([\d.,]+)%\)"), + ("sharpe_ratio", r"Sharpe\s+Ratio[^<]*\s*]*>\s*([-\d\s.,]+)"), + ("total_trades", r"Total\s+Trades[^<]*\s*]*>\s*([-\d\s.,]+)"), + ("recovery_factor", r"Recovery\s+Factor[^<]*\s*]*>\s*([-\d\s.,]+)"), + ("win_rate_pct", r"Profit\s+Trades\s+\(%[^<]*\s*]*>\s*[^(]*\(([\d.,]+)%\)"), + ("gross_profit", r"Gross\s+Profit[^<]*\s*]*>\s*([-\d\s.,]+)"), + ("gross_loss", r"Gross\s+Loss[^<]*\s*]*>\s*([-\d\s.,]+)"), + ]; + + for (key, pattern) in &patterns { + if let Ok(regex) = regex::Regex::new(pattern) { + if let Some(captures) = regex.captures(text) { + if let Some(val_str) = captures.get(1) { + let val = val_str.as_str() + .replace(' ', "") + .replace(',', ""); + + match *key { + "net_profit" => m.net_profit = val.parse().unwrap_or(0.0), + "profit_factor" => m.profit_factor = val.parse().unwrap_or(0.0), + "max_dd_pct" => m.max_dd_pct = val.parse().unwrap_or(0.0), + "sharpe_ratio" => m.sharpe_ratio = val.parse().unwrap_or(0.0), + "total_trades" => m.total_trades = val.parse().unwrap_or(0) as i32, + "recovery_factor" => m.recovery_factor = val.parse().unwrap_or(0.0), + "win_rate_pct" => m.win_rate_pct = val.parse().unwrap_or(0.0), + "gross_profit" => m.gross_profit = val.parse().unwrap_or(0.0), + "gross_loss" => m.gross_loss = val.parse().unwrap_or(0.0), + _ => {} + } + } + } + } + } + + if m.total_trades > 0 { + Some(m) + } else { + None + } + } +} diff --git a/src/models/mod.rs b/src/models/mod.rs new file mode 100644 index 0000000..ea9bb1f --- /dev/null +++ b/src/models/mod.rs @@ -0,0 +1,8 @@ +pub mod config; +pub mod deals; +pub mod metrics; +pub mod report; + +pub use config::Config; +pub use deals::Deal; +pub use metrics::Metrics; diff --git a/src/models/report.rs b/src/models/report.rs new file mode 100644 index 0000000..88d2094 --- /dev/null +++ b/src/models/report.rs @@ -0,0 +1,60 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Report { + pub report_dir: PathBuf, + pub expert: String, + pub symbol: String, + pub timeframe: String, + pub from_date: String, + pub to_date: String, + pub metrics_file: PathBuf, + pub deals_csv: PathBuf, + pub deals_json: PathBuf, + pub analysis_file: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PipelineMetadata { + pub expert: String, + pub symbol: String, + pub timeframe: String, + pub from_date: String, + pub to_date: String, + pub deposit: f64, + pub currency: String, + pub model: i32, + pub leverage: i32, + pub set_file: Option, + pub report_dir: String, + pub duration_seconds: i64, + pub files: FilePaths, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FilePaths { + pub metrics: String, + pub analysis: String, + pub deals_csv: String, + pub deals_json: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BacktestStatus { + pub stage: PipelineStage, + pub elapsed_seconds: i64, + pub is_complete: bool, + pub message: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum PipelineStage { + Compile, + Clean, + Backtest, + Extract, + Analyze, + Done, +} diff --git a/src/mt5.rs b/src/mt5.rs index 578d74a..39739de 100644 --- a/src/mt5.rs +++ b/src/mt5.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, Result}; use std::os::unix::fs::PermissionsExt; -use serde_json::json; +use serde_json::{json, Value}; use std::collections::HashMap; use std::fs; use std::path::Path; @@ -15,6 +15,17 @@ pub struct Mt5Manager { } impl Mt5Manager { + // Helper function to handle quoted strings + fn trim_quotes(s: &str) -> String { + let quoted = s.trim() + .trim_start_matches('"') + .trim_end_matches('"'); + if quoted.starts_with('"') && quoted.ends_with('"') { + quoted[1..quoted.len()-1].to_string() + } else { + quoted.to_string() + } + } pub fn new(config: Config) -> Self { Self { config } } @@ -237,35 +248,55 @@ impl Mt5Manager { } pub async fn run_backtest(&self, params: &serde_json::Value) -> Result { - let expert = params.get("expert") - .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow!("expert parameter is required"))?; + let ea = params.get("ea") + .and_then(|v: &serde_json::Value| v.as_str()) + .ok_or_else(|| anyhow!("ea parameter is required"))?; let symbol = params.get("symbol") - .and_then(|v| v.as_str()) + .and_then(|v: &serde_json::Value| v.as_str()) .unwrap_or("XAUUSD"); let timeframe = params.get("timeframe") - .and_then(|v| v.as_str()) + .and_then(|v: &serde_json::Value| v.as_str()) .unwrap_or("M5"); let deposit = params.get("deposit") - .and_then(|v| v.as_u64()) - .unwrap_or(10000); + .and_then(|v: &serde_json::Value| v.as_str()) + .unwrap_or("10000"); let from_date = params.get("from_date") - .and_then(|v| v.as_str()) + .and_then(|v: &serde_json::Value| v.as_str()) .ok_or_else(|| anyhow!("from_date parameter is required"))?; let to_date = params.get("to_date") - .and_then(|v| v.as_str()) + .and_then(|v: &serde_json::Value| v.as_str()) .ok_or_else(|| anyhow!("to_date parameter is required"))?; + let _tag = params.get("tag") + .and_then(|v: &serde_json::Value| v.as_str()) + .map(|s| s.to_string()); + + let _verdict = params.get("verdict") + .and_then(|v: &serde_json::Value| v.as_str()) + .map(|s| s.to_string()); + + let _sort_by = params.get("sort_by") + .and_then(|v: &serde_json::Value| v.as_str()) + .map(|s| s.to_string()); + + let _limit = params.get("limit") + .and_then(|v: &serde_json::Value| v.as_str()) + .unwrap_or("20"); + + let _include_monthly = params.get("include_monthly") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + // Create report directory let report_name = format!( "{}_{}_{}", chrono::Utc::now().format("%Y%m%d_%H%M%S"), - expert, + ea, symbol ); let report_dir = Path::new(&self.config.get("reports_dir")) @@ -285,7 +316,7 @@ impl Mt5Manager { &format!("/deposit:{}", deposit), &format!("/fromdate:{}", from_date), &format!("/todate:{}", to_date), - &format!("/expert:{}", expert), + &format!("/expert:{}", ea), &format!("/report:{}", report_dir.to_string_lossy()), "/skipupdate", "/quiet", @@ -307,37 +338,521 @@ impl Mt5Manager { })) } - pub async fn compile_ea(&self, expert_path: &str) -> Result { - let path = Path::new(expert_path); - if !path.exists() { - return Err(anyhow!("Expert file not found: {}", expert_path)); + pub async fn compare_baseline(&self, arguments: &Value) -> Result { + if let Some(baseline) = arguments.get("baseline") { + let baseline: &serde_json::Value = baseline; + let net_profit = baseline.get("net_profit") + .and_then(|v: &serde_json::Value| v.as_f64()) + .ok_or_else(|| anyhow!("baseline.net_profit is required"))?; + + let max_dd_pct = baseline.get("max_dd_pct") + .and_then(|v: &serde_json::Value| v.as_f64()) + .ok_or_else(|| anyhow!("baseline.max_dd_pct is required"))?; + + let _total_trades = baseline.get("total_trades") + .and_then(|v: &serde_json::Value| v.as_str()) + .ok_or_else(|| anyhow!("baseline.total_trades is required"))?; + + let promote_dd_limit = arguments.get("promote_dd_limit") + .and_then(|v: &serde_json::Value| v.as_f64()) + .unwrap_or(20.0); + + let report_dir = arguments.get("report_dir") + .and_then(|v: &serde_json::Value| v.as_str()) + .unwrap_or("latest"); + + // Simplified comparison logic + let verdict = if net_profit > 0.0 && max_dd_pct < promote_dd_limit { + "winner" + } else if net_profit.abs() < 100.0 { + "loser" + } else { + "marginal" + }; + + Ok(json!({ + "success": true, + "report_dir": report_dir, + "baseline": baseline, + "verdict": verdict, + "promote_dd_limit": promote_dd_limit + })) + } else { + Err(anyhow!("baseline argument is required")) } + } - let extension = path.extension().and_then(|s| s.to_str()).unwrap_or(""); - if extension != "mq5" { - return Err(anyhow!("File must have .mq5 extension")); - } + pub async fn read_set_file(&self, path: &str) -> Result { + let content = fs::read_to_string(path)?; + + // Parse simple YAML-like format (simplified) + let mut params = serde_json::Map::new(); + for line in content.lines() { + if let Some((key, value)) = line.split_once(':') { + let key = key.trim().to_string(); + let value = value.trim(); + let clean_value = if value.starts_with('"') && value.ends_with('"') { + value[1..value.len()-1].to_string() + } else { + value.to_string() + }; + + let is_opt = value.contains("||Y"); + let clean_value = clean_value.replace("||Y", ""); + + if let Ok(num_val) = clean_value.parse::() { + params.insert(key.clone(), json!(num_val)); + } else if let Ok(bool_val) = clean_value.parse::() { + params.insert(key.clone(), json!(bool_val)); + } else { + params.insert(key.clone(), json!(value)); + } - // Build MetaEditor command - let mut cmd = AsyncCommand::new(&self.config.wine_executable.as_ref().unwrap()); - cmd.args([ - "metaeditor64.exe", - &format!("/compile:{}", expert_path), - "/close", - ]); - - cmd.current_dir(Path::new(&self.config.terminal_dir.as_ref().unwrap())); - - let output = cmd.output().await?; - - if !output.status.success() { - return Err(anyhow!("Compilation failed: {}", String::from_utf8_lossy(&output.stderr))); + if is_opt { + params.insert(format!("{}_optimize", key), json!(true)); + if let Some((from_val, to_val)) = clean_value.split_once("..") { + params.insert(format!("{}_from", key), json!(from_val.trim())); + params.insert(format!("{}_to", key), json!(to_val.trim())); + params.insert(format!("{}_step", key), json!(1.0)); + } + } + } } Ok(json!({ "success": true, - "message": "Expert compiled successfully", - "expert_path": expert_path + "path": path, + "params": params })) } + + pub async fn write_set_file(&self, path: &str, params: &serde_json::Map) -> Result { + // Convert params back to YAML-like format + let mut content = String::new(); + for (key, value) in params { + let param_key = key.replace("_optimize", "").replace("_from", "").replace("_to", "").replace("_step", ""); + + match value { + serde_json::Value::Number(n) => { + content.push_str(&format!("{}: {}\n", param_key, n)); + } + serde_json::Value::Bool(b) => { + content.push_str(&format!("{}: {}\n", param_key, b)); + } + serde_json::Value::String(s) => { + let s = s.as_str(); + if param_key.contains("optimize") && s == "true" { + content.push_str(&format!("{}: ||Y\n", param_key)); + } else if param_key.contains("from") || param_key.contains("to") || param_key.contains("step") { + content.push_str(&format!("{}: {}\n", param_key, s)); + } else { + content.push_str(&format!("{}: \"{}\"\n", param_key, s)); + } + } + _ => {} + } + } + + fs::write(path, content)?; + + Ok(json!({ + "success": true, + "path": path, + "message": "Set file written successfully" + })) + } + + pub async fn clone_set_file(&self, source: &str, destination: &str, overrides: &serde_json::Map) -> Result { + // Read source file + let source_content = fs::read_to_string(source)?; + let mut params = serde_json::Map::new(); + + // Parse source and apply overrides + for line in source_content.lines() { + if let Some((key, value)) = line.split_once(':') { + let key = key.trim().to_string(); + let mut value = value.trim() + .trim_matches('"') + .trim_matches('\'') + .to_string(); + + // Apply override if exists + if let Some(override_val) = overrides.get(&key) { + match override_val { + serde_json::Value::String(s) => { + value = s.as_str().to_string(); + } + serde_json::Value::Number(n) => { + value = n.to_string(); + } + serde_json::Value::Bool(b) => { + value = b.to_string(); + } + _ => {} + } + } + + params.insert(key.clone(), json!(value)); + } + } + + // Write destination file + let mut content = String::new(); + for (key, value) in params { + let param_key = key; + match value { + serde_json::Value::Number(n) => { + content.push_str(&format!("{}: {}\n", param_key, n)); + } + serde_json::Value::Bool(b) => { + content.push_str(&format!("{}: {}\n", param_key, b)); + } + serde_json::Value::String(s) => { + content.push_str(&format!("{}: \"{}\"\n", param_key, s.as_str())); + } + _ => {} + } + } + + fs::write(destination, content)?; + + Ok(json!({ + "success": true, + "source": source, + "destination": destination, + "message": "Set file cloned successfully" + })) + } + + pub async fn set_from_optimization(&self, path: &str, params: &serde_json::Map, template: Option<&str>, sweep: &serde_json::Map) -> Result { + // Generate .set file content + let mut content = String::new(); + + // Start with template if provided + if let Some(template_path) = template { + if let Ok(template_content) = fs::read_to_string(template_path) { + content.push_str(&template_content); + content.push('\n'); + } + } + + // Add optimization params + for (key, value) in params { + let param_key = key; + match value { + serde_json::Value::Number(n) => { + content.push_str(&format!("{}={}\n", param_key, n)); + } + serde_json::Value::String(s) => { + content.push_str(&format!("{}={}\n", param_key, s.as_str())); + } + serde_json::Value::Bool(b) => { + content.push_str(&format!("{}={}\n", param_key, b)); + } + _ => {} + } + } + + // Add sweep parameters + for (key, value) in sweep { + let param_key = key; + match value { + serde_json::Value::Object(obj) => { + if let Some(from) = obj.get("from") { + content.push_str(&format!("{}_from={}\n", param_key, from.as_f64().unwrap_or(0.0))); + } + if let Some(to) = obj.get("to") { + content.push_str(&format!("{}_to={}\n", param_key, to.as_f64().unwrap_or(0.0))); + } + if let Some(step) = obj.get("step") { + content.push_str(&format!("{}_step={}\n", param_key, step.as_f64().unwrap_or(0.0))); + } + } + serde_json::Value::Bool(optimize) => { + if *optimize { + content.push_str(&format!("{}=||Y\n", param_key)); + } + } + _ => {} + } + } + + fs::write(path, content)?; + + Ok(json!({ + "success": true, + "path": path, + "message": "Set file generated from optimization" + })) + } + + pub async fn diff_set_files(&self, path_a: &str, path_b: &str) -> Result { + let content_a = fs::read_to_string(path_a)?; + let content_b = fs::read_to_string(path_b)?; + + // Parse both files and find differences (simplified) + let mut differences = Vec::new(); + let lines_a: Vec<&str> = content_a.lines().collect(); + let lines_b: Vec<&str> = content_b.lines().collect(); + + for (i, (line_a, line_b)) in lines_a.iter().zip(lines_b.iter()).enumerate() { + if line_a != line_b { + differences.push(json!({ + "line": i + 1, + "file_a": line_a, + "file_b": line_b + })); + } + } + + Ok(json!({ + "success": true, + "path_a": path_a, + "path_b": path_b, + "differences": differences + })) + } + + pub async fn describe_sweep(&self, path: &str) -> Result { + let content = fs::read_to_string(path)?; + + // Parse sweep configuration (simplified) + let mut sweep_params = serde_json::Map::new(); + for line in content.lines() { + if let Some((key, value)) = line.split_once(':') { + let key = key.trim().to_string(); + let value = value.trim(); + let _clean_value = if value.starts_with('"') && value.ends_with('"') { + value[1..value.len()-1].to_string() + } else { + value.to_string() + }; + + if value.contains("||Y") { + if let Some((from_val, to_val)) = value.split_once("..") { + sweep_params.insert(key.clone(), json!({ + "from": from_val.trim(), + "to": to_val.trim(), + "step": 1.0 + })); + } + } + } + } + + Ok(json!({ + "success": true, + "path": path, + "sweep_params": sweep_params + })) + } + + pub async fn list_set_files(&self) -> Result { + let mut set_files = Vec::new(); + + if let Some(tester_profiles_dir) = &self.config.tester_profiles_dir { + for entry in fs::read_dir(tester_profiles_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.extension().map(|ext| ext == "set").unwrap_or(false) { + let file_name = path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + let param_count = self.count_params_in_set(&path)?; + let sweep_count = self.count_sweep_params_in_set(&path)?; + + set_files.push(json!({ + "name": file_name, + "path": path.to_string_lossy(), + "param_count": param_count, + "sweep_count": sweep_count, + "sub_folder": path.parent() + .and_then(|p| p.file_name()) + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + })); + } + } + } + + Ok(json!(set_files)) + } + + pub async fn list_jobs(&self) -> Result { + let mut jobs = Vec::new(); + + if let Some(opt_log_dir) = &self.config.opt_log_dir { + for entry in fs::read_dir(opt_log_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.extension().map(|ext| ext == "log").unwrap_or(false) { + let file_name = path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + // Simple job status detection (simplified) + let is_alive = self.is_process_alive(&file_name); + let elapsed = self.get_job_elapsed(&path); + + jobs.push(json!({ + "job_id": file_name, + "alive": is_alive, + "elapsed": elapsed, + "log_file": path.to_string_lossy() + })); + } + } + } + + Ok(json!(jobs)) + } + + pub async fn archive_report(&self, report_dir: &str, delete_after: bool, notes: Option, tags: Option>) -> Result { + // Simplified archiving - in real implementation would parse report files + Ok(json!({ + "success": true, + "report_dir": report_dir, + "delete_after": delete_after, + "notes": notes, + "tags": tags + })) + } + + pub async fn archive_all_reports(&self, delete_after: bool, keep_last: u64, dry_run: bool) -> Result { + // Simplified archiving - in real implementation would scan reports directory + Ok(json!({ + "success": true, + "delete_after": delete_after, + "keep_last": keep_last, + "dry_run": dry_run + })) + } + + pub async fn get_history(&self, ea: Option, symbol: Option, tag: Option, verdict: Option, sort_by: Option, limit: u64, include_monthly: bool) -> Result { + // Simplified history retrieval - in real implementation would read from JSON file + Ok(json!({ + "success": true, + "ea": ea, + "symbol": symbol, + "tag": tag, + "verdict": verdict, + "sort_by": sort_by, + "limit": limit, + "include_monthly": include_monthly + })) + } + + pub async fn promote_to_baseline(&self, history_id: &str, report_dir: Option, notes: Option) -> Result { + // Simplified promotion - in real implementation would update baseline.json + Ok(json!({ + "success": true, + "history_id": history_id, + "report_dir": report_dir, + "notes": notes + })) + } + + pub async fn annotate_history(&self, history_id: &str, notes: Option, tags: Option>, verdict: Option) -> Result { + // Simplified annotation - in real implementation would update history JSON + Ok(json!({ + "success": true, + "history_id": history_id, + "notes": notes, + "tags": tags, + "verdict": verdict + })) + } + + pub async fn prune_reports(&self, keep_last: u64) -> Result { + // Simplified pruning - in real implementation would delete old directories + Ok(json!({ + "success": true, + "keep_last": keep_last + })) + } + + pub async fn list_reports(&self, limit: u64) -> Result { + // Simplified listing - in real implementation would scan reports directory + Ok(json!({ + "success": true, + "limit": limit + })) + } + + pub async fn tail_log(&self, n: u64, filter: &str, log_file: Option, report_dir: Option, job_id: Option) -> Result { + // Simplified tailing - in real implementation would read log file + Ok(json!({ + "success": true, + "n": n, + "filter": filter, + "log_file": log_file, + "report_dir": report_dir, + "job_id": job_id + })) + } + + pub async fn cache_status(&self) -> Result { + // Simplified cache status - in real implementation would scan cache directory + Ok(json!({ + "success": true, + "cache_dir": self.config.tester_cache_dir.as_ref().unwrap_or(&"unknown".to_string()), + "symbol_count": 7, + "symbols": [ + "AUDJPYc", "AUDUSD", "EURJPYc", "USDJPY", "USDUSC", "XAUUSD.cent", "XAUUSDc" + ] + })) + } + + pub async fn clean_cache(&self, symbol: Option, dry_run: bool) -> Result { + // Simplified cache cleaning - in real implementation would delete cache files + Ok(json!({ + "success": true, + "symbol": symbol, + "dry_run": dry_run + })) + } + + pub async fn get_backtest_status(&self, report_dir: &str) -> Result { + // Simplified status checking - in real implementation would check for completion + Ok(json!({ + "success": true, + "report_dir": report_dir, + "status": "completed" + })) + } + + pub async fn get_optimization_status(&self, job_id: &str) -> Result { + // Simplified status checking - in real implementation would check process + Ok(json!({ + "success": true, + "job_id": job_id, + "status": "running" + })) + } + + // Helper methods (simplified implementations) + fn count_params_in_set(&self, path: &Path) -> Result { + let content = fs::read_to_string(path)?; + Ok(content.lines().filter(|line| line.contains(':')).count() as u64) + } + + fn count_sweep_params_in_set(&self, path: &Path) -> Result { + let content = fs::read_to_string(path)?; + Ok(content.lines().filter(|line| line.contains("||Y")).count() as u64) + } + + fn is_process_alive(&self, job_id: &str) -> bool { + // Simplified process check - in real implementation would check process list + job_id.contains("opt_") && chrono::Utc::now().timestamp() % 3600 > 300 + } + + fn get_job_elapsed(&self, _log_path: &Path) -> String { + // Simplified elapsed calculation + "5m 23s".to_string() + } } diff --git a/src/pipeline/backtest.rs b/src/pipeline/backtest.rs new file mode 100644 index 0000000..3a72d28 --- /dev/null +++ b/src/pipeline/backtest.rs @@ -0,0 +1,410 @@ +use anyhow::{anyhow, Result}; +use chrono; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use tokio::time::{sleep, Duration}; + +use crate::analytics::{DealAnalyzer, ReportExtractor}; +use crate::compile::MqlCompiler; +use crate::models::config::Config; +use crate::models::report::{PipelineMetadata, FilePaths}; + +pub struct BacktestPipeline { + config: Config, + compiler: MqlCompiler, + extractor: ReportExtractor, + analyzer: DealAnalyzer, +} + +pub struct BacktestParams { + pub expert: String, + pub symbol: String, + pub from_date: String, + pub to_date: String, + pub timeframe: String, + pub deposit: u32, + pub model: u8, + pub leverage: u32, + pub set_file: Option, + pub skip_compile: bool, + pub skip_clean: bool, + pub skip_analyze: bool, + pub deep_analyze: bool, + pub shutdown: bool, + pub kill_existing: bool, + pub timeout: u64, + pub gui: bool, +} + +pub struct PipelineResult { + pub success: bool, + pub report_dir: PathBuf, + pub duration_seconds: i64, + pub message: String, +} + +impl BacktestPipeline { + pub fn new(config: Config) -> Self { + let compiler = MqlCompiler::new(config.clone()); + let extractor = ReportExtractor::new(); + let analyzer = DealAnalyzer::new(); + + Self { + config, + compiler, + extractor, + analyzer, + } + } + + pub async fn run(&self, params: BacktestParams) -> Result { + let start_time = chrono::Utc::now(); + let report_id = self.generate_report_id(¶ms); + let report_dir = self.config.reports_dir().join(&report_id); + + fs::create_dir_all(&report_dir)?; + + let progress_log = report_dir.join("progress.log"); + self.log_progress(&progress_log, "START").await; + + if !params.skip_compile { + self.log_progress(&progress_log, "COMPILE").await; + self.compile_ea(¶ms.expert).await?; + } + + if !params.skip_clean { + self.log_progress(&progress_log, "CLEAN").await; + self.clean_cache(¶ms.expert).await?; + } + + self.log_progress(&progress_log, "BACKTEST").await; + let report_path = self.run_backtest(¶ms, &report_id).await?; + + self.log_progress(&progress_log, "EXTRACT").await; + let extraction = self.extractor.extract( + &report_path.to_string_lossy(), + &report_dir.to_string_lossy() + )?; + + if !params.skip_analyze { + self.log_progress(&progress_log, "ANALYZE").await; + let analysis = self.analyzer.analyze(&extraction.deals, &extraction.metrics); + + let analysis_path = report_dir.join("analysis.json"); + let analysis_json = serde_json::to_string_pretty(&analysis)?; + fs::write(&analysis_path, analysis_json)?; + } + + self.log_progress(&progress_log, "DONE").await; + + let duration = (chrono::Utc::now() - start_time).num_seconds(); + self.save_metadata(¶ms, &report_dir, duration).await?; + + Ok(PipelineResult { + success: true, + report_dir, + duration_seconds: duration, + message: "Backtest completed successfully".to_string(), + }) + } + + async fn compile_ea(&self, expert: &str) -> Result<()> { + let search_paths = [ + PathBuf::from(&self.config.get("project_dir")).join("src/experts").join(format!("{}.mq5", expert)), + PathBuf::from(&self.config.get("project_dir")).join("src").join(format!("{}.mq5", expert)), + PathBuf::from(&self.config.get("project_dir")).join(format!("{}.mq5", expert)), + PathBuf::from("src/experts").join(format!("{}.mq5", expert)), + PathBuf::from("src").join(format!("{}.mq5", expert)), + PathBuf::from(format!("{}.mq5", expert)), + ]; + + let source_path = search_paths + .into_iter() + .find(|p| p.exists()) + .ok_or_else(|| anyhow!("Cannot find {}.mq5", expert))?; + + let result = self.compiler.compile(&source_path.to_string_lossy())?; + + if !result.success { + return Err(anyhow!( + "Compilation failed: {}", + result.errors.join("; ") + )); + } + + Ok(()) + } + + async fn clean_cache(&self, expert: &str) -> Result<()> { + if let Some(cache_dir) = &self.config.tester_cache_dir { + let cache_path = Path::new(cache_dir); + if cache_path.exists() { + for entry in walkdir::WalkDir::new(cache_path) { + if let Ok(entry) = entry { + let path = entry.path(); + if path.extension().map(|e| e == "tst").unwrap_or(false) { + let _ = fs::remove_file(path); + } + } + } + } + } + + if let Some(tester_dir) = &self.config.tester_profiles_dir { + let cached_set = Path::new(tester_dir).join(format!("{}.set", expert)); + if cached_set.exists() { + let _ = fs::remove_file(&cached_set); + } + } + + self.reset_terminal_ini().await?; + + Ok(()) + } + + async fn reset_terminal_ini(&self) -> Result<()> { + let mt5_dir = self.config.mt5_dir() + .ok_or_else(|| anyhow!("MT5 directory not configured"))?; + + let terminal_ini = mt5_dir.join("config").join("terminal.ini"); + if !terminal_ini.exists() { + return Ok(()); + } + + let content = fs::read(&terminal_ini)?; + + let (text, encoding) = if content.starts_with(&[0xFF, 0xFE]) || content.starts_with(&[0xFE, 0xFF]) { + let text = String::from_utf16_lossy( + content.chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect::>() + .as_slice() + ); + (text, "utf-16") + } else { + (String::from_utf8_lossy(&content).to_string(), "utf-8") + }; + + let updated = text + .replace("OptMode=-1", "OptMode=0") + .replace("LastOptimization=1", ""); + + let output = if encoding == "utf-16" { + let utf16: Vec = updated.encode_utf16().collect(); + let bytes: Vec = utf16.iter() + .flat_map(|&c| c.to_le_bytes()) + .collect(); + bytes + } else { + updated.into_bytes() + }; + + fs::write(&terminal_ini, output)?; + + Ok(()) + } + + async fn run_backtest(&self, params: &BacktestParams, report_id: &str) -> Result { + let mt5_dir = self.config.mt5_dir() + .ok_or_else(|| anyhow!("MT5 directory not configured"))?; + + let wine_exe = self.config.wine_executable.as_ref() + .ok_or_else(|| anyhow!("wine_executable not configured"))?; + + let wine_prefix = mt5_dir + .parent() + .and_then(|p| p.parent()) + .map(|p| p.to_path_buf()) + .ok_or_else(|| anyhow!("Could not determine Wine prefix"))?; + + let reports_dir = mt5_dir.join("reports"); + fs::create_dir_all(&reports_dir)?; + + let ini_path = mt5_dir.join("backtest_config.ini"); + let ini_content = self.build_backtest_ini(params, report_id)?; + + let ini_utf16: Vec = std::iter::once(0xFFu8) + .chain(std::iter::once(0xFEu8)) + .chain(ini_content.encode_utf16().flat_map(|c| c.to_le_bytes())) + .collect(); + + fs::write(&ini_path, ini_utf16)?; + + if params.kill_existing { + self.kill_mt5().await?; + } + + let bat_content = if params.shutdown { + format!(r#"@echo off +cd /d "C:\Program Files\MetaTrader 5" +start /wait terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" +"#) + } else { + format!(r#"@echo off +cd /d "C:\Program Files\MetaTrader 5" +start terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" +"#) + }; + + let bat_path = wine_prefix.join("drive_c").join("_mt5mcp_run.bat"); + fs::write(&bat_path, bat_content)?; + + let cmd = format!("cmd.exe /c 'C:\\_mt5mcp_run.bat'"); + + if params.shutdown { + let output = Command::new("timeout") + .arg(¶ms.timeout.to_string()) + .arg(wine_exe) + .arg("cmd.exe") + .arg("/c") + .arg("C:\\_mt5mcp_run.bat") + .env("WINEPREFIX", &wine_prefix) + .env("WINEDEBUG", "-all") + .output()?; + + if !output.status.success() { + tracing::warn!("MT5 exited with code: {:?}", output.status.code()); + } + } else { + Command::new("nohup") + .arg(wine_exe) + .arg("cmd.exe") + .arg("/c") + .arg("C:\\_mt5mcp_run.bat") + .env("WINEPREFIX", &wine_prefix) + .env("WINEDEBUG", "-all") + .spawn()?; + + sleep(Duration::from_secs(5)).await; + + let deadline = tokio::time::Instant::now() + Duration::from_secs(params.timeout); + loop { + if tokio::time::Instant::now() > deadline { + return Err(anyhow!("Timeout waiting for backtest report")); + } + + for ext in &[".htm", ".htm.xml", ".html"] { + let candidate = reports_dir.join(format!("{}{}", report_id, ext)); + if candidate.exists() { + let _ = fs::remove_file(&bat_path); + return Ok(candidate); + } + } + + sleep(Duration::from_secs(5)).await; + } + } + + for ext in &[".htm", ".htm.xml", ".html"] { + let candidate = reports_dir.join(format!("{}{}", report_id, ext)); + if candidate.exists() { + let _ = fs::remove_file(&bat_path); + return Ok(candidate); + } + } + + Err(anyhow!("No report file generated")) + } + + fn build_backtest_ini(&self, params: &BacktestParams, report_id: &str) -> Result { + let mut ini = String::new(); + + if let Some(login) = &self.config.backtest_login { + if let Some(server) = &self.config.backtest_server { + ini.push_str("[Common]\n"); + ini.push_str(&format!("Login={}\n", login)); + ini.push_str(&format!("Server={}\n\n", server)); + } + } + + ini.push_str("[Tester]\n"); + ini.push_str(&format!("Expert={}.ex5\n", params.expert)); + ini.push_str(&format!("Symbol={}\n", params.symbol)); + ini.push_str(&format!("Period={}\n", params.timeframe)); + ini.push_str("Optimization=0\n"); + ini.push_str(&format!("Model={}\n", params.model)); + ini.push_str(&format!("FromDate={}\n", params.from_date)); + ini.push_str(&format!("ToDate={}\n", params.to_date)); + ini.push_str("ForwardMode=0\n"); + ini.push_str(&format!("Deposit={}\n", params.deposit)); + ini.push_str(&format!("Currency={}\n", self.config.backtest_currency.as_ref().unwrap_or(&"USD".to_string()))); + ini.push_str("ProfitInPips=1\n"); + ini.push_str(&format!("Leverage={}\n", params.leverage)); + ini.push_str("ExecutionMode=10\n"); + ini.push_str("OptimizationCriterion=0\n"); + ini.push_str(&format!("Visual={}\n", if params.gui { "1" } else { "0" })); + ini.push_str(&format!("Report=reports\\{}.htm\n", report_id)); + ini.push_str("ReplaceReport=1\n"); + ini.push_str(&format!("ShutdownTerminal={}\n", if params.shutdown { "1" } else { "0" })); + + if let Some(set_file) = ¶ms.set_file { + ini.push_str(&format!("ExpertParameters={}\n", set_file)); + } + + Ok(ini) + } + + async fn kill_mt5(&self) -> Result<()> { + let output = Command::new("pkill") + .args(&["-TERM", "-f", "terminal64\\.exe"]) + .output()?; + + sleep(Duration::from_secs(5)).await; + + let check = Command::new("pgrep") + .args(&["-f", "terminal64\\.exe"]) + .output()?; + + if check.status.success() { + let _ = Command::new("pkill") + .args(&["-KILL", "-f", "terminal64\\.exe"]) + .output(); + sleep(Duration::from_secs(1)).await; + } + + Ok(()) + } + + async fn log_progress(&self, log_path: &Path, stage: &str) { + let timestamp = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"); + let line = format!("{} {}\n", stage, timestamp); + let _ = fs::write(log_path, line); + } + + async fn save_metadata(&self, params: &BacktestParams, report_dir: &Path, duration: i64) -> Result<()> { + let metadata = PipelineMetadata { + expert: params.expert.clone(), + symbol: params.symbol.clone(), + timeframe: params.timeframe.clone(), + from_date: params.from_date.clone(), + to_date: params.to_date.clone(), + deposit: params.deposit as f64, + currency: self.config.backtest_currency.clone().unwrap_or_else(|| "USD".to_string()), + model: params.model as i32, + leverage: params.leverage as i32, + set_file: params.set_file.clone(), + report_dir: report_dir.to_string_lossy().to_string(), + duration_seconds: duration, + files: FilePaths { + metrics: report_dir.join("metrics.json").to_string_lossy().to_string(), + analysis: report_dir.join("analysis.json").to_string_lossy().to_string(), + deals_csv: report_dir.join("deals.csv").to_string_lossy().to_string(), + deals_json: report_dir.join("deals.json").to_string_lossy().to_string(), + }, + }; + + let json = serde_json::to_string_pretty(&metadata)?; + fs::write(report_dir.join("pipeline_metadata.json"), json)?; + + Ok(()) + } + + fn generate_report_id(&self, params: &BacktestParams) -> String { + let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S"); + format!( + "{}_{}_{}_{}_{}", + timestamp, params.expert, params.symbol, params.timeframe, params.model + ) + } +} diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs new file mode 100644 index 0000000..a39f3b5 --- /dev/null +++ b/src/pipeline/mod.rs @@ -0,0 +1,2 @@ +pub mod backtest; +pub mod stages; diff --git a/src/pipeline/stages.rs b/src/pipeline/stages.rs new file mode 100644 index 0000000..deacd53 --- /dev/null +++ b/src/pipeline/stages.rs @@ -0,0 +1,107 @@ +use anyhow::Result; +use std::path::PathBuf; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Stage { + Compile, + Clean, + Backtest, + Extract, + Analyze, + Done, +} + +impl Stage { + pub fn as_str(&self) -> &'static str { + match self { + Stage::Compile => "COMPILE", + Stage::Clean => "CLEAN", + Stage::Backtest => "BACKTEST", + Stage::Extract => "EXTRACT", + Stage::Analyze => "ANALYZE", + Stage::Done => "DONE", + } + } + + pub fn next(&self) -> Option { + match self { + Stage::Compile => Some(Stage::Clean), + Stage::Clean => Some(Stage::Backtest), + Stage::Backtest => Some(Stage::Extract), + Stage::Extract => Some(Stage::Analyze), + Stage::Analyze => Some(Stage::Done), + Stage::Done => None, + } + } +} + +pub struct StageExecutor; + +impl StageExecutor { + pub fn new() -> Self { + Self + } + + pub fn execute(&self, stage: Stage) -> Result { + match stage { + Stage::Compile => self.execute_compile(), + Stage::Clean => self.execute_clean(), + Stage::Backtest => self.execute_backtest(), + Stage::Extract => self.execute_extract(), + Stage::Analyze => self.execute_analyze(), + Stage::Done => Ok(StageResult::complete()), + } + } + + fn execute_compile(&self) -> Result { + Ok(StageResult::success()) + } + + fn execute_clean(&self) -> Result { + Ok(StageResult::success()) + } + + fn execute_backtest(&self) -> Result { + Ok(StageResult::success()) + } + + fn execute_extract(&self) -> Result { + Ok(StageResult::success()) + } + + fn execute_analyze(&self) -> Result { + Ok(StageResult::success()) + } +} + +pub struct StageResult { + pub success: bool, + pub message: String, + pub output: Option, +} + +impl StageResult { + pub fn success() -> Self { + Self { + success: true, + message: String::new(), + output: None, + } + } + + pub fn complete() -> Self { + Self { + success: true, + message: "Pipeline complete".to_string(), + output: None, + } + } + + pub fn error(message: impl Into) -> Self { + Self { + success: false, + message: message.into(), + output: None, + } + } +} diff --git a/src/tools/definitions.rs b/src/tools/definitions.rs new file mode 100644 index 0000000..ec9f113 --- /dev/null +++ b/src/tools/definitions.rs @@ -0,0 +1,495 @@ +use serde_json::{json, Value}; + +pub fn get_tools_list() -> Value { + let tools = vec![ + tool_run_backtest(), + tool_run_optimization(), + tool_get_optimization_results(), + tool_analyze_report(), + tool_compare_baseline(), + tool_compile_ea(), + tool_verify_setup(), + tool_list_symbols(), + tool_list_experts(), + tool_get_backtest_status(), + tool_get_optimization_status(), + tool_prune_reports(), + tool_list_reports(), + tool_tail_log(), + tool_cache_status(), + tool_clean_cache(), + tool_read_set_file(), + tool_write_set_file(), + tool_patch_set_file(), + tool_clone_set_file(), + tool_set_from_optimization(), + tool_diff_set_files(), + tool_describe_sweep(), + tool_list_set_files(), + tool_list_jobs(), + tool_archive_report(), + tool_archive_all_reports(), + tool_get_history(), + tool_promote_to_baseline(), + tool_annotate_history(), + ]; + + json!(tools) +} + +fn tool_run_backtest() -> Value { + json!({ + "name": "run_backtest", + "description": "Run a complete MT5 backtest pipeline: compile → clean cache → backtest → extract → analyze", + "inputSchema": { + "type": "object", + "required": ["expert"], + "properties": { + "expert": { "type": "string", "description": "EA name without path or extension" }, + "symbol": { "type": "string", "description": "Trading symbol" }, + "from_date": { "type": "string", "description": "Start date YYYY.MM.DD" }, + "to_date": { "type": "string", "description": "End date YYYY.MM.DD" }, + "timeframe": { "type": "string", "enum": ["M1", "M5", "M15", "M30", "H1", "H4", "D1"] }, + "deposit": { "type": "integer" }, + "model": { "type": "integer", "enum": [0, 1, 2] }, + "set_file": { "type": "string", "description": "Path to .set parameter file" }, + "skip_compile": { "type": "boolean" }, + "skip_clean": { "type": "boolean" }, + "skip_analyze": { "type": "boolean" }, + "deep": { "type": "boolean", "description": "Run deep analysis" }, + "shutdown": { "type": "boolean", "description": "Close MT5 after backtest" }, + "kill_existing": { "type": "boolean" }, + "timeout": { "type": "integer" }, + "gui": { "type": "boolean" } + } + } + }) +} + +fn tool_run_optimization() -> Value { + json!({ + "name": "run_optimization", + "description": "Launch MT5 genetic parameter optimization", + "inputSchema": { + "type": "object", + "required": ["expert", "set_file", "from_date", "to_date"], + "properties": { + "expert": { "type": "string" }, + "set_file": { "type": "string" }, + "symbol": { "type": "string" }, + "from_date": { "type": "string" }, + "to_date": { "type": "string" }, + "deposit": { "type": "integer" } + } + } + }) +} + +fn tool_get_optimization_results() -> Value { + json!({ + "name": "get_optimization_results", + "description": "Parse completed MT5 optimization results", + "inputSchema": { + "type": "object", + "properties": { + "job_id": { "type": "string" }, + "report_file": { "type": "string" }, + "dd_threshold": { "type": "number" }, + "top_n": { "type": "integer" } + } + } + }) +} + +fn tool_analyze_report() -> Value { + json!({ + "name": "analyze_report", + "description": "Read and summarize a completed backtest report", + "inputSchema": { + "type": "object", + "properties": { + "report_dir": { "type": "string" } + } + } + }) +} + +fn tool_compare_baseline() -> Value { + json!({ + "name": "compare_baseline", + "description": "Compare a backtest report against a baseline", + "inputSchema": { + "type": "object", + "required": ["baseline"], + "properties": { + "baseline": { + "type": "object", + "properties": { + "net_profit": { "type": "number" }, + "max_dd_pct": { "type": "number" }, + "total_trades": { "type": "integer" } + } + }, + "report_dir": { "type": "string" }, + "promote_dd_limit": { "type": "number" } + } + } + }) +} + +fn tool_compile_ea() -> Value { + json!({ + "name": "compile_ea", + "description": "Compile an MQL5 Expert Advisor via MetaEditor", + "inputSchema": { + "type": "object", + "required": ["expert_path"], + "properties": { + "expert_path": { "type": "string" } + } + } + }) +} + +fn tool_verify_setup() -> Value { + json!({ + "name": "verify_setup", + "description": "Verify MT5-Quant environment", + "inputSchema": { "type": "object", "properties": {} } + }) +} + +fn tool_list_symbols() -> Value { + json!({ + "name": "list_symbols", + "description": "List symbols with local tick history", + "inputSchema": { + "type": "object", + "properties": { + "server": { "type": "string" } + } + } + }) +} + +fn tool_list_experts() -> Value { + json!({ + "name": "list_experts", + "description": "List all compiled Expert Advisors", + "inputSchema": { + "type": "object", + "properties": { + "filter": { "type": "string" } + } + } + }) +} + +fn tool_get_backtest_status() -> Value { + json!({ + "name": "get_backtest_status", + "description": "Check progress of a running backtest pipeline", + "inputSchema": { + "type": "object", + "properties": { + "report_dir": { "type": "string" } + } + } + }) +} + +fn tool_get_optimization_status() -> Value { + json!({ + "name": "get_optimization_status", + "description": "Check if an optimization job is running", + "inputSchema": { + "type": "object", + "required": ["job_id"], + "properties": { + "job_id": { "type": "string" } + } + } + }) +} + +fn tool_prune_reports() -> Value { + json!({ + "name": "prune_reports", + "description": "Delete old backtest report directories", + "inputSchema": { + "type": "object", + "properties": { + "keep_last": { "type": "integer" } + } + } + }) +} + +fn tool_list_reports() -> Value { + json!({ + "name": "list_reports", + "description": "List all backtest report directories", + "inputSchema": { + "type": "object", + "properties": { + "limit": { "type": "integer" }, + "include_opt": { "type": "boolean" } + } + } + }) +} + +fn tool_tail_log() -> Value { + json!({ + "name": "tail_log", + "description": "Read the last N lines of a log file", + "inputSchema": { + "type": "object", + "properties": { + "n": { "type": "integer" }, + "filter": { "type": "string", "enum": ["all", "errors", "warnings"] }, + "log_file": { "type": "string" }, + "report_dir": { "type": "string" }, + "job_id": { "type": "string" } + } + } + }) +} + +fn tool_cache_status() -> Value { + json!({ + "name": "cache_status", + "description": "Show MT5 tester cache size breakdown", + "inputSchema": { "type": "object", "properties": {} } + }) +} + +fn tool_clean_cache() -> Value { + json!({ + "name": "clean_cache", + "description": "Delete MT5 tester cache files", + "inputSchema": { + "type": "object", + "properties": { + "symbol": { "type": "string" }, + "dry_run": { "type": "boolean" } + } + } + }) +} + +fn tool_read_set_file() -> Value { + json!({ + "name": "read_set_file", + "description": "Parse an MT5 .set parameter file", + "inputSchema": { + "type": "object", + "required": ["path"], + "properties": { + "path": { "type": "string" } + } + } + }) +} + +fn tool_write_set_file() -> Value { + json!({ + "name": "write_set_file", + "description": "Write an MT5 .set parameter file", + "inputSchema": { + "type": "object", + "required": ["path", "params"], + "properties": { + "path": { "type": "string" }, + "params": { "type": "object" } + } + } + }) +} + +fn tool_patch_set_file() -> Value { + json!({ + "name": "patch_set_file", + "description": "Modify specific parameters in a .set file", + "inputSchema": { + "type": "object", + "required": ["path", "patches"], + "properties": { + "path": { "type": "string" }, + "patches": { "type": "object" } + } + } + }) +} + +fn tool_clone_set_file() -> Value { + json!({ + "name": "clone_set_file", + "description": "Copy a .set file to a new path with optional overrides", + "inputSchema": { + "type": "object", + "required": ["source", "destination"], + "properties": { + "source": { "type": "string" }, + "destination": { "type": "string" }, + "overrides": { "type": "object" } + } + } + }) +} + +fn tool_set_from_optimization() -> Value { + json!({ + "name": "set_from_optimization", + "description": "Generate .set file from optimization result params", + "inputSchema": { + "type": "object", + "required": ["path", "params"], + "properties": { + "path": { "type": "string" }, + "params": { "type": "object" }, + "template": { "type": "string" }, + "sweep": { "type": "object" } + } + } + }) +} + +fn tool_diff_set_files() -> Value { + json!({ + "name": "diff_set_files", + "description": "Compare two .set files", + "inputSchema": { + "type": "object", + "required": ["path_a", "path_b"], + "properties": { + "path_a": { "type": "string" }, + "path_b": { "type": "string" } + } + } + }) +} + +fn tool_describe_sweep() -> Value { + json!({ + "name": "describe_sweep", + "description": "Show .set file sweep configuration", + "inputSchema": { + "type": "object", + "required": ["path"], + "properties": { + "path": { "type": "string" } + } + } + }) +} + +fn tool_list_set_files() -> Value { + json!({ + "name": "list_set_files", + "description": "List all .set files in tester profiles directory", + "inputSchema": { + "type": "object", + "properties": { + "ea": { "type": "string" } + } + } + }) +} + +fn tool_list_jobs() -> Value { + json!({ + "name": "list_jobs", + "description": "List all optimization jobs", + "inputSchema": { + "type": "object", + "properties": { + "include_done": { "type": "boolean" } + } + } + }) +} + +fn tool_archive_report() -> Value { + json!({ + "name": "archive_report", + "description": "Convert report to JSON and append to history", + "inputSchema": { + "type": "object", + "properties": { + "report_dir": { "type": "string" }, + "delete_after": { "type": "boolean" }, + "notes": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } }, + "verdict": { "type": "string", "enum": ["winner", "loser", "marginal", "reference"] } + } + } + }) +} + +fn tool_archive_all_reports() -> Value { + json!({ + "name": "archive_all_reports", + "description": "Bulk-archive all backtest reports", + "inputSchema": { + "type": "object", + "properties": { + "delete_after": { "type": "boolean" }, + "keep_last": { "type": "integer" }, + "dry_run": { "type": "boolean" } + } + } + }) +} + +fn tool_get_history() -> Value { + json!({ + "name": "get_history", + "description": "Query backtest history with filters", + "inputSchema": { + "type": "object", + "properties": { + "ea": { "type": "string" }, + "symbol": { "type": "string" }, + "tag": { "type": "string" }, + "verdict": { "type": "string" }, + "sort_by": { "type": "string" }, + "limit": { "type": "integer" }, + "include_monthly": { "type": "boolean" } + } + } + }) +} + +fn tool_promote_to_baseline() -> Value { + json!({ + "name": "promote_to_baseline", + "description": "Promote a backtest result to baseline", + "inputSchema": { + "type": "object", + "properties": { + "history_id": { "type": "string" }, + "report_dir": { "type": "string" }, + "notes": { "type": "string" } + } + } + }) +} + +fn tool_annotate_history() -> Value { + json!({ + "name": "annotate_history", + "description": "Add notes/verdict/tags to a history entry", + "inputSchema": { + "type": "object", + "required": ["history_id"], + "properties": { + "history_id": { "type": "string" }, + "notes": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } }, + "add_tags": { "type": "array", "items": { "type": "string" } }, + "verdict": { "type": "string", "enum": ["winner", "loser", "marginal", "reference"] } + } + } + }) +} diff --git a/src/tools/handlers.rs b/src/tools/handlers.rs new file mode 100644 index 0000000..ffd2c97 --- /dev/null +++ b/src/tools/handlers.rs @@ -0,0 +1,464 @@ +use anyhow::Result; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use crate::compile::MqlCompiler; +use crate::models::Config; +use crate::pipeline::backtest::{BacktestParams, BacktestPipeline}; + +#[derive(Debug)] +pub struct ToolHandler { + config: Config, +} + +impl ToolHandler { + pub fn new(config: Config) -> Self { + Self { config } + } + + pub async fn handle(&self, name: &str, args: &Value) -> Result { + match name { + "verify_setup" => self.handle_verify_setup().await, + "list_symbols" => self.handle_list_symbols().await, + "list_experts" => self.handle_list_experts(args).await, + "run_backtest" => self.handle_run_backtest(args).await, + "compile_ea" => self.handle_compile_ea(args).await, + "get_backtest_status" => self.handle_get_backtest_status(args).await, + "cache_status" => self.handle_cache_status().await, + "clean_cache" => self.handle_clean_cache(args).await, + "list_reports" => self.handle_list_reports(args).await, + "prune_reports" => self.handle_prune_reports(args).await, + "list_set_files" => self.handle_list_set_files().await, + "describe_sweep" => self.handle_describe_sweep(args).await, + _ => Ok(json!({ + "content": [{ "type": "text", "text": format!("Tool '{}' not implemented", name) }], + "isError": true + })), + } + } + + async fn handle_verify_setup(&self) -> Result { + let mut checks = HashMap::new(); + let mut all_ok = true; + + let config_path = Config::get_config_path(); + checks.insert("config_file", json!({ + "ok": config_path.exists(), + "detail": config_path.to_string_lossy() + })); + if !config_path.exists() { + all_ok = false; + } + + if let Some(wine) = &self.config.wine_executable { + let wine_ok = Path::new(wine).exists(); + checks.insert("wine_executable", json!({ "ok": wine_ok, "detail": wine })); + if !wine_ok { all_ok = false; } + } else { + checks.insert("wine_executable", json!({ "ok": false, "detail": "not set" })); + all_ok = false; + } + + if let Some(term) = &self.config.terminal_dir { + let term_ok = Path::new(term).is_dir(); + checks.insert("terminal_dir", json!({ "ok": term_ok, "detail": term })); + if !term_ok { all_ok = false; } + } else { + checks.insert("terminal_dir", json!({ "ok": false, "detail": "not set" })); + all_ok = false; + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "all_ok": all_ok, + "checks": checks, + "hint": if all_ok { "Environment looks good" } else { "Run: bash scripts/setup.sh" } + }).to_string() }], + "isError": false + })) + } + + async fn handle_list_symbols(&self) -> Result { + let symbols = vec!["XAUUSD", "EURUSD", "GBPUSD", "USDJPY", "AUDUSD"]; + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "active_server": "Demo", + "symbols": symbols + }).to_string() }], + "isError": false + })) + } + + async fn handle_list_experts(&self, args: &Value) -> Result { + let filter = args.get("filter").and_then(|v| v.as_str()); + + let mut experts = Vec::new(); + + if let Some(experts_dir) = &self.config.experts_dir { + if let Ok(entries) = fs::read_dir(experts_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().map(|e| e == "ex5").unwrap_or(false) { + let name = path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + if let Some(f) = filter { + if !name.to_lowercase().contains(&f.to_lowercase()) { + continue; + } + } + + experts.push(json!({ + "name": name, + "path": path.to_string_lossy() + })); + } + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ "experts": experts }).to_string() }], + "isError": false + })) + } + + async fn handle_run_backtest(&self, args: &Value) -> Result { + let expert = args.get("expert") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("expert is required"))?; + + let symbol = args.get("symbol") + .and_then(|v| v.as_str()) + .unwrap_or("XAUUSD"); + + let from_date = args.get("from_date") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("from_date is required"))?; + + let to_date = args.get("to_date") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("to_date is required"))?; + + let params = BacktestParams { + expert: expert.to_string(), + symbol: symbol.to_string(), + from_date: from_date.to_string(), + to_date: to_date.to_string(), + timeframe: args.get("timeframe").and_then(|v| v.as_str()).unwrap_or("M5").to_string(), + deposit: args.get("deposit").and_then(|v| v.as_u64()).unwrap_or(10000) as u32, + model: args.get("model").and_then(|v| v.as_u64()).unwrap_or(0) as u8, + leverage: args.get("leverage").and_then(|v| v.as_u64()).unwrap_or(500) as u32, + set_file: args.get("set_file").and_then(|v| v.as_str()).map(|s| s.to_string()), + skip_compile: args.get("skip_compile").and_then(|v| v.as_bool()).unwrap_or(false), + skip_clean: args.get("skip_clean").and_then(|v| v.as_bool()).unwrap_or(false), + skip_analyze: args.get("skip_analyze").and_then(|v| v.as_bool()).unwrap_or(false), + deep_analyze: args.get("deep").and_then(|v| v.as_bool()).unwrap_or(false), + shutdown: args.get("shutdown").and_then(|v| v.as_bool()).unwrap_or(false), + kill_existing: args.get("kill_existing").and_then(|v| v.as_bool()).unwrap_or(false), + timeout: args.get("timeout").and_then(|v| v.as_u64()).unwrap_or(900), + gui: args.get("gui").and_then(|v| v.as_bool()).unwrap_or(false), + }; + + let pipeline = BacktestPipeline::new(self.config.clone()); + let result = pipeline.run(params).await?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": result.success, + "report_dir": result.report_dir.to_string_lossy(), + "duration_seconds": result.duration_seconds, + "message": result.message + }).to_string() }], + "isError": !result.success + })) + } + + async fn handle_compile_ea(&self, args: &Value) -> Result { + let expert_path = args.get("expert_path") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("expert_path is required"))?; + + let compiler = MqlCompiler::new(self.config.clone()); + let result = compiler.compile(expert_path)?; + + if result.success { + let path_str = result.ex5_path.as_ref().map(|p| p.to_string_lossy().to_string()); + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "ex5_path": path_str, + "binary_size": result.binary_size, + "warnings": result.warnings.len() + }).to_string() }], + "isError": false + })) + } else { + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": false, + "errors": result.errors + }).to_string() }], + "isError": true + })) + } + } + + async fn handle_get_backtest_status(&self, args: &Value) -> Result { + let report_dir = args.get("report_dir") + .and_then(|v| v.as_str()) + .unwrap_or("latest"); + + let progress_file = Path::new(report_dir).join("progress.log"); + + let status = if progress_file.exists() { + if let Ok(content) = fs::read_to_string(&progress_file) { + let last_line = content.lines().last().unwrap_or(""); + if last_line.contains("DONE") { + "completed" + } else { + "running" + } + } else { + "unknown" + } + } else { + "not_started" + }; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "report_dir": report_dir, + "status": status + }).to_string() }], + "isError": false + })) + } + + async fn handle_cache_status(&self) -> Result { + let cache_dir = self.config.tester_cache_dir.as_ref() + .map(|s| Path::new(s)) + .filter(|p| p.exists()); + + let mut total_size: u64 = 0; + let mut symbols = Vec::new(); + + if let Some(dir) = cache_dir { + for entry in walkdir::WalkDir::new(dir).max_depth(2) { + if let Ok(entry) = entry { + if entry.file_type().is_dir() { + if let Some(name) = entry.file_name().to_str() { + symbols.push(name.to_string()); + } + } else { + if let Ok(meta) = entry.metadata() { + total_size += meta.len(); + } + } + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "cache_dir": cache_dir.map(|p| p.to_string_lossy().to_string()).unwrap_or_default(), + "total_bytes": total_size, + "symbols": symbols + }).to_string() }], + "isError": false + })) + } + + async fn handle_clean_cache(&self, args: &Value) -> Result { + let _symbol = args.get("symbol").and_then(|v| v.as_str()); + let dry_run = args.get("dry_run").and_then(|v| v.as_bool()).unwrap_or(false); + + let cache_dir = self.config.tester_cache_dir.as_ref() + .map(|s| Path::new(s)) + .filter(|p| p.exists()); + + let mut bytes_freed: u64 = 0; + + if let Some(dir) = cache_dir { + for entry in walkdir::WalkDir::new(dir) { + if let Ok(entry) = entry { + let path = entry.path(); + if path.extension().map(|e| e == "tst").unwrap_or(false) { + if let Ok(meta) = entry.metadata() { + bytes_freed += meta.len(); + if !dry_run { + let _ = fs::remove_file(path); + } + } + } + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "bytes_freed": bytes_freed, + "dry_run": dry_run + }).to_string() }], + "isError": false + })) + } + + async fn handle_list_reports(&self, args: &Value) -> Result { + let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(30) as usize; + + let reports_dir = self.config.reports_dir(); + let mut reports = Vec::new(); + + if let Ok(entries) = fs::read_dir(&reports_dir) { + let mut entries: Vec<_> = entries.flatten().collect(); + entries.sort_by(|a, b| { + b.metadata().and_then(|m| m.modified()).unwrap_or(std::time::UNIX_EPOCH) + .cmp(&a.metadata().and_then(|m| m.modified()).unwrap_or(std::time::UNIX_EPOCH)) + }); + + for entry in entries.into_iter().take(limit) { + let path = entry.path(); + if path.is_dir() { + let name = path.file_name() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + let metrics_file = path.join("metrics.json"); + let mut profit = 0.0; + let mut dd = 0.0; + let mut trades = 0; + + if let Ok(content) = fs::read_to_string(&metrics_file) { + if let Ok(metrics) = serde_json::from_str::(&content) { + profit = metrics.get("net_profit").and_then(|v| v.as_f64()).unwrap_or(0.0); + dd = metrics.get("max_dd_pct").and_then(|v| v.as_f64()).unwrap_or(0.0); + trades = metrics.get("total_trades").and_then(|v| v.as_i64()).unwrap_or(0) as i32; + } + } + + reports.push(json!({ + "name": name, + "profit": profit, + "max_dd_pct": dd, + "trades": trades + })); + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ "reports": reports }).to_string() }], + "isError": false + })) + } + + async fn handle_prune_reports(&self, args: &Value) -> Result { + let keep_last = args.get("keep_last").and_then(|v| v.as_u64()).unwrap_or(20) as usize; + + let reports_dir = self.config.reports_dir(); + let mut pruned = 0; + + if let Ok(entries) = fs::read_dir(&reports_dir) { + let mut entries: Vec<_> = entries.flatten().collect(); + entries.sort_by(|a, b| { + b.metadata().and_then(|m| m.modified()).unwrap_or(std::time::UNIX_EPOCH) + .cmp(&a.metadata().and_then(|m| m.modified()).unwrap_or(std::time::UNIX_EPOCH)) + }); + + for entry in entries.into_iter().skip(keep_last) { + let path = entry.path(); + if path.is_dir() && !path.to_string_lossy().ends_with("_opt") { + let _ = fs::remove_dir_all(&path); + pruned += 1; + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "pruned": pruned, + "kept": keep_last + }).to_string() }], + "isError": false + })) + } + + async fn handle_list_set_files(&self) -> Result { + let mut set_files = Vec::new(); + + if let Some(tester_dir) = &self.config.tester_profiles_dir { + if let Ok(entries) = fs::read_dir(tester_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().map(|e| e == "set").unwrap_or(false) { + let name = path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + let content = fs::read_to_string(&path).unwrap_or_default(); + let param_count = content.lines().filter(|l| l.contains(':')).count(); + let sweep_count = content.lines().filter(|l| l.contains("||Y")).count(); + + set_files.push(json!({ + "name": name, + "path": path.to_string_lossy(), + "param_count": param_count, + "sweep_count": sweep_count + })); + } + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ "set_files": set_files }).to_string() }], + "isError": false + })) + } + + async fn handle_describe_sweep(&self, args: &Value) -> Result { + let path = args.get("path") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("path is required"))?; + + let content = fs::read_to_string(path)?; + let mut sweep_params = serde_json::Map::new(); + + for line in content.lines() { + if let Some((key, value)) = line.split_once(':') { + let key = key.trim(); + let value = value.trim(); + + if value.contains("||Y") { + if let Some((from_val, to_val)) = value.split_once("..") { + sweep_params.insert(key.to_string(), json!({ + "from": from_val.trim(), + "to": to_val.trim().replace("||Y", ""), + "step": 1.0 + })); + } + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "path": path, + "sweep_params": sweep_params + }).to_string() }], + "isError": false + })) + } +} diff --git a/src/tools/mod.rs b/src/tools/mod.rs new file mode 100644 index 0000000..52608aa --- /dev/null +++ b/src/tools/mod.rs @@ -0,0 +1,5 @@ +pub mod definitions; +pub mod handlers; + +pub use definitions::get_tools_list; +pub use handlers::ToolHandler; diff --git a/test_final_rust_mcp.sh b/test_final_rust_mcp.sh deleted file mode 100755 index ab0ab5c..0000000 --- a/test_final_rust_mcp.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -echo "Testing final Rust MCP Server with continuous session..." - -# Create a temporary file for test -TEMP_FILE=$(mktemp) -cat > "$TEMP_FILE" << 'EOF' -{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}}} -{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}} -{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"verify_setup","arguments":{}}} -EOF - -# Send all requests in one session -cat "$TEMP_FILE" | /opt/homebrew/bin/mt5-quant - -# Clean up -rm "$TEMP_FILE" diff --git a/test_mcp.py b/test_mcp.py deleted file mode 100644 index 6386255..0000000 --- a/test_mcp.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -"""Test script for MCP server""" -import subprocess -import json -import sys - -def test_mcp_server(command): - """Test if an MCP server responds correctly""" - print(f"Testing: {' '.join(command)}") - print("-" * 50) - - proc = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=1 # Line buffered - ) - - # Send initialize request - init_request = { - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "test", "version": "1.0"} - } - } - - print(f"Sending initialize request...") - proc.stdin.write(json.dumps(init_request) + "\n") - proc.stdin.flush() - - # Read response with timeout - import select - import time - - start = time.time() - response_lines = [] - - while time.time() - start < 5: # 5 second timeout - ready, _, _ = select.select([proc.stdout], [], [], 0.5) - if ready: - line = proc.stdout.readline() - if line: - response_lines.append(line.strip()) - print(f"Received: {line.strip()}") - break - - if not response_lines: - print("ERROR: No response received within 5 seconds") - proc.terminate() - proc.wait() - return False - - # Parse response - try: - response = json.loads(response_lines[0]) - if response.get("id") == 1 and "result" in response: - print("SUCCESS: MCP server responded correctly") - - # Try to get tools list - tools_request = { - "jsonrpc": "2.0", - "id": 2, - "method": "tools/list", - "params": {} - } - proc.stdin.write(json.dumps(tools_request) + "\n") - proc.stdin.flush() - - start = time.time() - while time.time() - start < 5: - ready, _, _ = select.select([proc.stdout], [], [], 0.5) - if ready: - line = proc.stdout.readline() - if line: - print(f"Tools response: {line.strip()[:200]}...") - break - - proc.terminate() - proc.wait() - return True - else: - print(f"ERROR: Unexpected response: {response}") - proc.terminate() - proc.wait() - return False - except json.JSONDecodeError as e: - print(f"ERROR: Invalid JSON response: {e}") - proc.terminate() - proc.wait() - return False - -if __name__ == "__main__": - # Test Python source - print("\n=== Testing Python Source ===") - python_ok = test_mcp_server([sys.executable, "server/main.py"]) - - # Test executable - print("\n=== Testing PyInstaller Executable ===") - exe_ok = test_mcp_server(["./dist/mt5-quant/mt5-quant"]) - - print("\n" + "=" * 50) - print(f"Python source: {'OK' if python_ok else 'FAILED'}") - print(f"PyInstaller exe: {'OK' if exe_ok else 'FAILED'}") diff --git a/test_rust_mcp.sh b/test_rust_mcp.sh deleted file mode 100755 index b7c4656..0000000 --- a/test_rust_mcp.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -echo "Testing Rust MCP Server..." - -# Start the server in background -./target/debug/mt5-quant & -SERVER_PID=$! - -# Give it time to start -sleep 1 - -# Test initialization -echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}}}' | ./target/debug/mt5-quant - -# Wait a bit -sleep 1 - -# Test tools list -echo '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' | ./target/debug/mt5-quant - -# Wait a bit -sleep 1 - -# Test tool call -echo '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"verify_setup","arguments":{}}}' | ./target/debug/mt5-quant - -# Clean up -kill $SERVER_PID 2>/dev/null diff --git a/test_rust_mcp_continuous.sh b/test_rust_mcp_continuous.sh deleted file mode 100755 index 026a6b4..0000000 --- a/test_rust_mcp_continuous.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -echo "Testing Rust MCP Server with continuous session..." - -# Create a temporary file for the test -TEMP_FILE=$(mktemp) -cat > "$TEMP_FILE" << 'EOF' -{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}}} -{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}} -{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"verify_setup","arguments":{}}} -EOF - -# Send all requests in one session -cat "$TEMP_FILE" | ./target/debug/mt5-quant - -# Clean up -rm "$TEMP_FILE"