Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3ad67d4c57 | |||
| 4c34ae236c | |||
| ecf5606b7a | |||
| 6bfb330b5d | |||
| 52a3d44b99 | |||
| caabc2c03b | |||
| 62c1ca773e | |||
| 3b9d4ce2dd | |||
| 12fba8ad4b | |||
| b6171d0c56 | |||
| 05214ff34b | |||
| 498126dfd4 | |||
| 147da213d6 | |||
| 804c1f8808 | |||
| f8b6a11bb8 | |||
| e82bb5466b | |||
| 50ea45f8cd | |||
| 4bc5ca22e9 | |||
| e44345a05f |
@@ -0,0 +1,4 @@
|
||||
# These are supported funding model platforms
|
||||
|
||||
github: masdevid
|
||||
custom: ["https://www.paypal.me/devidhw", "https://saweria.co/depod"]
|
||||
@@ -0,0 +1,11 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
@@ -88,10 +88,36 @@ jobs:
|
||||
name: linux-binary
|
||||
path: dist/mcp-mt5-quant-linux-x64.tar.gz
|
||||
|
||||
# ── Cargo Publish ────────────────────────────────────────────────────────────
|
||||
|
||||
cargo-publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache Cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: cargo-publish-${{ hashFiles('Cargo.lock') }}
|
||||
restore-keys: cargo-publish-
|
||||
|
||||
- name: Publish to crates.io
|
||||
run: cargo publish --no-verify
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
|
||||
# ── GitHub Release ───────────────────────────────────────────────────────────
|
||||
|
||||
release:
|
||||
needs: [build-macos, build-linux]
|
||||
needs: [build-macos, build-linux, cargo-publish]
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
@@ -1,5 +1,31 @@
|
||||
# Changelog
|
||||
|
||||
## [1.32.4] — 2026-04-25
|
||||
|
||||
### Fixed
|
||||
- **HTML report extraction**: `ShutdownTerminal=1` does not cause `terminal64.exe` to exit on Wine/macOS.
|
||||
Inactivity watchdog now waits 30 s for the report file after the tester log goes quiet, then kills
|
||||
MT5 unconditionally — no longer depends on natural process exit.
|
||||
- **Duplicate deals**: tester agent log writes each deal twice; deduplicated via `HashSet<deal_number>`
|
||||
in `parse_journal_deals`.
|
||||
- **Entry direction inference**: all journal deals were marked `entry="in"` because no direction info
|
||||
exists in the log. Fixed with a per-symbol position tracker (signed lot accumulation) to infer
|
||||
"in" / "out" correctly.
|
||||
- **`list_deals` returning 0 results**: `is_closed_trade()` was gating on `profit != 0.0`; journal
|
||||
deals legitimately have zero profit. Removed the profit gate.
|
||||
- **Wrong tester log selected**: `Agent-0.0.0.0` logs only contain startup lines, not deal lines.
|
||||
`find_active_tester_agent_log` now prefers `Agent-127.0.0.1` and tiebreaks by file size.
|
||||
|
||||
### Added
|
||||
- `launch_backtest` exposes `shutdown` (default `true`) and `inactivity_kill_secs` (default: disabled;
|
||||
set to e.g. `120` to enable the inactivity watchdog) as explicit tool parameters.
|
||||
- Report DB stores deals in SQLite; all analytics tools resolve by `report_id` / `report_dir` / latest.
|
||||
|
||||
### Verified
|
||||
- 1-month DPS21/XAUUSD.cent/M5 backtest produces full HTML report: win_rate=70%,
|
||||
profit_factor=0.77, sharpe=-3.61, max_dd=6.93%. All 17 analytics tools return real data.
|
||||
|
||||
|
||||
## [1.31.5] — 2026-04-22
|
||||
|
||||
- feat: add check_update and update tools with background auto-check
|
||||
|
||||
Generated
+1
-1
@@ -481,7 +481,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mt5-quant"
|
||||
version = "1.32.0"
|
||||
version = "1.32.4"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
|
||||
+3
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mt5-quant"
|
||||
version = "1.32.0"
|
||||
version = "1.32.4"
|
||||
edition = "2021"
|
||||
description = "MCP server for MT5 strategy development on macOS/Linux"
|
||||
authors = ["masdevid <masdevid@example.com>"]
|
||||
@@ -11,6 +11,8 @@ keywords = ["mt5", "mql5", "trading", "mcp", "backtest"]
|
||||
categories = ["finance", "development-tools"]
|
||||
homepage = "https://github.com/masdevid/mt5-quant"
|
||||
documentation = "https://github.com/masdevid/mt5-quant"
|
||||
|
||||
[package.metadata]
|
||||
maintenance = { status = "actively-developed" }
|
||||
|
||||
[[bin]]
|
||||
|
||||
+12
-4
@@ -185,10 +185,18 @@ Fire-and-forget mode: compile → clean → launch MT5 backtest, return immediat
|
||||
deposit?: number; // Initial deposit (default: 10000)
|
||||
model?: 0 | 1 | 2; // Tick model (default: 0)
|
||||
set_file?: string; // Path to .set parameter file
|
||||
skip_compile?: boolean; // Skip compilation
|
||||
skip_clean?: boolean; // Skip cache cleaning
|
||||
timeout?: number; // Max time in seconds (default: 900)
|
||||
gui?: boolean; // Enable MT5 visualization
|
||||
skip_compile?: boolean; // Skip compilation
|
||||
skip_clean?: boolean; // Skip cache cleaning
|
||||
timeout?: number; // Max time in seconds (default: 900)
|
||||
gui?: boolean; // Enable MT5 visualization
|
||||
shutdown?: boolean; // Shut down MT5 after test (default: true).
|
||||
// NOTE: on Wine/macOS terminal64.exe may not exit naturally
|
||||
// even with ShutdownTerminal=1. Use inactivity_kill_secs too.
|
||||
inactivity_kill_secs?: number; // Kill MT5 if tester log hasn't grown for N seconds
|
||||
// (default: disabled / not set). Recommended: 120.
|
||||
// After silence, pipeline polls for HTML report for 30s,
|
||||
// then kills MT5 unconditionally. If HTML present → extracted;
|
||||
// otherwise falls back to journal extraction (no P&L data).
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -69,6 +69,42 @@ Set `MT5_MCP_HOME` or ensure config exists at:
|
||||
|
||||
4. **Date range has no trades** — try wider range or confirm symbol was active.
|
||||
|
||||
## Backtest Completes But Always Shows "journal-only" (No HTML Report)
|
||||
|
||||
**Symptom:** Every backtest produces `status: completed_no_html` and all deal profits are `0.0`.
|
||||
|
||||
**Cause:** On Wine/macOS, `ShutdownTerminal=1` does **not** cause `terminal64.exe` to exit after the
|
||||
test completes. The tester agent (MetaTester 5) closes normally, but the terminal process stays alive
|
||||
indefinitely. Without process exit, MT5 never writes the HTML report.
|
||||
|
||||
**How the pipeline handles this:** The inactivity watchdog detects that the tester log has stopped
|
||||
growing (test done), waits 30 seconds polling for the HTML file, then kills `terminal64.exe`
|
||||
unconditionally. If the HTML appears during the wait it is extracted; otherwise journal extraction
|
||||
provides deal counts and final balance (but no per-deal P&L).
|
||||
|
||||
**To maximise HTML report chances:** Pass `inactivity_kill_secs=120` explicitly (it is disabled by
|
||||
default). With `shutdown=true` (default) this gives MT5 120s of quiet time + 30s of HTML-wait before
|
||||
the kill.
|
||||
|
||||
```
|
||||
launch_backtest(expert: "MyEA", shutdown: true, inactivity_kill_secs: 120)
|
||||
```
|
||||
|
||||
**Fallback data available from journal:**
|
||||
- Deal count, volume, prices, timestamps ✓
|
||||
- Final balance (pips) ✓
|
||||
- Per-deal profit/loss ✗ (always 0.0)
|
||||
- Win rate, profit factor, Sharpe, drawdown ✗ (require HTML)
|
||||
|
||||
## `list_deals` Returns 0 Filtered Results
|
||||
|
||||
Analytics tools filter deals with `is_closed_trade()` which checks `entry = "out"`. If all deals
|
||||
show `entry = "in"`, the position tracker that infers direction from signed lot accumulation may
|
||||
not have run (old binary in memory).
|
||||
|
||||
**Fix:** Restart the MCP server (restart Claude Code / your IDE) after installing a new binary.
|
||||
The server process caches the binary in memory — `install` doesn't hot-reload it.
|
||||
|
||||
## MetaEditor Compile Errors
|
||||
|
||||
Check `<terminal_dir>/MQL5/Logs/`:
|
||||
|
||||
+4
-4
@@ -7,13 +7,13 @@
|
||||
"url": "https://github.com/masdevid/mt5-quant",
|
||||
"source": "github"
|
||||
},
|
||||
"version": "1.31.5",
|
||||
"version": "1.32.3",
|
||||
"packages": [
|
||||
{
|
||||
"registryType": "mcpb",
|
||||
"version": "1.31.5",
|
||||
"identifier": "https://github.com/masdevid/mt5-quant/releases/download/v1.31.5/mcp-mt5-quant-macos-arm64.tar.gz",
|
||||
"fileSha256": "97e05d3bc97270866d5736660bd19f071fd0fabbc474ba481825d69ec2b4bdf5",
|
||||
"version": "1.32.3",
|
||||
"identifier": "https://github.com/masdevid/mt5-quant/releases/download/v1.32.3/mcp-mt5-quant-macos-arm64.tar.gz",
|
||||
"fileSha256": "4c0b396f48c64334455e344644eaf6eaa6783a80fbf148688193e0108eadb949",
|
||||
"transport": {
|
||||
"type": "stdio"
|
||||
},
|
||||
|
||||
+82
-45
@@ -77,59 +77,96 @@ impl ReportExtractor {
|
||||
fn parse_deals_html(&self, text: &str) -> Result<Vec<Deal>> {
|
||||
let mut deals = Vec::new();
|
||||
|
||||
let re = regex::Regex::new(r"<tr[^>]*>.*?Deal.*?Time.*?Type.*?Direction.*?</tr>(.*)")
|
||||
.map_err(|e| anyhow!("Regex error: {}", e))?;
|
||||
// (?s) = dotall: makes '.' match '\n' so the regex works on multiline HTML.
|
||||
// MT5 HTML column order: Time | Deal | Symbol | Type | Direction |
|
||||
// Volume | Price | Order | Commission | Swap | Profit | Balance | Comment
|
||||
//
|
||||
// Strategy: locate the deals table by finding the header <tr> that contains
|
||||
// the column names, then parse every subsequent <tr> as a data row.
|
||||
// We try two header patterns to handle different MT5 versions/locales.
|
||||
|
||||
if let Some(captures) = re.captures(text) {
|
||||
let section = captures.get(1).map(|m| m.as_str()).unwrap_or("");
|
||||
let row_re = regex::Regex::new(r"(?s)<tr[^>]*>(.*?)</tr>")
|
||||
.map_err(|e| anyhow!("Row regex error: {}", e))?;
|
||||
let cell_re = regex::Regex::new(r"(?s)<td[^>]*>(.*?)</td>")
|
||||
.map_err(|e| anyhow!("Cell regex error: {}", e))?;
|
||||
|
||||
let row_re = regex::Regex::new(r"<tr[^>]*>(.*?)</tr>")
|
||||
.map_err(|e| anyhow!("Regex error: {}", e))?;
|
||||
// Collect all <tr> blocks once, then find the deals header and parse from there.
|
||||
let rows: Vec<&str> = row_re.captures_iter(text)
|
||||
.filter_map(|cap| cap.get(0).map(|m| m.as_str()))
|
||||
.collect();
|
||||
|
||||
for row_caps in row_re.captures_iter(section) {
|
||||
let row = row_caps.get(1).map(|m| m.as_str()).unwrap_or("");
|
||||
|
||||
let cell_re = regex::Regex::new(r"<td[^>]*>(.*?)</td>")
|
||||
.map_err(|e| anyhow!("Regex error: {}", e))?;
|
||||
|
||||
let cells: Vec<String> = cell_re.captures_iter(row)
|
||||
.filter_map(|cap| cap.get(1))
|
||||
.map(|m| Self::strip_tags(m.as_str()))
|
||||
.map(|s| s.replace(',', ""))
|
||||
.collect();
|
||||
// Find the header row index: it must contain both "Deal" and "Symbol" (case-insensitive).
|
||||
let header_idx = rows.iter().position(|row| {
|
||||
let lower = row.to_lowercase();
|
||||
(lower.contains(">deal<") || lower.contains(">deal </")) &&
|
||||
(lower.contains(">symbol<") || lower.contains(">symbol </"))
|
||||
});
|
||||
|
||||
if cells.len() < 3 || cells[0].is_empty() {
|
||||
continue;
|
||||
let start_idx = match header_idx {
|
||||
Some(i) => i + 1,
|
||||
None => {
|
||||
// Fallback: look for any row containing Time+Volume+Profit headers
|
||||
let alt = rows.iter().position(|row| {
|
||||
let lower = row.to_lowercase();
|
||||
lower.contains(">time<") && lower.contains(">volume<") && lower.contains(">profit<")
|
||||
});
|
||||
match alt {
|
||||
Some(i) => i + 1,
|
||||
None => {
|
||||
tracing::warn!("parse_deals_html: no deals table header found");
|
||||
return Ok(deals);
|
||||
}
|
||||
}
|
||||
|
||||
if cells.iter().take(5).any(|c| {
|
||||
let c_lower = c.trim().to_lowercase();
|
||||
c_lower == "balance" || c_lower == "credit"
|
||||
}) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let deal = Deal {
|
||||
time: cells.get(0).cloned().unwrap_or_default(),
|
||||
deal: cells.get(1).cloned().unwrap_or_default(),
|
||||
symbol: cells.get(2).cloned().unwrap_or_default(),
|
||||
deal_type: cells.get(3).cloned().unwrap_or_default(),
|
||||
entry: cells.get(4).cloned().unwrap_or_default(),
|
||||
volume: cells.get(5).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
price: cells.get(6).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
order: cells.get(7).cloned().unwrap_or_default(),
|
||||
commission: cells.get(8).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
swap: cells.get(9).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
profit: cells.get(10).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
balance: cells.get(11).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
comment: cells.get(12).cloned().unwrap_or_default(),
|
||||
magic: cells.get(13).cloned(),
|
||||
};
|
||||
|
||||
deals.push(deal);
|
||||
}
|
||||
};
|
||||
|
||||
for row in &rows[start_idx..] {
|
||||
let cells: Vec<String> = cell_re.captures_iter(row)
|
||||
.filter_map(|cap| cap.get(1))
|
||||
.map(|m| Self::strip_tags(m.as_str()))
|
||||
.map(|s| s.replace(',', ""))
|
||||
.collect();
|
||||
|
||||
if cells.len() < 3 || cells[0].trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip balance/credit operation rows (Type column is index 3 in MT5 HTML)
|
||||
let type_cell = cells.get(3).map(|s| s.trim().to_lowercase()).unwrap_or_default();
|
||||
if type_cell == "balance" || type_cell == "credit" {
|
||||
continue;
|
||||
}
|
||||
// Also skip sub-header rows that repeat column names
|
||||
if type_cell == "type" || cells.get(1).map(|s| s.trim().to_lowercase()).as_deref() == Some("deal") {
|
||||
continue;
|
||||
}
|
||||
// Skip rows with no deal number (e.g. totals row)
|
||||
let deal_num = cells.get(1).map(|s| s.trim().to_string()).unwrap_or_default();
|
||||
if deal_num.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let deal = Deal {
|
||||
time: cells.get(0).cloned().unwrap_or_default(),
|
||||
deal: deal_num,
|
||||
symbol: cells.get(2).cloned().unwrap_or_default(),
|
||||
deal_type: cells.get(3).cloned().unwrap_or_default(),
|
||||
entry: cells.get(4).cloned().unwrap_or_default(),
|
||||
volume: cells.get(5).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
price: cells.get(6).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
order: cells.get(7).cloned().unwrap_or_default(),
|
||||
commission: cells.get(8).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
swap: cells.get(9).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
profit: cells.get(10).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
balance: cells.get(11).and_then(|s| s.parse().ok()).unwrap_or(0.0),
|
||||
comment: cells.get(12).cloned().unwrap_or_default(),
|
||||
magic: cells.get(13).cloned(),
|
||||
};
|
||||
|
||||
deals.push(deal);
|
||||
}
|
||||
|
||||
tracing::info!("parse_deals_html: extracted {} deals", deals.len());
|
||||
Ok(deals)
|
||||
}
|
||||
|
||||
|
||||
@@ -90,8 +90,8 @@ impl MqlCompiler {
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
|
||||
let ex5_path = staged_mq5.with_extension("ex5");
|
||||
if !ex5_path.exists() {
|
||||
let staged_ex5 = staged_mq5.with_extension("ex5");
|
||||
if !staged_ex5.exists() {
|
||||
let final_errors = if errors.is_empty() {
|
||||
vec![format!("Compilation failed. Log:\n{}", &log_text[log_text.len().saturating_sub(500)..])]
|
||||
} else {
|
||||
@@ -107,11 +107,37 @@ impl MqlCompiler {
|
||||
});
|
||||
}
|
||||
|
||||
let binary_size = fs::metadata(&ex5_path)?.len();
|
||||
let binary_size = fs::metadata(&staged_ex5)?.len();
|
||||
|
||||
// Deploy compiled output to the real MQL5/Experts/{ea_name}/ directory so
|
||||
// MT5 can actually load it. The temp dir is only needed to avoid Wine path
|
||||
// issues with spaces; the real experts dir is the authoritative location.
|
||||
let final_ex5_path = if let Some(experts_dir) = self.config.experts_dir.as_ref() {
|
||||
let real_experts = PathBuf::from(experts_dir);
|
||||
let real_ea_dir = real_experts.join(ea_name);
|
||||
fs::create_dir_all(&real_ea_dir)?;
|
||||
|
||||
// Sync source files (.mq5 + .mqh) into the real experts dir so that
|
||||
// future compiles from the experts dir also work correctly.
|
||||
if let Err(e) = self.sync_project_to_experts(source_path, &real_experts) {
|
||||
tracing::warn!("Could not sync source to experts dir: {}", e);
|
||||
}
|
||||
|
||||
// Copy the compiled .ex5 to the real experts dir.
|
||||
let dest_ex5 = real_ea_dir.join(format!("{}.ex5", ea_name));
|
||||
fs::copy(&staged_ex5, &dest_ex5)?;
|
||||
tracing::info!("Deployed {} → {}", staged_ex5.display(), dest_ex5.display());
|
||||
dest_ex5
|
||||
} else {
|
||||
// No experts_dir configured — fall back to the staged path so callers
|
||||
// still get a valid path even if MT5 won't find it automatically.
|
||||
tracing::warn!("experts_dir not configured; .ex5 left in staging dir");
|
||||
staged_ex5
|
||||
};
|
||||
|
||||
Ok(CompileResult {
|
||||
success: errors.is_empty(),
|
||||
ex5_path: Some(ex5_path),
|
||||
ex5_path: Some(final_ex5_path),
|
||||
errors,
|
||||
warnings,
|
||||
binary_size,
|
||||
|
||||
@@ -148,6 +148,7 @@ async fn run_test_launch(ea: Option<String>, startup_delay: Option<u64>) -> Resu
|
||||
timeout: 900,
|
||||
gui: false,
|
||||
startup_delay_secs: delay,
|
||||
inactivity_kill_secs: None,
|
||||
};
|
||||
|
||||
let pipeline = BacktestPipeline::new(config);
|
||||
|
||||
@@ -4,6 +4,7 @@ use tokio::sync::{mpsc, Mutex};
|
||||
|
||||
use crate::{models::Config as ModelsConfig, tools::ToolHandler, McpError, McpRequest, McpResponse};
|
||||
|
||||
#[allow(dead_code)]
|
||||
type NotificationCallback = Arc<dyn Fn(&str, serde_json::Value) + Send + Sync>;
|
||||
|
||||
/// Auto-verify result stored after first initialization
|
||||
@@ -59,6 +60,7 @@ impl McpServer {
|
||||
*handler_guard = Some(new_handler);
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub async fn get_notification_sender(&self) -> Option<mpsc::UnboundedSender<Notification>> {
|
||||
let guard = self.notification_tx.lock().await;
|
||||
guard.clone()
|
||||
|
||||
+87
-28
@@ -508,29 +508,50 @@ impl Config {
|
||||
self.terminal_dir.as_ref().map(|d| Path::new(d).to_path_buf())
|
||||
}
|
||||
|
||||
/// Scan Bases/*/history/ for symbol directories that contain at least one .hcc file.
|
||||
/// Returns deduplicated, sorted list of symbol names available for backtesting.
|
||||
/// If server_filter is provided, only scans that specific server's directory.
|
||||
/// Scan the tester's own history store for symbols with downloaded data.
|
||||
///
|
||||
/// MT5 maintains two separate history trees:
|
||||
/// • `Bases/{server}/history/` — live-trading tick/bar data (NOT usable by tester)
|
||||
/// • `Tester/bases/{server}/history/` — data the Strategy Tester actually reads
|
||||
///
|
||||
/// Scanning `Bases/` (the old approach) returned symbols that exist for live trading
|
||||
/// but may have no tester data, causing the tester to fail with "symbol does not exist".
|
||||
/// This function scans `Tester/bases/` instead, which is the authoritative source.
|
||||
///
|
||||
/// Falls back to `Bases/` only when `Tester/bases/` is absent (first-run / no backtests yet).
|
||||
///
|
||||
/// If `server_filter` is provided only that server's directory is scanned.
|
||||
pub fn discover_symbols(&self, server_filter: Option<&str>) -> Vec<String> {
|
||||
let mt5_dir = match self.mt5_dir() {
|
||||
Some(d) => d,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
|
||||
let bases_dir = mt5_dir.join("Bases");
|
||||
if !bases_dir.is_dir() {
|
||||
return Vec::new();
|
||||
}
|
||||
// Prefer the tester's own data store; fall back to live-trading Bases/ when absent.
|
||||
let tester_bases = mt5_dir.join("Tester").join("bases");
|
||||
let bases_dir = if tester_bases.is_dir() {
|
||||
tester_bases
|
||||
} else {
|
||||
let fallback = mt5_dir.join("Bases");
|
||||
if !fallback.is_dir() {
|
||||
return Vec::new();
|
||||
}
|
||||
tracing::warn!(
|
||||
"Tester/bases/ not found — falling back to Bases/ for symbol discovery. \
|
||||
Run at least one backtest to populate tester data."
|
||||
);
|
||||
fallback
|
||||
};
|
||||
|
||||
let mut symbols = std::collections::HashSet::new();
|
||||
|
||||
// Bases/{server}/history/{symbol}/{year}.hcc
|
||||
// {bases_dir}/{server}/history/{symbol}/ — directory presence = data available
|
||||
// (the tester uses .hst/.hcc files; existence of the directory is sufficient)
|
||||
if let Ok(servers) = fs::read_dir(&bases_dir) {
|
||||
for server in servers.filter_map(|e| e.ok()) {
|
||||
let server_name_os = server.file_name();
|
||||
let server_name = server_name_os.to_str().unwrap_or("");
|
||||
|
||||
// Skip non-directory entries and filter by server if specified
|
||||
|
||||
if server_name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
@@ -539,7 +560,7 @@ impl Config {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
let history_dir = server.path().join("history");
|
||||
if !history_dir.is_dir() {
|
||||
continue;
|
||||
@@ -550,23 +571,8 @@ impl Config {
|
||||
if !sym_path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
// Only include if at least one .hcc file exists (has downloaded data)
|
||||
let has_data = fs::read_dir(&sym_path)
|
||||
.ok()
|
||||
.map(|entries| {
|
||||
entries.filter_map(|e| e.ok()).any(|e| {
|
||||
e.path().extension()
|
||||
.and_then(|x| x.to_str())
|
||||
.map(|x| x == "hcc")
|
||||
.unwrap_or(false)
|
||||
})
|
||||
})
|
||||
.unwrap_or(false);
|
||||
|
||||
if has_data {
|
||||
if let Some(name) = sym_path.file_name().and_then(|n| n.to_str()) {
|
||||
symbols.insert(name.to_string());
|
||||
}
|
||||
if let Some(name) = sym_path.file_name().and_then(|n| n.to_str()) {
|
||||
symbols.insert(name.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -578,6 +584,59 @@ impl Config {
|
||||
sorted
|
||||
}
|
||||
|
||||
/// Find the closest available tester symbol to the one requested.
|
||||
///
|
||||
/// Matching priority (first hit wins):
|
||||
/// 1. Exact match → `XAUUSD.cent` == `XAUUSD.cent`
|
||||
/// 2. Case-insensitive exact match → `xauusd.cent` → `XAUUSD.cent`
|
||||
/// 3. Strip/add common cent suffixes → `XAUUSDc` ↔ `XAUUSD.cent`
|
||||
/// 4. Prefix match on the base ticker → `XAUUSD` matches `XAUUSD.cent`
|
||||
pub fn resolve_symbol<'a>(requested: &str, available: &'a [String]) -> Option<&'a str> {
|
||||
if available.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// 1. Exact
|
||||
if let Some(s) = available.iter().find(|s| s.as_str() == requested) {
|
||||
return Some(s.as_str());
|
||||
}
|
||||
|
||||
// 2. Case-insensitive exact
|
||||
let req_lower = requested.to_lowercase();
|
||||
if let Some(s) = available.iter().find(|s| s.to_lowercase() == req_lower) {
|
||||
return Some(s.as_str());
|
||||
}
|
||||
|
||||
// 3. Cent-suffix normalisation: build a normalised "base" for both sides
|
||||
// Strip known cent suffixes: `.cent`, `c` (trailing, uppercase only), `.c`
|
||||
fn base_ticker(sym: &str) -> &str {
|
||||
let s = sym.trim_end_matches(".cent")
|
||||
.trim_end_matches(".c");
|
||||
// Strip trailing lowercase 'c' only when the rest is all-uppercase
|
||||
// (so "XAUUSDc" → "XAUUSD", but "Misc" stays "Misc")
|
||||
if s.ends_with('c') && s[..s.len()-1].chars().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) {
|
||||
&s[..s.len()-1]
|
||||
} else {
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
let req_base = base_ticker(requested).to_lowercase();
|
||||
if let Some(s) = available.iter().find(|s| base_ticker(s).to_lowercase() == req_base) {
|
||||
return Some(s.as_str());
|
||||
}
|
||||
|
||||
// 4. Prefix match: available symbol starts with the requested string (or vice-versa)
|
||||
if let Some(s) = available.iter().find(|s| {
|
||||
let sl = s.to_lowercase();
|
||||
sl.starts_with(&req_lower) || req_lower.starts_with(sl.as_str())
|
||||
}) {
|
||||
return Some(s.as_str());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Get the currently active MT5 account from common.ini
|
||||
pub fn current_account(&self) -> Option<CurrentAccount> {
|
||||
self.mt5_dir().and_then(|d| CurrentAccount::from_common_ini(&d))
|
||||
|
||||
@@ -14,8 +14,6 @@ pub struct Report {
|
||||
pub from_date: String,
|
||||
pub to_date: String,
|
||||
pub metrics_file: PathBuf,
|
||||
pub deals_csv: PathBuf,
|
||||
pub deals_json: PathBuf,
|
||||
pub analysis_file: Option<PathBuf>,
|
||||
}
|
||||
|
||||
@@ -42,8 +40,6 @@ pub struct PipelineMetadata {
|
||||
pub struct FilePaths {
|
||||
pub metrics: String,
|
||||
pub analysis: String,
|
||||
pub deals_csv: String,
|
||||
pub deals_json: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -83,6 +79,9 @@ pub struct BacktestJob {
|
||||
pub mt5_pid: Option<u32>,
|
||||
pub expected_report_path: String,
|
||||
pub timeout_seconds: u64,
|
||||
/// Set by background monitor: "running"|"completed"|"completed_no_html"|"failed"|"timeout"
|
||||
#[serde(default)]
|
||||
pub status: Option<String>,
|
||||
}
|
||||
|
||||
impl BacktestJob {
|
||||
@@ -105,6 +104,7 @@ impl BacktestJob {
|
||||
mt5_pid: None,
|
||||
expected_report_path,
|
||||
timeout_seconds,
|
||||
status: Some("running".to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+611
-63
@@ -44,6 +44,9 @@ pub struct BacktestParams {
|
||||
pub timeout: u64,
|
||||
pub gui: bool,
|
||||
pub startup_delay_secs: u64,
|
||||
/// Kill MT5 if tester agent log hasn't grown for this many seconds.
|
||||
/// 0 = disabled. Useful to abort EAs that stop trading mid-backtest.
|
||||
pub inactivity_kill_secs: Option<u64>,
|
||||
}
|
||||
|
||||
pub struct PipelineResult {
|
||||
@@ -193,7 +196,7 @@ impl BacktestPipeline {
|
||||
}
|
||||
|
||||
self.log_progress(&progress_log, "BACKTEST").await;
|
||||
|
||||
|
||||
// Get MT5 paths
|
||||
let mt5_dir = self.config.mt5_dir()
|
||||
.ok_or_else(|| anyhow!("MT5 directory not configured"))?;
|
||||
@@ -208,15 +211,16 @@ impl BacktestPipeline {
|
||||
let reports_dir = mt5_dir.join("reports");
|
||||
fs::create_dir_all(&reports_dir)?;
|
||||
|
||||
// Write params via /config: (triggers tester) and terminal.ini (redundancy).
|
||||
// Kill first — MT5 writes terminal.ini on exit, which would clobber
|
||||
// the backtest params we're about to write.
|
||||
self.kill_mt5().await?;
|
||||
|
||||
// Write params *after* MT5 is dead so nothing can overwrite them.
|
||||
let ini_content = self.build_backtest_ini(¶ms, &report_id)?;
|
||||
let config_host = wine_prefix.join("drive_c").join("backtest_config.ini");
|
||||
fs::write(&config_host, ini_content.as_bytes())?;
|
||||
self.update_terminal_ini(¶ms, &report_id)?;
|
||||
|
||||
// Kill any running MT5
|
||||
self.kill_mt5().await?;
|
||||
|
||||
// Launch MT5 (fire and forget)
|
||||
let mut cmd = self.build_wine_launch(wine_exe, &wine_prefix)?;
|
||||
let child = cmd.stdin(std::process::Stdio::null())
|
||||
@@ -258,6 +262,28 @@ impl BacktestPipeline {
|
||||
let timeout_secs = params.timeout;
|
||||
let report_id_clone = report_id.clone();
|
||||
let notification_callback = self.notification_callback.clone();
|
||||
let config_clone = self.config.clone();
|
||||
let params_clone = BacktestParams {
|
||||
expert: params.expert.clone(),
|
||||
symbol: params.symbol.clone(),
|
||||
from_date: params.from_date.clone(),
|
||||
to_date: params.to_date.clone(),
|
||||
timeframe: params.timeframe.clone(),
|
||||
deposit: params.deposit,
|
||||
model: params.model,
|
||||
leverage: params.leverage,
|
||||
set_file: params.set_file.clone(),
|
||||
skip_compile: params.skip_compile,
|
||||
skip_clean: params.skip_clean,
|
||||
skip_analyze: params.skip_analyze,
|
||||
deep_analyze: params.deep_analyze,
|
||||
shutdown: params.shutdown,
|
||||
kill_existing: params.kill_existing,
|
||||
timeout: params.timeout,
|
||||
gui: params.gui,
|
||||
startup_delay_secs: params.startup_delay_secs,
|
||||
inactivity_kill_secs: params.inactivity_kill_secs,
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
Self::monitor_backtest_completion(
|
||||
report_dir_clone,
|
||||
@@ -265,12 +291,81 @@ impl BacktestPipeline {
|
||||
timeout_secs,
|
||||
report_id_clone,
|
||||
notification_callback,
|
||||
config_clone,
|
||||
params_clone,
|
||||
).await;
|
||||
});
|
||||
|
||||
Ok(job)
|
||||
}
|
||||
|
||||
/// Extract deals from a completed report and store them in the DB.
|
||||
/// Returns true if extraction and DB registration succeeded.
|
||||
async fn extract_and_store(
|
||||
report_path: &Path,
|
||||
report_dir: &Path,
|
||||
report_id: &str,
|
||||
config: &Config,
|
||||
params: &BacktestParams,
|
||||
) -> bool {
|
||||
let extractor = ReportExtractor::new();
|
||||
let start_time = chrono::Utc::now();
|
||||
match extractor.extract(
|
||||
&report_path.to_string_lossy(),
|
||||
&report_dir.to_string_lossy(),
|
||||
) {
|
||||
Ok(extraction) => {
|
||||
let duration = (chrono::Utc::now() - start_time).num_seconds();
|
||||
let db = ReportDb::new(&Config::db_path());
|
||||
if db.init().is_err() {
|
||||
tracing::warn!("launch_backtest: failed to init DB for {}", report_id);
|
||||
return false;
|
||||
}
|
||||
let entry = ReportEntry {
|
||||
id: report_id.to_string(),
|
||||
expert: params.expert.clone(),
|
||||
symbol: params.symbol.clone(),
|
||||
timeframe: params.timeframe.clone(),
|
||||
model: params.model as i64,
|
||||
from_date: params.from_date.clone(),
|
||||
to_date: params.to_date.clone(),
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
set_file_original: params.set_file.clone(),
|
||||
set_snapshot_path: None,
|
||||
report_dir: report_dir.to_string_lossy().to_string(),
|
||||
charts_dir: None,
|
||||
net_profit: Some(extraction.metrics.net_profit),
|
||||
profit_factor: Some(extraction.metrics.profit_factor),
|
||||
max_dd_pct: Some(extraction.metrics.max_dd_pct),
|
||||
sharpe_ratio: Some(extraction.metrics.sharpe_ratio),
|
||||
total_trades: Some(extraction.metrics.total_trades as i64),
|
||||
win_rate_pct: Some(extraction.metrics.win_rate_pct),
|
||||
recovery_factor: Some(extraction.metrics.recovery_factor),
|
||||
deposit: Some(params.deposit as f64),
|
||||
currency: config.backtest_currency.clone(),
|
||||
leverage: Some(params.leverage as i64),
|
||||
duration_seconds: Some(duration),
|
||||
tags: Vec::new(),
|
||||
notes: None,
|
||||
verdict: None,
|
||||
};
|
||||
if let Err(e) = db.insert(&entry) {
|
||||
tracing::warn!("launch_backtest: failed to register report in DB: {}", e);
|
||||
return false;
|
||||
}
|
||||
if let Err(e) = db.insert_deals(report_id, &extraction.deals) {
|
||||
tracing::warn!("launch_backtest: failed to store deals in DB: {}", e);
|
||||
}
|
||||
tracing::info!("launch_backtest: extracted {} deals for {}", extraction.deals.len(), report_id);
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("launch_backtest: extraction failed for {}: {}", report_id, e);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Background task to monitor backtest completion and update status file.
|
||||
async fn monitor_backtest_completion(
|
||||
report_dir: PathBuf,
|
||||
@@ -278,20 +373,54 @@ impl BacktestPipeline {
|
||||
timeout_secs: u64,
|
||||
report_id: String,
|
||||
notification_callback: Option<NotificationCallback>,
|
||||
config: Config,
|
||||
params: BacktestParams,
|
||||
) {
|
||||
let start = tokio::time::Instant::now();
|
||||
let deadline = start + Duration::from_secs(timeout_secs);
|
||||
let grace_period = Duration::from_secs(30);
|
||||
let poll_start = std::time::SystemTime::now();
|
||||
|
||||
// Inactivity watchdog: kill MT5 if tester agent log hasn't grown for this long.
|
||||
// Catches EAs that stall (no ticks processed) or flat periods with zero trades.
|
||||
let inactivity_threshold = Duration::from_secs(
|
||||
params.inactivity_kill_secs.unwrap_or(0)
|
||||
);
|
||||
let mut last_log_size: u64 = 0;
|
||||
let mut last_log_activity = tokio::time::Instant::now();
|
||||
let inactivity_enabled = inactivity_threshold.as_secs() > 0;
|
||||
|
||||
loop {
|
||||
let _elapsed = start.elapsed().as_secs();
|
||||
|
||||
// Check for report file
|
||||
for ext in &[".htm", ".htm.xml", ".html"] {
|
||||
let candidate = expected_report.with_extension(ext.trim_start_matches('.'));
|
||||
// Check for report file (exact name first)
|
||||
for ext in &["htm", "htm.xml", "html"] {
|
||||
let candidate = if *ext == "htm" {
|
||||
expected_report.clone()
|
||||
} else {
|
||||
// Build alternate extension path without touching expected_report extension
|
||||
let stem = expected_report.file_stem()
|
||||
.map(|s| s.to_string_lossy().to_string())
|
||||
.unwrap_or_default();
|
||||
expected_report.with_file_name(format!("{}.{}", stem, ext))
|
||||
};
|
||||
if candidate.exists() {
|
||||
tracing::info!("Backtest {} completed: report found at {}", report_id, candidate.display());
|
||||
let extracted = Self::extract_and_store(&candidate, &report_dir, &report_id, &config, ¶ms).await;
|
||||
if extracted {
|
||||
let _ = fs::remove_file(&candidate);
|
||||
} else {
|
||||
tracing::warn!("Backtest {}: extraction failed, keeping report file at {}", report_id, candidate.display());
|
||||
}
|
||||
// When ShutdownTerminal=0 (shutdown=false), MT5 stays running after the
|
||||
// test so the report can be written reliably. Kill it ourselves now that
|
||||
// extraction is done so we don't leave zombie Wine processes.
|
||||
if !params.shutdown && Self::is_mt5_running() {
|
||||
tracing::info!("Backtest {}: killing MT5 after report extraction (shutdown=false)", report_id);
|
||||
let _ = std::process::Command::new("pkill")
|
||||
.args(["-TERM", "-f", "terminal64\\.exe"])
|
||||
.output();
|
||||
}
|
||||
Self::update_job_status(&report_dir, "completed", Some(candidate.to_string_lossy().to_string())).await;
|
||||
if let Some(ref callback) = notification_callback {
|
||||
callback("backtest_completed", json!({
|
||||
@@ -304,14 +433,142 @@ impl BacktestPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
// Inactivity watchdog: check if tester agent log is growing.
|
||||
// The agent log is written incrementally as the tester processes ticks.
|
||||
// If it stops growing for `inactivity_threshold` seconds → the test is done
|
||||
// (or EA is stuck). Either way, we wait 30 s for MT5 to write the HTML
|
||||
// report, then kill it unconditionally.
|
||||
//
|
||||
// NOTE: ShutdownTerminal=1 is supposed to make MT5 exit after the test,
|
||||
// but on Wine/macOS this is unreliable — terminal64.exe often stays alive
|
||||
// indefinitely. So we always kill after the HTML-wait window rather than
|
||||
// depending on natural exit. The 30 s window is long enough for MT5 to flush
|
||||
// the report file; if it isn't there by then, it won't appear.
|
||||
if inactivity_enabled && Self::is_mt5_running() {
|
||||
if let Some(log_path) = Self::find_active_tester_agent_log(&config) {
|
||||
let current_size = fs::metadata(&log_path)
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0);
|
||||
if current_size > last_log_size {
|
||||
last_log_size = current_size;
|
||||
last_log_activity = tokio::time::Instant::now();
|
||||
} else if last_log_activity.elapsed() >= inactivity_threshold && last_log_size > 0 {
|
||||
tracing::info!(
|
||||
"Backtest {}: tester log inactive for {}s — waiting 30s for HTML report, then killing MT5",
|
||||
report_id, inactivity_threshold.as_secs()
|
||||
);
|
||||
// Poll for the HTML during the grace window (30 s, 1 s intervals).
|
||||
let reports_parent = expected_report.parent();
|
||||
let mut html_found: Option<std::path::PathBuf> = None;
|
||||
for _wait in 0u32..30 {
|
||||
// Check exact expected path first.
|
||||
for ext in &["htm", "htm.xml", "html"] {
|
||||
let candidate = if *ext == "htm" {
|
||||
expected_report.clone()
|
||||
} else {
|
||||
let stem = expected_report.file_stem()
|
||||
.map(|s| s.to_string_lossy().to_string())
|
||||
.unwrap_or_default();
|
||||
expected_report.with_file_name(format!("{}.{}", stem, ext))
|
||||
};
|
||||
if candidate.exists() {
|
||||
html_found = Some(candidate);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if html_found.is_some() { break; }
|
||||
// Also scan for any newly created report.
|
||||
if let Some(parent) = reports_parent {
|
||||
if let Some(path) = Self::find_newest_report(parent, poll_start) {
|
||||
html_found = Some(path);
|
||||
break;
|
||||
}
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
|
||||
// Kill MT5 now — it either wrote the report or it won't.
|
||||
tracing::info!("Backtest {}: killing MT5 after inactivity+HTML-wait window", report_id);
|
||||
let _ = std::process::Command::new("pkill")
|
||||
.args(["-TERM", "-f", "terminal64\\.exe"])
|
||||
.output();
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
let _ = std::process::Command::new("pkill")
|
||||
.args(["-KILL", "-f", "terminal64\\.exe"])
|
||||
.output();
|
||||
|
||||
if let Some(path) = html_found {
|
||||
tracing::info!("Backtest {}: HTML report found during wait: {}", report_id, path.display());
|
||||
let extracted = Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await;
|
||||
if extracted { let _ = fs::remove_file(&path); }
|
||||
Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await;
|
||||
if let Some(ref callback) = notification_callback {
|
||||
callback("backtest_completed", json!({
|
||||
"report_id": report_id,
|
||||
"status": "completed"
|
||||
}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// No HTML — fall back to journal extraction.
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
if let Some(log) = Self::find_active_tester_agent_log(&config) {
|
||||
if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await {
|
||||
Self::update_job_status(&report_dir, "completed_no_html", None).await;
|
||||
if let Some(ref callback) = notification_callback {
|
||||
callback("backtest_completed", json!({
|
||||
"report_id": report_id,
|
||||
"status": "completed_no_html",
|
||||
"reason": "extracted from journal after inactivity kill (no HTML produced)"
|
||||
}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
Self::update_job_status(&report_dir, "timeout_inactive", None).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check process liveness after grace period
|
||||
let in_grace = start.elapsed() <= grace_period;
|
||||
let mt5_alive = Self::is_mt5_running();
|
||||
|
||||
if !in_grace && !mt5_alive {
|
||||
// MT5 exited without report - check for any newer report
|
||||
if let Some(path) = Self::find_newest_report(expected_report.parent().unwrap(), poll_start) {
|
||||
tracing::info!("Backtest {} completed: found fallback report {}", report_id, path.display());
|
||||
// MT5 exited. Poll for the .htm report for up to 10 s (check every 1 s).
|
||||
// Wine on macOS can take several seconds to flush the file after the
|
||||
// process exits — a single fixed wait often misses the window.
|
||||
let reports_parent = match expected_report.parent() {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
tracing::error!("Backtest {}: expected_report path has no parent", report_id);
|
||||
Self::update_job_status(&report_dir, "failed", None).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let mut found_report: Option<std::path::PathBuf> = None;
|
||||
for attempt in 1u32..=10 {
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
if let Some(path) = Self::find_newest_report(reports_parent, poll_start) {
|
||||
tracing::info!(
|
||||
"Backtest {}: found report after {}s — {}",
|
||||
report_id, attempt, path.display()
|
||||
);
|
||||
found_report = Some(path);
|
||||
break;
|
||||
}
|
||||
tracing::debug!("Backtest {}: no report yet ({}s elapsed after MT5 exit)", report_id, attempt);
|
||||
}
|
||||
if let Some(path) = found_report {
|
||||
tracing::info!("Backtest {} completed: found report {}", report_id, path.display());
|
||||
let extracted = Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await;
|
||||
if extracted {
|
||||
let _ = fs::remove_file(&path);
|
||||
} else {
|
||||
tracing::warn!("Backtest {}: extraction failed, keeping report at {}", report_id, path.display());
|
||||
}
|
||||
Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await;
|
||||
if let Some(ref callback) = notification_callback {
|
||||
callback("backtest_completed", json!({
|
||||
@@ -320,22 +577,46 @@ impl BacktestPipeline {
|
||||
"status": "completed"
|
||||
}));
|
||||
}
|
||||
} else {
|
||||
tracing::warn!("Backtest {} failed: MT5 exited without producing a report", report_id);
|
||||
Self::update_job_status(&report_dir, "failed", None).await;
|
||||
if let Some(ref callback) = notification_callback {
|
||||
callback("backtest_failed", json!({
|
||||
"report_id": report_id,
|
||||
"status": "failed",
|
||||
"reason": "MT5 exited without producing a report"
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
// No HTML report found — fallback to journal extraction.
|
||||
tracing::warn!("Backtest {}: no HTML report found, trying journal extraction", report_id);
|
||||
if let Some(log) = Self::find_active_tester_agent_log(&config) {
|
||||
if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await {
|
||||
Self::update_job_status(&report_dir, "completed_no_html", None).await;
|
||||
if let Some(ref callback) = notification_callback {
|
||||
callback("backtest_completed", json!({
|
||||
"report_id": report_id,
|
||||
"status": "completed_no_html",
|
||||
"reason": "extracted from tester journal (HTML report not produced)"
|
||||
}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::warn!("Backtest {} failed: MT5 exited without producing a report", report_id);
|
||||
Self::update_job_status(&report_dir, "failed", None).await;
|
||||
if let Some(ref callback) = notification_callback {
|
||||
callback("backtest_failed", json!({
|
||||
"report_id": report_id,
|
||||
"status": "failed",
|
||||
"reason": "MT5 exited without producing a report or recoverable journal"
|
||||
}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if tokio::time::Instant::now() > deadline {
|
||||
tracing::warn!("Backtest {} timed out after {} seconds", report_id, timeout_secs);
|
||||
// Last-chance journal extraction on timeout
|
||||
if let Some(log) = Self::find_active_tester_agent_log(&config) {
|
||||
if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await {
|
||||
Self::update_job_status(&report_dir, "completed_no_html", None).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
Self::update_job_status(&report_dir, "timeout", None).await;
|
||||
if let Some(ref callback) = notification_callback {
|
||||
callback("backtest_timeout", json!({
|
||||
@@ -351,6 +632,251 @@ impl BacktestPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the best tester agent log file for today.
|
||||
///
|
||||
/// Selection priority:
|
||||
/// 1. Local agents (127.0.0.1) preferred over external/cloud agents (0.0.0.0)
|
||||
/// — 0.0.0.0 logs only contain startup info, never actual deal lines.
|
||||
/// 2. Among equal-priority agents, pick the **largest** file (most content).
|
||||
pub fn find_active_tester_agent_log(config: &Config) -> Option<PathBuf> {
|
||||
let mt5_dir = config.mt5_dir()?;
|
||||
let tester_dir = mt5_dir.join("Tester");
|
||||
let today = chrono::Utc::now().format("%Y%m%d").to_string();
|
||||
|
||||
// (priority, size, path) — higher priority = more preferred
|
||||
// priority 1 = local (127.0.0.1), priority 0 = other (0.0.0.0 / any)
|
||||
let mut best: Option<(u8, u64, PathBuf)> = None;
|
||||
|
||||
if let Ok(agents) = fs::read_dir(&tester_dir) {
|
||||
for agent in agents.filter_map(|e| e.ok()) {
|
||||
let agent_name = agent.file_name();
|
||||
let agent_str = agent_name.to_string_lossy();
|
||||
|
||||
// Only consider Agent-* directories
|
||||
if !agent_str.starts_with("Agent-") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let logs_dir = agent.path().join("logs");
|
||||
let candidate = logs_dir.join(format!("{}.log", today));
|
||||
if !candidate.exists() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let meta = match fs::metadata(&candidate) {
|
||||
Ok(m) => m,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let size = meta.len();
|
||||
|
||||
// Prefer local agents (127.0.0.1) — they log actual deal execution
|
||||
let priority: u8 = if agent_str.contains("127.0.0.1") { 1 } else { 0 };
|
||||
|
||||
let is_better = match &best {
|
||||
None => true,
|
||||
Some((bp, bs, _)) => priority > *bp || (priority == *bp && size > *bs),
|
||||
};
|
||||
|
||||
if is_better {
|
||||
best = Some((priority, size, candidate));
|
||||
}
|
||||
}
|
||||
}
|
||||
best.map(|(_, _, p)| p)
|
||||
}
|
||||
|
||||
/// Read the most recent tester agent log and return its lines (UTF-16 or UTF-8).
|
||||
pub fn read_tester_agent_log(log_path: &Path) -> Option<Vec<String>> {
|
||||
let bytes = fs::read(log_path).ok()?;
|
||||
let text = if bytes.len() >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE {
|
||||
// UTF-16 LE with BOM
|
||||
let words: Vec<u16> = bytes[2..].chunks_exact(2)
|
||||
.map(|c| u16::from_le_bytes([c[0], c[1]]))
|
||||
.collect();
|
||||
String::from_utf16_lossy(&words).to_string()
|
||||
} else {
|
||||
String::from_utf8_lossy(&bytes).to_string()
|
||||
};
|
||||
Some(text.lines().map(|l| l.to_string()).collect())
|
||||
}
|
||||
|
||||
/// Parse deal entries from a tester agent log.
|
||||
/// Returns (deals_parsed, final_balance_pips, sim_progress_line).
|
||||
///
|
||||
/// Entry direction (in/out) is inferred via a per-symbol position tracker:
|
||||
/// - No open position → "in"
|
||||
/// - Same direction as existing position → "in" (grid/martingale add)
|
||||
/// - Opposite direction → "out" (closing)
|
||||
/// Profit/balance are unavailable in the journal; they remain 0.0.
|
||||
pub fn parse_journal_deals(lines: &[String]) -> (Vec<crate::models::deals::Deal>, f64, String) {
|
||||
use regex::Regex;
|
||||
use std::collections::HashMap;
|
||||
|
||||
// Format: "... YYYY.MM.DD HH:MM:SS deal #N buy/sell VOLUME SYM at PRICE done ..."
|
||||
let deal_re = Regex::new(
|
||||
r"(\d{4}\.\d{2}\.\d{2} \d{2}:\d{2}:\d{2})\s+deal #(\d+) (buy|sell) ([\d.]+) (\S+) at ([\d.]+) done"
|
||||
).unwrap();
|
||||
let balance_re = Regex::new(r"final balance ([\d.]+) pips").unwrap();
|
||||
let progress_re = Regex::new(r"Test passed in (.+)").unwrap();
|
||||
|
||||
let mut deals = Vec::new();
|
||||
let mut final_balance = 0.0f64;
|
||||
let mut progress_str = String::new();
|
||||
|
||||
// signed lots per symbol: positive = net long, negative = net short
|
||||
let mut position: HashMap<String, f64> = HashMap::new();
|
||||
// MT5 tester logs each deal TWICE (dual-agent logging) — deduplicate by deal number
|
||||
let mut seen_deals: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||
|
||||
for line in lines {
|
||||
if let Some(cap) = deal_re.captures(line) {
|
||||
let sim_time = cap[1].to_string();
|
||||
let deal_num = cap[2].to_string();
|
||||
let direction = cap[3].to_string(); // "buy" | "sell"
|
||||
let volume: f64 = cap[4].parse().unwrap_or(0.0);
|
||||
let symbol = cap[5].to_string();
|
||||
let price: f64 = cap[6].parse().unwrap_or(0.0);
|
||||
|
||||
// Skip duplicate deal entries (MT5 writes each deal twice)
|
||||
if !seen_deals.insert(deal_num.clone()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let signed = if direction == "buy" { volume } else { -volume };
|
||||
let current = position.get(&symbol).copied().unwrap_or(0.0);
|
||||
|
||||
// Determine entry type by comparing new direction against open position
|
||||
let entry_type = if current.abs() < 1e-9 {
|
||||
// flat → opening a new position
|
||||
"in"
|
||||
} else if (current > 0.0 && direction == "buy")
|
||||
|| (current < 0.0 && direction == "sell")
|
||||
{
|
||||
// same direction as existing → adding (grid/martingale)
|
||||
"in"
|
||||
} else {
|
||||
// opposite direction → closing / partial close
|
||||
"out"
|
||||
};
|
||||
|
||||
// Update tracked position
|
||||
let new_pos = current + signed;
|
||||
if new_pos.abs() < 1e-9 {
|
||||
position.remove(&symbol);
|
||||
} else {
|
||||
position.insert(symbol.clone(), new_pos);
|
||||
}
|
||||
|
||||
deals.push(crate::models::deals::Deal {
|
||||
time: sim_time,
|
||||
deal: deal_num,
|
||||
symbol,
|
||||
deal_type: direction,
|
||||
entry: entry_type.to_string(),
|
||||
volume,
|
||||
price,
|
||||
order: String::new(),
|
||||
commission: 0.0,
|
||||
swap: 0.0,
|
||||
profit: 0.0, // not available in journal
|
||||
balance: 0.0, // not available in journal
|
||||
comment: String::new(),
|
||||
magic: None,
|
||||
});
|
||||
}
|
||||
if let Some(cap) = balance_re.captures(line) {
|
||||
final_balance = cap[1].parse().unwrap_or(0.0);
|
||||
}
|
||||
if let Some(cap) = progress_re.captures(line) {
|
||||
progress_str = cap[1].to_string();
|
||||
}
|
||||
}
|
||||
|
||||
(deals, final_balance, progress_str)
|
||||
}
|
||||
|
||||
/// Fallback: extract deals from the tester agent journal log when no HTML report exists.
|
||||
/// Stores partial deal data (no per-deal P&L) and records the final balance only.
|
||||
async fn extract_from_journal(
|
||||
log_path: &Path,
|
||||
report_dir: &Path,
|
||||
report_id: &str,
|
||||
config: &Config,
|
||||
params: &BacktestParams,
|
||||
) -> bool {
|
||||
let lines = match Self::read_tester_agent_log(log_path) {
|
||||
Some(l) => l,
|
||||
None => {
|
||||
tracing::warn!("Journal extraction: could not read log {}", log_path.display());
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
let (deals, final_balance_pips, progress) = Self::parse_journal_deals(&lines);
|
||||
if deals.is_empty() {
|
||||
tracing::warn!("Journal extraction: no deals found in {}", log_path.display());
|
||||
return false;
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Journal extraction: {} deals, final balance {} pips, {}",
|
||||
deals.len(), final_balance_pips, progress
|
||||
);
|
||||
|
||||
// Save journal summary to report_dir
|
||||
let summary_path = report_dir.join("journal_extraction.json");
|
||||
let summary = json!({
|
||||
"source": "tester_agent_log",
|
||||
"log_path": log_path.to_string_lossy(),
|
||||
"total_deals": deals.len(),
|
||||
"final_balance_pips": final_balance_pips,
|
||||
"progress": progress,
|
||||
"note": "No HTML report was produced. Deals extracted from tester agent log. profit/balance fields are 0 (not available in log format)."
|
||||
});
|
||||
let _ = fs::write(&summary_path, serde_json::to_string_pretty(&summary).unwrap_or_default());
|
||||
|
||||
// Register in DB with partial metrics
|
||||
let db = crate::storage::ReportDb::new(&Config::db_path());
|
||||
if db.init().is_err() {
|
||||
return false;
|
||||
}
|
||||
let entry = crate::storage::ReportEntry {
|
||||
id: report_id.to_string(),
|
||||
expert: params.expert.clone(),
|
||||
symbol: params.symbol.clone(),
|
||||
timeframe: params.timeframe.clone(),
|
||||
model: params.model as i64,
|
||||
from_date: params.from_date.clone(),
|
||||
to_date: params.to_date.clone(),
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
set_file_original: params.set_file.clone(),
|
||||
set_snapshot_path: None,
|
||||
report_dir: report_dir.to_string_lossy().to_string(),
|
||||
charts_dir: None,
|
||||
net_profit: Some(final_balance_pips - params.deposit as f64),
|
||||
profit_factor: None,
|
||||
max_dd_pct: None,
|
||||
sharpe_ratio: None,
|
||||
total_trades: Some(deals.len() as i64 / 2), // open+close pairs
|
||||
win_rate_pct: None,
|
||||
recovery_factor: None,
|
||||
deposit: Some(params.deposit as f64),
|
||||
currency: config.backtest_currency.clone(),
|
||||
leverage: Some(params.leverage as i64),
|
||||
duration_seconds: None,
|
||||
tags: vec!["journal-only".to_string()],
|
||||
notes: Some(format!("Extracted from journal: {} deals, final balance {} pips. No HTML report.", deals.len(), final_balance_pips)),
|
||||
verdict: None,
|
||||
};
|
||||
if db.insert(&entry).is_err() {
|
||||
return false;
|
||||
}
|
||||
if let Err(e) = db.insert_deals(report_id, &deals) {
|
||||
tracing::warn!("Journal extraction: failed to store deals: {}", e);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Update job status in job.json file.
|
||||
async fn update_job_status(report_dir: &Path, status: &str, report_path: Option<String>) {
|
||||
let job_path = report_dir.join("job.json");
|
||||
@@ -594,15 +1120,16 @@ impl BacktestPipeline {
|
||||
let reports_dir = mt5_dir.join("reports");
|
||||
fs::create_dir_all(&reports_dir)?;
|
||||
|
||||
// Write params via /config: (triggers tester auto-start) and terminal.ini (redundancy).
|
||||
// Kill first — MT5 writes terminal.ini on exit, which would clobber
|
||||
// the backtest params we're about to write.
|
||||
self.kill_mt5().await?;
|
||||
|
||||
// Write params *after* MT5 is dead so nothing can overwrite them.
|
||||
let ini_content = self.build_backtest_ini(params, report_id)?;
|
||||
let config_host = wine_prefix.join("drive_c").join("backtest_config.ini");
|
||||
fs::write(&config_host, ini_content.as_bytes())?;
|
||||
self.update_terminal_ini(params, report_id)?;
|
||||
|
||||
// Always kill any running MT5 — Wine allows only one instance per prefix.
|
||||
self.kill_mt5().await?;
|
||||
|
||||
// Record launch time before sleeping so find_newest_report doesn't miss
|
||||
// reports written during the startup wait.
|
||||
let poll_start = std::time::SystemTime::now();
|
||||
@@ -647,8 +1174,13 @@ impl BacktestPipeline {
|
||||
tracing::info!("poll t+{}s: in_grace={} mt5_alive={}", elapsed, in_grace, mt5_alive);
|
||||
|
||||
if !in_grace && !mt5_alive {
|
||||
// MT5 writes the .htm report file right before exiting. There is a
|
||||
// short window where the process is gone but the file hasn't been
|
||||
// flushed to the directory. Wait 3 s to let Wine/macOS finish the
|
||||
// write before we scan — this prevents false "no report" failures.
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
if let Some(path) = Self::find_newest_report(&reports_dir, poll_start) {
|
||||
tracing::info!("poll: MT5 exited, found fallback report {}", path.display());
|
||||
tracing::info!("poll: MT5 exited, found report {}", path.display());
|
||||
return Ok(path);
|
||||
}
|
||||
return Err(anyhow!(
|
||||
@@ -711,12 +1243,12 @@ impl BacktestPipeline {
|
||||
};
|
||||
|
||||
let set_file_line = params.set_file.as_ref()
|
||||
.map(|p| format!("ExpertParameters={}\n", p))
|
||||
.map(|p| format!("ExpertParameters={}\n", Self::ini_safe(p)))
|
||||
.unwrap_or_default();
|
||||
|
||||
let updates: &[(&str, String)] = &[
|
||||
("Expert", expert_path),
|
||||
("Symbol", params.symbol.clone()),
|
||||
("Expert", Self::ini_safe(&expert_path)),
|
||||
("Symbol", Self::ini_safe(¶ms.symbol)),
|
||||
("Period", period.to_string()),
|
||||
("DateRange", "3".into()),
|
||||
("DateFrom", from_ts.to_string()),
|
||||
@@ -743,6 +1275,11 @@ impl BacktestPipeline {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Strip CR/LF from a user-supplied INI value to prevent newline injection.
|
||||
fn ini_safe(value: &str) -> String {
|
||||
value.replace(['\n', '\r'], "")
|
||||
}
|
||||
|
||||
fn patch_ini_section(text: &str, section: &str, updates: &[(&str, String)]) -> String {
|
||||
let section_header = format!("[{}]", section);
|
||||
let mut result = String::with_capacity(text.len() + 256);
|
||||
@@ -846,23 +1383,16 @@ impl BacktestPipeline {
|
||||
);
|
||||
|
||||
let script_path = std::env::temp_dir().join("mt5_backtest_launch.sh");
|
||||
|
||||
// Only rewrite script if it doesn't exist or content differs (optimization)
|
||||
let needs_write = !script_path.exists() || fs::read_to_string(&script_path).map(|existing| existing != script).unwrap_or(true);
|
||||
|
||||
if needs_write {
|
||||
fs::write(&script_path, &script)?;
|
||||
// chmod +x
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(&script_path,
|
||||
fs::Permissions::from_mode(0o755))?;
|
||||
}
|
||||
tracing::debug!("Created/updated launch script: {}", script_path.display());
|
||||
} else {
|
||||
tracing::debug!("Reusing existing launch script: {}", script_path.display());
|
||||
|
||||
// Always rewrite: script content changes per backtest (DYLD paths are
|
||||
// dynamic) and we must ensure +x permissions are set every time.
|
||||
fs::write(&script_path, &script)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(&script_path, fs::Permissions::from_mode(0o755))?;
|
||||
}
|
||||
tracing::debug!("Wrote launch script: {}", script_path.display());
|
||||
|
||||
tracing::info!("Launching MT5 via shell script (terminal.ini mode): {}", script_path.display());
|
||||
let mut cmd = Command::new("/bin/sh");
|
||||
@@ -913,9 +1443,9 @@ impl BacktestPipeline {
|
||||
|
||||
ini.push_str("[Tester]\n");
|
||||
// Expert path is relative to MQL5/Experts/ in the /config: format (no "Experts\" prefix).
|
||||
ini.push_str(&format!("Expert={}\n", self.resolve_backtest_ini_expert_path(¶ms.expert)));
|
||||
ini.push_str(&format!("Symbol={}\n", params.symbol));
|
||||
ini.push_str(&format!("Period={}\n", params.timeframe));
|
||||
ini.push_str(&format!("Expert={}\n", Self::ini_safe(&self.resolve_backtest_ini_expert_path(¶ms.expert))));
|
||||
ini.push_str(&format!("Symbol={}\n", Self::ini_safe(¶ms.symbol)));
|
||||
ini.push_str(&format!("Period={}\n", Self::ini_safe(¶ms.timeframe)));
|
||||
ini.push_str("Optimization=0\n");
|
||||
ini.push_str(&format!("Model={}\n", params.model));
|
||||
ini.push_str(&format!("FromDate={}\n", params.from_date));
|
||||
@@ -932,7 +1462,7 @@ impl BacktestPipeline {
|
||||
ini.push_str(&format!("ShutdownTerminal={}\n", if params.shutdown { "1" } else { "0" }));
|
||||
|
||||
if let Some(set_file) = ¶ms.set_file {
|
||||
ini.push_str(&format!("ExpertParameters={}\n", set_file));
|
||||
ini.push_str(&format!("ExpertParameters={}\n", Self::ini_safe(set_file)));
|
||||
}
|
||||
|
||||
Ok(ini)
|
||||
@@ -955,22 +1485,42 @@ impl BacktestPipeline {
|
||||
|
||||
tracing::info!("Stopping existing MT5 instance...");
|
||||
// SIGKILL immediately — MT5 holds no state we care about preserving.
|
||||
let mut killed_any = false;
|
||||
for pat in &patterns {
|
||||
let result = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output();
|
||||
if result.map(|o| o.status.success()).unwrap_or(false) {
|
||||
killed_any = true;
|
||||
let _ = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output();
|
||||
}
|
||||
// Also kill wineserver so the Wine prefix is fully reset before relaunch.
|
||||
// If wineserver is still alive when the new MT5 spawns, the new Wine
|
||||
// instance may attach to the dying server and never enter tester mode.
|
||||
let _ = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output();
|
||||
|
||||
// Poll until wineserver is actually gone (max 10 s) rather than sleeping
|
||||
// a fixed amount. On macOS wineserver cleanup varies from 1 s to 6+ s.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
let ws_alive = Command::new("pgrep")
|
||||
.args(["-f", "wineserver"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false);
|
||||
let mt5_alive = patterns.iter().any(|pat| {
|
||||
Command::new("pgrep")
|
||||
.args(["-f", pat.as_str()])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
});
|
||||
if !ws_alive && !mt5_alive {
|
||||
tracing::info!("MT5 and wineserver fully exited");
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
tracing::warn!("wineserver still alive after 10 s — proceeding anyway");
|
||||
break;
|
||||
}
|
||||
}
|
||||
let wineserver_result = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output();
|
||||
if wineserver_result.map(|o| o.status.success()).unwrap_or(false) {
|
||||
killed_any = true;
|
||||
}
|
||||
|
||||
// Only sleep if we actually killed something - skip if no processes were running
|
||||
if killed_any {
|
||||
sleep(Duration::from_secs(2)).await; // Reduced from 3s to 2s
|
||||
}
|
||||
// Brief extra pause to let the kernel release sockets and shared memory.
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1047,8 +1597,6 @@ impl BacktestPipeline {
|
||||
files: FilePaths {
|
||||
metrics: report_dir.join("metrics.json").to_string_lossy().to_string(),
|
||||
analysis: report_dir.join("analysis.json").to_string_lossy().to_string(),
|
||||
deals_csv: report_dir.join("deals.csv").to_string_lossy().to_string(),
|
||||
deals_json: report_dir.join("deals.json").to_string_lossy().to_string(),
|
||||
},
|
||||
no_trades,
|
||||
};
|
||||
|
||||
+33
-22
@@ -564,26 +564,39 @@ impl ReportDb {
|
||||
/// Search reports by tags (at least one tag must match)
|
||||
pub fn search_by_tags(&self, tags: &[String], limit: usize) -> Result<Vec<ReportEntry>> {
|
||||
let conn = self.connect()?;
|
||||
let mut sql = "SELECT id, expert, symbol, timeframe, model, from_date, to_date, \
|
||||
let base_sql = "SELECT id, expert, symbol, timeframe, model, from_date, to_date, \
|
||||
created_at, set_file_original, set_snapshot_path, report_dir, charts_dir, \
|
||||
net_profit, profit_factor, max_dd_pct, sharpe_ratio, total_trades, \
|
||||
win_rate_pct, recovery_factor, deposit, currency, leverage, \
|
||||
duration_seconds, tags, notes, verdict \
|
||||
FROM reports WHERE 1=1"
|
||||
.to_string();
|
||||
FROM reports WHERE 1=1";
|
||||
|
||||
// Build OR conditions for tags - use JSON1 extension for tag matching
|
||||
if !tags.is_empty() {
|
||||
let tag_conditions: Vec<String> = tags.iter()
|
||||
.map(|tag| format!("tags LIKE '%{}%'", tag.replace("'", "''")))
|
||||
let (sql, params): (String, Vec<rusqlite::types::Value>) = if tags.is_empty() {
|
||||
(
|
||||
format!("{} ORDER BY created_at DESC LIMIT ?1", base_sql),
|
||||
vec![(limit as i64).into()],
|
||||
)
|
||||
} else {
|
||||
let placeholders = tags.iter().enumerate()
|
||||
.map(|(i, _)| format!("tags LIKE ?{}", i + 1))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" OR ");
|
||||
let sql = format!(
|
||||
"{} AND ({}) ORDER BY created_at DESC LIMIT ?{}",
|
||||
base_sql,
|
||||
placeholders,
|
||||
tags.len() + 1
|
||||
);
|
||||
let mut p: Vec<rusqlite::types::Value> = tags.iter()
|
||||
.map(|t| format!("%{}%", t).into())
|
||||
.collect();
|
||||
sql.push_str(&format!(" AND ({})", tag_conditions.join(" OR ")));
|
||||
}
|
||||
sql.push_str(&format!(" ORDER BY created_at DESC LIMIT {}", limit));
|
||||
p.push((limit as i64).into());
|
||||
(sql, p)
|
||||
};
|
||||
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let entries: Vec<ReportEntry> = stmt
|
||||
.query_map([], |row| {
|
||||
.query_map(rusqlite::params_from_iter(params.iter()), |row| {
|
||||
let tags_str: String = row.get(23).unwrap_or_else(|_| "[]".to_string());
|
||||
let tags: Vec<String> = serde_json::from_str(&tags_str).unwrap_or_default();
|
||||
Ok(ReportEntry {
|
||||
@@ -624,19 +637,18 @@ impl ReportDb {
|
||||
/// Search reports by notes text (case-insensitive LIKE)
|
||||
pub fn search_by_notes(&self, query: &str, limit: usize) -> Result<Vec<ReportEntry>> {
|
||||
let conn = self.connect()?;
|
||||
let pattern = format!("%{}%", query.replace("'", "''"));
|
||||
let pattern = format!("%{}%", query);
|
||||
let mut stmt = conn.prepare(
|
||||
&format!("SELECT id, expert, symbol, timeframe, model, from_date, to_date, \
|
||||
"SELECT id, expert, symbol, timeframe, model, from_date, to_date, \
|
||||
created_at, set_file_original, set_snapshot_path, report_dir, charts_dir, \
|
||||
net_profit, profit_factor, max_dd_pct, sharpe_ratio, total_trades, \
|
||||
win_rate_pct, recovery_factor, deposit, currency, leverage, \
|
||||
duration_seconds, tags, notes, verdict \
|
||||
FROM reports WHERE notes LIKE '{}' ORDER BY created_at DESC LIMIT {}",
|
||||
pattern, limit)
|
||||
FROM reports WHERE notes LIKE ?1 ORDER BY created_at DESC LIMIT ?2"
|
||||
)?;
|
||||
|
||||
let entries: Vec<ReportEntry> = stmt
|
||||
.query_map([], |row| {
|
||||
.query_map(rusqlite::params![pattern, limit as i64], |row| {
|
||||
let tags_str: String = row.get(23).unwrap_or_else(|_| "[]".to_string());
|
||||
let tags: Vec<String> = serde_json::from_str(&tags_str).unwrap_or_default();
|
||||
Ok(ReportEntry {
|
||||
@@ -677,20 +689,19 @@ impl ReportDb {
|
||||
/// Find reports by set file (original or snapshot)
|
||||
pub fn search_by_set_file(&self, set_file: &str, limit: usize) -> Result<Vec<ReportEntry>> {
|
||||
let conn = self.connect()?;
|
||||
let pattern = format!("%{}%", set_file.replace("'", "''"));
|
||||
let pattern = format!("%{}%", set_file);
|
||||
let mut stmt = conn.prepare(
|
||||
&format!("SELECT id, expert, symbol, timeframe, model, from_date, to_date, \
|
||||
"SELECT id, expert, symbol, timeframe, model, from_date, to_date, \
|
||||
created_at, set_file_original, set_snapshot_path, report_dir, charts_dir, \
|
||||
net_profit, profit_factor, max_dd_pct, sharpe_ratio, total_trades, \
|
||||
win_rate_pct, recovery_factor, deposit, currency, leverage, \
|
||||
duration_seconds, tags, notes, verdict \
|
||||
FROM reports WHERE (set_file_original LIKE '{}' OR set_snapshot_path LIKE '{}') \
|
||||
ORDER BY created_at DESC LIMIT {}",
|
||||
pattern, pattern, limit)
|
||||
FROM reports WHERE (set_file_original LIKE ?1 OR set_snapshot_path LIKE ?1) \
|
||||
ORDER BY created_at DESC LIMIT ?2"
|
||||
)?;
|
||||
|
||||
let entries: Vec<ReportEntry> = stmt
|
||||
.query_map([], |row| {
|
||||
.query_map(rusqlite::params![pattern, limit as i64], |row| {
|
||||
let tags_str: String = row.get(23).unwrap_or_else(|_| "[]".to_string());
|
||||
let tags: Vec<String> = serde_json::from_str(&tags_str).unwrap_or_default();
|
||||
Ok(ReportEntry {
|
||||
|
||||
@@ -99,8 +99,10 @@ pub fn tool_launch_backtest() -> Value {
|
||||
"skip_compile": { "type": "boolean" },
|
||||
"skip_clean": { "type": "boolean" },
|
||||
"timeout": { "type": "integer", "description": "Max time in seconds to wait for backtest (default: 900)" },
|
||||
"shutdown": { "type": "boolean", "description": "Shut down MT5 after test (default: true — required for HTML report to be written)" },
|
||||
"gui": { "type": "boolean", "description": "Enable visualization during backtest" },
|
||||
"startup_delay_secs": { "type": "integer", "description": "Seconds to wait for MT5 initialization (default: 10)" }
|
||||
"startup_delay_secs": { "type": "integer", "description": "Seconds to wait for MT5 initialization (default: 10)" },
|
||||
"inactivity_kill_secs": { "type": "integer", "description": "Kill MT5 if tester log hasn't grown for this many seconds (0 = disabled). Use to abort EAs that stop trading mid-test." }
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -119,6 +121,22 @@ pub fn tool_get_backtest_status() -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tool_get_tester_log() -> Value {
|
||||
json!({
|
||||
"name": "get_tester_log",
|
||||
"description": "Read the active MT5 tester agent journal log. Returns parsed deals, final balance, test progress, and raw log tail. Works during a backtest (if the log is being written) or after completion. Use this to inspect what trades occurred, check for EA activity, or debug issues when the HTML report wasn't produced.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tail_lines": {
|
||||
"type": "integer",
|
||||
"description": "Number of log tail lines to return (default: 100)"
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tool_cache_status() -> Value {
|
||||
json!({
|
||||
"name": "cache_status",
|
||||
|
||||
@@ -18,6 +18,7 @@ pub fn get_tools_list() -> Value {
|
||||
backtest::tool_run_backtest_only(), // Minimal: skip compile, do clean + backtest + extract only
|
||||
backtest::tool_launch_backtest(), // Fire-and-forget: compile + clean + launch MT5
|
||||
backtest::tool_get_backtest_status(), // Poll for completion
|
||||
backtest::tool_get_tester_log(), // Live journal reading mid-backtest or after
|
||||
backtest::tool_cache_status(),
|
||||
backtest::tool_clean_cache(),
|
||||
// Optimization
|
||||
|
||||
@@ -289,7 +289,7 @@ pub fn tool_get_wine_prefix_info() -> Value {
|
||||
pub fn tool_get_backtest_crash_info() -> Value {
|
||||
json!({
|
||||
"name": "get_backtest_crash_info",
|
||||
"description": "Investigate backtest crashes/failures. Checks for incomplete markers, missing deals.csv, error logs. Can scan recent reports.",
|
||||
"description": "Investigate backtest crashes/failures. Checks for incomplete markers, missing metrics.json, DB deal count, error logs. Can scan recent reports.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
@@ -225,7 +225,9 @@ fn deal_to_json(d: &Deal) -> Value {
|
||||
}
|
||||
|
||||
fn is_closed_trade(d: &Deal) -> bool {
|
||||
d.entry.to_lowercase().contains("out") && d.profit != 0.0
|
||||
// Accept any "out" entry. Profit may legitimately be 0.0 for journal-extracted
|
||||
// deals (where per-deal P&L is unavailable), so we don't gate on profit != 0.0.
|
||||
d.entry.to_lowercase().contains("out")
|
||||
}
|
||||
|
||||
pub async fn handle_list_deals(_config: &Config, args: &Value) -> Result<Value> {
|
||||
|
||||
+239
-79
@@ -74,66 +74,87 @@ pub async fn handle_run_backtest(config: &Config, args: &Value) -> Result<Value>
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
|
||||
let symbol = if requested_symbol.is_empty() {
|
||||
// Use config default or first available symbol
|
||||
if let Some(default) = config.backtest_symbol.clone() {
|
||||
if preflight.available_symbols.contains(&default) {
|
||||
default
|
||||
} else if let Some(first) = preflight.available_symbols.first() {
|
||||
tracing::warn!("Default symbol {} not found for server {}; using {}", default, active_server, first);
|
||||
first.clone()
|
||||
} else {
|
||||
// No tester data at all → hard fail with helpful context
|
||||
let no_symbols_error = || json!({
|
||||
"content": [{ "type": "text", "text": json!({
|
||||
"error": format!("No symbols available for backtesting on server '{}'.", active_server),
|
||||
"account": { "login": active_login, "server": active_server },
|
||||
"hint": "Open MT5 → View → Strategy Tester → download history for at least one symbol.",
|
||||
"pre_check": "no_symbols"
|
||||
}).to_string() }],
|
||||
"isError": true
|
||||
});
|
||||
|
||||
let symbol: String = if requested_symbol.is_empty() {
|
||||
// No symbol requested — use config default or first available tester symbol.
|
||||
let candidate = config.backtest_symbol.as_deref().unwrap_or("");
|
||||
if candidate.is_empty() {
|
||||
preflight.available_symbols.first()
|
||||
.cloned()
|
||||
.ok_or(()
|
||||
).unwrap_or_else(|_| return String::new())
|
||||
} else {
|
||||
match Config::resolve_symbol(candidate, &preflight.available_symbols) {
|
||||
Some(resolved) => {
|
||||
if resolved != candidate {
|
||||
tracing::warn!(
|
||||
"Config symbol '{}' not in tester data for '{}'; using '{}' instead",
|
||||
candidate, active_server, resolved
|
||||
);
|
||||
}
|
||||
resolved.to_string()
|
||||
}
|
||||
None => {
|
||||
// Config default has no tester data — pick first available
|
||||
preflight.available_symbols.first()
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Caller specified a symbol — resolve it against actual tester data.
|
||||
if preflight.available_symbols.is_empty() {
|
||||
return Ok(no_symbols_error());
|
||||
}
|
||||
match Config::resolve_symbol(requested_symbol, &preflight.available_symbols) {
|
||||
Some(resolved) if resolved == requested_symbol => {
|
||||
// Exact match — use as-is
|
||||
resolved.to_string()
|
||||
}
|
||||
Some(resolved) => {
|
||||
// Fuzzy match — proceed with the corrected symbol, surface the substitution
|
||||
tracing::warn!(
|
||||
"Symbol '{}' not in tester data for '{}'; substituting '{}'",
|
||||
requested_symbol, active_server, resolved
|
||||
);
|
||||
resolved.to_string()
|
||||
}
|
||||
None => {
|
||||
// No match at all — fail with full context so the caller can act
|
||||
return Ok(json!({
|
||||
"content": [{ "type": "text", "text": json!({
|
||||
"error": format!("No symbols available for backtesting on server '{}'.", active_server),
|
||||
"account": {
|
||||
"login": active_login,
|
||||
"server": active_server
|
||||
},
|
||||
"hint": "Download historical data in MT5 Strategy Tester for this server.",
|
||||
"pre_check": "no_symbols",
|
||||
"suggestion": "Use get_active_account to see available symbols."
|
||||
"error": format!(
|
||||
"Symbol '{}' has no tester data on server '{}' and no close match was found.",
|
||||
requested_symbol, active_server
|
||||
),
|
||||
"account": { "login": active_login, "server": active_server },
|
||||
"requested_symbol": requested_symbol,
|
||||
"available_symbols": preflight.available_symbols,
|
||||
"hint": "The tester data for this symbol hasn't been downloaded yet. \
|
||||
Open MT5 → Strategy Tester → select the symbol and click Download.",
|
||||
"pre_check": "symbol_not_available"
|
||||
}).to_string() }],
|
||||
"isError": true
|
||||
}));
|
||||
}
|
||||
} else if let Some(first) = preflight.available_symbols.first() {
|
||||
first.clone()
|
||||
} else {
|
||||
return Ok(json!({
|
||||
"content": [{ "type": "text", "text": json!({
|
||||
"error": format!("No symbols available for backtesting on server '{}'.", active_server),
|
||||
"account": {
|
||||
"login": active_login,
|
||||
"server": active_server
|
||||
},
|
||||
"hint": "Download historical data in MT5 Strategy Tester for this server.",
|
||||
"pre_check": "no_symbols",
|
||||
"suggestion": "Use get_active_account to see available symbols."
|
||||
}).to_string() }],
|
||||
"isError": true
|
||||
}));
|
||||
}
|
||||
} else {
|
||||
if !preflight.available_symbols.is_empty() && !preflight.available_symbols.contains(&requested_symbol.to_string()) {
|
||||
return Ok(json!({
|
||||
"content": [{ "type": "text", "text": json!({
|
||||
"error": format!("Symbol '{}' is not available for server '{}'.", requested_symbol, active_server),
|
||||
"account": {
|
||||
"login": active_login,
|
||||
"server": active_server
|
||||
},
|
||||
"requested_symbol": requested_symbol,
|
||||
"available_symbols": preflight.available_symbols,
|
||||
"hint": "The symbol may not have history data for this account's server. Use list_symbols to see available symbols.",
|
||||
"pre_check": "symbol_not_available",
|
||||
"suggestion": "Either switch to a different MT5 account with this symbol's data, or download history for this symbol on the current server."
|
||||
}).to_string() }],
|
||||
"isError": true
|
||||
}));
|
||||
}
|
||||
requested_symbol.to_string()
|
||||
};
|
||||
|
||||
// Guard against the empty-string edge case (no symbols at all)
|
||||
if symbol.is_empty() {
|
||||
return Ok(no_symbols_error());
|
||||
}
|
||||
|
||||
// EA existence check with context
|
||||
if !preflight.ea_exists {
|
||||
@@ -172,11 +193,12 @@ pub async fn handle_run_backtest(config: &Config, args: &Value) -> Result<Value>
|
||||
skip_clean: args.get("skip_clean").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
skip_analyze: args.get("skip_analyze").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
deep_analyze: args.get("deep").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
shutdown: args.get("shutdown").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
shutdown: args.get("shutdown").and_then(|v| v.as_bool()).unwrap_or(true),
|
||||
kill_existing: args.get("kill_existing").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
timeout: args.get("timeout").and_then(|v| v.as_u64()).unwrap_or(900),
|
||||
gui: args.get("gui").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
startup_delay_secs: args.get("startup_delay_secs").and_then(|v| v.as_u64()).unwrap_or(0),
|
||||
inactivity_kill_secs: args.get("inactivity_kill_secs").and_then(|v| v.as_u64()),
|
||||
};
|
||||
|
||||
let pipeline = BacktestPipeline::new(config.clone());
|
||||
@@ -193,24 +215,26 @@ pub async fn handle_run_backtest(config: &Config, args: &Value) -> Result<Value>
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn handle_run_backtest_quick(config: &Config, args: &Value) -> Result<Value> {
|
||||
// Quick backtest: skip compile, do clean → backtest → extract → analyze
|
||||
pub async fn handle_run_backtest_quick(handler: &crate::tools::handlers::ToolHandler, args: &Value) -> Result<Value> {
|
||||
// Quick backtest: skip compile, clean → launch → background monitor → return job.
|
||||
// Uses the fire-and-forget launch path so the MCP response is returned immediately
|
||||
// and the result is available via get_backtest_status / get_latest_report once done.
|
||||
// (The synchronous blocking path exceeds MCP request timeouts for any backtest >2 min.)
|
||||
let mut args = args.clone();
|
||||
if let Some(obj) = args.as_object_mut() {
|
||||
obj.insert("skip_compile".to_string(), json!(true));
|
||||
// keep skip_analyze as false (default) to run analysis
|
||||
}
|
||||
handle_run_backtest(config, &args).await
|
||||
handle_launch_backtest(handler, &args).await
|
||||
}
|
||||
|
||||
pub async fn handle_run_backtest_only(config: &Config, args: &Value) -> Result<Value> {
|
||||
// Backtest only: skip compile, skip analyze - just backtest and extract
|
||||
pub async fn handle_run_backtest_only(handler: &crate::tools::handlers::ToolHandler, args: &Value) -> Result<Value> {
|
||||
// Backtest only: skip compile and analyze — launch and return job immediately.
|
||||
let mut args = args.clone();
|
||||
if let Some(obj) = args.as_object_mut() {
|
||||
obj.insert("skip_compile".to_string(), json!(true));
|
||||
obj.insert("skip_analyze".to_string(), json!(true));
|
||||
}
|
||||
handle_run_backtest(config, &args).await
|
||||
handle_launch_backtest(handler, &args).await
|
||||
}
|
||||
|
||||
pub async fn handle_launch_backtest(handler: &crate::tools::handlers::ToolHandler, args: &Value) -> Result<Value> {
|
||||
@@ -232,17 +256,50 @@ pub async fn handle_launch_backtest(handler: &crate::tools::handlers::ToolHandle
|
||||
}));
|
||||
}
|
||||
|
||||
// Get symbol
|
||||
// Get symbol — resolve against actual tester data (same logic as handle_run_backtest)
|
||||
let active_server = preflight.server.as_deref().unwrap_or("unknown");
|
||||
let requested_symbol = args.get("symbol")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
|
||||
let symbol = if requested_symbol.is_empty() {
|
||||
handler.config.backtest_symbol.clone()
|
||||
.or_else(|| preflight.available_symbols.first().cloned())
|
||||
.unwrap_or_else(|| "EURUSD".to_string())
|
||||
let symbol: String = if requested_symbol.is_empty() {
|
||||
let candidate = handler.config.backtest_symbol.as_deref().unwrap_or("");
|
||||
if candidate.is_empty() {
|
||||
preflight.available_symbols.first().cloned().unwrap_or_default()
|
||||
} else {
|
||||
Config::resolve_symbol(candidate, &preflight.available_symbols)
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| preflight.available_symbols.first().cloned())
|
||||
.unwrap_or_else(|| candidate.to_string())
|
||||
}
|
||||
} else {
|
||||
requested_symbol.to_string()
|
||||
match Config::resolve_symbol(requested_symbol, &preflight.available_symbols) {
|
||||
Some(resolved) => {
|
||||
if resolved != requested_symbol {
|
||||
tracing::warn!(
|
||||
"launch_backtest: symbol '{}' not in tester data for '{}'; using '{}'",
|
||||
requested_symbol, active_server, resolved
|
||||
);
|
||||
}
|
||||
resolved.to_string()
|
||||
}
|
||||
None if preflight.available_symbols.is_empty() => requested_symbol.to_string(),
|
||||
None => {
|
||||
return Ok(json!({
|
||||
"content": [{ "type": "text", "text": json!({
|
||||
"error": format!(
|
||||
"Symbol '{}' has no tester data on server '{}' and no close match was found.",
|
||||
requested_symbol, active_server
|
||||
),
|
||||
"requested_symbol": requested_symbol,
|
||||
"available_symbols": preflight.available_symbols,
|
||||
"hint": "Open MT5 → Strategy Tester → select the symbol and click Download.",
|
||||
"pre_check": "symbol_not_available"
|
||||
}).to_string() }],
|
||||
"isError": true
|
||||
}));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// EA existence check
|
||||
@@ -281,11 +338,17 @@ pub async fn handle_launch_backtest(handler: &crate::tools::handlers::ToolHandle
|
||||
skip_clean: args.get("skip_clean").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
skip_analyze: true, // Not needed for launch mode
|
||||
deep_analyze: false,
|
||||
shutdown: false, // Don't shutdown so we can poll
|
||||
// On Wine/macOS, ShutdownTerminal=1 does NOT reliably cause terminal64.exe to exit.
|
||||
// When inactivity_kill_secs is set, the monitor waits that many seconds after the
|
||||
// tester log goes quiet, then polls for the HTML report for 30 s, then kills MT5.
|
||||
// If no inactivity_kill_secs is given (default None → disabled), the monitor relies
|
||||
// solely on timeout (900 s) or natural MT5 exit for completion detection.
|
||||
shutdown: args.get("shutdown").and_then(|v| v.as_bool()).unwrap_or(true),
|
||||
kill_existing: false,
|
||||
timeout: args.get("timeout").and_then(|v| v.as_u64()).unwrap_or(900),
|
||||
gui: args.get("gui").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
startup_delay_secs: args.get("startup_delay_secs").and_then(|v| v.as_u64()).unwrap_or(0),
|
||||
inactivity_kill_secs: args.get("inactivity_kill_secs").and_then(|v| v.as_u64()),
|
||||
};
|
||||
|
||||
let pipeline = if let Some(ref callback) = handler.notification_callback {
|
||||
@@ -348,16 +411,27 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul
|
||||
// Check if MT5 is running
|
||||
let mt5_running = is_mt5_running();
|
||||
|
||||
// Check if report file exists
|
||||
// Check if report file exists (still on disk — it's deleted after extraction)
|
||||
let report_found = job.as_ref()
|
||||
.map(|j| Path::new(&j.expected_report_path).exists())
|
||||
.unwrap_or(false);
|
||||
|
||||
|
||||
// Check for completed artifacts
|
||||
let metrics_exists = report_path.join("metrics.json").exists();
|
||||
let deals_exists = report_path.join("deals.csv").exists();
|
||||
let is_complete = stage == "DONE" || (report_found && metrics_exists);
|
||||
|
||||
|
||||
// Read the authoritative status written by the background monitor into job.json.
|
||||
// This avoids false "failed" when the HTML was extracted+deleted (report_found=false)
|
||||
// or when journal extraction ran instead of HTML extraction.
|
||||
let job_status = job.as_ref()
|
||||
.and_then(|j| j.status.as_deref())
|
||||
.unwrap_or("");
|
||||
let monitor_says_complete = matches!(job_status, "completed" | "completed_no_html");
|
||||
let monitor_says_failed = matches!(job_status, "failed" | "timeout" | "timeout_inactive");
|
||||
|
||||
let is_complete = monitor_says_complete
|
||||
|| stage == "DONE"
|
||||
|| (report_found && metrics_exists);
|
||||
|
||||
// Calculate elapsed time if job exists
|
||||
let elapsed_seconds = job.as_ref()
|
||||
.and_then(|j| {
|
||||
@@ -366,26 +440,32 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul
|
||||
.map(|t| (chrono::Utc::now() - t.with_timezone(&chrono::Utc)).num_seconds())
|
||||
})
|
||||
.unwrap_or(0);
|
||||
|
||||
// Determine status message
|
||||
|
||||
// Determine status message — trust monitor's job.json first
|
||||
let status_msg = if is_complete {
|
||||
"completed"
|
||||
} else if monitor_says_failed {
|
||||
if job_status == "timeout" || job_status == "timeout_inactive" { "timeout" } else { "failed" }
|
||||
} else if stage == "BACKTEST" && mt5_running {
|
||||
"running"
|
||||
} else if stage == "BACKTEST" && !mt5_running && !report_found {
|
||||
} else if stage == "BACKTEST" && !mt5_running {
|
||||
"failed"
|
||||
} else if progress_lines > 0 {
|
||||
"in_progress"
|
||||
} else {
|
||||
"not_started"
|
||||
};
|
||||
|
||||
|
||||
let message = if is_complete {
|
||||
"Backtest completed successfully"
|
||||
if job_status == "completed_no_html" {
|
||||
"Backtest completed (extracted from tester journal — no HTML report)"
|
||||
} else {
|
||||
"Backtest completed successfully"
|
||||
}
|
||||
} else if stage == "BACKTEST" && mt5_running {
|
||||
"MT5 is running the backtest"
|
||||
} else if stage == "BACKTEST" && !mt5_running {
|
||||
"MT5 process exited but report not found - backtest may have failed"
|
||||
"MT5 process exited — report not yet found"
|
||||
} else {
|
||||
&format!("Backtest is at stage: {}", stage)
|
||||
};
|
||||
@@ -400,7 +480,6 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul
|
||||
"mt5_running": mt5_running,
|
||||
"report_found": report_found,
|
||||
"metrics_extracted": metrics_exists,
|
||||
"deals_extracted": deals_exists,
|
||||
"elapsed_seconds": elapsed_seconds,
|
||||
"message": message,
|
||||
"job": job.map(|j| {
|
||||
@@ -435,6 +514,87 @@ fn is_mt5_running() -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
/// Read the active tester agent log for live/post-test deal inspection.
|
||||
/// Returns the last N lines and a parsed deal summary.
|
||||
pub async fn handle_get_tester_log(config: &Config, args: &Value) -> Result<Value> {
|
||||
use crate::pipeline::backtest::BacktestPipeline;
|
||||
|
||||
let tail_lines = args.get("tail_lines").and_then(|v| v.as_u64()).unwrap_or(100) as usize;
|
||||
|
||||
let log_path = match BacktestPipeline::find_active_tester_agent_log(config) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
return Ok(json!({
|
||||
"content": [{ "type": "text", "text": json!({
|
||||
"error": "No tester agent log found for today.",
|
||||
"hint": "Run a backtest first. The log appears after the tester starts."
|
||||
}).to_string() }],
|
||||
"isError": true
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
let lines = match BacktestPipeline::read_tester_agent_log(&log_path) {
|
||||
Some(l) => l,
|
||||
None => {
|
||||
return Ok(json!({
|
||||
"content": [{ "type": "text", "text": json!({
|
||||
"error": format!("Could not read log at {}", log_path.display())
|
||||
}).to_string() }],
|
||||
"isError": true
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
let (deals, final_balance_pips, progress) = BacktestPipeline::parse_journal_deals(&lines);
|
||||
|
||||
// Collect tail lines
|
||||
let tail: Vec<&str> = lines.iter()
|
||||
.rev()
|
||||
.take(tail_lines)
|
||||
.rev()
|
||||
.map(|s| s.as_str())
|
||||
.collect();
|
||||
|
||||
// Detect last sim timestamp for progress estimation
|
||||
let last_sim_time = lines.iter().rev()
|
||||
.find_map(|l| {
|
||||
let parts: Vec<&str> = l.split_whitespace().collect();
|
||||
// Format: XX 0 HH:MM:SS.mmm Core NN YYYY.MM.DD HH:MM:SS ...
|
||||
if parts.len() >= 6 {
|
||||
let date = parts[4];
|
||||
let time = parts[5];
|
||||
if date.contains('.') && time.contains(':') {
|
||||
return Some(format!("{} {}", date, time));
|
||||
}
|
||||
}
|
||||
None
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
Ok(json!({
|
||||
"content": [{ "type": "text", "text": json!({
|
||||
"log_path": log_path.to_string_lossy(),
|
||||
"total_lines": lines.len(),
|
||||
"deals_found": deals.len(),
|
||||
"final_balance_pips": final_balance_pips,
|
||||
"progress": progress,
|
||||
"last_sim_time": last_sim_time,
|
||||
"is_complete": !progress.is_empty(),
|
||||
"tail_lines": tail,
|
||||
"deals_summary": deals.iter().map(|d| json!({
|
||||
"deal": d.deal,
|
||||
"time": d.time,
|
||||
"type": d.deal_type,
|
||||
"volume": d.volume,
|
||||
"price": d.price,
|
||||
"symbol": d.symbol,
|
||||
})).collect::<Vec<_>>()
|
||||
}).to_string() }],
|
||||
"isError": false
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn handle_cache_status(config: &Config) -> Result<Value> {
|
||||
let cache_dir = config.tester_cache_dir.as_ref()
|
||||
.map(|s| Path::new(s))
|
||||
|
||||
@@ -70,10 +70,11 @@ impl ToolHandler {
|
||||
|
||||
// Backtest handlers - Granular pipeline options
|
||||
"run_backtest" => backtest::handle_run_backtest(&self.config, args).await, // Full: compile + clean + backtest + extract + analyze
|
||||
"run_backtest_quick" => backtest::handle_run_backtest_quick(&self.config, args).await, // Quick: skip compile, do backtest + extract + analyze
|
||||
"run_backtest_only" => backtest::handle_run_backtest_only(&self.config, args).await, // Minimal: skip compile, do backtest + extract only
|
||||
"run_backtest_quick" => backtest::handle_run_backtest_quick(self, args).await, // Quick: skip compile, fire-and-forget launch
|
||||
"run_backtest_only" => backtest::handle_run_backtest_only(self, args).await, // Minimal: skip compile+analyze, fire-and-forget launch
|
||||
"launch_backtest" => backtest::handle_launch_backtest(self, args).await, // Fire-and-forget mode
|
||||
"get_backtest_status" => backtest::handle_get_backtest_status(&self.config, args).await,
|
||||
"get_tester_log" => backtest::handle_get_tester_log(&self.config, args).await,
|
||||
"cache_status" => backtest::handle_cache_status(&self.config).await,
|
||||
"clean_cache" => backtest::handle_clean_cache(&self.config, args).await,
|
||||
|
||||
@@ -179,6 +180,7 @@ pub(crate) fn dir_size(path: &Path) -> u64 {
|
||||
.sum()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn past_complete_month() -> (String, String) {
|
||||
let now = chrono::Utc::now();
|
||||
let today = chrono::NaiveDate::from_ymd_opt(now.year(), now.month(), 1)
|
||||
|
||||
@@ -3,6 +3,7 @@ use serde_json::{json, Value};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use crate::models::Config;
|
||||
use crate::storage::ReportDb;
|
||||
|
||||
/// Validate that `user_path` resolves to a location within `allowed_base`.
|
||||
/// Returns the canonicalized absolute path on success.
|
||||
@@ -785,7 +786,6 @@ pub async fn handle_export_report(_config: &Config, args: &Value) -> Result<Valu
|
||||
|
||||
let path = Path::new(report_dir);
|
||||
let metrics_path = path.join("metrics.json");
|
||||
let _deals_path = path.join("deals.csv");
|
||||
|
||||
// Read metrics
|
||||
let metrics: Value = if metrics_path.exists() {
|
||||
@@ -1640,21 +1640,33 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
|
||||
}
|
||||
}
|
||||
|
||||
// Check if deals.csv is missing or empty
|
||||
let deals_csv = path.join("deals.csv");
|
||||
if !deals_csv.exists() {
|
||||
result["crashes_found"].as_array_mut().unwrap().push(json!({
|
||||
"report_dir": dir,
|
||||
"type": "missing_deals",
|
||||
"reason": "deals.csv not found - backtest likely failed",
|
||||
}));
|
||||
} else if let Ok(meta) = deals_csv.metadata() {
|
||||
if meta.len() < 100 {
|
||||
result["crashes_found"].as_array_mut().unwrap().push(json!({
|
||||
"report_dir": dir,
|
||||
"type": "empty_deals",
|
||||
"reason": "deals.csv is nearly empty - no trades were made",
|
||||
}));
|
||||
// Check deal count in DB
|
||||
let db = ReportDb::new(&Config::db_path());
|
||||
if db.init().is_ok() {
|
||||
match db.get_by_report_dir(dir) {
|
||||
Ok(Some(entry)) => {
|
||||
let deal_count = db.get_deals(&entry.id)
|
||||
.map(|d| d.len())
|
||||
.unwrap_or(0);
|
||||
if deal_count == 0 {
|
||||
result["crashes_found"].as_array_mut().unwrap().push(json!({
|
||||
"report_dir": dir,
|
||||
"type": "empty_deals",
|
||||
"reason": "No deals stored in DB - EA did not trade or extraction failed",
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(None) | Err(_) => {
|
||||
// Not in DB at all means extraction never completed
|
||||
let metrics_exists = path.join("metrics.json").exists();
|
||||
if !metrics_exists {
|
||||
result["crashes_found"].as_array_mut().unwrap().push(json!({
|
||||
"report_dir": dir,
|
||||
"type": "missing_deals",
|
||||
"reason": "Report not found in DB and no metrics.json - backtest likely crashed before extraction",
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1676,10 +1688,10 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
|
||||
if let Ok(modified) = meta.modified() {
|
||||
if modified >= cutoff {
|
||||
// Check for failure indicators
|
||||
let has_deals = path.join("deals.csv").exists();
|
||||
let has_metrics = path.join("metrics.json").exists();
|
||||
let has_incomplete = path.join(".incomplete").exists();
|
||||
|
||||
if !has_deals || has_incomplete {
|
||||
|
||||
if !has_metrics || has_incomplete {
|
||||
failures += 1;
|
||||
}
|
||||
}
|
||||
@@ -1701,7 +1713,7 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
|
||||
|
||||
if types.contains(&"missing_deals".to_string()) {
|
||||
result["common_patterns"].as_array_mut().unwrap().push(
|
||||
json!("Missing deals.csv suggests MT5 crashed during backtest")
|
||||
json!("Report not in DB and no metrics.json suggests MT5 crashed before extraction completed")
|
||||
);
|
||||
}
|
||||
if types.contains(&"incomplete".to_string()) {
|
||||
|
||||
Reference in New Issue
Block a user