diff --git a/Cargo.lock b/Cargo.lock index a76639b..7b98855 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -481,7 +481,7 @@ dependencies = [ [[package]] name = "mt5-quant" -version = "1.32.3" +version = "1.32.4" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index 9490c1c..c9b641f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mt5-quant" -version = "1.32.3" +version = "1.32.4" edition = "2021" description = "MCP server for MT5 strategy development on macOS/Linux" authors = ["masdevid "] diff --git a/src/analytics/extract.rs b/src/analytics/extract.rs index df30c5c..7bd68a2 100644 --- a/src/analytics/extract.rs +++ b/src/analytics/extract.rs @@ -77,59 +77,96 @@ impl ReportExtractor { fn parse_deals_html(&self, text: &str) -> Result> { let mut deals = Vec::new(); - let re = regex::Regex::new(r"]*>.*?Deal.*?Time.*?Type.*?Direction.*?(.*)") - .map_err(|e| anyhow!("Regex error: {}", e))?; + // (?s) = dotall: makes '.' match '\n' so the regex works on multiline HTML. + // MT5 HTML column order: Time | Deal | Symbol | Type | Direction | + // Volume | Price | Order | Commission | Swap | Profit | Balance | Comment + // + // Strategy: locate the deals table by finding the header that contains + // the column names, then parse every subsequent as a data row. + // We try two header patterns to handle different MT5 versions/locales. - if let Some(captures) = re.captures(text) { - let section = captures.get(1).map(|m| m.as_str()).unwrap_or(""); + let row_re = regex::Regex::new(r"(?s)]*>(.*?)") + .map_err(|e| anyhow!("Row regex error: {}", e))?; + let cell_re = regex::Regex::new(r"(?s)]*>(.*?)") + .map_err(|e| anyhow!("Cell regex error: {}", e))?; - let row_re = regex::Regex::new(r"]*>(.*?)") - .map_err(|e| anyhow!("Regex error: {}", e))?; + // Collect all blocks once, then find the deals header and parse from there. + let rows: Vec<&str> = row_re.captures_iter(text) + .filter_map(|cap| cap.get(0).map(|m| m.as_str())) + .collect(); - for row_caps in row_re.captures_iter(section) { - let row = row_caps.get(1).map(|m| m.as_str()).unwrap_or(""); - - let cell_re = regex::Regex::new(r"]*>(.*?)") - .map_err(|e| anyhow!("Regex error: {}", e))?; - - let cells: Vec = cell_re.captures_iter(row) - .filter_map(|cap| cap.get(1)) - .map(|m| Self::strip_tags(m.as_str())) - .map(|s| s.replace(',', "")) - .collect(); + // Find the header row index: it must contain both "Deal" and "Symbol" (case-insensitive). + let header_idx = rows.iter().position(|row| { + let lower = row.to_lowercase(); + (lower.contains(">deal<") || lower.contains(">deal symbol<") || lower.contains(">symbol i + 1, + None => { + // Fallback: look for any row containing Time+Volume+Profit headers + let alt = rows.iter().position(|row| { + let lower = row.to_lowercase(); + lower.contains(">time<") && lower.contains(">volume<") && lower.contains(">profit<") + }); + match alt { + Some(i) => i + 1, + None => { + tracing::warn!("parse_deals_html: no deals table header found"); + return Ok(deals); + } } - - if cells.iter().take(5).any(|c| { - let c_lower = c.trim().to_lowercase(); - c_lower == "balance" || c_lower == "credit" - }) { - continue; - } - - let deal = Deal { - time: cells.get(0).cloned().unwrap_or_default(), - deal: cells.get(1).cloned().unwrap_or_default(), - symbol: cells.get(2).cloned().unwrap_or_default(), - deal_type: cells.get(3).cloned().unwrap_or_default(), - entry: cells.get(4).cloned().unwrap_or_default(), - volume: cells.get(5).and_then(|s| s.parse().ok()).unwrap_or(0.0), - price: cells.get(6).and_then(|s| s.parse().ok()).unwrap_or(0.0), - order: cells.get(7).cloned().unwrap_or_default(), - commission: cells.get(8).and_then(|s| s.parse().ok()).unwrap_or(0.0), - swap: cells.get(9).and_then(|s| s.parse().ok()).unwrap_or(0.0), - profit: cells.get(10).and_then(|s| s.parse().ok()).unwrap_or(0.0), - balance: cells.get(11).and_then(|s| s.parse().ok()).unwrap_or(0.0), - comment: cells.get(12).cloned().unwrap_or_default(), - magic: cells.get(13).cloned(), - }; - - deals.push(deal); } + }; + + for row in &rows[start_idx..] { + let cells: Vec = cell_re.captures_iter(row) + .filter_map(|cap| cap.get(1)) + .map(|m| Self::strip_tags(m.as_str())) + .map(|s| s.replace(',', "")) + .collect(); + + if cells.len() < 3 || cells[0].trim().is_empty() { + continue; + } + + // Skip balance/credit operation rows (Type column is index 3 in MT5 HTML) + let type_cell = cells.get(3).map(|s| s.trim().to_lowercase()).unwrap_or_default(); + if type_cell == "balance" || type_cell == "credit" { + continue; + } + // Also skip sub-header rows that repeat column names + if type_cell == "type" || cells.get(1).map(|s| s.trim().to_lowercase()).as_deref() == Some("deal") { + continue; + } + // Skip rows with no deal number (e.g. totals row) + let deal_num = cells.get(1).map(|s| s.trim().to_string()).unwrap_or_default(); + if deal_num.is_empty() { + continue; + } + + let deal = Deal { + time: cells.get(0).cloned().unwrap_or_default(), + deal: deal_num, + symbol: cells.get(2).cloned().unwrap_or_default(), + deal_type: cells.get(3).cloned().unwrap_or_default(), + entry: cells.get(4).cloned().unwrap_or_default(), + volume: cells.get(5).and_then(|s| s.parse().ok()).unwrap_or(0.0), + price: cells.get(6).and_then(|s| s.parse().ok()).unwrap_or(0.0), + order: cells.get(7).cloned().unwrap_or_default(), + commission: cells.get(8).and_then(|s| s.parse().ok()).unwrap_or(0.0), + swap: cells.get(9).and_then(|s| s.parse().ok()).unwrap_or(0.0), + profit: cells.get(10).and_then(|s| s.parse().ok()).unwrap_or(0.0), + balance: cells.get(11).and_then(|s| s.parse().ok()).unwrap_or(0.0), + comment: cells.get(12).cloned().unwrap_or_default(), + magic: cells.get(13).cloned(), + }; + + deals.push(deal); } + tracing::info!("parse_deals_html: extracted {} deals", deals.len()); Ok(deals) } diff --git a/src/compile/mql_compiler.rs b/src/compile/mql_compiler.rs index 1980d9b..9be6db4 100644 --- a/src/compile/mql_compiler.rs +++ b/src/compile/mql_compiler.rs @@ -90,8 +90,8 @@ impl MqlCompiler { .map(|s| s.to_string()) .collect(); - let ex5_path = staged_mq5.with_extension("ex5"); - if !ex5_path.exists() { + let staged_ex5 = staged_mq5.with_extension("ex5"); + if !staged_ex5.exists() { let final_errors = if errors.is_empty() { vec![format!("Compilation failed. Log:\n{}", &log_text[log_text.len().saturating_sub(500)..])] } else { @@ -107,11 +107,37 @@ impl MqlCompiler { }); } - let binary_size = fs::metadata(&ex5_path)?.len(); + let binary_size = fs::metadata(&staged_ex5)?.len(); + + // Deploy compiled output to the real MQL5/Experts/{ea_name}/ directory so + // MT5 can actually load it. The temp dir is only needed to avoid Wine path + // issues with spaces; the real experts dir is the authoritative location. + let final_ex5_path = if let Some(experts_dir) = self.config.experts_dir.as_ref() { + let real_experts = PathBuf::from(experts_dir); + let real_ea_dir = real_experts.join(ea_name); + fs::create_dir_all(&real_ea_dir)?; + + // Sync source files (.mq5 + .mqh) into the real experts dir so that + // future compiles from the experts dir also work correctly. + if let Err(e) = self.sync_project_to_experts(source_path, &real_experts) { + tracing::warn!("Could not sync source to experts dir: {}", e); + } + + // Copy the compiled .ex5 to the real experts dir. + let dest_ex5 = real_ea_dir.join(format!("{}.ex5", ea_name)); + fs::copy(&staged_ex5, &dest_ex5)?; + tracing::info!("Deployed {} → {}", staged_ex5.display(), dest_ex5.display()); + dest_ex5 + } else { + // No experts_dir configured — fall back to the staged path so callers + // still get a valid path even if MT5 won't find it automatically. + tracing::warn!("experts_dir not configured; .ex5 left in staging dir"); + staged_ex5 + }; Ok(CompileResult { success: errors.is_empty(), - ex5_path: Some(ex5_path), + ex5_path: Some(final_ex5_path), errors, warnings, binary_size, diff --git a/src/main.rs b/src/main.rs index 28033b7..ee6944e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -148,6 +148,7 @@ async fn run_test_launch(ea: Option, startup_delay: Option) -> Resu timeout: 900, gui: false, startup_delay_secs: delay, + inactivity_kill_secs: None, }; let pipeline = BacktestPipeline::new(config); diff --git a/src/models/config.rs b/src/models/config.rs index 2a844c0..f1d751c 100644 --- a/src/models/config.rs +++ b/src/models/config.rs @@ -508,29 +508,50 @@ impl Config { self.terminal_dir.as_ref().map(|d| Path::new(d).to_path_buf()) } - /// Scan Bases/*/history/ for symbol directories that contain at least one .hcc file. - /// Returns deduplicated, sorted list of symbol names available for backtesting. - /// If server_filter is provided, only scans that specific server's directory. + /// Scan the tester's own history store for symbols with downloaded data. + /// + /// MT5 maintains two separate history trees: + /// • `Bases/{server}/history/` — live-trading tick/bar data (NOT usable by tester) + /// • `Tester/bases/{server}/history/` — data the Strategy Tester actually reads + /// + /// Scanning `Bases/` (the old approach) returned symbols that exist for live trading + /// but may have no tester data, causing the tester to fail with "symbol does not exist". + /// This function scans `Tester/bases/` instead, which is the authoritative source. + /// + /// Falls back to `Bases/` only when `Tester/bases/` is absent (first-run / no backtests yet). + /// + /// If `server_filter` is provided only that server's directory is scanned. pub fn discover_symbols(&self, server_filter: Option<&str>) -> Vec { let mt5_dir = match self.mt5_dir() { Some(d) => d, None => return Vec::new(), }; - let bases_dir = mt5_dir.join("Bases"); - if !bases_dir.is_dir() { - return Vec::new(); - } + // Prefer the tester's own data store; fall back to live-trading Bases/ when absent. + let tester_bases = mt5_dir.join("Tester").join("bases"); + let bases_dir = if tester_bases.is_dir() { + tester_bases + } else { + let fallback = mt5_dir.join("Bases"); + if !fallback.is_dir() { + return Vec::new(); + } + tracing::warn!( + "Tester/bases/ not found — falling back to Bases/ for symbol discovery. \ + Run at least one backtest to populate tester data." + ); + fallback + }; let mut symbols = std::collections::HashSet::new(); - // Bases/{server}/history/{symbol}/{year}.hcc + // {bases_dir}/{server}/history/{symbol}/ — directory presence = data available + // (the tester uses .hst/.hcc files; existence of the directory is sufficient) if let Ok(servers) = fs::read_dir(&bases_dir) { for server in servers.filter_map(|e| e.ok()) { let server_name_os = server.file_name(); let server_name = server_name_os.to_str().unwrap_or(""); - - // Skip non-directory entries and filter by server if specified + if server_name.is_empty() { continue; } @@ -539,7 +560,7 @@ impl Config { continue; } } - + let history_dir = server.path().join("history"); if !history_dir.is_dir() { continue; @@ -550,23 +571,8 @@ impl Config { if !sym_path.is_dir() { continue; } - // Only include if at least one .hcc file exists (has downloaded data) - let has_data = fs::read_dir(&sym_path) - .ok() - .map(|entries| { - entries.filter_map(|e| e.ok()).any(|e| { - e.path().extension() - .and_then(|x| x.to_str()) - .map(|x| x == "hcc") - .unwrap_or(false) - }) - }) - .unwrap_or(false); - - if has_data { - if let Some(name) = sym_path.file_name().and_then(|n| n.to_str()) { - symbols.insert(name.to_string()); - } + if let Some(name) = sym_path.file_name().and_then(|n| n.to_str()) { + symbols.insert(name.to_string()); } } } @@ -578,6 +584,59 @@ impl Config { sorted } + /// Find the closest available tester symbol to the one requested. + /// + /// Matching priority (first hit wins): + /// 1. Exact match → `XAUUSD.cent` == `XAUUSD.cent` + /// 2. Case-insensitive exact match → `xauusd.cent` → `XAUUSD.cent` + /// 3. Strip/add common cent suffixes → `XAUUSDc` ↔ `XAUUSD.cent` + /// 4. Prefix match on the base ticker → `XAUUSD` matches `XAUUSD.cent` + pub fn resolve_symbol<'a>(requested: &str, available: &'a [String]) -> Option<&'a str> { + if available.is_empty() { + return None; + } + + // 1. Exact + if let Some(s) = available.iter().find(|s| s.as_str() == requested) { + return Some(s.as_str()); + } + + // 2. Case-insensitive exact + let req_lower = requested.to_lowercase(); + if let Some(s) = available.iter().find(|s| s.to_lowercase() == req_lower) { + return Some(s.as_str()); + } + + // 3. Cent-suffix normalisation: build a normalised "base" for both sides + // Strip known cent suffixes: `.cent`, `c` (trailing, uppercase only), `.c` + fn base_ticker(sym: &str) -> &str { + let s = sym.trim_end_matches(".cent") + .trim_end_matches(".c"); + // Strip trailing lowercase 'c' only when the rest is all-uppercase + // (so "XAUUSDc" → "XAUUSD", but "Misc" stays "Misc") + if s.ends_with('c') && s[..s.len()-1].chars().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) { + &s[..s.len()-1] + } else { + s + } + } + + let req_base = base_ticker(requested).to_lowercase(); + if let Some(s) = available.iter().find(|s| base_ticker(s).to_lowercase() == req_base) { + return Some(s.as_str()); + } + + // 4. Prefix match: available symbol starts with the requested string (or vice-versa) + if let Some(s) = available.iter().find(|s| { + let sl = s.to_lowercase(); + sl.starts_with(&req_lower) || req_lower.starts_with(sl.as_str()) + }) { + return Some(s.as_str()); + } + + None + } + /// Get the currently active MT5 account from common.ini pub fn current_account(&self) -> Option { self.mt5_dir().and_then(|d| CurrentAccount::from_common_ini(&d)) diff --git a/src/models/report.rs b/src/models/report.rs index 669533b..3ef9f50 100644 --- a/src/models/report.rs +++ b/src/models/report.rs @@ -79,6 +79,9 @@ pub struct BacktestJob { pub mt5_pid: Option, pub expected_report_path: String, pub timeout_seconds: u64, + /// Set by background monitor: "running"|"completed"|"completed_no_html"|"failed"|"timeout" + #[serde(default)] + pub status: Option, } impl BacktestJob { @@ -101,6 +104,7 @@ impl BacktestJob { mt5_pid: None, expected_report_path, timeout_seconds, + status: Some("running".to_string()), } } } diff --git a/src/pipeline/backtest.rs b/src/pipeline/backtest.rs index 39f3561..52a25d5 100644 --- a/src/pipeline/backtest.rs +++ b/src/pipeline/backtest.rs @@ -44,6 +44,9 @@ pub struct BacktestParams { pub timeout: u64, pub gui: bool, pub startup_delay_secs: u64, + /// Kill MT5 if tester agent log hasn't grown for this many seconds. + /// 0 = disabled. Useful to abort EAs that stop trading mid-backtest. + pub inactivity_kill_secs: Option, } pub struct PipelineResult { @@ -193,7 +196,7 @@ impl BacktestPipeline { } self.log_progress(&progress_log, "BACKTEST").await; - + // Get MT5 paths let mt5_dir = self.config.mt5_dir() .ok_or_else(|| anyhow!("MT5 directory not configured"))?; @@ -208,15 +211,16 @@ impl BacktestPipeline { let reports_dir = mt5_dir.join("reports"); fs::create_dir_all(&reports_dir)?; - // Write params via /config: (triggers tester) and terminal.ini (redundancy). + // Kill first — MT5 writes terminal.ini on exit, which would clobber + // the backtest params we're about to write. + self.kill_mt5().await?; + + // Write params *after* MT5 is dead so nothing can overwrite them. let ini_content = self.build_backtest_ini(¶ms, &report_id)?; let config_host = wine_prefix.join("drive_c").join("backtest_config.ini"); fs::write(&config_host, ini_content.as_bytes())?; self.update_terminal_ini(¶ms, &report_id)?; - // Kill any running MT5 - self.kill_mt5().await?; - // Launch MT5 (fire and forget) let mut cmd = self.build_wine_launch(wine_exe, &wine_prefix)?; let child = cmd.stdin(std::process::Stdio::null()) @@ -278,6 +282,7 @@ impl BacktestPipeline { timeout: params.timeout, gui: params.gui, startup_delay_secs: params.startup_delay_secs, + inactivity_kill_secs: params.inactivity_kill_secs, }; tokio::spawn(async move { Self::monitor_backtest_completion( @@ -376,12 +381,29 @@ impl BacktestPipeline { let grace_period = Duration::from_secs(30); let poll_start = std::time::SystemTime::now(); + // Inactivity watchdog: kill MT5 if tester agent log hasn't grown for this long. + // Catches EAs that stall (no ticks processed) or flat periods with zero trades. + let inactivity_threshold = Duration::from_secs( + params.inactivity_kill_secs.unwrap_or(0) + ); + let mut last_log_size: u64 = 0; + let mut last_log_activity = tokio::time::Instant::now(); + let inactivity_enabled = inactivity_threshold.as_secs() > 0; + loop { let _elapsed = start.elapsed().as_secs(); - // Check for report file - for ext in &[".htm", ".htm.xml", ".html"] { - let candidate = expected_report.with_extension(ext.trim_start_matches('.')); + // Check for report file (exact name first) + for ext in &["htm", "htm.xml", "html"] { + let candidate = if *ext == "htm" { + expected_report.clone() + } else { + // Build alternate extension path without touching expected_report extension + let stem = expected_report.file_stem() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_default(); + expected_report.with_file_name(format!("{}.{}", stem, ext)) + }; if candidate.exists() { tracing::info!("Backtest {} completed: report found at {}", report_id, candidate.display()); let extracted = Self::extract_and_store(&candidate, &report_dir, &report_id, &config, ¶ms).await; @@ -390,6 +412,15 @@ impl BacktestPipeline { } else { tracing::warn!("Backtest {}: extraction failed, keeping report file at {}", report_id, candidate.display()); } + // When ShutdownTerminal=0 (shutdown=false), MT5 stays running after the + // test so the report can be written reliably. Kill it ourselves now that + // extraction is done so we don't leave zombie Wine processes. + if !params.shutdown && Self::is_mt5_running() { + tracing::info!("Backtest {}: killing MT5 after report extraction (shutdown=false)", report_id); + let _ = std::process::Command::new("pkill") + .args(["-TERM", "-f", "terminal64\\.exe"]) + .output(); + } Self::update_job_status(&report_dir, "completed", Some(candidate.to_string_lossy().to_string())).await; if let Some(ref callback) = notification_callback { callback("backtest_completed", json!({ @@ -402,12 +433,113 @@ impl BacktestPipeline { } } + // Inactivity watchdog: check if tester agent log is growing. + // The agent log is written incrementally as the tester processes ticks. + // If it stops growing for `inactivity_threshold` seconds → the test is done + // (or EA is stuck). Either way, we wait 30 s for MT5 to write the HTML + // report, then kill it unconditionally. + // + // NOTE: ShutdownTerminal=1 is supposed to make MT5 exit after the test, + // but on Wine/macOS this is unreliable — terminal64.exe often stays alive + // indefinitely. So we always kill after the HTML-wait window rather than + // depending on natural exit. The 30 s window is long enough for MT5 to flush + // the report file; if it isn't there by then, it won't appear. + if inactivity_enabled && Self::is_mt5_running() { + if let Some(log_path) = Self::find_active_tester_agent_log(&config) { + let current_size = fs::metadata(&log_path) + .map(|m| m.len()) + .unwrap_or(0); + if current_size > last_log_size { + last_log_size = current_size; + last_log_activity = tokio::time::Instant::now(); + } else if last_log_activity.elapsed() >= inactivity_threshold && last_log_size > 0 { + tracing::info!( + "Backtest {}: tester log inactive for {}s — waiting 30s for HTML report, then killing MT5", + report_id, inactivity_threshold.as_secs() + ); + // Poll for the HTML during the grace window (30 s, 1 s intervals). + let reports_parent = expected_report.parent(); + let mut html_found: Option = None; + for _wait in 0u32..30 { + // Check exact expected path first. + for ext in &["htm", "htm.xml", "html"] { + let candidate = if *ext == "htm" { + expected_report.clone() + } else { + let stem = expected_report.file_stem() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_default(); + expected_report.with_file_name(format!("{}.{}", stem, ext)) + }; + if candidate.exists() { + html_found = Some(candidate); + break; + } + } + if html_found.is_some() { break; } + // Also scan for any newly created report. + if let Some(parent) = reports_parent { + if let Some(path) = Self::find_newest_report(parent, poll_start) { + html_found = Some(path); + break; + } + } + sleep(Duration::from_secs(1)).await; + } + + // Kill MT5 now — it either wrote the report or it won't. + tracing::info!("Backtest {}: killing MT5 after inactivity+HTML-wait window", report_id); + let _ = std::process::Command::new("pkill") + .args(["-TERM", "-f", "terminal64\\.exe"]) + .output(); + sleep(Duration::from_secs(2)).await; + let _ = std::process::Command::new("pkill") + .args(["-KILL", "-f", "terminal64\\.exe"]) + .output(); + + if let Some(path) = html_found { + tracing::info!("Backtest {}: HTML report found during wait: {}", report_id, path.display()); + let extracted = Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await; + if extracted { let _ = fs::remove_file(&path); } + Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await; + if let Some(ref callback) = notification_callback { + callback("backtest_completed", json!({ + "report_id": report_id, + "status": "completed" + })); + } + return; + } + + // No HTML — fall back to journal extraction. + sleep(Duration::from_secs(1)).await; + if let Some(log) = Self::find_active_tester_agent_log(&config) { + if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await { + Self::update_job_status(&report_dir, "completed_no_html", None).await; + if let Some(ref callback) = notification_callback { + callback("backtest_completed", json!({ + "report_id": report_id, + "status": "completed_no_html", + "reason": "extracted from journal after inactivity kill (no HTML produced)" + })); + } + return; + } + } + Self::update_job_status(&report_dir, "timeout_inactive", None).await; + return; + } + } + } + // Check process liveness after grace period let in_grace = start.elapsed() <= grace_period; let mt5_alive = Self::is_mt5_running(); if !in_grace && !mt5_alive { - // MT5 exited without report - check for any newer report + // MT5 exited. Poll for the .htm report for up to 10 s (check every 1 s). + // Wine on macOS can take several seconds to flush the file after the + // process exits — a single fixed wait often misses the window. let reports_parent = match expected_report.parent() { Some(p) => p, None => { @@ -416,13 +548,26 @@ impl BacktestPipeline { return; } }; - if let Some(path) = Self::find_newest_report(reports_parent, poll_start) { - tracing::info!("Backtest {} completed: found fallback report {}", report_id, path.display()); + let mut found_report: Option = None; + for attempt in 1u32..=10 { + sleep(Duration::from_secs(1)).await; + if let Some(path) = Self::find_newest_report(reports_parent, poll_start) { + tracing::info!( + "Backtest {}: found report after {}s — {}", + report_id, attempt, path.display() + ); + found_report = Some(path); + break; + } + tracing::debug!("Backtest {}: no report yet ({}s elapsed after MT5 exit)", report_id, attempt); + } + if let Some(path) = found_report { + tracing::info!("Backtest {} completed: found report {}", report_id, path.display()); let extracted = Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await; if extracted { let _ = fs::remove_file(&path); } else { - tracing::warn!("Backtest {}: extraction failed, keeping fallback report at {}", report_id, path.display()); + tracing::warn!("Backtest {}: extraction failed, keeping report at {}", report_id, path.display()); } Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await; if let Some(ref callback) = notification_callback { @@ -432,22 +577,46 @@ impl BacktestPipeline { "status": "completed" })); } - } else { - tracing::warn!("Backtest {} failed: MT5 exited without producing a report", report_id); - Self::update_job_status(&report_dir, "failed", None).await; - if let Some(ref callback) = notification_callback { - callback("backtest_failed", json!({ - "report_id": report_id, - "status": "failed", - "reason": "MT5 exited without producing a report" - })); + return; + } + + // No HTML report found — fallback to journal extraction. + tracing::warn!("Backtest {}: no HTML report found, trying journal extraction", report_id); + if let Some(log) = Self::find_active_tester_agent_log(&config) { + if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await { + Self::update_job_status(&report_dir, "completed_no_html", None).await; + if let Some(ref callback) = notification_callback { + callback("backtest_completed", json!({ + "report_id": report_id, + "status": "completed_no_html", + "reason": "extracted from tester journal (HTML report not produced)" + })); + } + return; } } + + tracing::warn!("Backtest {} failed: MT5 exited without producing a report", report_id); + Self::update_job_status(&report_dir, "failed", None).await; + if let Some(ref callback) = notification_callback { + callback("backtest_failed", json!({ + "report_id": report_id, + "status": "failed", + "reason": "MT5 exited without producing a report or recoverable journal" + })); + } return; } if tokio::time::Instant::now() > deadline { tracing::warn!("Backtest {} timed out after {} seconds", report_id, timeout_secs); + // Last-chance journal extraction on timeout + if let Some(log) = Self::find_active_tester_agent_log(&config) { + if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await { + Self::update_job_status(&report_dir, "completed_no_html", None).await; + return; + } + } Self::update_job_status(&report_dir, "timeout", None).await; if let Some(ref callback) = notification_callback { callback("backtest_timeout", json!({ @@ -463,6 +632,251 @@ impl BacktestPipeline { } } + /// Find the best tester agent log file for today. + /// + /// Selection priority: + /// 1. Local agents (127.0.0.1) preferred over external/cloud agents (0.0.0.0) + /// — 0.0.0.0 logs only contain startup info, never actual deal lines. + /// 2. Among equal-priority agents, pick the **largest** file (most content). + pub fn find_active_tester_agent_log(config: &Config) -> Option { + let mt5_dir = config.mt5_dir()?; + let tester_dir = mt5_dir.join("Tester"); + let today = chrono::Utc::now().format("%Y%m%d").to_string(); + + // (priority, size, path) — higher priority = more preferred + // priority 1 = local (127.0.0.1), priority 0 = other (0.0.0.0 / any) + let mut best: Option<(u8, u64, PathBuf)> = None; + + if let Ok(agents) = fs::read_dir(&tester_dir) { + for agent in agents.filter_map(|e| e.ok()) { + let agent_name = agent.file_name(); + let agent_str = agent_name.to_string_lossy(); + + // Only consider Agent-* directories + if !agent_str.starts_with("Agent-") { + continue; + } + + let logs_dir = agent.path().join("logs"); + let candidate = logs_dir.join(format!("{}.log", today)); + if !candidate.exists() { + continue; + } + + let meta = match fs::metadata(&candidate) { + Ok(m) => m, + Err(_) => continue, + }; + let size = meta.len(); + + // Prefer local agents (127.0.0.1) — they log actual deal execution + let priority: u8 = if agent_str.contains("127.0.0.1") { 1 } else { 0 }; + + let is_better = match &best { + None => true, + Some((bp, bs, _)) => priority > *bp || (priority == *bp && size > *bs), + }; + + if is_better { + best = Some((priority, size, candidate)); + } + } + } + best.map(|(_, _, p)| p) + } + + /// Read the most recent tester agent log and return its lines (UTF-16 or UTF-8). + pub fn read_tester_agent_log(log_path: &Path) -> Option> { + let bytes = fs::read(log_path).ok()?; + let text = if bytes.len() >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE { + // UTF-16 LE with BOM + let words: Vec = bytes[2..].chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect(); + String::from_utf16_lossy(&words).to_string() + } else { + String::from_utf8_lossy(&bytes).to_string() + }; + Some(text.lines().map(|l| l.to_string()).collect()) + } + + /// Parse deal entries from a tester agent log. + /// Returns (deals_parsed, final_balance_pips, sim_progress_line). + /// + /// Entry direction (in/out) is inferred via a per-symbol position tracker: + /// - No open position → "in" + /// - Same direction as existing position → "in" (grid/martingale add) + /// - Opposite direction → "out" (closing) + /// Profit/balance are unavailable in the journal; they remain 0.0. + pub fn parse_journal_deals(lines: &[String]) -> (Vec, f64, String) { + use regex::Regex; + use std::collections::HashMap; + + // Format: "... YYYY.MM.DD HH:MM:SS deal #N buy/sell VOLUME SYM at PRICE done ..." + let deal_re = Regex::new( + r"(\d{4}\.\d{2}\.\d{2} \d{2}:\d{2}:\d{2})\s+deal #(\d+) (buy|sell) ([\d.]+) (\S+) at ([\d.]+) done" + ).unwrap(); + let balance_re = Regex::new(r"final balance ([\d.]+) pips").unwrap(); + let progress_re = Regex::new(r"Test passed in (.+)").unwrap(); + + let mut deals = Vec::new(); + let mut final_balance = 0.0f64; + let mut progress_str = String::new(); + + // signed lots per symbol: positive = net long, negative = net short + let mut position: HashMap = HashMap::new(); + // MT5 tester logs each deal TWICE (dual-agent logging) — deduplicate by deal number + let mut seen_deals: std::collections::HashSet = std::collections::HashSet::new(); + + for line in lines { + if let Some(cap) = deal_re.captures(line) { + let sim_time = cap[1].to_string(); + let deal_num = cap[2].to_string(); + let direction = cap[3].to_string(); // "buy" | "sell" + let volume: f64 = cap[4].parse().unwrap_or(0.0); + let symbol = cap[5].to_string(); + let price: f64 = cap[6].parse().unwrap_or(0.0); + + // Skip duplicate deal entries (MT5 writes each deal twice) + if !seen_deals.insert(deal_num.clone()) { + continue; + } + + let signed = if direction == "buy" { volume } else { -volume }; + let current = position.get(&symbol).copied().unwrap_or(0.0); + + // Determine entry type by comparing new direction against open position + let entry_type = if current.abs() < 1e-9 { + // flat → opening a new position + "in" + } else if (current > 0.0 && direction == "buy") + || (current < 0.0 && direction == "sell") + { + // same direction as existing → adding (grid/martingale) + "in" + } else { + // opposite direction → closing / partial close + "out" + }; + + // Update tracked position + let new_pos = current + signed; + if new_pos.abs() < 1e-9 { + position.remove(&symbol); + } else { + position.insert(symbol.clone(), new_pos); + } + + deals.push(crate::models::deals::Deal { + time: sim_time, + deal: deal_num, + symbol, + deal_type: direction, + entry: entry_type.to_string(), + volume, + price, + order: String::new(), + commission: 0.0, + swap: 0.0, + profit: 0.0, // not available in journal + balance: 0.0, // not available in journal + comment: String::new(), + magic: None, + }); + } + if let Some(cap) = balance_re.captures(line) { + final_balance = cap[1].parse().unwrap_or(0.0); + } + if let Some(cap) = progress_re.captures(line) { + progress_str = cap[1].to_string(); + } + } + + (deals, final_balance, progress_str) + } + + /// Fallback: extract deals from the tester agent journal log when no HTML report exists. + /// Stores partial deal data (no per-deal P&L) and records the final balance only. + async fn extract_from_journal( + log_path: &Path, + report_dir: &Path, + report_id: &str, + config: &Config, + params: &BacktestParams, + ) -> bool { + let lines = match Self::read_tester_agent_log(log_path) { + Some(l) => l, + None => { + tracing::warn!("Journal extraction: could not read log {}", log_path.display()); + return false; + } + }; + + let (deals, final_balance_pips, progress) = Self::parse_journal_deals(&lines); + if deals.is_empty() { + tracing::warn!("Journal extraction: no deals found in {}", log_path.display()); + return false; + } + + tracing::info!( + "Journal extraction: {} deals, final balance {} pips, {}", + deals.len(), final_balance_pips, progress + ); + + // Save journal summary to report_dir + let summary_path = report_dir.join("journal_extraction.json"); + let summary = json!({ + "source": "tester_agent_log", + "log_path": log_path.to_string_lossy(), + "total_deals": deals.len(), + "final_balance_pips": final_balance_pips, + "progress": progress, + "note": "No HTML report was produced. Deals extracted from tester agent log. profit/balance fields are 0 (not available in log format)." + }); + let _ = fs::write(&summary_path, serde_json::to_string_pretty(&summary).unwrap_or_default()); + + // Register in DB with partial metrics + let db = crate::storage::ReportDb::new(&Config::db_path()); + if db.init().is_err() { + return false; + } + let entry = crate::storage::ReportEntry { + id: report_id.to_string(), + expert: params.expert.clone(), + symbol: params.symbol.clone(), + timeframe: params.timeframe.clone(), + model: params.model as i64, + from_date: params.from_date.clone(), + to_date: params.to_date.clone(), + created_at: chrono::Utc::now().to_rfc3339(), + set_file_original: params.set_file.clone(), + set_snapshot_path: None, + report_dir: report_dir.to_string_lossy().to_string(), + charts_dir: None, + net_profit: Some(final_balance_pips - params.deposit as f64), + profit_factor: None, + max_dd_pct: None, + sharpe_ratio: None, + total_trades: Some(deals.len() as i64 / 2), // open+close pairs + win_rate_pct: None, + recovery_factor: None, + deposit: Some(params.deposit as f64), + currency: config.backtest_currency.clone(), + leverage: Some(params.leverage as i64), + duration_seconds: None, + tags: vec!["journal-only".to_string()], + notes: Some(format!("Extracted from journal: {} deals, final balance {} pips. No HTML report.", deals.len(), final_balance_pips)), + verdict: None, + }; + if db.insert(&entry).is_err() { + return false; + } + if let Err(e) = db.insert_deals(report_id, &deals) { + tracing::warn!("Journal extraction: failed to store deals: {}", e); + } + true + } + /// Update job status in job.json file. async fn update_job_status(report_dir: &Path, status: &str, report_path: Option) { let job_path = report_dir.join("job.json"); @@ -706,15 +1120,16 @@ impl BacktestPipeline { let reports_dir = mt5_dir.join("reports"); fs::create_dir_all(&reports_dir)?; - // Write params via /config: (triggers tester auto-start) and terminal.ini (redundancy). + // Kill first — MT5 writes terminal.ini on exit, which would clobber + // the backtest params we're about to write. + self.kill_mt5().await?; + + // Write params *after* MT5 is dead so nothing can overwrite them. let ini_content = self.build_backtest_ini(params, report_id)?; let config_host = wine_prefix.join("drive_c").join("backtest_config.ini"); fs::write(&config_host, ini_content.as_bytes())?; self.update_terminal_ini(params, report_id)?; - // Always kill any running MT5 — Wine allows only one instance per prefix. - self.kill_mt5().await?; - // Record launch time before sleeping so find_newest_report doesn't miss // reports written during the startup wait. let poll_start = std::time::SystemTime::now(); @@ -759,8 +1174,13 @@ impl BacktestPipeline { tracing::info!("poll t+{}s: in_grace={} mt5_alive={}", elapsed, in_grace, mt5_alive); if !in_grace && !mt5_alive { + // MT5 writes the .htm report file right before exiting. There is a + // short window where the process is gone but the file hasn't been + // flushed to the directory. Wait 3 s to let Wine/macOS finish the + // write before we scan — this prevents false "no report" failures. + sleep(Duration::from_secs(3)).await; if let Some(path) = Self::find_newest_report(&reports_dir, poll_start) { - tracing::info!("poll: MT5 exited, found fallback report {}", path.display()); + tracing::info!("poll: MT5 exited, found report {}", path.display()); return Ok(path); } return Err(anyhow!( @@ -963,23 +1383,16 @@ impl BacktestPipeline { ); let script_path = std::env::temp_dir().join("mt5_backtest_launch.sh"); - - // Only rewrite script if it doesn't exist or content differs (optimization) - let needs_write = !script_path.exists() || fs::read_to_string(&script_path).map(|existing| existing != script).unwrap_or(true); - - if needs_write { - fs::write(&script_path, &script)?; - // chmod +x - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&script_path, - fs::Permissions::from_mode(0o755))?; - } - tracing::debug!("Created/updated launch script: {}", script_path.display()); - } else { - tracing::debug!("Reusing existing launch script: {}", script_path.display()); + + // Always rewrite: script content changes per backtest (DYLD paths are + // dynamic) and we must ensure +x permissions are set every time. + fs::write(&script_path, &script)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&script_path, fs::Permissions::from_mode(0o755))?; } + tracing::debug!("Wrote launch script: {}", script_path.display()); tracing::info!("Launching MT5 via shell script (terminal.ini mode): {}", script_path.display()); let mut cmd = Command::new("/bin/sh"); @@ -1072,22 +1485,42 @@ impl BacktestPipeline { tracing::info!("Stopping existing MT5 instance..."); // SIGKILL immediately — MT5 holds no state we care about preserving. - let mut killed_any = false; for pat in &patterns { - let result = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output(); - if result.map(|o| o.status.success()).unwrap_or(false) { - killed_any = true; + let _ = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output(); + } + // Also kill wineserver so the Wine prefix is fully reset before relaunch. + // If wineserver is still alive when the new MT5 spawns, the new Wine + // instance may attach to the dying server and never enter tester mode. + let _ = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output(); + + // Poll until wineserver is actually gone (max 10 s) rather than sleeping + // a fixed amount. On macOS wineserver cleanup varies from 1 s to 6+ s. + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + sleep(Duration::from_millis(500)).await; + let ws_alive = Command::new("pgrep") + .args(["-f", "wineserver"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + let mt5_alive = patterns.iter().any(|pat| { + Command::new("pgrep") + .args(["-f", pat.as_str()]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + }); + if !ws_alive && !mt5_alive { + tracing::info!("MT5 and wineserver fully exited"); + break; + } + if tokio::time::Instant::now() >= deadline { + tracing::warn!("wineserver still alive after 10 s — proceeding anyway"); + break; } } - let wineserver_result = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output(); - if wineserver_result.map(|o| o.status.success()).unwrap_or(false) { - killed_any = true; - } - - // Only sleep if we actually killed something - skip if no processes were running - if killed_any { - sleep(Duration::from_secs(2)).await; // Reduced from 3s to 2s - } + // Brief extra pause to let the kernel release sockets and shared memory. + sleep(Duration::from_millis(500)).await; Ok(()) } diff --git a/src/tools/definitions/backtest.rs b/src/tools/definitions/backtest.rs index e7997db..3116bfe 100644 --- a/src/tools/definitions/backtest.rs +++ b/src/tools/definitions/backtest.rs @@ -99,8 +99,10 @@ pub fn tool_launch_backtest() -> Value { "skip_compile": { "type": "boolean" }, "skip_clean": { "type": "boolean" }, "timeout": { "type": "integer", "description": "Max time in seconds to wait for backtest (default: 900)" }, + "shutdown": { "type": "boolean", "description": "Shut down MT5 after test (default: true — required for HTML report to be written)" }, "gui": { "type": "boolean", "description": "Enable visualization during backtest" }, - "startup_delay_secs": { "type": "integer", "description": "Seconds to wait for MT5 initialization (default: 10)" } + "startup_delay_secs": { "type": "integer", "description": "Seconds to wait for MT5 initialization (default: 10)" }, + "inactivity_kill_secs": { "type": "integer", "description": "Kill MT5 if tester log hasn't grown for this many seconds (0 = disabled). Use to abort EAs that stop trading mid-test." } } } }) @@ -119,6 +121,22 @@ pub fn tool_get_backtest_status() -> Value { }) } +pub fn tool_get_tester_log() -> Value { + json!({ + "name": "get_tester_log", + "description": "Read the active MT5 tester agent journal log. Returns parsed deals, final balance, test progress, and raw log tail. Works during a backtest (if the log is being written) or after completion. Use this to inspect what trades occurred, check for EA activity, or debug issues when the HTML report wasn't produced.", + "inputSchema": { + "type": "object", + "properties": { + "tail_lines": { + "type": "integer", + "description": "Number of log tail lines to return (default: 100)" + } + } + } + }) +} + pub fn tool_cache_status() -> Value { json!({ "name": "cache_status", diff --git a/src/tools/definitions/mod.rs b/src/tools/definitions/mod.rs index 7b1196c..8f35578 100644 --- a/src/tools/definitions/mod.rs +++ b/src/tools/definitions/mod.rs @@ -18,6 +18,7 @@ pub fn get_tools_list() -> Value { backtest::tool_run_backtest_only(), // Minimal: skip compile, do clean + backtest + extract only backtest::tool_launch_backtest(), // Fire-and-forget: compile + clean + launch MT5 backtest::tool_get_backtest_status(), // Poll for completion + backtest::tool_get_tester_log(), // Live journal reading mid-backtest or after backtest::tool_cache_status(), backtest::tool_clean_cache(), // Optimization diff --git a/src/tools/handlers/analysis.rs b/src/tools/handlers/analysis.rs index 1d9e4b5..0345639 100644 --- a/src/tools/handlers/analysis.rs +++ b/src/tools/handlers/analysis.rs @@ -225,7 +225,9 @@ fn deal_to_json(d: &Deal) -> Value { } fn is_closed_trade(d: &Deal) -> bool { - d.entry.to_lowercase().contains("out") && d.profit != 0.0 + // Accept any "out" entry. Profit may legitimately be 0.0 for journal-extracted + // deals (where per-deal P&L is unavailable), so we don't gate on profit != 0.0. + d.entry.to_lowercase().contains("out") } pub async fn handle_list_deals(_config: &Config, args: &Value) -> Result { diff --git a/src/tools/handlers/backtest.rs b/src/tools/handlers/backtest.rs index e4390bb..ddc4368 100644 --- a/src/tools/handlers/backtest.rs +++ b/src/tools/handlers/backtest.rs @@ -74,66 +74,87 @@ pub async fn handle_run_backtest(config: &Config, args: &Value) -> Result .and_then(|v| v.as_str()) .unwrap_or(""); - let symbol = if requested_symbol.is_empty() { - // Use config default or first available symbol - if let Some(default) = config.backtest_symbol.clone() { - if preflight.available_symbols.contains(&default) { - default - } else if let Some(first) = preflight.available_symbols.first() { - tracing::warn!("Default symbol {} not found for server {}; using {}", default, active_server, first); - first.clone() - } else { + // No tester data at all → hard fail with helpful context + let no_symbols_error = || json!({ + "content": [{ "type": "text", "text": json!({ + "error": format!("No symbols available for backtesting on server '{}'.", active_server), + "account": { "login": active_login, "server": active_server }, + "hint": "Open MT5 → View → Strategy Tester → download history for at least one symbol.", + "pre_check": "no_symbols" + }).to_string() }], + "isError": true + }); + + let symbol: String = if requested_symbol.is_empty() { + // No symbol requested — use config default or first available tester symbol. + let candidate = config.backtest_symbol.as_deref().unwrap_or(""); + if candidate.is_empty() { + preflight.available_symbols.first() + .cloned() + .ok_or(() + ).unwrap_or_else(|_| return String::new()) + } else { + match Config::resolve_symbol(candidate, &preflight.available_symbols) { + Some(resolved) => { + if resolved != candidate { + tracing::warn!( + "Config symbol '{}' not in tester data for '{}'; using '{}' instead", + candidate, active_server, resolved + ); + } + resolved.to_string() + } + None => { + // Config default has no tester data — pick first available + preflight.available_symbols.first() + .cloned() + .unwrap_or_default() + } + } + } + } else { + // Caller specified a symbol — resolve it against actual tester data. + if preflight.available_symbols.is_empty() { + return Ok(no_symbols_error()); + } + match Config::resolve_symbol(requested_symbol, &preflight.available_symbols) { + Some(resolved) if resolved == requested_symbol => { + // Exact match — use as-is + resolved.to_string() + } + Some(resolved) => { + // Fuzzy match — proceed with the corrected symbol, surface the substitution + tracing::warn!( + "Symbol '{}' not in tester data for '{}'; substituting '{}'", + requested_symbol, active_server, resolved + ); + resolved.to_string() + } + None => { + // No match at all — fail with full context so the caller can act return Ok(json!({ "content": [{ "type": "text", "text": json!({ - "error": format!("No symbols available for backtesting on server '{}'.", active_server), - "account": { - "login": active_login, - "server": active_server - }, - "hint": "Download historical data in MT5 Strategy Tester for this server.", - "pre_check": "no_symbols", - "suggestion": "Use get_active_account to see available symbols." + "error": format!( + "Symbol '{}' has no tester data on server '{}' and no close match was found.", + requested_symbol, active_server + ), + "account": { "login": active_login, "server": active_server }, + "requested_symbol": requested_symbol, + "available_symbols": preflight.available_symbols, + "hint": "The tester data for this symbol hasn't been downloaded yet. \ + Open MT5 → Strategy Tester → select the symbol and click Download.", + "pre_check": "symbol_not_available" }).to_string() }], "isError": true })); } - } else if let Some(first) = preflight.available_symbols.first() { - first.clone() - } else { - return Ok(json!({ - "content": [{ "type": "text", "text": json!({ - "error": format!("No symbols available for backtesting on server '{}'.", active_server), - "account": { - "login": active_login, - "server": active_server - }, - "hint": "Download historical data in MT5 Strategy Tester for this server.", - "pre_check": "no_symbols", - "suggestion": "Use get_active_account to see available symbols." - }).to_string() }], - "isError": true - })); } - } else { - if !preflight.available_symbols.is_empty() && !preflight.available_symbols.contains(&requested_symbol.to_string()) { - return Ok(json!({ - "content": [{ "type": "text", "text": json!({ - "error": format!("Symbol '{}' is not available for server '{}'.", requested_symbol, active_server), - "account": { - "login": active_login, - "server": active_server - }, - "requested_symbol": requested_symbol, - "available_symbols": preflight.available_symbols, - "hint": "The symbol may not have history data for this account's server. Use list_symbols to see available symbols.", - "pre_check": "symbol_not_available", - "suggestion": "Either switch to a different MT5 account with this symbol's data, or download history for this symbol on the current server." - }).to_string() }], - "isError": true - })); - } - requested_symbol.to_string() }; + + // Guard against the empty-string edge case (no symbols at all) + if symbol.is_empty() { + return Ok(no_symbols_error()); + } // EA existence check with context if !preflight.ea_exists { @@ -172,11 +193,12 @@ pub async fn handle_run_backtest(config: &Config, args: &Value) -> Result skip_clean: args.get("skip_clean").and_then(|v| v.as_bool()).unwrap_or(false), skip_analyze: args.get("skip_analyze").and_then(|v| v.as_bool()).unwrap_or(false), deep_analyze: args.get("deep").and_then(|v| v.as_bool()).unwrap_or(false), - shutdown: args.get("shutdown").and_then(|v| v.as_bool()).unwrap_or(false), + shutdown: args.get("shutdown").and_then(|v| v.as_bool()).unwrap_or(true), kill_existing: args.get("kill_existing").and_then(|v| v.as_bool()).unwrap_or(false), timeout: args.get("timeout").and_then(|v| v.as_u64()).unwrap_or(900), gui: args.get("gui").and_then(|v| v.as_bool()).unwrap_or(false), startup_delay_secs: args.get("startup_delay_secs").and_then(|v| v.as_u64()).unwrap_or(0), + inactivity_kill_secs: args.get("inactivity_kill_secs").and_then(|v| v.as_u64()), }; let pipeline = BacktestPipeline::new(config.clone()); @@ -193,24 +215,26 @@ pub async fn handle_run_backtest(config: &Config, args: &Value) -> Result })) } -pub async fn handle_run_backtest_quick(config: &Config, args: &Value) -> Result { - // Quick backtest: skip compile, do clean → backtest → extract → analyze +pub async fn handle_run_backtest_quick(handler: &crate::tools::handlers::ToolHandler, args: &Value) -> Result { + // Quick backtest: skip compile, clean → launch → background monitor → return job. + // Uses the fire-and-forget launch path so the MCP response is returned immediately + // and the result is available via get_backtest_status / get_latest_report once done. + // (The synchronous blocking path exceeds MCP request timeouts for any backtest >2 min.) let mut args = args.clone(); if let Some(obj) = args.as_object_mut() { obj.insert("skip_compile".to_string(), json!(true)); - // keep skip_analyze as false (default) to run analysis } - handle_run_backtest(config, &args).await + handle_launch_backtest(handler, &args).await } -pub async fn handle_run_backtest_only(config: &Config, args: &Value) -> Result { - // Backtest only: skip compile, skip analyze - just backtest and extract +pub async fn handle_run_backtest_only(handler: &crate::tools::handlers::ToolHandler, args: &Value) -> Result { + // Backtest only: skip compile and analyze — launch and return job immediately. let mut args = args.clone(); if let Some(obj) = args.as_object_mut() { obj.insert("skip_compile".to_string(), json!(true)); obj.insert("skip_analyze".to_string(), json!(true)); } - handle_run_backtest(config, &args).await + handle_launch_backtest(handler, &args).await } pub async fn handle_launch_backtest(handler: &crate::tools::handlers::ToolHandler, args: &Value) -> Result { @@ -232,17 +256,50 @@ pub async fn handle_launch_backtest(handler: &crate::tools::handlers::ToolHandle })); } - // Get symbol + // Get symbol — resolve against actual tester data (same logic as handle_run_backtest) + let active_server = preflight.server.as_deref().unwrap_or("unknown"); let requested_symbol = args.get("symbol") .and_then(|v| v.as_str()) .unwrap_or(""); - let symbol = if requested_symbol.is_empty() { - handler.config.backtest_symbol.clone() - .or_else(|| preflight.available_symbols.first().cloned()) - .unwrap_or_else(|| "EURUSD".to_string()) + let symbol: String = if requested_symbol.is_empty() { + let candidate = handler.config.backtest_symbol.as_deref().unwrap_or(""); + if candidate.is_empty() { + preflight.available_symbols.first().cloned().unwrap_or_default() + } else { + Config::resolve_symbol(candidate, &preflight.available_symbols) + .map(|s| s.to_string()) + .or_else(|| preflight.available_symbols.first().cloned()) + .unwrap_or_else(|| candidate.to_string()) + } } else { - requested_symbol.to_string() + match Config::resolve_symbol(requested_symbol, &preflight.available_symbols) { + Some(resolved) => { + if resolved != requested_symbol { + tracing::warn!( + "launch_backtest: symbol '{}' not in tester data for '{}'; using '{}'", + requested_symbol, active_server, resolved + ); + } + resolved.to_string() + } + None if preflight.available_symbols.is_empty() => requested_symbol.to_string(), + None => { + return Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "error": format!( + "Symbol '{}' has no tester data on server '{}' and no close match was found.", + requested_symbol, active_server + ), + "requested_symbol": requested_symbol, + "available_symbols": preflight.available_symbols, + "hint": "Open MT5 → Strategy Tester → select the symbol and click Download.", + "pre_check": "symbol_not_available" + }).to_string() }], + "isError": true + })); + } + } }; // EA existence check @@ -281,11 +338,16 @@ pub async fn handle_launch_backtest(handler: &crate::tools::handlers::ToolHandle skip_clean: args.get("skip_clean").and_then(|v| v.as_bool()).unwrap_or(false), skip_analyze: true, // Not needed for launch mode deep_analyze: false, - shutdown: false, // Don't shutdown so we can poll + // ShutdownTerminal=1 (default): MT5 writes the HTML report and exits cleanly. + // The background monitor's post-exit scan finds the report within 10s. + // The inactivity watchdog is intentionally skipped when shutdown=true so it + // doesn't race with MT5's report write right before natural exit. + shutdown: args.get("shutdown").and_then(|v| v.as_bool()).unwrap_or(true), kill_existing: false, timeout: args.get("timeout").and_then(|v| v.as_u64()).unwrap_or(900), gui: args.get("gui").and_then(|v| v.as_bool()).unwrap_or(false), startup_delay_secs: args.get("startup_delay_secs").and_then(|v| v.as_u64()).unwrap_or(0), + inactivity_kill_secs: args.get("inactivity_kill_secs").and_then(|v| v.as_u64()), }; let pipeline = if let Some(ref callback) = handler.notification_callback { @@ -348,15 +410,27 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul // Check if MT5 is running let mt5_running = is_mt5_running(); - // Check if report file exists + // Check if report file exists (still on disk — it's deleted after extraction) let report_found = job.as_ref() .map(|j| Path::new(&j.expected_report_path).exists()) .unwrap_or(false); - + // Check for completed artifacts let metrics_exists = report_path.join("metrics.json").exists(); - let is_complete = stage == "DONE" || (report_found && metrics_exists); - + + // Read the authoritative status written by the background monitor into job.json. + // This avoids false "failed" when the HTML was extracted+deleted (report_found=false) + // or when journal extraction ran instead of HTML extraction. + let job_status = job.as_ref() + .and_then(|j| j.status.as_deref()) + .unwrap_or(""); + let monitor_says_complete = matches!(job_status, "completed" | "completed_no_html"); + let monitor_says_failed = matches!(job_status, "failed" | "timeout" | "timeout_inactive"); + + let is_complete = monitor_says_complete + || stage == "DONE" + || (report_found && metrics_exists); + // Calculate elapsed time if job exists let elapsed_seconds = job.as_ref() .and_then(|j| { @@ -365,26 +439,32 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul .map(|t| (chrono::Utc::now() - t.with_timezone(&chrono::Utc)).num_seconds()) }) .unwrap_or(0); - - // Determine status message + + // Determine status message — trust monitor's job.json first let status_msg = if is_complete { "completed" + } else if monitor_says_failed { + if job_status == "timeout" || job_status == "timeout_inactive" { "timeout" } else { "failed" } } else if stage == "BACKTEST" && mt5_running { "running" - } else if stage == "BACKTEST" && !mt5_running && !report_found { + } else if stage == "BACKTEST" && !mt5_running { "failed" } else if progress_lines > 0 { "in_progress" } else { "not_started" }; - + let message = if is_complete { - "Backtest completed successfully" + if job_status == "completed_no_html" { + "Backtest completed (extracted from tester journal — no HTML report)" + } else { + "Backtest completed successfully" + } } else if stage == "BACKTEST" && mt5_running { "MT5 is running the backtest" } else if stage == "BACKTEST" && !mt5_running { - "MT5 process exited but report not found - backtest may have failed" + "MT5 process exited — report not yet found" } else { &format!("Backtest is at stage: {}", stage) }; @@ -433,6 +513,87 @@ fn is_mt5_running() -> bool { }) } +/// Read the active tester agent log for live/post-test deal inspection. +/// Returns the last N lines and a parsed deal summary. +pub async fn handle_get_tester_log(config: &Config, args: &Value) -> Result { + use crate::pipeline::backtest::BacktestPipeline; + + let tail_lines = args.get("tail_lines").and_then(|v| v.as_u64()).unwrap_or(100) as usize; + + let log_path = match BacktestPipeline::find_active_tester_agent_log(config) { + Some(p) => p, + None => { + return Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "error": "No tester agent log found for today.", + "hint": "Run a backtest first. The log appears after the tester starts." + }).to_string() }], + "isError": true + })); + } + }; + + let lines = match BacktestPipeline::read_tester_agent_log(&log_path) { + Some(l) => l, + None => { + return Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "error": format!("Could not read log at {}", log_path.display()) + }).to_string() }], + "isError": true + })); + } + }; + + let (deals, final_balance_pips, progress) = BacktestPipeline::parse_journal_deals(&lines); + + // Collect tail lines + let tail: Vec<&str> = lines.iter() + .rev() + .take(tail_lines) + .rev() + .map(|s| s.as_str()) + .collect(); + + // Detect last sim timestamp for progress estimation + let last_sim_time = lines.iter().rev() + .find_map(|l| { + let parts: Vec<&str> = l.split_whitespace().collect(); + // Format: XX 0 HH:MM:SS.mmm Core NN YYYY.MM.DD HH:MM:SS ... + if parts.len() >= 6 { + let date = parts[4]; + let time = parts[5]; + if date.contains('.') && time.contains(':') { + return Some(format!("{} {}", date, time)); + } + } + None + }) + .unwrap_or_default(); + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "log_path": log_path.to_string_lossy(), + "total_lines": lines.len(), + "deals_found": deals.len(), + "final_balance_pips": final_balance_pips, + "progress": progress, + "last_sim_time": last_sim_time, + "is_complete": !progress.is_empty(), + "tail_lines": tail, + "deals_summary": deals.iter().map(|d| json!({ + "deal": d.deal, + "time": d.time, + "type": d.deal_type, + "volume": d.volume, + "price": d.price, + "symbol": d.symbol, + })).collect::>() + }).to_string() }], + "isError": false + })) +} + pub async fn handle_cache_status(config: &Config) -> Result { let cache_dir = config.tester_cache_dir.as_ref() .map(|s| Path::new(s)) diff --git a/src/tools/handlers/mod.rs b/src/tools/handlers/mod.rs index cc37e80..3dad05c 100644 --- a/src/tools/handlers/mod.rs +++ b/src/tools/handlers/mod.rs @@ -70,10 +70,11 @@ impl ToolHandler { // Backtest handlers - Granular pipeline options "run_backtest" => backtest::handle_run_backtest(&self.config, args).await, // Full: compile + clean + backtest + extract + analyze - "run_backtest_quick" => backtest::handle_run_backtest_quick(&self.config, args).await, // Quick: skip compile, do backtest + extract + analyze - "run_backtest_only" => backtest::handle_run_backtest_only(&self.config, args).await, // Minimal: skip compile, do backtest + extract only + "run_backtest_quick" => backtest::handle_run_backtest_quick(self, args).await, // Quick: skip compile, fire-and-forget launch + "run_backtest_only" => backtest::handle_run_backtest_only(self, args).await, // Minimal: skip compile+analyze, fire-and-forget launch "launch_backtest" => backtest::handle_launch_backtest(self, args).await, // Fire-and-forget mode "get_backtest_status" => backtest::handle_get_backtest_status(&self.config, args).await, + "get_tester_log" => backtest::handle_get_tester_log(&self.config, args).await, "cache_status" => backtest::handle_cache_status(&self.config).await, "clean_cache" => backtest::handle_clean_cache(&self.config, args).await,