use anyhow::{anyhow, Result}; use chrono; use serde_json::json; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::Arc; use tokio::time::{sleep, Duration}; use crate::analytics::{DealAnalyzer, ReportExtractor}; use crate::compile::MqlCompiler; use crate::models::config::Config; use crate::models::report::{PipelineMetadata, FilePaths, BacktestJob}; use crate::storage::{ReportDb, ReportEntry}; type NotificationCallback = Arc; pub struct BacktestPipeline { config: Config, compiler: MqlCompiler, extractor: ReportExtractor, analyzer: DealAnalyzer, notification_callback: Option, } pub struct BacktestParams { pub expert: String, pub symbol: String, pub from_date: String, pub to_date: String, pub timeframe: String, pub deposit: u32, pub model: u8, pub leverage: u32, pub set_file: Option, pub skip_compile: bool, pub skip_clean: bool, pub skip_analyze: bool, #[allow(dead_code)] pub deep_analyze: bool, pub shutdown: bool, #[allow(dead_code)] pub kill_existing: bool, pub timeout: u64, pub gui: bool, pub startup_delay_secs: u64, /// Kill MT5 if tester agent log hasn't grown for this many seconds. /// 0 = disabled. Useful to abort EAs that stop trading mid-backtest. pub inactivity_kill_secs: Option, } pub struct PipelineResult { pub success: bool, pub report_dir: PathBuf, pub duration_seconds: i64, pub message: String, } impl BacktestPipeline { pub fn new(config: Config) -> Self { let compiler = MqlCompiler::new(config.clone()); let extractor = ReportExtractor::new(); let analyzer = DealAnalyzer::new(); Self { config, compiler, extractor, analyzer, notification_callback: None, } } pub fn with_notification_callback(config: Config, callback: NotificationCallback) -> Self { let compiler = MqlCompiler::new(config.clone()); let extractor = ReportExtractor::new(); let analyzer = DealAnalyzer::new(); Self { config, compiler, extractor, analyzer, notification_callback: Some(callback), } } pub async fn run(&self, params: BacktestParams) -> Result { let start_time = chrono::Utc::now(); let report_id = self.generate_report_id(¶ms); let report_dir = self.config.reports_dir().join(&report_id); fs::create_dir_all(&report_dir)?; let progress_log = report_dir.join("progress.log"); self.log_progress(&progress_log, "START").await; if !params.skip_compile { self.log_progress(&progress_log, "COMPILE").await; self.compile_ea(¶ms.expert, params.timeout).await?; } if !params.skip_clean { self.log_progress(&progress_log, "CLEAN").await; self.clean_cache(¶ms.expert).await?; } self.log_progress(&progress_log, "BACKTEST").await; let report_path = self.run_backtest(¶ms, &report_id).await?; self.log_progress(&progress_log, "EXTRACT").await; let extraction = self.extractor.extract( &report_path.to_string_lossy(), &report_dir.to_string_lossy(), )?; // Handle case where EA didn't trade - no deals generated if extraction.deals.is_empty() { tracing::warn!("Backtest completed but no deals were generated - EA did not trade during this period"); let warning_path = report_dir.join("NO_TRADES_WARNING.txt"); let _ = fs::write(&warning_path, "Warning: No deals were generated during this backtest.\nThe EA did not execute any trades during the specified date range.\n"); } // Move equity chart images to OS temp dir, then delete the HTML report. let charts_dir = self.relocate_charts(&report_path, &report_id).await; let _ = fs::remove_file(&report_path); // Snapshot the set file alongside the extracted data. let set_snapshot = self.snapshot_set_file(¶ms, &report_dir).await; if !params.skip_analyze { self.log_progress(&progress_log, "ANALYZE").await; let analysis = self.analyzer.analyze(&extraction.deals, &extraction.metrics); let analysis_path = report_dir.join("analysis.json"); fs::write(&analysis_path, serde_json::to_string_pretty(&analysis)?)?; } self.log_progress(&progress_log, "DONE").await; let duration = (chrono::Utc::now() - start_time).num_seconds(); self.save_metadata(¶ms, &report_dir, duration, extraction.deals.is_empty()).await?; // Register in the SQLite report registry and store deals. let db = self.register_in_db( &report_id, ¶ms, &report_dir, charts_dir.as_deref(), set_snapshot.as_deref(), &extraction.metrics, duration, ) .await; if let Some(db) = db { if let Err(e) = db.insert_deals(&report_id, &extraction.deals) { tracing::warn!("Failed to store deals in DB: {}", e); } } let message = if extraction.deals.is_empty() { "Backtest completed successfully, but EA did not execute any trades during this period".to_string() } else { "Backtest completed successfully".to_string() }; Ok(PipelineResult { success: true, report_dir, duration_seconds: duration, message, }) } /// Launch backtest in fire-and-forget mode: compile, clean, launch MT5, return immediately. /// Returns a BacktestJob that can be used with get_backtest_status to poll for completion. pub async fn launch_backtest(&self, params: BacktestParams) -> Result { let _start_time = chrono::Utc::now(); let report_id = self.generate_report_id(¶ms); let report_dir = self.config.reports_dir().join(&report_id); fs::create_dir_all(&report_dir)?; let progress_log = report_dir.join("progress.log"); self.log_progress(&progress_log, "START").await; if !params.skip_compile { self.log_progress(&progress_log, "COMPILE").await; self.compile_ea(¶ms.expert, params.timeout).await?; } if !params.skip_clean { self.log_progress(&progress_log, "CLEAN").await; self.clean_cache(¶ms.expert).await?; } self.log_progress(&progress_log, "BACKTEST").await; // Get MT5 paths let mt5_dir = self.config.mt5_dir() .ok_or_else(|| anyhow!("MT5 directory not configured"))?; let wine_exe = self.config.wine_executable.as_ref() .ok_or_else(|| anyhow!("wine_executable not configured"))?; let wine_prefix = mt5_dir .parent() .and_then(|p| p.parent()) .and_then(|p| p.parent()) .map(|p| p.to_path_buf()) .ok_or_else(|| anyhow!("Could not determine Wine prefix from terminal_dir"))?; let reports_dir = mt5_dir.join("reports"); fs::create_dir_all(&reports_dir)?; // Kill first — MT5 writes terminal.ini on exit, which would clobber // the backtest params we're about to write. self.kill_mt5().await?; // Write params *after* MT5 is dead so nothing can overwrite them. let ini_content = self.build_backtest_ini(¶ms, &report_id)?; let config_host = wine_prefix.join("drive_c").join("backtest_config.ini"); fs::write(&config_host, ini_content.as_bytes())?; self.update_terminal_ini(¶ms, &report_id)?; // Launch MT5 (fire and forget) let mut cmd = self.build_wine_launch(wine_exe, &wine_prefix)?; let child = cmd.stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .spawn()?; let pid = child.id(); tracing::info!("MT5 launched with PID {:?} for backtest {}", pid, report_id); // Create and save the job tracking file let expected_report = reports_dir.join(format!("{}.htm", report_id)); let job = BacktestJob::new( report_id.clone(), report_dir.to_string_lossy().to_string(), params.expert.clone(), params.symbol.clone(), params.timeframe.clone(), expected_report.to_string_lossy().to_string(), params.timeout, ); // Save job info for polling let job_path = report_dir.join("job.json"); fs::write(&job_path, serde_json::to_string_pretty(&job)?)?; // Save initial metadata self.save_metadata(¶ms, &report_dir, 0, false).await?; // Register in DB as "running" let db = ReportDb::new(&Config::db_path()); if let Err(e) = db.init() { tracing::warn!("Failed to init report DB: {}", e); } // Spawn background task to monitor completion and update status let report_dir_clone = report_dir.clone(); let expected_report_clone = expected_report.clone(); let timeout_secs = params.timeout; let report_id_clone = report_id.clone(); let notification_callback = self.notification_callback.clone(); let config_clone = self.config.clone(); let params_clone = BacktestParams { expert: params.expert.clone(), symbol: params.symbol.clone(), from_date: params.from_date.clone(), to_date: params.to_date.clone(), timeframe: params.timeframe.clone(), deposit: params.deposit, model: params.model, leverage: params.leverage, set_file: params.set_file.clone(), skip_compile: params.skip_compile, skip_clean: params.skip_clean, skip_analyze: params.skip_analyze, deep_analyze: params.deep_analyze, shutdown: params.shutdown, kill_existing: params.kill_existing, timeout: params.timeout, gui: params.gui, startup_delay_secs: params.startup_delay_secs, inactivity_kill_secs: params.inactivity_kill_secs, }; tokio::spawn(async move { Self::monitor_backtest_completion( report_dir_clone, expected_report_clone, timeout_secs, report_id_clone, notification_callback, config_clone, params_clone, ).await; }); Ok(job) } /// Extract deals from a completed report and store them in the DB. /// Returns true if extraction and DB registration succeeded. async fn extract_and_store( report_path: &Path, report_dir: &Path, report_id: &str, config: &Config, params: &BacktestParams, ) -> bool { let extractor = ReportExtractor::new(); let start_time = chrono::Utc::now(); match extractor.extract( &report_path.to_string_lossy(), &report_dir.to_string_lossy(), ) { Ok(extraction) => { let duration = (chrono::Utc::now() - start_time).num_seconds(); let db = ReportDb::new(&Config::db_path()); if db.init().is_err() { tracing::warn!("launch_backtest: failed to init DB for {}", report_id); return false; } let entry = ReportEntry { id: report_id.to_string(), expert: params.expert.clone(), symbol: params.symbol.clone(), timeframe: params.timeframe.clone(), model: params.model as i64, from_date: params.from_date.clone(), to_date: params.to_date.clone(), created_at: chrono::Utc::now().to_rfc3339(), set_file_original: params.set_file.clone(), set_snapshot_path: None, report_dir: report_dir.to_string_lossy().to_string(), charts_dir: None, net_profit: Some(extraction.metrics.net_profit), profit_factor: Some(extraction.metrics.profit_factor), max_dd_pct: Some(extraction.metrics.max_dd_pct), sharpe_ratio: Some(extraction.metrics.sharpe_ratio), total_trades: Some(extraction.metrics.total_trades as i64), win_rate_pct: Some(extraction.metrics.win_rate_pct), recovery_factor: Some(extraction.metrics.recovery_factor), deposit: Some(params.deposit as f64), currency: config.backtest_currency.clone(), leverage: Some(params.leverage as i64), duration_seconds: Some(duration), tags: Vec::new(), notes: None, verdict: None, }; if let Err(e) = db.insert(&entry) { tracing::warn!("launch_backtest: failed to register report in DB: {}", e); return false; } if let Err(e) = db.insert_deals(report_id, &extraction.deals) { tracing::warn!("launch_backtest: failed to store deals in DB: {}", e); } tracing::info!("launch_backtest: extracted {} deals for {}", extraction.deals.len(), report_id); true } Err(e) => { tracing::warn!("launch_backtest: extraction failed for {}: {}", report_id, e); false } } } /// Background task to monitor backtest completion and update status file. async fn monitor_backtest_completion( report_dir: PathBuf, expected_report: PathBuf, timeout_secs: u64, report_id: String, notification_callback: Option, config: Config, params: BacktestParams, ) { let start = tokio::time::Instant::now(); let deadline = start + Duration::from_secs(timeout_secs); let grace_period = Duration::from_secs(30); let poll_start = std::time::SystemTime::now(); // Inactivity watchdog: kill MT5 if tester agent log hasn't grown for this long. // Catches EAs that stall (no ticks processed) or flat periods with zero trades. let inactivity_threshold = Duration::from_secs( params.inactivity_kill_secs.unwrap_or(0) ); let mut last_log_size: u64 = 0; let mut last_log_activity = tokio::time::Instant::now(); let inactivity_enabled = inactivity_threshold.as_secs() > 0; loop { let _elapsed = start.elapsed().as_secs(); // Check for report file (exact name first) for ext in &["htm", "htm.xml", "html"] { let candidate = if *ext == "htm" { expected_report.clone() } else { // Build alternate extension path without touching expected_report extension let stem = expected_report.file_stem() .map(|s| s.to_string_lossy().to_string()) .unwrap_or_default(); expected_report.with_file_name(format!("{}.{}", stem, ext)) }; if candidate.exists() { tracing::info!("Backtest {} completed: report found at {}", report_id, candidate.display()); let extracted = Self::extract_and_store(&candidate, &report_dir, &report_id, &config, ¶ms).await; if extracted { let _ = fs::remove_file(&candidate); } else { tracing::warn!("Backtest {}: extraction failed, keeping report file at {}", report_id, candidate.display()); } // When ShutdownTerminal=0 (shutdown=false), MT5 stays running after the // test so the report can be written reliably. Kill it ourselves now that // extraction is done so we don't leave zombie Wine processes. if !params.shutdown && Self::is_mt5_running() { tracing::info!("Backtest {}: killing MT5 after report extraction (shutdown=false)", report_id); let _ = std::process::Command::new("pkill") .args(["-TERM", "-f", "terminal64\\.exe"]) .output(); } Self::update_job_status(&report_dir, "completed", Some(candidate.to_string_lossy().to_string())).await; if let Some(ref callback) = notification_callback { callback("backtest_completed", json!({ "report_id": report_id, "report_path": candidate.to_string_lossy().to_string(), "status": "completed" })); } return; } } // Inactivity watchdog: check if tester agent log is growing. // The agent log is written incrementally as the tester processes ticks. // If it stops growing for `inactivity_threshold` seconds → the test is done // (or EA is stuck). Either way, we wait 30 s for MT5 to write the HTML // report, then kill it unconditionally. // // NOTE: ShutdownTerminal=1 is supposed to make MT5 exit after the test, // but on Wine/macOS this is unreliable — terminal64.exe often stays alive // indefinitely. So we always kill after the HTML-wait window rather than // depending on natural exit. The 30 s window is long enough for MT5 to flush // the report file; if it isn't there by then, it won't appear. if inactivity_enabled && Self::is_mt5_running() { if let Some(log_path) = Self::find_active_tester_agent_log(&config) { let current_size = fs::metadata(&log_path) .map(|m| m.len()) .unwrap_or(0); if current_size > last_log_size { last_log_size = current_size; last_log_activity = tokio::time::Instant::now(); } else if last_log_activity.elapsed() >= inactivity_threshold && last_log_size > 0 { tracing::info!( "Backtest {}: tester log inactive for {}s — waiting 30s for HTML report, then killing MT5", report_id, inactivity_threshold.as_secs() ); // Poll for the HTML during the grace window (30 s, 1 s intervals). let reports_parent = expected_report.parent(); let mut html_found: Option = None; for _wait in 0u32..30 { // Check exact expected path first. for ext in &["htm", "htm.xml", "html"] { let candidate = if *ext == "htm" { expected_report.clone() } else { let stem = expected_report.file_stem() .map(|s| s.to_string_lossy().to_string()) .unwrap_or_default(); expected_report.with_file_name(format!("{}.{}", stem, ext)) }; if candidate.exists() { html_found = Some(candidate); break; } } if html_found.is_some() { break; } // Also scan for any newly created report. if let Some(parent) = reports_parent { if let Some(path) = Self::find_newest_report(parent, poll_start) { html_found = Some(path); break; } } sleep(Duration::from_secs(1)).await; } // Kill MT5 now — it either wrote the report or it won't. tracing::info!("Backtest {}: killing MT5 after inactivity+HTML-wait window", report_id); let _ = std::process::Command::new("pkill") .args(["-TERM", "-f", "terminal64\\.exe"]) .output(); sleep(Duration::from_secs(2)).await; let _ = std::process::Command::new("pkill") .args(["-KILL", "-f", "terminal64\\.exe"]) .output(); if let Some(path) = html_found { tracing::info!("Backtest {}: HTML report found during wait: {}", report_id, path.display()); let extracted = Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await; if extracted { let _ = fs::remove_file(&path); } Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await; if let Some(ref callback) = notification_callback { callback("backtest_completed", json!({ "report_id": report_id, "status": "completed" })); } return; } // No HTML — fall back to journal extraction. sleep(Duration::from_secs(1)).await; if let Some(log) = Self::find_active_tester_agent_log(&config) { if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await { Self::update_job_status(&report_dir, "completed_no_html", None).await; if let Some(ref callback) = notification_callback { callback("backtest_completed", json!({ "report_id": report_id, "status": "completed_no_html", "reason": "extracted from journal after inactivity kill (no HTML produced)" })); } return; } } Self::update_job_status(&report_dir, "timeout_inactive", None).await; return; } } } // Check process liveness after grace period let in_grace = start.elapsed() <= grace_period; let mt5_alive = Self::is_mt5_running(); if !in_grace && !mt5_alive { // MT5 exited. Poll for the .htm report for up to 10 s (check every 1 s). // Wine on macOS can take several seconds to flush the file after the // process exits — a single fixed wait often misses the window. let reports_parent = match expected_report.parent() { Some(p) => p, None => { tracing::error!("Backtest {}: expected_report path has no parent", report_id); Self::update_job_status(&report_dir, "failed", None).await; return; } }; let mut found_report: Option = None; for attempt in 1u32..=10 { sleep(Duration::from_secs(1)).await; if let Some(path) = Self::find_newest_report(reports_parent, poll_start) { tracing::info!( "Backtest {}: found report after {}s — {}", report_id, attempt, path.display() ); found_report = Some(path); break; } tracing::debug!("Backtest {}: no report yet ({}s elapsed after MT5 exit)", report_id, attempt); } if let Some(path) = found_report { tracing::info!("Backtest {} completed: found report {}", report_id, path.display()); let extracted = Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await; if extracted { let _ = fs::remove_file(&path); } else { tracing::warn!("Backtest {}: extraction failed, keeping report at {}", report_id, path.display()); } Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await; if let Some(ref callback) = notification_callback { callback("backtest_completed", json!({ "report_id": report_id, "report_path": path.to_string_lossy().to_string(), "status": "completed" })); } return; } // No HTML report found — fallback to journal extraction. tracing::warn!("Backtest {}: no HTML report found, trying journal extraction", report_id); if let Some(log) = Self::find_active_tester_agent_log(&config) { if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await { Self::update_job_status(&report_dir, "completed_no_html", None).await; if let Some(ref callback) = notification_callback { callback("backtest_completed", json!({ "report_id": report_id, "status": "completed_no_html", "reason": "extracted from tester journal (HTML report not produced)" })); } return; } } tracing::warn!("Backtest {} failed: MT5 exited without producing a report", report_id); Self::update_job_status(&report_dir, "failed", None).await; if let Some(ref callback) = notification_callback { callback("backtest_failed", json!({ "report_id": report_id, "status": "failed", "reason": "MT5 exited without producing a report or recoverable journal" })); } return; } if tokio::time::Instant::now() > deadline { tracing::warn!("Backtest {} timed out after {} seconds", report_id, timeout_secs); // Last-chance journal extraction on timeout if let Some(log) = Self::find_active_tester_agent_log(&config) { if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await { Self::update_job_status(&report_dir, "completed_no_html", None).await; return; } } Self::update_job_status(&report_dir, "timeout", None).await; if let Some(ref callback) = notification_callback { callback("backtest_timeout", json!({ "report_id": report_id, "status": "timeout", "timeout_seconds": timeout_secs })); } return; } sleep(Duration::from_secs(2)).await; } } /// Find the best tester agent log file for today. /// /// Selection priority: /// 1. Local agents (127.0.0.1) preferred over external/cloud agents (0.0.0.0) /// — 0.0.0.0 logs only contain startup info, never actual deal lines. /// 2. Among equal-priority agents, pick the **largest** file (most content). pub fn find_active_tester_agent_log(config: &Config) -> Option { let mt5_dir = config.mt5_dir()?; let tester_dir = mt5_dir.join("Tester"); let today = chrono::Utc::now().format("%Y%m%d").to_string(); // (priority, size, path) — higher priority = more preferred // priority 1 = local (127.0.0.1), priority 0 = other (0.0.0.0 / any) let mut best: Option<(u8, u64, PathBuf)> = None; if let Ok(agents) = fs::read_dir(&tester_dir) { for agent in agents.filter_map(|e| e.ok()) { let agent_name = agent.file_name(); let agent_str = agent_name.to_string_lossy(); // Only consider Agent-* directories if !agent_str.starts_with("Agent-") { continue; } let logs_dir = agent.path().join("logs"); let candidate = logs_dir.join(format!("{}.log", today)); if !candidate.exists() { continue; } let meta = match fs::metadata(&candidate) { Ok(m) => m, Err(_) => continue, }; let size = meta.len(); // Prefer local agents (127.0.0.1) — they log actual deal execution let priority: u8 = if agent_str.contains("127.0.0.1") { 1 } else { 0 }; let is_better = match &best { None => true, Some((bp, bs, _)) => priority > *bp || (priority == *bp && size > *bs), }; if is_better { best = Some((priority, size, candidate)); } } } best.map(|(_, _, p)| p) } /// Read the most recent tester agent log and return its lines (UTF-16 or UTF-8). pub fn read_tester_agent_log(log_path: &Path) -> Option> { let bytes = fs::read(log_path).ok()?; let text = if bytes.len() >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE { // UTF-16 LE with BOM let words: Vec = bytes[2..].chunks_exact(2) .map(|c| u16::from_le_bytes([c[0], c[1]])) .collect(); String::from_utf16_lossy(&words).to_string() } else { String::from_utf8_lossy(&bytes).to_string() }; Some(text.lines().map(|l| l.to_string()).collect()) } /// Parse deal entries from a tester agent log. /// Returns (deals_parsed, final_balance_pips, sim_progress_line). /// /// Entry direction (in/out) is inferred via a per-symbol position tracker: /// - No open position → "in" /// - Same direction as existing position → "in" (grid/martingale add) /// - Opposite direction → "out" (closing) /// Profit/balance are unavailable in the journal; they remain 0.0. pub fn parse_journal_deals(lines: &[String]) -> (Vec, f64, String) { use regex::Regex; use std::collections::HashMap; // Format: "... YYYY.MM.DD HH:MM:SS deal #N buy/sell VOLUME SYM at PRICE done ..." let deal_re = Regex::new( r"(\d{4}\.\d{2}\.\d{2} \d{2}:\d{2}:\d{2})\s+deal #(\d+) (buy|sell) ([\d.]+) (\S+) at ([\d.]+) done" ).unwrap(); let balance_re = Regex::new(r"final balance ([\d.]+) pips").unwrap(); let progress_re = Regex::new(r"Test passed in (.+)").unwrap(); let mut deals = Vec::new(); let mut final_balance = 0.0f64; let mut progress_str = String::new(); // signed lots per symbol: positive = net long, negative = net short let mut position: HashMap = HashMap::new(); // MT5 tester logs each deal TWICE (dual-agent logging) — deduplicate by deal number let mut seen_deals: std::collections::HashSet = std::collections::HashSet::new(); for line in lines { if let Some(cap) = deal_re.captures(line) { let sim_time = cap[1].to_string(); let deal_num = cap[2].to_string(); let direction = cap[3].to_string(); // "buy" | "sell" let volume: f64 = cap[4].parse().unwrap_or(0.0); let symbol = cap[5].to_string(); let price: f64 = cap[6].parse().unwrap_or(0.0); // Skip duplicate deal entries (MT5 writes each deal twice) if !seen_deals.insert(deal_num.clone()) { continue; } let signed = if direction == "buy" { volume } else { -volume }; let current = position.get(&symbol).copied().unwrap_or(0.0); // Determine entry type by comparing new direction against open position let entry_type = if current.abs() < 1e-9 { // flat → opening a new position "in" } else if (current > 0.0 && direction == "buy") || (current < 0.0 && direction == "sell") { // same direction as existing → adding (grid/martingale) "in" } else { // opposite direction → closing / partial close "out" }; // Update tracked position let new_pos = current + signed; if new_pos.abs() < 1e-9 { position.remove(&symbol); } else { position.insert(symbol.clone(), new_pos); } deals.push(crate::models::deals::Deal { time: sim_time, deal: deal_num, symbol, deal_type: direction, entry: entry_type.to_string(), volume, price, order: String::new(), commission: 0.0, swap: 0.0, profit: 0.0, // not available in journal balance: 0.0, // not available in journal comment: String::new(), magic: None, }); } if let Some(cap) = balance_re.captures(line) { final_balance = cap[1].parse().unwrap_or(0.0); } if let Some(cap) = progress_re.captures(line) { progress_str = cap[1].to_string(); } } (deals, final_balance, progress_str) } /// Fallback: extract deals from the tester agent journal log when no HTML report exists. /// Stores partial deal data (no per-deal P&L) and records the final balance only. async fn extract_from_journal( log_path: &Path, report_dir: &Path, report_id: &str, config: &Config, params: &BacktestParams, ) -> bool { let lines = match Self::read_tester_agent_log(log_path) { Some(l) => l, None => { tracing::warn!("Journal extraction: could not read log {}", log_path.display()); return false; } }; let (deals, final_balance_pips, progress) = Self::parse_journal_deals(&lines); if deals.is_empty() { tracing::warn!("Journal extraction: no deals found in {}", log_path.display()); return false; } tracing::info!( "Journal extraction: {} deals, final balance {} pips, {}", deals.len(), final_balance_pips, progress ); // Save journal summary to report_dir let summary_path = report_dir.join("journal_extraction.json"); let summary = json!({ "source": "tester_agent_log", "log_path": log_path.to_string_lossy(), "total_deals": deals.len(), "final_balance_pips": final_balance_pips, "progress": progress, "note": "No HTML report was produced. Deals extracted from tester agent log. profit/balance fields are 0 (not available in log format)." }); let _ = fs::write(&summary_path, serde_json::to_string_pretty(&summary).unwrap_or_default()); // Register in DB with partial metrics let db = crate::storage::ReportDb::new(&Config::db_path()); if db.init().is_err() { return false; } let entry = crate::storage::ReportEntry { id: report_id.to_string(), expert: params.expert.clone(), symbol: params.symbol.clone(), timeframe: params.timeframe.clone(), model: params.model as i64, from_date: params.from_date.clone(), to_date: params.to_date.clone(), created_at: chrono::Utc::now().to_rfc3339(), set_file_original: params.set_file.clone(), set_snapshot_path: None, report_dir: report_dir.to_string_lossy().to_string(), charts_dir: None, net_profit: Some(final_balance_pips - params.deposit as f64), profit_factor: None, max_dd_pct: None, sharpe_ratio: None, total_trades: Some(deals.len() as i64 / 2), // open+close pairs win_rate_pct: None, recovery_factor: None, deposit: Some(params.deposit as f64), currency: config.backtest_currency.clone(), leverage: Some(params.leverage as i64), duration_seconds: None, tags: vec!["journal-only".to_string()], notes: Some(format!("Extracted from journal: {} deals, final balance {} pips. No HTML report.", deals.len(), final_balance_pips)), verdict: None, }; if db.insert(&entry).is_err() { return false; } if let Err(e) = db.insert_deals(report_id, &deals) { tracing::warn!("Journal extraction: failed to store deals: {}", e); } true } /// Update job status in job.json file. async fn update_job_status(report_dir: &Path, status: &str, report_path: Option) { let job_path = report_dir.join("job.json"); if let Ok(job_json) = fs::read_to_string(&job_path) { if let Ok(mut job) = serde_json::from_str::(&job_json) { job["status"] = serde_json::Value::String(status.to_string()); job["completed_at"] = serde_json::Value::String(chrono::Utc::now().to_rfc3339()); if let Some(path) = report_path { job["actual_report_path"] = serde_json::Value::String(path); } if let Ok(updated) = serde_json::to_string_pretty(&job) { let _ = fs::write(&job_path, updated); } } } } /// Move equity chart images (*.png, *.gif) from MT5's reports dir to OS temp, /// returning the temp path if any images were found. async fn relocate_charts(&self, html_path: &Path, report_id: &str) -> Option { let reports_dir = html_path.parent()?; let charts_dir = Config::charts_temp_dir(report_id); let image_exts = ["png", "gif", "jpg", "jpeg"]; let entries = fs::read_dir(reports_dir).ok()?; let mut found = false; for entry in entries.filter_map(|e| e.ok()) { let path = entry.path(); let name = path.file_name()?.to_string_lossy().to_string(); let is_chart = name.starts_with(report_id) && path .extension() .and_then(|e| e.to_str()) .map(|e| image_exts.contains(&e)) .unwrap_or(false); if is_chart { if !found { if fs::create_dir_all(&charts_dir).is_err() { return None; } } let dest = charts_dir.join(entry.file_name()); let _ = fs::rename(&path, &dest); found = true; } } if found { Some(charts_dir) } else { None } } /// Copy the set file into the report dir as set_snapshot.set. async fn snapshot_set_file( &self, params: &BacktestParams, report_dir: &Path, ) -> Option { let set_src = params.set_file.as_ref()?; let src_path = Path::new(set_src); if !src_path.exists() { return None; } let dest = report_dir.join("set_snapshot.set"); fs::copy(src_path, &dest).ok()?; Some(dest) } async fn register_in_db( &self, report_id: &str, params: &BacktestParams, report_dir: &Path, charts_dir: Option<&Path>, set_snapshot: Option<&Path>, metrics: &crate::models::metrics::Metrics, duration: i64, ) -> Option { let db = ReportDb::new(&Config::db_path()); if let Err(e) = db.init() { tracing::warn!("Failed to init report DB: {}", e); return None; } let entry = ReportEntry { id: report_id.to_string(), expert: params.expert.clone(), symbol: params.symbol.clone(), timeframe: params.timeframe.clone(), model: params.model as i64, from_date: params.from_date.clone(), to_date: params.to_date.clone(), created_at: chrono::Utc::now().to_rfc3339(), set_file_original: params.set_file.clone(), set_snapshot_path: set_snapshot.map(|p| p.to_string_lossy().to_string()), report_dir: report_dir.to_string_lossy().to_string(), charts_dir: charts_dir.map(|p| p.to_string_lossy().to_string()), net_profit: Some(metrics.net_profit), profit_factor: Some(metrics.profit_factor), max_dd_pct: Some(metrics.max_dd_pct), sharpe_ratio: Some(metrics.sharpe_ratio), total_trades: Some(metrics.total_trades as i64), win_rate_pct: Some(metrics.win_rate_pct), recovery_factor: Some(metrics.recovery_factor), deposit: Some(params.deposit as f64), currency: self.config.backtest_currency.clone(), leverage: Some(params.leverage as i64), duration_seconds: Some(duration), tags: Vec::new(), notes: None, verdict: None, }; if let Err(e) = db.insert(&entry) { tracing::warn!("Failed to register report in DB: {}", e); return None; } Some(db) } async fn compile_ea(&self, expert: &str, timeout_secs: u64) -> Result<()> { let mut search_paths = vec![ PathBuf::from(&self.config.get("project_dir")).join("src/experts").join(format!("{}.mq5", expert)), PathBuf::from(&self.config.get("project_dir")).join("src").join(format!("{}.mq5", expert)), PathBuf::from(&self.config.get("project_dir")).join(format!("{}.mq5", expert)), PathBuf::from("src/experts").join(format!("{}.mq5", expert)), PathBuf::from("src").join(format!("{}.mq5", expert)), PathBuf::from(format!("{}.mq5", expert)), ]; // Also search in MT5 Experts dir: Experts/{expert}/{expert}.mq5 and Experts/{expert}.mq5 if let Some(experts_dir) = &self.config.experts_dir { search_paths.push(PathBuf::from(experts_dir).join(expert).join(format!("{}.mq5", expert))); search_paths.push(PathBuf::from(experts_dir).join(format!("{}.mq5", expert))); } let source_path = search_paths .into_iter() .find(|p| p.exists()) .ok_or_else(|| anyhow!("Cannot find {}.mq5 — searched project_dir and MT5 Experts dir", expert))?; let timeout = std::time::Duration::from_secs(timeout_secs.min(300)); // Max 5 min for compile let result = self.compiler.compile_with_timeout(&source_path.to_string_lossy(), timeout).await?; if !result.success { return Err(anyhow!( "Compilation failed: {}", result.errors.join("; ") )); } Ok(()) } async fn clean_cache(&self, expert: &str) -> Result<()> { if let Some(cache_dir) = &self.config.tester_cache_dir { let cache_path = Path::new(cache_dir); if cache_path.exists() { for entry in walkdir::WalkDir::new(cache_path) { if let Ok(entry) = entry { let path = entry.path(); if path.extension().map(|e| e == "tst").unwrap_or(false) { let _ = fs::remove_file(path); } } } } } if let Some(tester_dir) = &self.config.tester_profiles_dir { let cached_set = Path::new(tester_dir).join(format!("{}.set", expert)); if cached_set.exists() { let _ = fs::remove_file(&cached_set); } } self.reset_terminal_ini().await?; Ok(()) } async fn reset_terminal_ini(&self) -> Result<()> { let mt5_dir = self.config.mt5_dir() .ok_or_else(|| anyhow!("MT5 directory not configured"))?; let terminal_ini = mt5_dir.join("config").join("terminal.ini"); if !terminal_ini.exists() { return Ok(()); } let content = fs::read(&terminal_ini)?; let (text, encoding) = if content.starts_with(&[0xFF, 0xFE]) || content.starts_with(&[0xFE, 0xFF]) { let text = String::from_utf16_lossy( content.chunks_exact(2) .map(|c| u16::from_le_bytes([c[0], c[1]])) .collect::>() .as_slice() ); (text, "utf-16") } else { (String::from_utf8_lossy(&content).to_string(), "utf-8") }; let updated = text .replace("OptMode=-1", "OptMode=0") .replace("LastOptimization=1", ""); let output = if encoding == "utf-16" { let utf16: Vec = updated.encode_utf16().collect(); let bytes: Vec = utf16.iter() .flat_map(|&c| c.to_le_bytes()) .collect(); bytes } else { updated.into_bytes() }; fs::write(&terminal_ini, output)?; Ok(()) } async fn run_backtest(&self, params: &BacktestParams, report_id: &str) -> Result { let mt5_dir = self.config.mt5_dir() .ok_or_else(|| anyhow!("MT5 directory not configured"))?; let wine_exe = self.config.wine_executable.as_ref() .ok_or_else(|| anyhow!("wine_executable not configured"))?; // mt5_dir = {prefix}/drive_c/Program Files/MetaTrader 5 // WINEPREFIX = three levels up let wine_prefix = mt5_dir .parent() // .../drive_c/Program Files .and_then(|p| p.parent()) // .../drive_c .and_then(|p| p.parent()) // .../ .map(|p| p.to_path_buf()) .ok_or_else(|| anyhow!("Could not determine Wine prefix from terminal_dir"))?; let reports_dir = mt5_dir.join("reports"); fs::create_dir_all(&reports_dir)?; // Kill first — MT5 writes terminal.ini on exit, which would clobber // the backtest params we're about to write. self.kill_mt5().await?; // Write params *after* MT5 is dead so nothing can overwrite them. let ini_content = self.build_backtest_ini(params, report_id)?; let config_host = wine_prefix.join("drive_c").join("backtest_config.ini"); fs::write(&config_host, ini_content.as_bytes())?; self.update_terminal_ini(params, report_id)?; // Record launch time before sleeping so find_newest_report doesn't miss // reports written during the startup wait. let poll_start = std::time::SystemTime::now(); let launch_instant = tokio::time::Instant::now(); // Build the launch command, adapting for the Wine runtime in use. let mut cmd = self.build_wine_launch(wine_exe, &wine_prefix)?; cmd.stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .spawn()?; // Give MT5 time to fully initialize before polling. // MT5 app startup (Wine init + network auth + tester) typically takes 10–15 s. // Configurable via startup_delay_secs parameter (default 10s for faster launches). let delay = if params.startup_delay_secs > 0 { params.startup_delay_secs } else { 10 }; sleep(Duration::from_secs(delay)).await; // Poll for the report file (MT5 writes it when the backtest completes). // Grace period: don't check process liveness for the first 30 s after launch — // MT5 may still be appearing in the process list while wineserver re-initializes. let grace_period = Duration::from_secs(30); let deadline = launch_instant + Duration::from_secs(params.timeout); loop { let elapsed = launch_instant.elapsed().as_secs(); // 1. Check for the exact expected report filename. for ext in &[".htm", ".htm.xml", ".html"] { let candidate = reports_dir.join(format!("{}{}", report_id, ext)); tracing::debug!("poll t+{}s: checking {}", elapsed, candidate.display()); if candidate.exists() { tracing::info!("poll t+{}s: found exact report {}", elapsed, candidate.display()); return Ok(candidate); } } // 2. Only check process liveness after the grace period — this prevents // a false "not running" when the new instance is still starting up. let in_grace = launch_instant.elapsed() <= grace_period; let mt5_alive = Self::is_mt5_running(); tracing::info!("poll t+{}s: in_grace={} mt5_alive={}", elapsed, in_grace, mt5_alive); if !in_grace && !mt5_alive { // MT5 writes the .htm report file right before exiting. There is a // short window where the process is gone but the file hasn't been // flushed to the directory. Wait 3 s to let Wine/macOS finish the // write before we scan — this prevents false "no report" failures. sleep(Duration::from_secs(3)).await; if let Some(path) = Self::find_newest_report(&reports_dir, poll_start) { tracing::info!("poll: MT5 exited, found report {}", path.display()); return Ok(path); } return Err(anyhow!( "MT5 exited without producing a report. \ The backtest may have been stopped mid-way or failed to start." )); } if tokio::time::Instant::now() > deadline { return Err(anyhow!("Timeout: no report after {} seconds", params.timeout)); } sleep(Duration::from_secs(2)).await; } } /// Write backtest params into terminal.ini [Tester] section. /// MT5 uses this when restarting — it reconnects via the saved session in common.ini /// rather than requiring fresh credentials. This is more reliable than /config: alone, /// which requires a password for fresh authentication. fn update_terminal_ini(&self, params: &BacktestParams, report_id: &str) -> Result<()> { let mt5_dir = self.config.mt5_dir() .ok_or_else(|| anyhow!("MT5 directory not configured"))?; // Portable mode uses config/ inside the install dir; non-portable uses the root. let terminal_ini = if mt5_dir.join("config").exists() { mt5_dir.join("config").join("terminal.ini") } else { mt5_dir.join("terminal.ini") }; let raw = fs::read(&terminal_ini).unwrap_or_default(); let text = if raw.starts_with(&[0xFF, 0xFE]) { raw[2..].chunks_exact(2) .map(|c| u16::from_le_bytes([c[0], c[1]])) .collect::>() .iter() .map(|&c| char::from_u32(c as u32).unwrap_or('?')) .collect::() } else { String::from_utf8_lossy(&raw).into_owned() }; let period = match params.timeframe.as_str() { "M1" => 1u32, "M5" => 5, "M15" => 15, "M30" => 30, "H1" => 60, "H4" => 240, "D1" => 1440, _ => 5, }; let from_ts = Self::date_str_to_unix(¶ms.from_date)?; let to_ts = Self::date_str_to_unix(¶ms.to_date)?; let currency = self.config.backtest_currency.as_deref().unwrap_or("USD"); let expert_path = if let Some(experts_dir) = &self.config.experts_dir { let nested = std::path::Path::new(experts_dir).join(¶ms.expert).join(format!("{}.mq5", params.expert)); if nested.exists() { format!("Experts\\{}\\{}.ex5", params.expert, params.expert) } else { format!("Experts\\{}.ex5", params.expert) } } else { format!("Experts\\{}.ex5", params.expert) }; let set_file_line = params.set_file.as_ref() .map(|p| format!("ExpertParameters={}\n", Self::ini_safe(p))) .unwrap_or_default(); let updates: &[(&str, String)] = &[ ("Expert", Self::ini_safe(&expert_path)), ("Symbol", Self::ini_safe(¶ms.symbol)), ("Period", period.to_string()), ("DateRange", "3".into()), ("DateFrom", from_ts.to_string()), ("DateTo", to_ts.to_string()), ("Visualization", "0".into()), ("Execution", "10".into()), ("Currency", currency.into()), ("Leverage", params.leverage.to_string()), ("Deposit", format!("{:.2}", params.deposit)), ("TicksMode", params.model.to_string()), ("PipsCalculation", "1".into()), ("OptMode", "0".into()), ("Report", format!("reports\\{}.htm", report_id)), ("ReplaceReport", "1".into()), ("ShutdownTerminal", if params.shutdown { "1" } else { "0" }.into()), ]; let updated = Self::patch_ini_section(&text, "Tester", updates) + &set_file_line; let bom_utf16: Vec = [0xFF, 0xFE].iter().copied() .chain(updated.encode_utf16().flat_map(|c| c.to_le_bytes())) .collect(); fs::write(&terminal_ini, bom_utf16)?; tracing::info!("terminal.ini [Tester] updated → {}", terminal_ini.display()); Ok(()) } /// Strip CR/LF from a user-supplied INI value to prevent newline injection. fn ini_safe(value: &str) -> String { value.replace(['\n', '\r'], "") } fn patch_ini_section(text: &str, section: &str, updates: &[(&str, String)]) -> String { let section_header = format!("[{}]", section); let mut result = String::with_capacity(text.len() + 256); let mut in_section = false; let mut pending: std::collections::HashMap<&str, &String> = updates.iter().map(|(k, v)| (*k, v)).collect(); for line in text.lines() { let trimmed = line.trim(); if trimmed == section_header { in_section = true; result.push_str(line); result.push('\n'); continue; } if trimmed.starts_with('[') && in_section { for (k, v) in &pending { result.push_str(&format!("{}={}\n", k, v)); } pending.clear(); in_section = false; } if in_section { if let Some((key, _)) = trimmed.split_once('=') { let key = key.trim(); if let Some(val) = pending.remove(key) { result.push_str(&format!("{}={}\n", key, val)); continue; } } } result.push_str(line); result.push('\n'); } if in_section { for (k, v) in &pending { result.push_str(&format!("{}={}\n", k, v)); } } result } fn date_str_to_unix(date: &str) -> Result { let parts: Vec = date.split('.').filter_map(|p| p.parse().ok()).collect(); if parts.len() != 3 { return Err(anyhow!("Invalid date format: {}", date)); } let dt = chrono::NaiveDate::from_ymd_opt(parts[0] as i32, parts[1], parts[2]) .ok_or_else(|| anyhow!("Invalid date: {}", date))? .and_hms_opt(0, 0, 0) .ok_or_else(|| anyhow!("Date conversion failed"))?; Ok(chrono::DateTime::::from_naive_utc_and_offset(dt, chrono::Utc).timestamp()) } /// Build the OS-appropriate command to launch MT5 with the backtest config. /// /// - macOS MT5.app bundle: use shell script to bypass SIP and set DYLD vars. /// Relies on terminal.ini for backtest config (more reliable than /config:). /// - macOS CrossOver / Linux Wine: standard WINEPREFIX + wine64 direct invocation. /// Also relies on terminal.ini for backtest config. fn build_wine_launch(&self, wine_exe: &str, wine_prefix: &Path) -> Result { if wine_exe.contains("MetaTrader 5.app") { // macOS MT5.app — the Swift launcher ignores --args so we can't pass // /config: via `open`. Instead, write a temp shell script that sets // DYLD_FALLBACK_LIBRARY_PATH and invokes wine64 directly. // Shell scripts bypass the SIP restriction that strips DYLD_* vars // when Rust spawns a codesigned binary as a direct child process. // NOTE: We rely on terminal.ini for config instead of /config: because // MT5.app's bundled wine64 doesn't reliably handle /config: arguments. let wine_bin = Path::new(wine_exe); let wine_root = wine_bin .parent() // bin/ .and_then(|p| p.parent()) // wine/ .map(|p| p.to_path_buf()) .ok_or_else(|| anyhow!("Cannot derive Wine root from wine_exe"))?; let ext_libs = wine_root.join("lib").join("external"); let wine_libs = wine_root.join("lib"); let dyld = format!("{}:{}:/usr/lib:/usr/local/lib", ext_libs.display(), wine_libs.display()); // Use host path for the exe; use /config: with backslash-escaped path let terminal_host = wine_prefix.join("drive_c") .join("Program Files").join("MetaTrader 5").join("terminal64.exe"); // /config: triggers the Strategy Tester to auto-start. // terminal.ini is also patched with the same params as a belt-and-suspenders. let config_win = r"C:\backtest_config.ini"; let script = format!( "#!/bin/sh\n\ export DYLD_FALLBACK_LIBRARY_PATH='{dyld}'\n\ export WINEPREFIX='{prefix}'\n\ export WINEDEBUG='-all'\n\ nohup '{wine}' '{terminal}' '/config:{config}' \ >/dev/null 2>&1 &\n", dyld = dyld, prefix = wine_prefix.display(), wine = wine_exe, terminal = terminal_host.display(), config = config_win, ); let script_path = std::env::temp_dir().join("mt5_backtest_launch.sh"); // Always rewrite: script content changes per backtest (DYLD paths are // dynamic) and we must ensure +x permissions are set every time. fs::write(&script_path, &script)?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; fs::set_permissions(&script_path, fs::Permissions::from_mode(0o755))?; } tracing::debug!("Wrote launch script: {}", script_path.display()); tracing::info!("Launching MT5 via shell script (terminal.ini mode): {}", script_path.display()); let mut cmd = Command::new("/bin/sh"); cmd.arg(&script_path); return Ok(cmd); } // CrossOver / Linux: invoke wine64 directly with /config: to trigger the tester. let terminal_win_path = r"C:\Program Files\MetaTrader 5\terminal64.exe"; let config_win = r"C:\backtest_config.ini"; let mut cmd = Command::new(wine_exe); cmd.arg(terminal_win_path) .arg(format!("/config:{}", config_win)) .env("WINEPREFIX", wine_prefix) .env("WINEDEBUG", "-all"); Ok(cmd) } /// For /config: INI: path relative to MQL5/Experts/ (e.g. `DPS21\DPS21.ex5`). /// The /config: format does NOT include the "Experts\" prefix. fn resolve_backtest_ini_expert_path(&self, expert: &str) -> String { if let Some(experts_dir) = &self.config.experts_dir { let nested_ex5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.ex5", expert)); let nested_mq5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.mq5", expert)); if nested_ex5.exists() || nested_mq5.exists() { return format!("{}\\{}.ex5", expert, expert); } } format!("{}.ex5", expert) } fn build_backtest_ini(&self, params: &BacktestParams, report_id: &str) -> Result { let mut ini = String::new(); // [Common] section: only written when explicit credentials are configured. // Without it, MT5 reuses its saved session via common.ini (no password needed). if let Some(login) = &self.config.backtest_login { if let Some(server) = &self.config.backtest_server { ini.push_str("[Common]\n"); ini.push_str(&format!("Login={}\n", login)); ini.push_str(&format!("Server={}\n", server)); if let Some(password) = &self.config.backtest_password { ini.push_str(&format!("Password={}\n", password)); } ini.push_str("\n"); } } ini.push_str("[Tester]\n"); // Expert path is relative to MQL5/Experts/ in the /config: format (no "Experts\" prefix). ini.push_str(&format!("Expert={}\n", Self::ini_safe(&self.resolve_backtest_ini_expert_path(¶ms.expert)))); ini.push_str(&format!("Symbol={}\n", Self::ini_safe(¶ms.symbol))); ini.push_str(&format!("Period={}\n", Self::ini_safe(¶ms.timeframe))); ini.push_str("Optimization=0\n"); ini.push_str(&format!("Model={}\n", params.model)); ini.push_str(&format!("FromDate={}\n", params.from_date)); ini.push_str(&format!("ToDate={}\n", params.to_date)); ini.push_str("ForwardMode=0\n"); ini.push_str(&format!("Deposit={}\n", params.deposit)); ini.push_str(&format!("Currency={}\n", self.config.backtest_currency.as_ref().unwrap_or(&"USD".to_string()))); ini.push_str("ProfitInPips=1\n"); ini.push_str(&format!("Leverage={}\n", params.leverage)); ini.push_str("Execution=10\n"); ini.push_str(&format!("Visual={}\n", if params.gui { "1" } else { "0" })); ini.push_str(&format!("Report=reports\\{}.htm\n", report_id)); ini.push_str("ReplaceReport=1\n"); ini.push_str(&format!("ShutdownTerminal={}\n", if params.shutdown { "1" } else { "0" })); if let Some(set_file) = ¶ms.set_file { ini.push_str(&format!("ExpertParameters={}\n", Self::ini_safe(set_file))); } Ok(ini) } async fn kill_mt5(&self) -> Result<()> { let patterns = Self::mt5_process_patterns(); let running = patterns.iter().any(|pat| { Command::new("pgrep") .args(["-f", pat.as_str()]) .output() .map(|o| o.status.success()) .unwrap_or(false) }); if !running { return Ok(()); } tracing::info!("Stopping existing MT5 instance..."); // SIGKILL immediately — MT5 holds no state we care about preserving. for pat in &patterns { let _ = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output(); } // Also kill wineserver so the Wine prefix is fully reset before relaunch. // If wineserver is still alive when the new MT5 spawns, the new Wine // instance may attach to the dying server and never enter tester mode. let _ = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output(); // Poll until wineserver is actually gone (max 10 s) rather than sleeping // a fixed amount. On macOS wineserver cleanup varies from 1 s to 6+ s. let deadline = tokio::time::Instant::now() + Duration::from_secs(10); loop { sleep(Duration::from_millis(500)).await; let ws_alive = Command::new("pgrep") .args(["-f", "wineserver"]) .output() .map(|o| o.status.success()) .unwrap_or(false); let mt5_alive = patterns.iter().any(|pat| { Command::new("pgrep") .args(["-f", pat.as_str()]) .output() .map(|o| o.status.success()) .unwrap_or(false) }); if !ws_alive && !mt5_alive { tracing::info!("MT5 and wineserver fully exited"); break; } if tokio::time::Instant::now() >= deadline { tracing::warn!("wineserver still alive after 10 s — proceeding anyway"); break; } } // Brief extra pause to let the kernel release sockets and shared memory. sleep(Duration::from_millis(500)).await; Ok(()) } fn is_mt5_running() -> bool { Self::mt5_process_patterns().iter().any(|pat| { Command::new("pgrep") .args(["-f", pat.as_str()]) .output() .map(|o| o.status.success()) .unwrap_or(false) }) } /// Scan `dir` for the newest .htm/.htm.xml/.html file written after `since`. fn find_newest_report(dir: &Path, since: std::time::SystemTime) -> Option { let entries = fs::read_dir(dir).ok()?; let mut candidates: Vec<(std::time::SystemTime, PathBuf)> = entries .filter_map(|e| e.ok()) .filter(|e| { let ext = e.path().extension() .and_then(|x| x.to_str()) .unwrap_or("") .to_lowercase(); matches!(ext.as_str(), "htm" | "xml" | "html") }) .filter_map(|e| { let mtime = e.metadata().ok()?.modified().ok()?; if mtime >= since { Some((mtime, e.path())) } else { None } }) .collect(); candidates.sort_by_key(|(t, _)| *t); candidates.into_iter().last().map(|(_, p)| p) } fn mt5_process_patterns() -> Vec { if cfg!(target_os = "macos") { // macOS: official MT5.app bundle (contains its own Wine runtime) // Also match Wine-hosted terminal64.exe for CrossOver installs vec![ "MetaTrader 5\\.app".to_string(), "terminal64\\.exe".to_string(), ] } else { // Linux: MT5 always runs as a Wine process vec![ "terminal64\\.exe".to_string(), "metatrader".to_string(), ] } } async fn log_progress(&self, log_path: &Path, stage: &str) { let timestamp = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"); let line = format!("{} {}\n", stage, timestamp); let _ = fs::write(log_path, line); } async fn save_metadata(&self, params: &BacktestParams, report_dir: &Path, duration: i64, no_trades: bool) -> Result<()> { let metadata = PipelineMetadata { expert: params.expert.clone(), symbol: params.symbol.clone(), timeframe: params.timeframe.clone(), from_date: params.from_date.clone(), to_date: params.to_date.clone(), deposit: params.deposit as f64, currency: self.config.backtest_currency.clone().unwrap_or_else(|| "USD".to_string()), model: params.model as i32, leverage: params.leverage as i32, set_file: params.set_file.clone(), report_dir: report_dir.to_string_lossy().to_string(), duration_seconds: duration, files: FilePaths { metrics: report_dir.join("metrics.json").to_string_lossy().to_string(), analysis: report_dir.join("analysis.json").to_string_lossy().to_string(), }, no_trades, }; let json = serde_json::to_string_pretty(&metadata)?; fs::write(report_dir.join("pipeline_metadata.json"), json)?; Ok(()) } fn generate_report_id(&self, params: &BacktestParams) -> String { let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S"); format!( "{}_{}_{}_{}_{}", timestamp, params.expert, params.symbol, params.timeframe, params.model ) } }