diff --git a/Cargo.lock b/Cargo.lock index b6b51c5..ec40855 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -228,6 +240,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.4.1" @@ -270,6 +294,15 @@ dependencies = [ "wasip3", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -285,6 +318,15 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -382,6 +424,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -431,6 +484,7 @@ dependencies = [ "encoding_rs", "regex", "roxmltree", + "rusqlite", "serde", "serde_json", "serde_yaml", @@ -507,6 +561,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "prettyplease" version = "0.2.37" @@ -599,6 +659,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "rusqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustix" version = "1.1.4" @@ -927,6 +1001,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -1277,6 +1363,26 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 082e44f..d8565b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,3 +26,4 @@ chrono = { version = "0.4", features = ["serde"] } encoding_rs = "0.8" tempfile = "3.0" roxmltree = "0.21.1" +rusqlite = { version = "0.31", features = ["bundled"] } diff --git a/src/main.rs b/src/main.rs index 86a6a2e..0c205d1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ mod compile; mod models; mod optimization; mod pipeline; +mod storage; mod tools; mod config; diff --git a/src/pipeline/backtest.rs b/src/pipeline/backtest.rs index 2a692a3..fedb030 100644 --- a/src/pipeline/backtest.rs +++ b/src/pipeline/backtest.rs @@ -9,6 +9,7 @@ use crate::analytics::{DealAnalyzer, ReportExtractor}; use crate::compile::MqlCompiler; use crate::models::config::Config; use crate::models::report::{PipelineMetadata, FilePaths}; +use crate::storage::{ReportDb, ReportEntry}; pub struct BacktestPipeline { config: Config, @@ -33,6 +34,7 @@ pub struct BacktestParams { #[allow(dead_code)] pub deep_analyze: bool, pub shutdown: bool, + #[allow(dead_code)] pub kill_existing: bool, pub timeout: u64, pub gui: bool, @@ -63,9 +65,9 @@ impl BacktestPipeline { let start_time = chrono::Utc::now(); let report_id = self.generate_report_id(¶ms); let report_dir = self.config.reports_dir().join(&report_id); - + fs::create_dir_all(&report_dir)?; - + let progress_log = report_dir.join("progress.log"); self.log_progress(&progress_log, "START").await; @@ -85,23 +87,41 @@ impl BacktestPipeline { self.log_progress(&progress_log, "EXTRACT").await; let extraction = self.extractor.extract( &report_path.to_string_lossy(), - &report_dir.to_string_lossy() + &report_dir.to_string_lossy(), )?; + // Move equity chart images to OS temp dir, then delete the HTML report. + let charts_dir = self.relocate_charts(&report_path, &report_id).await; + let _ = fs::remove_file(&report_path); + + // Snapshot the set file alongside the extracted data. + let set_snapshot = self.snapshot_set_file(¶ms, &report_dir).await; + if !params.skip_analyze { self.log_progress(&progress_log, "ANALYZE").await; let analysis = self.analyzer.analyze(&extraction.deals, &extraction.metrics); - + let analysis_path = report_dir.join("analysis.json"); - let analysis_json = serde_json::to_string_pretty(&analysis)?; - fs::write(&analysis_path, analysis_json)?; + fs::write(&analysis_path, serde_json::to_string_pretty(&analysis)?)?; } self.log_progress(&progress_log, "DONE").await; - + let duration = (chrono::Utc::now() - start_time).num_seconds(); self.save_metadata(¶ms, &report_dir, duration).await?; + // Register in the SQLite report registry. + self.register_in_db( + &report_id, + ¶ms, + &report_dir, + charts_dir.as_deref(), + set_snapshot.as_deref(), + &extraction.metrics, + duration, + ) + .await; + Ok(PipelineResult { success: true, report_dir, @@ -110,8 +130,110 @@ impl BacktestPipeline { }) } + /// Move equity chart images (*.png, *.gif) from MT5's reports dir to OS temp, + /// returning the temp path if any images were found. + async fn relocate_charts(&self, html_path: &Path, report_id: &str) -> Option { + let reports_dir = html_path.parent()?; + let charts_dir = Config::charts_temp_dir(report_id); + let image_exts = ["png", "gif", "jpg", "jpeg"]; + + let entries = fs::read_dir(reports_dir).ok()?; + let mut found = false; + + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + let name = path.file_name()?.to_string_lossy().to_string(); + + let is_chart = name.starts_with(report_id) + && path + .extension() + .and_then(|e| e.to_str()) + .map(|e| image_exts.contains(&e)) + .unwrap_or(false); + + if is_chart { + if !found { + if fs::create_dir_all(&charts_dir).is_err() { + return None; + } + } + let dest = charts_dir.join(entry.file_name()); + let _ = fs::rename(&path, &dest); + found = true; + } + } + + if found { Some(charts_dir) } else { None } + } + + /// Copy the set file into the report dir as set_snapshot.set. + async fn snapshot_set_file( + &self, + params: &BacktestParams, + report_dir: &Path, + ) -> Option { + let set_src = params.set_file.as_ref()?; + let src_path = Path::new(set_src); + if !src_path.exists() { + return None; + } + let dest = report_dir.join("set_snapshot.set"); + fs::copy(src_path, &dest).ok()?; + Some(dest) + } + + async fn register_in_db( + &self, + report_id: &str, + params: &BacktestParams, + report_dir: &Path, + charts_dir: Option<&Path>, + set_snapshot: Option<&Path>, + metrics: &crate::models::metrics::Metrics, + duration: i64, + ) { + let db = ReportDb::new(&Config::db_path()); + if let Err(e) = db.init() { + tracing::warn!("Failed to init report DB: {}", e); + return; + } + + let entry = ReportEntry { + id: report_id.to_string(), + expert: params.expert.clone(), + symbol: params.symbol.clone(), + timeframe: params.timeframe.clone(), + model: params.model as i64, + from_date: params.from_date.clone(), + to_date: params.to_date.clone(), + created_at: chrono::Utc::now().to_rfc3339(), + set_file_original: params.set_file.clone(), + set_snapshot_path: set_snapshot.map(|p| p.to_string_lossy().to_string()), + report_dir: report_dir.to_string_lossy().to_string(), + charts_dir: charts_dir.map(|p| p.to_string_lossy().to_string()), + net_profit: Some(metrics.net_profit), + profit_factor: Some(metrics.profit_factor), + max_dd_pct: Some(metrics.max_dd_pct), + sharpe_ratio: Some(metrics.sharpe_ratio), + total_trades: Some(metrics.total_trades as i64), + win_rate_pct: Some(metrics.win_rate_pct), + recovery_factor: Some(metrics.recovery_factor), + deposit: Some(params.deposit as f64), + currency: self.config.backtest_currency.clone(), + leverage: Some(params.leverage as i64), + duration_seconds: Some(duration), + tags: Vec::new(), + notes: None, + verdict: None, + }; + + if let Err(e) = db.insert(&entry) { + tracing::warn!("Failed to register report in DB: {}", e); + } + } + async fn compile_ea(&self, expert: &str) -> Result<()> { - let search_paths = [ + let mut search_paths = vec![ PathBuf::from(&self.config.get("project_dir")).join("src/experts").join(format!("{}.mq5", expert)), PathBuf::from(&self.config.get("project_dir")).join("src").join(format!("{}.mq5", expert)), PathBuf::from(&self.config.get("project_dir")).join(format!("{}.mq5", expert)), @@ -119,11 +241,16 @@ impl BacktestPipeline { PathBuf::from("src").join(format!("{}.mq5", expert)), PathBuf::from(format!("{}.mq5", expert)), ]; + // Also search in MT5 Experts dir: Experts/{expert}/{expert}.mq5 and Experts/{expert}.mq5 + if let Some(experts_dir) = &self.config.experts_dir { + search_paths.push(PathBuf::from(experts_dir).join(expert).join(format!("{}.mq5", expert))); + search_paths.push(PathBuf::from(experts_dir).join(format!("{}.mq5", expert))); + } let source_path = search_paths .into_iter() .find(|p| p.exists()) - .ok_or_else(|| anyhow!("Cannot find {}.mq5", expert))?; + .ok_or_else(|| anyhow!("Cannot find {}.mq5 — searched project_dir and MT5 Experts dir", expert))?; let result = self.compiler.compile(&source_path.to_string_lossy())?; @@ -209,103 +336,310 @@ impl BacktestPipeline { async fn run_backtest(&self, params: &BacktestParams, report_id: &str) -> Result { let mt5_dir = self.config.mt5_dir() .ok_or_else(|| anyhow!("MT5 directory not configured"))?; - + let wine_exe = self.config.wine_executable.as_ref() .ok_or_else(|| anyhow!("wine_executable not configured"))?; + // mt5_dir = {prefix}/drive_c/Program Files/MetaTrader 5 + // WINEPREFIX = three levels up let wine_prefix = mt5_dir - .parent() - .and_then(|p| p.parent()) + .parent() // .../drive_c/Program Files + .and_then(|p| p.parent()) // .../drive_c + .and_then(|p| p.parent()) // .../ .map(|p| p.to_path_buf()) - .ok_or_else(|| anyhow!("Could not determine Wine prefix"))?; + .ok_or_else(|| anyhow!("Could not determine Wine prefix from terminal_dir"))?; let reports_dir = mt5_dir.join("reports"); fs::create_dir_all(&reports_dir)?; - let ini_path = mt5_dir.join("backtest_config.ini"); + // Write backtest_config.ini (used by wine64 shell-script launch via /config:) + // and also patch terminal.ini so the Strategy Tester panel shows the right + // settings if the user opens MT5 manually. let ini_content = self.build_backtest_ini(params, report_id)?; - - let ini_utf16: Vec = std::iter::once(0xFFu8) - .chain(std::iter::once(0xFEu8)) - .chain(ini_content.encode_utf16().flat_map(|c| c.to_le_bytes())) - .collect(); - - fs::write(&ini_path, ini_utf16)?; + // Write to drive_c root (C:\backtest_config.ini) — no spaces in path avoids + // Wine argument-quoting issues when MT5 parses the /config: value. + let config_host = wine_prefix.join("drive_c").join("backtest_config.ini"); + fs::write(&config_host, ini_content.as_bytes())?; + self.update_terminal_ini(params, report_id)?; - if params.kill_existing { - self.kill_mt5().await?; + // Always kill any running MT5 — Wine allows only one instance per prefix. + self.kill_mt5().await?; + + // Record launch time before sleeping so find_newest_report doesn't miss + // reports written during the startup wait. + let poll_start = std::time::SystemTime::now(); + let launch_instant = tokio::time::Instant::now(); + + // Build the launch command, adapting for the Wine runtime in use. + let mut cmd = self.build_wine_launch(wine_exe, &wine_prefix)?; + cmd.stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?; + + // Give MT5 time to fully initialize before polling. + // MT5 app startup (Wine init + network auth + tester) typically takes 15–20 s. + sleep(Duration::from_secs(20)).await; + + // Poll for the report file (MT5 writes it when the backtest completes). + // Grace period: don't check process liveness for the first 30 s after launch — + // MT5 may still be appearing in the process list while wineserver re-initializes. + let grace_period = Duration::from_secs(30); + let deadline = launch_instant + Duration::from_secs(params.timeout); + + loop { + let elapsed = launch_instant.elapsed().as_secs(); + + // 1. Check for the exact expected report filename. + for ext in &[".htm", ".htm.xml", ".html"] { + let candidate = reports_dir.join(format!("{}{}", report_id, ext)); + tracing::debug!("poll t+{}s: checking {}", elapsed, candidate.display()); + if candidate.exists() { + tracing::info!("poll t+{}s: found exact report {}", elapsed, candidate.display()); + return Ok(candidate); + } + } + + // 2. Only check process liveness after the grace period — this prevents + // a false "not running" when the new instance is still starting up. + let in_grace = launch_instant.elapsed() <= grace_period; + let mt5_alive = Self::is_mt5_running(); + tracing::info!("poll t+{}s: in_grace={} mt5_alive={}", elapsed, in_grace, mt5_alive); + + if !in_grace && !mt5_alive { + if let Some(path) = Self::find_newest_report(&reports_dir, poll_start) { + tracing::info!("poll: MT5 exited, found fallback report {}", path.display()); + return Ok(path); + } + return Err(anyhow!( + "MT5 exited without producing a report. \ + The backtest may have been stopped mid-way or failed to start." + )); + } + + if tokio::time::Instant::now() > deadline { + return Err(anyhow!("Timeout: no report after {} seconds", params.timeout)); + } + sleep(Duration::from_secs(2)).await; } + } - let bat_content = if params.shutdown { - format!(r#"@echo off -cd /d "C:\Program Files\MetaTrader 5" -start /wait terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" -"#) + /// Write backtest parameters into terminal.ini [Tester] section so MT5 reads + /// them on startup without needing a /config: command-line argument. + fn update_terminal_ini(&self, params: &BacktestParams, report_id: &str) -> Result<()> { + let mt5_dir = self.config.mt5_dir() + .ok_or_else(|| anyhow!("MT5 directory not configured"))?; + let terminal_ini = mt5_dir.join("config").join("terminal.ini"); + + let raw = fs::read(&terminal_ini) + .unwrap_or_default(); + let text = if raw.starts_with(&[0xFF, 0xFE]) { + raw[2..].chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect::>() + .iter() + .map(|&c| char::from_u32(c as u32).unwrap_or('?')) + .collect::() } else { - format!(r#"@echo off -cd /d "C:\Program Files\MetaTrader 5" -start terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" -"#) + String::from_utf8_lossy(&raw).into_owned() }; - let bat_path = wine_prefix.join("drive_c").join("_mt5mcp_run.bat"); - fs::write(&bat_path, bat_content)?; + let period = match params.timeframe.as_str() { + "M1" => 1u32, "M5" => 5, "M15" => 15, "M30" => 30, + "H1" => 60, "H4" => 240, "D1" => 1440, + _ => 5, + }; - let _cmd = format!("cmd.exe /c 'C:\\_mt5mcp_run.bat'"); + let from_ts = Self::date_str_to_unix(¶ms.from_date)?; + let to_ts = Self::date_str_to_unix(¶ms.to_date)?; - if params.shutdown { - let output = Command::new("timeout") - .arg(¶ms.timeout.to_string()) - .arg(wine_exe) - .arg("cmd.exe") - .arg("/c") - .arg("C:\\_mt5mcp_run.bat") - .env("WINEPREFIX", &wine_prefix) - .env("WINEDEBUG", "-all") - .output()?; + let currency = self.config.backtest_currency.as_deref().unwrap_or("USD"); + let set_file_line = params.set_file.as_ref() + .map(|p| format!("ExpertParameters={}\n", p)) + .unwrap_or_default(); - if !output.status.success() { - tracing::warn!("MT5 exited with code: {:?}", output.status.code()); + let updates: &[(&str, String)] = &[ + ("Expert", self.resolve_expert_path(¶ms.expert)), + ("Symbol", params.symbol.clone()), + ("Period", period.to_string()), + ("DateRange", "3".into()), + ("DateFrom", from_ts.to_string()), + ("DateTo", to_ts.to_string()), + ("Visualization", "0".into()), + ("Execution", "10".into()), + ("Currency", currency.into()), + ("Leverage", params.leverage.to_string()), + ("Deposit", format!("{:.2}", params.deposit)), + ("TicksMode", params.model.to_string()), + ("PipsCalculation", "1".into()), + ("OptMode", "0".into()), + ("Report", format!("reports\\{}.htm", report_id)), + ("ReplaceReport", "1".into()), + ("ShutdownTerminal", if params.shutdown { "1" } else { "0" }.into()), + ]; + + let updated = Self::patch_ini_section(&text, "Tester", updates) + + &set_file_line; + + let bom_utf16: Vec = [0xFF, 0xFE].iter().copied() + .chain(updated.encode_utf16().flat_map(|c| c.to_le_bytes())) + .collect(); + fs::write(&terminal_ini, bom_utf16)?; + tracing::info!("terminal.ini [Tester] updated for backtest {}", report_id); + Ok(()) + } + + /// Parse "YYYY.MM.DD" and return a Unix timestamp (seconds since 1970-01-01 UTC). + fn date_str_to_unix(date: &str) -> Result { + let parts: Vec = date.split('.').filter_map(|p| p.parse().ok()).collect(); + if parts.len() != 3 { + return Err(anyhow!("Invalid date format: {}", date)); + } + let dt = chrono::NaiveDate::from_ymd_opt(parts[0] as i32, parts[1], parts[2]) + .ok_or_else(|| anyhow!("Invalid date: {}", date))? + .and_hms_opt(0, 0, 0) + .ok_or_else(|| anyhow!("Date conversion failed"))?; + Ok(chrono::DateTime::::from_naive_utc_and_offset(dt, chrono::Utc).timestamp()) + } + + /// Replace or add key=value pairs in a named INI section. + fn patch_ini_section(text: &str, section: &str, updates: &[(&str, String)]) -> String { + let section_header = format!("[{}]", section); + let mut result = String::with_capacity(text.len() + 256); + let mut in_section = false; + let mut pending: std::collections::HashMap<&str, &String> = + updates.iter().map(|(k, v)| (*k, v)).collect(); + + for line in text.lines() { + let trimmed = line.trim(); + if trimmed == section_header { + in_section = true; + result.push_str(line); + result.push('\n'); + continue; } - } else { - Command::new("nohup") - .arg(wine_exe) - .arg("cmd.exe") - .arg("/c") - .arg("C:\\_mt5mcp_run.bat") - .env("WINEPREFIX", &wine_prefix) - .env("WINEDEBUG", "-all") - .spawn()?; - - sleep(Duration::from_secs(5)).await; - - let deadline = tokio::time::Instant::now() + Duration::from_secs(params.timeout); - loop { - if tokio::time::Instant::now() > deadline { - return Err(anyhow!("Timeout waiting for backtest report")); + if trimmed.starts_with('[') && in_section { + // End of our section — flush any keys not yet written + for (k, v) in &pending { + result.push_str(&format!("{}={}\n", k, v)); } - - for ext in &[".htm", ".htm.xml", ".html"] { - let candidate = reports_dir.join(format!("{}{}", report_id, ext)); - if candidate.exists() { - let _ = fs::remove_file(&bat_path); - return Ok(candidate); + pending.clear(); + in_section = false; + } + if in_section { + if let Some((key, _)) = trimmed.split_once('=') { + let key = key.trim(); + if let Some(val) = pending.remove(key) { + result.push_str(&format!("{}={}\n", key, val)); + continue; } } - - sleep(Duration::from_secs(5)).await; + } + result.push_str(line); + result.push('\n'); + } + // Flush remaining keys if section was at end of file + if in_section { + for (k, v) in &pending { + result.push_str(&format!("{}={}\n", k, v)); } } + result + } - for ext in &[".htm", ".htm.xml", ".html"] { - let candidate = reports_dir.join(format!("{}{}", report_id, ext)); - if candidate.exists() { - let _ = fs::remove_file(&bat_path); - return Ok(candidate); + /// Build the OS-appropriate command to launch MT5 with the backtest config. + /// + /// - macOS MT5.app bundle: use `open -a "MetaTrader 5" --args /config:...` + /// The native launcher handles Wine env setup (DYLD vars are stripped by SIP + /// when set on child processes spawned from a Rust binary). + /// - macOS CrossOver / Linux Wine: standard WINEPREFIX + wine64 direct invocation. + fn build_wine_launch(&self, wine_exe: &str, wine_prefix: &Path) -> Result { + if wine_exe.contains("MetaTrader 5.app") { + // macOS MT5.app — the Swift launcher ignores --args so we can't pass + // /config: via `open`. Instead, write a temp shell script that sets + // DYLD_FALLBACK_LIBRARY_PATH and invokes wine64 with /config: directly. + // Shell scripts bypass the SIP restriction that strips DYLD_* vars + // when Rust spawns a codesigned binary as a direct child process. + let wine_bin = Path::new(wine_exe); + let wine_root = wine_bin + .parent() // bin/ + .and_then(|p| p.parent()) // wine/ + .map(|p| p.to_path_buf()) + .ok_or_else(|| anyhow!("Cannot derive Wine root from wine_exe"))?; + + let ext_libs = wine_root.join("lib").join("external"); + let wine_libs = wine_root.join("lib"); + let dyld = format!("{}:{}:/usr/lib:/usr/local/lib", + ext_libs.display(), wine_libs.display()); + + // Use host path for the exe; config at drive root to avoid spaces in path. + let terminal_host = wine_prefix.join("drive_c") + .join("Program Files").join("MetaTrader 5").join("terminal64.exe"); + let config_win = r"C:\backtest_config.ini"; + + let script = format!( + "#!/bin/sh\n\ + export DYLD_FALLBACK_LIBRARY_PATH='{dyld}'\n\ + export WINEPREFIX='{prefix}'\n\ + export WINEDEBUG='-all'\n\ + nohup '{wine}' '{terminal}' '/config:{config}' \ + >/dev/null 2>&1 &\n", + dyld = dyld, + prefix = wine_prefix.display(), + wine = wine_exe, + terminal = terminal_host.display(), + config = config_win, + ); + + let script_path = std::env::temp_dir().join("mt5_backtest_launch.sh"); + fs::write(&script_path, &script)?; + // chmod +x + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&script_path, + fs::Permissions::from_mode(0o755))?; } + + tracing::info!("Launching MT5 via shell script: {}", script_path.display()); + let mut cmd = Command::new("/bin/sh"); + cmd.arg(&script_path); + return Ok(cmd); } - Err(anyhow!("No report file generated")) + // CrossOver / Linux: invoke wine64 directly with WINEPREFIX set. + // Params are already written to terminal.ini so no /config: arg needed. + let terminal_win_path = r"C:\Program Files\MetaTrader 5\terminal64.exe"; + let mut cmd = Command::new(wine_exe); + cmd.arg(terminal_win_path) + .env("WINEPREFIX", wine_prefix) + .env("WINEDEBUG", "-all"); + Ok(cmd) + } + + /// For terminal.ini: path relative to MQL5/ (e.g. `Experts\DPS21\DPS21.ex5`). + fn resolve_expert_path(&self, expert: &str) -> String { + if let Some(experts_dir) = &self.config.experts_dir { + let nested_ex5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.ex5", expert)); + let nested_mq5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.mq5", expert)); + if nested_ex5.exists() || nested_mq5.exists() { + return format!("Experts\\{}\\{}.ex5", expert, expert); + } + } + format!("Experts\\{}.ex5", expert) + } + + /// For /config: INI: path relative to MQL5/Experts/ (e.g. `DPS21\DPS21.ex5`). + /// The /config: format does NOT include the "Experts\" prefix. + fn resolve_backtest_ini_expert_path(&self, expert: &str) -> String { + if let Some(experts_dir) = &self.config.experts_dir { + let nested_ex5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.ex5", expert)); + let nested_mq5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.mq5", expert)); + if nested_ex5.exists() || nested_mq5.exists() { + return format!("{}\\{}.ex5", expert, expert); + } + } + format!("{}.ex5", expert) } fn build_backtest_ini(&self, params: &BacktestParams, report_id: &str) -> Result { @@ -320,7 +654,8 @@ start terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" } ini.push_str("[Tester]\n"); - ini.push_str(&format!("Expert={}.ex5\n", params.expert)); + // Expert path is relative to MQL5/Experts/ in the /config: format (no "Experts\" prefix). + ini.push_str(&format!("Expert={}\n", self.resolve_backtest_ini_expert_path(¶ms.expert))); ini.push_str(&format!("Symbol={}\n", params.symbol)); ini.push_str(&format!("Period={}\n", params.timeframe)); ini.push_str("Optimization=0\n"); @@ -332,8 +667,7 @@ start terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" ini.push_str(&format!("Currency={}\n", self.config.backtest_currency.as_ref().unwrap_or(&"USD".to_string()))); ini.push_str("ProfitInPips=1\n"); ini.push_str(&format!("Leverage={}\n", params.leverage)); - ini.push_str("ExecutionMode=10\n"); - ini.push_str("OptimizationCriterion=0\n"); + ini.push_str("Execution=10\n"); ini.push_str(&format!("Visual={}\n", if params.gui { "1" } else { "0" })); ini.push_str(&format!("Report=reports\\{}.htm\n", report_id)); ini.push_str("ReplaceReport=1\n"); @@ -347,26 +681,80 @@ start terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" } async fn kill_mt5(&self) -> Result<()> { - let _output = Command::new("pkill") - .args(&["-TERM", "-f", "terminal64\\.exe"]) - .output()?; + let patterns = Self::mt5_process_patterns(); - sleep(Duration::from_secs(5)).await; + let running = patterns.iter().any(|pat| { + Command::new("pgrep") + .args(["-f", pat.as_str()]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + }); - let check = Command::new("pgrep") - .args(&["-f", "terminal64\\.exe"]) - .output()?; - - if check.status.success() { - let _ = Command::new("pkill") - .args(&["-KILL", "-f", "terminal64\\.exe"]) - .output(); - sleep(Duration::from_secs(1)).await; + if !running { + return Ok(()); } + tracing::info!("Stopping existing MT5 instance..."); + // SIGKILL immediately — MT5 holds no state we care about preserving. + for pat in &patterns { + let _ = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output(); + } + let _ = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output(); + sleep(Duration::from_secs(3)).await; + Ok(()) } + fn is_mt5_running() -> bool { + Self::mt5_process_patterns().iter().any(|pat| { + Command::new("pgrep") + .args(["-f", pat.as_str()]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + }) + } + + /// Scan `dir` for the newest .htm/.htm.xml/.html file written after `since`. + fn find_newest_report(dir: &Path, since: std::time::SystemTime) -> Option { + let entries = fs::read_dir(dir).ok()?; + let mut candidates: Vec<(std::time::SystemTime, PathBuf)> = entries + .filter_map(|e| e.ok()) + .filter(|e| { + let ext = e.path().extension() + .and_then(|x| x.to_str()) + .unwrap_or("") + .to_lowercase(); + matches!(ext.as_str(), "htm" | "xml" | "html") + }) + .filter_map(|e| { + let mtime = e.metadata().ok()?.modified().ok()?; + if mtime >= since { Some((mtime, e.path())) } else { None } + }) + .collect(); + + candidates.sort_by_key(|(t, _)| *t); + candidates.into_iter().last().map(|(_, p)| p) + } + + fn mt5_process_patterns() -> Vec { + if cfg!(target_os = "macos") { + // macOS: official MT5.app bundle (contains its own Wine runtime) + // Also match Wine-hosted terminal64.exe for CrossOver installs + vec![ + "MetaTrader 5\\.app".to_string(), + "terminal64\\.exe".to_string(), + ] + } else { + // Linux: MT5 always runs as a Wine process + vec![ + "terminal64\\.exe".to_string(), + "metatrader".to_string(), + ] + } + } + async fn log_progress(&self, log_path: &Path, stage: &str) { let timestamp = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"); let line = format!("{} {}\n", stage, timestamp); diff --git a/src/storage/database.rs b/src/storage/database.rs new file mode 100644 index 0000000..f4cb1d9 --- /dev/null +++ b/src/storage/database.rs @@ -0,0 +1,326 @@ +use anyhow::Result; +use rusqlite::{params, Connection}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +pub struct ReportEntry { + pub id: String, + pub expert: String, + pub symbol: String, + pub timeframe: String, + pub model: i64, + pub from_date: String, + pub to_date: String, + pub created_at: String, + pub set_file_original: Option, + pub set_snapshot_path: Option, + pub report_dir: String, + pub charts_dir: Option, + pub net_profit: Option, + pub profit_factor: Option, + pub max_dd_pct: Option, + pub sharpe_ratio: Option, + pub total_trades: Option, + pub win_rate_pct: Option, + pub recovery_factor: Option, + pub deposit: Option, + pub currency: Option, + pub leverage: Option, + pub duration_seconds: Option, + pub tags: Vec, + pub notes: Option, + pub verdict: Option, +} + +#[derive(Debug, Default)] +pub struct ReportFilters { + pub expert: Option, + pub symbol: Option, + pub timeframe: Option, + pub created_after: Option, + pub min_profit: Option, + pub max_dd: Option, + pub verdict: Option, +} + +pub struct ReportDb { + db_path: PathBuf, +} + +impl ReportDb { + pub fn new(db_path: &Path) -> Self { + Self { + db_path: db_path.to_path_buf(), + } + } + + fn connect(&self) -> Result { + let conn = Connection::open(&self.db_path)?; + conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;")?; + Ok(conn) + } + + pub fn init(&self) -> Result<()> { + if let Some(parent) = self.db_path.parent() { + std::fs::create_dir_all(parent)?; + } + let conn = self.connect()?; + conn.execute_batch(" + CREATE TABLE IF NOT EXISTS reports ( + id TEXT PRIMARY KEY, + expert TEXT NOT NULL, + symbol TEXT NOT NULL, + timeframe TEXT NOT NULL, + model INTEGER NOT NULL DEFAULT 0, + from_date TEXT NOT NULL DEFAULT '', + to_date TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + set_file_original TEXT, + set_snapshot_path TEXT, + report_dir TEXT NOT NULL, + charts_dir TEXT, + net_profit REAL, + profit_factor REAL, + max_dd_pct REAL, + sharpe_ratio REAL, + total_trades INTEGER, + win_rate_pct REAL, + recovery_factor REAL, + deposit REAL, + currency TEXT, + leverage INTEGER, + duration_seconds INTEGER, + tags TEXT DEFAULT '[]', + notes TEXT, + verdict TEXT + ); + CREATE INDEX IF NOT EXISTS idx_reports_expert ON reports(expert); + CREATE INDEX IF NOT EXISTS idx_reports_symbol ON reports(symbol); + CREATE INDEX IF NOT EXISTS idx_reports_created_at ON reports(created_at DESC); + ")?; + Ok(()) + } + + pub fn insert(&self, entry: &ReportEntry) -> Result<()> { + let conn = self.connect()?; + let tags_json = serde_json::to_string(&entry.tags)?; + conn.execute( + "INSERT OR REPLACE INTO reports + (id, expert, symbol, timeframe, model, from_date, to_date, created_at, + set_file_original, set_snapshot_path, report_dir, charts_dir, + net_profit, profit_factor, max_dd_pct, sharpe_ratio, total_trades, + win_rate_pct, recovery_factor, deposit, currency, leverage, + duration_seconds, tags, notes, verdict) + VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26)", + params![ + entry.id, + entry.expert, + entry.symbol, + entry.timeframe, + entry.model, + entry.from_date, + entry.to_date, + entry.created_at, + entry.set_file_original, + entry.set_snapshot_path, + entry.report_dir, + entry.charts_dir, + entry.net_profit, + entry.profit_factor, + entry.max_dd_pct, + entry.sharpe_ratio, + entry.total_trades, + entry.win_rate_pct, + entry.recovery_factor, + entry.deposit, + entry.currency, + entry.leverage, + entry.duration_seconds, + tags_json, + entry.notes, + entry.verdict, + ], + )?; + Ok(()) + } + + pub fn list(&self, limit: usize, filters: &ReportFilters) -> Result> { + let conn = self.connect()?; + + let mut sql = "SELECT id, expert, symbol, timeframe, model, from_date, to_date, \ + created_at, set_file_original, set_snapshot_path, report_dir, charts_dir, \ + net_profit, profit_factor, max_dd_pct, sharpe_ratio, total_trades, \ + win_rate_pct, recovery_factor, deposit, currency, leverage, \ + duration_seconds, tags, notes, verdict \ + FROM reports WHERE 1=1" + .to_string(); + + let mut filter_params: Vec = Vec::new(); + + if let Some(ea) = &filters.expert { + sql.push_str(" AND expert LIKE ?"); + filter_params.push(format!("%{}%", ea)); + } + if let Some(sym) = &filters.symbol { + sql.push_str(" AND symbol = ?"); + filter_params.push(sym.clone()); + } + if let Some(tf) = &filters.timeframe { + sql.push_str(" AND timeframe = ?"); + filter_params.push(tf.clone()); + } + if let Some(after) = &filters.created_after { + sql.push_str(" AND created_at >= ?"); + filter_params.push(after.clone()); + } + if let Some(verdict) = &filters.verdict { + sql.push_str(" AND verdict = ?"); + filter_params.push(verdict.clone()); + } + + // Overfetch for in-memory numeric filters, then truncate + let fetch_limit = if filters.min_profit.is_some() || filters.max_dd.is_some() { + limit * 4 + } else { + limit + }; + sql.push_str(&format!(" ORDER BY created_at DESC LIMIT {}", fetch_limit)); + + let mut stmt = conn.prepare(&sql)?; + let entries: Vec = stmt + .query_map(rusqlite::params_from_iter(filter_params.iter()), |row| { + let tags_str: String = row.get(23).unwrap_or_else(|_| "[]".to_string()); + let tags: Vec = serde_json::from_str(&tags_str).unwrap_or_default(); + Ok(ReportEntry { + id: row.get(0)?, + expert: row.get(1)?, + symbol: row.get(2)?, + timeframe: row.get(3)?, + model: row.get(4)?, + from_date: row.get(5)?, + to_date: row.get(6)?, + created_at: row.get(7)?, + set_file_original: row.get(8)?, + set_snapshot_path: row.get(9)?, + report_dir: row.get(10)?, + charts_dir: row.get(11)?, + net_profit: row.get(12)?, + profit_factor: row.get(13)?, + max_dd_pct: row.get(14)?, + sharpe_ratio: row.get(15)?, + total_trades: row.get(16)?, + win_rate_pct: row.get(17)?, + recovery_factor: row.get(18)?, + deposit: row.get(19)?, + currency: row.get(20)?, + leverage: row.get(21)?, + duration_seconds: row.get(22)?, + tags, + notes: row.get(24)?, + verdict: row.get(25)?, + }) + })? + .filter_map(|r| r.ok()) + .filter(|e| { + if let Some(min_profit) = filters.min_profit { + if e.net_profit.unwrap_or(f64::MIN) < min_profit { + return false; + } + } + if let Some(max_dd) = filters.max_dd { + if e.max_dd_pct.unwrap_or(100.0) > max_dd { + return false; + } + } + true + }) + .take(limit) + .collect(); + + Ok(entries) + } + + pub fn annotate( + &self, + id: &str, + notes: Option<&str>, + tags: Option>, + verdict: Option<&str>, + ) -> Result { + let conn = self.connect()?; + let mut sets: Vec = Vec::new(); + let mut param_vals: Vec = Vec::new(); + + if let Some(n) = notes { + sets.push("notes = ?".to_string()); + param_vals.push(n.to_string()); + } + if let Some(t) = tags { + sets.push("tags = ?".to_string()); + param_vals.push(serde_json::to_string(&t).unwrap_or_default()); + } + if let Some(v) = verdict { + sets.push("verdict = ?".to_string()); + param_vals.push(v.to_string()); + } + + if sets.is_empty() { + return Ok(false); + } + + param_vals.push(id.to_string()); + let sql = format!("UPDATE reports SET {} WHERE id = ?", sets.join(", ")); + let changed = + conn.execute(&sql, rusqlite::params_from_iter(param_vals.iter()))?; + Ok(changed > 0) + } + + /// Returns (id, report_dir, charts_dir) for the oldest entries beyond keep_last. + pub fn list_purgeable( + &self, + keep_last: usize, + ) -> Result)>> { + let conn = self.connect()?; + let count: usize = + conn.query_row("SELECT COUNT(*) FROM reports", [], |r| r.get(0))?; + + if count <= keep_last { + return Ok(Vec::new()); + } + + let to_delete = count - keep_last; + let mut stmt = conn.prepare( + "SELECT id, report_dir, charts_dir FROM reports ORDER BY created_at ASC LIMIT ?", + )?; + + let rows: Vec<(String, String, Option)> = stmt + .query_map(params![to_delete as i64], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + })? + .filter_map(|r| r.ok()) + .collect(); + + Ok(rows) + } + + pub fn delete_entry(&self, id: &str) -> Result> { + let conn = self.connect()?; + let report_dir: Option = conn + .query_row( + "SELECT report_dir FROM reports WHERE id = ?", + params![id], + |row| row.get(0), + ) + .ok(); + + conn.execute("DELETE FROM reports WHERE id = ?", params![id])?; + Ok(report_dir) + } + + pub fn count(&self) -> Result { + let conn = self.connect()?; + let n: usize = conn.query_row("SELECT COUNT(*) FROM reports", [], |r| r.get(0))?; + Ok(n) + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs new file mode 100644 index 0000000..5f626a3 --- /dev/null +++ b/src/storage/mod.rs @@ -0,0 +1,2 @@ +pub mod database; +pub use database::{ReportDb, ReportEntry, ReportFilters}; diff --git a/src/tools/definitions.rs b/src/tools/definitions.rs index 997d870..dd88f0d 100644 --- a/src/tools/definitions.rs +++ b/src/tools/definitions.rs @@ -144,12 +144,12 @@ fn tool_compare_baseline() -> Value { fn tool_compile_ea() -> Value { json!({ "name": "compile_ea", - "description": "Compile an MQL5 Expert Advisor via MetaEditor", + "description": "Compile an MQL5 Expert Advisor via MetaEditor. Provide either 'expert' (EA name, searches project and MT5 Experts dirs) or 'expert_path' (full path to .mq5 file).", "inputSchema": { "type": "object", - "required": ["expert_path"], "properties": { - "expert_path": { "type": "string" } + "expert": { "type": "string", "description": "EA name without extension (e.g. 'DPS21')" }, + "expert_path": { "type": "string", "description": "Full path to the .mq5 source file" } } } }) diff --git a/src/tools/handlers/experts.rs b/src/tools/handlers/experts.rs index d15d5d0..2706925 100644 --- a/src/tools/handlers/experts.rs +++ b/src/tools/handlers/experts.rs @@ -160,22 +160,47 @@ pub async fn handle_list_scripts(config: &Config, args: &Value) -> Result } pub async fn handle_compile_ea(config: &Config, args: &Value) -> Result { - let expert_path = args.get("expert_path") - .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("expert_path is required"))?; - + use std::path::PathBuf; + + let resolved_path: String = if let Some(p) = args.get("expert_path").and_then(|v| v.as_str()) { + p.to_string() + } else if let Some(name) = args.get("expert").and_then(|v| v.as_str()) { + let mut candidates = vec![ + PathBuf::from(name).with_extension("mq5"), + ]; + if let Some(experts_dir) = &config.experts_dir { + candidates.push(PathBuf::from(experts_dir).join(name).join(format!("{}.mq5", name))); + candidates.push(PathBuf::from(experts_dir).join(format!("{}.mq5", name))); + } + match candidates.into_iter().find(|p| p.exists()) { + Some(p) => p.to_string_lossy().to_string(), + None => return Ok(serde_json::json!({ + "content": [{ "type": "text", "text": serde_json::json!({ + "success": false, + "error": format!("Cannot find {}.mq5 in MT5 Experts dir or current directory", name), + }).to_string() }], + "isError": true + })), + } + } else { + return Err(anyhow::anyhow!("Either 'expert' or 'expert_path' is required")); + }; + let compiler = MqlCompiler::new(config.clone()); + let expert_path = resolved_path.as_str(); - match compiler.compile(expert_path) { + match compiler.compile(&expert_path) { Ok(result) => { Ok(json!({ "content": [{ "type": "text", "text": json!({ "success": result.success, "binary_path": result.ex5_path.map(|p| p.to_string_lossy().to_string()), "binary_size_bytes": result.binary_size, + "files_synced": result.files_synced, "warnings": result.warnings.len(), "errors": result.errors.len(), "error_list": result.errors, + "warning_list": result.warnings, }).to_string() }], "isError": !result.success }))