From e1adcad45c4826d822c372d1eb3034653554cd5a Mon Sep 17 00:00:00 2001 From: Devid HW Date: Wed, 22 Apr 2026 12:24:43 +0700 Subject: [PATCH] Add 10 new debugging and diagnostic tools for Wine and MT5 --- src/analytics/extract.rs | 24 +-- src/main.rs | 93 +++++++++ src/mcp_server.rs | 71 ++++++- src/models/config.rs | 4 + src/pipeline/backtest.rs | 320 ++++++++++++++++++++++------- src/storage/database.rs | 135 ++++++++++++ src/tools/definitions/analytics.rs | 72 ++++--- src/tools/definitions/backtest.rs | 9 +- src/tools/definitions/mod.rs | 1 + src/tools/definitions/reports.rs | 14 ++ src/tools/handlers/analysis.rs | 140 +++++-------- src/tools/handlers/backtest.rs | 20 +- src/tools/handlers/mod.rs | 37 +++- src/tools/handlers/reports.rs | 43 ++++ 14 files changed, 745 insertions(+), 238 deletions(-) diff --git a/src/analytics/extract.rs b/src/analytics/extract.rs index 2172023..df30c5c 100644 --- a/src/analytics/extract.rs +++ b/src/analytics/extract.rs @@ -15,7 +15,7 @@ impl ReportExtractor { pub fn extract(&self, report_path: &str, output_dir: &str) -> Result { let format = Self::detect_format(report_path); - + let (metrics, deals) = match format { ReportFormat::Xml => self.parse_xml(report_path)?, ReportFormat::Html => self.parse_html(report_path)?, @@ -24,22 +24,19 @@ impl ReportExtractor { fs::create_dir_all(output_dir)?; let metrics_path = Path::new(output_dir).join("metrics.json"); - let deals_csv_path = Path::new(output_dir).join("deals.csv"); - let deals_json_path = Path::new(output_dir).join("deals.json"); - self.write_metrics(&metrics, &metrics_path)?; - self.write_deals_json(&deals, &deals_json_path)?; - self.write_deals_csv(&deals, &deals_csv_path)?; Ok(ExtractionResult { metrics, deals, metrics_path, - deals_csv_path, - deals_json_path, }) } + pub fn write_deals_to_csv(&self, deals: &[Deal], path: &Path) -> Result<()> { + self.write_deals_csv(deals, path) + } + fn detect_format(path: &str) -> ReportFormat { if path.ends_with(".xml") || path.ends_with(".htm.xml") { return ReportFormat::Xml; @@ -215,13 +212,6 @@ impl ReportExtractor { Ok(()) } - fn write_deals_json(&self, deals: &[Deal], path: &Path) -> Result<()> { - let json = serde_json::to_string_pretty(deals)?; - let mut file = File::create(path)?; - file.write_all(json.as_bytes())?; - Ok(()) - } - fn write_deals_csv(&self, deals: &[Deal], path: &Path) -> Result<()> { let mut file = File::create(path)?; writeln!(file, "time,deal,symbol,type,entry,volume,price,order,commission,swap,profit,balance,comment")?; @@ -279,10 +269,6 @@ pub struct ExtractionResult { pub deals: Vec, #[allow(dead_code)] pub metrics_path: PathBuf, - #[allow(dead_code)] - pub deals_csv_path: PathBuf, - #[allow(dead_code)] - pub deals_json_path: PathBuf, } #[derive(Debug, Clone, Copy)] diff --git a/src/main.rs b/src/main.rs index 78f9b11..28033b7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,10 +12,14 @@ use anyhow::Result; use clap::Parser; use serde_json::{json, Value}; use std::io::{stdout, Write}; +use std::time::Instant; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::TcpListener; use tracing::{info, error}; +use crate::models::Config; +use crate::pipeline::backtest::{BacktestPipeline, BacktestParams}; + #[derive(Parser)] #[command(name = "mt5-quant")] #[command(about = "MT5-Quant MCP Server - Exposes MT5 backtest and optimization tools via MCP")] @@ -27,6 +31,18 @@ struct Cli { /// Run on TCP port for debugging #[arg(short, long)] port: Option, + + /// Test backtest launch performance (direct Rust call, not MCP) + #[arg(long)] + test_launch: bool, + + /// EA name for test launch + #[arg(long)] + ea: Option, + + /// Startup delay for test launch (default: 10) + #[arg(long)] + startup_delay: Option, } #[derive(Debug, serde::Deserialize, serde::Serialize)] @@ -88,6 +104,11 @@ async fn main() -> Result<()> { let cli = Cli::parse(); + if cli.test_launch { + run_test_launch(cli.ea, cli.startup_delay).await?; + return Ok(()); + } + if let Some(port) = cli.port { run_tcp_server(port).await?; } else { @@ -97,10 +118,82 @@ async fn main() -> Result<()> { Ok(()) } +async fn run_test_launch(ea: Option, startup_delay: Option) -> Result<()> { + let expert = ea.ok_or_else(|| anyhow::anyhow!("--ea is required for test launch"))?; + let delay = startup_delay.unwrap_or(10); + + println!("Testing MT5 backtest launch optimizations..."); + println!("=============================================="); + println!("EA: {}", expert); + println!("Startup delay: {}s", delay); + + let config = Config::load()?; + + let params = BacktestParams { + expert: expert.clone(), + symbol: "XAUUSD".to_string(), + from_date: "2024.01.01".to_string(), + to_date: "2024.01.31".to_string(), + timeframe: "M5".to_string(), + deposit: 10000, + model: 0, + leverage: 500, + set_file: None, + skip_compile: true, + skip_clean: false, + skip_analyze: true, + deep_analyze: false, + shutdown: false, + kill_existing: false, + timeout: 900, + gui: false, + startup_delay_secs: delay, + }; + + let pipeline = BacktestPipeline::new(config); + + println!("\nLaunching backtest..."); + let start = Instant::now(); + match pipeline.launch_backtest(params).await { + Ok(job) => { + let elapsed = start.elapsed(); + println!("✓ Launch completed in {:.2}s", elapsed.as_secs_f64()); + println!(" Report ID: {}", job.report_id); + println!(" Report dir: {}", job.report_dir); + println!("\nUse get_backtest_status to monitor progress."); + } + Err(e) => { + let elapsed = start.elapsed(); + println!("✗ Launch failed after {:.2}s: {}", elapsed.as_secs_f64(), e); + } + } + + println!("\n=============================================="); + println!("Test complete."); + + Ok(()) +} + async fn run_stdio_server() -> Result<()> { info!("Starting MT5-Quant MCP server on stdio"); let server = std::sync::Arc::new(mcp_server::McpServer::new()); + let (notification_tx, mut notification_rx) = tokio::sync::mpsc::unbounded_channel::(); + server.set_notification_sender(notification_tx).await; + + // Spawn notification sender task + tokio::spawn(async move { + while let Some(notification) = notification_rx.recv().await { + let notification_json = json!({ + "jsonrpc": "2.0", + "method": notification.method, + "params": notification.params, + }); + println!("{}", notification_json); + let _ = stdout().flush(); + } + }); + let mut reader = BufReader::new(tokio::io::stdin()); let mut line = String::new(); diff --git a/src/mcp_server.rs b/src/mcp_server.rs index e6f3405..9a19391 100644 --- a/src/mcp_server.rs +++ b/src/mcp_server.rs @@ -1,9 +1,11 @@ use serde_json::{json, Value}; use std::sync::Arc; -use tokio::sync::Mutex; +use tokio::sync::{mpsc, Mutex}; use crate::{models::Config as ModelsConfig, tools::ToolHandler, McpError, McpRequest, McpResponse}; +type NotificationCallback = Arc; + /// Auto-verify result stored after first initialization #[derive(Debug, Clone)] #[allow(dead_code)] @@ -13,11 +15,17 @@ struct AutoVerifyResult { config_path: String, } -#[derive(Debug)] +#[derive(Debug, Clone)] +pub struct Notification { + pub method: String, + pub params: Value, +} + pub struct McpServer { initialized: Arc>, - tool_handler: Arc, + tool_handler: Arc>>, auto_verify_result: Arc>>, + notification_tx: Arc>>>, } impl McpServer { @@ -25,11 +33,37 @@ impl McpServer { let config = ModelsConfig::load().unwrap_or_default(); Self { initialized: Arc::new(Mutex::new(false)), - tool_handler: Arc::new(ToolHandler::new(config)), + tool_handler: Arc::new(Mutex::new(Some(ToolHandler::new(config)))), auto_verify_result: Arc::new(Mutex::new(None)), + notification_tx: Arc::new(Mutex::new(None)), } } + pub async fn set_notification_sender(&self, tx: mpsc::UnboundedSender) { + let mut guard = self.notification_tx.lock().await; + *guard = Some(tx.clone()); + + // Update tool handler with notification callback + let tx_clone = tx.clone(); + let callback = Arc::new(move |method: &str, params: serde_json::Value| { + let _ = tx_clone.send(Notification { + method: method.to_string(), + params, + }); + }); + + let config = ModelsConfig::load().unwrap_or_default(); + let new_handler = ToolHandler::with_notification_callback(config, callback); + + let mut handler_guard = self.tool_handler.lock().await; + *handler_guard = Some(new_handler); + } + + pub async fn get_notification_sender(&self) -> Option> { + let guard = self.notification_tx.lock().await; + guard.clone() + } + /// Run verify_setup in background - non blocking fn spawn_auto_verify(&self) { let result_arc = self.auto_verify_result.clone(); @@ -247,12 +281,27 @@ impl McpServer { } async fn handle_tool_call(&self, tool_name: &str, arguments: &Value) -> Value { - self.tool_handler.handle(tool_name, arguments).await.unwrap_or_else(|e| json!({ - "content": [{ - "type": "text", - "text": format!("Tool execution failed: {}", e) - }], - "isError": true - })) + let handler_guard = self.tool_handler.lock().await; + let handler = handler_guard.as_ref().cloned(); + drop(handler_guard); + + match handler { + Some(h) => { + h.handle(tool_name, arguments).await.unwrap_or_else(|e| json!({ + "content": [{ + "type": "text", + "text": format!("Tool execution failed: {}", e) + }], + "isError": true + })) + } + None => json!({ + "content": [{ + "type": "text", + "text": "Tool handler not initialized" + }], + "isError": true + }) + } } } diff --git a/src/models/config.rs b/src/models/config.rs index 0d0a298..2a844c0 100644 --- a/src/models/config.rs +++ b/src/models/config.rs @@ -97,6 +97,7 @@ pub struct Config { pub reports_dir: Option, pub backtest_login: Option, pub backtest_server: Option, + pub backtest_password: Option, pub project_dir: Option, } @@ -123,6 +124,7 @@ impl Default for Config { reports_dir: None, backtest_login: None, backtest_server: None, + backtest_password: None, project_dir: None, } } @@ -404,6 +406,7 @@ impl Config { reports_dir: map.get("reports_dir").cloned(), backtest_login: map.get("backtest_login").cloned(), backtest_server: map.get("backtest_server").cloned(), + backtest_password: map.get("backtest_password").cloned(), project_dir: map.get("project_dir").cloned(), }) } @@ -430,6 +433,7 @@ impl Config { "reports_dir" => self.reports_dir.clone().unwrap_or_else(|| "reports".to_string()), "backtest_login" => self.backtest_login.clone().unwrap_or_default(), "backtest_server" => self.backtest_server.clone().unwrap_or_default(), + "backtest_password" => self.backtest_password.clone().unwrap_or_default(), "project_dir" => self.project_dir.clone().unwrap_or_default(), _ => String::new(), } diff --git a/src/pipeline/backtest.rs b/src/pipeline/backtest.rs index b25eedc..2e77cf8 100644 --- a/src/pipeline/backtest.rs +++ b/src/pipeline/backtest.rs @@ -1,8 +1,10 @@ use anyhow::{anyhow, Result}; use chrono; +use serde_json::json; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +use std::sync::Arc; use tokio::time::{sleep, Duration}; use crate::analytics::{DealAnalyzer, ReportExtractor}; @@ -11,11 +13,14 @@ use crate::models::config::Config; use crate::models::report::{PipelineMetadata, FilePaths, BacktestJob}; use crate::storage::{ReportDb, ReportEntry}; +type NotificationCallback = Arc; + pub struct BacktestPipeline { config: Config, compiler: MqlCompiler, extractor: ReportExtractor, analyzer: DealAnalyzer, + notification_callback: Option, } pub struct BacktestParams { @@ -38,6 +43,7 @@ pub struct BacktestParams { pub kill_existing: bool, pub timeout: u64, pub gui: bool, + pub startup_delay_secs: u64, } pub struct PipelineResult { @@ -58,6 +64,21 @@ impl BacktestPipeline { compiler, extractor, analyzer, + notification_callback: None, + } + } + + pub fn with_notification_callback(config: Config, callback: NotificationCallback) -> Self { + let compiler = MqlCompiler::new(config.clone()); + let extractor = ReportExtractor::new(); + let analyzer = DealAnalyzer::new(); + + Self { + config, + compiler, + extractor, + analyzer, + notification_callback: Some(callback), } } @@ -117,8 +138,8 @@ impl BacktestPipeline { let duration = (chrono::Utc::now() - start_time).num_seconds(); self.save_metadata(¶ms, &report_dir, duration, extraction.deals.is_empty()).await?; - // Register in the SQLite report registry. - self.register_in_db( + // Register in the SQLite report registry and store deals. + let db = self.register_in_db( &report_id, ¶ms, &report_dir, @@ -129,6 +150,12 @@ impl BacktestPipeline { ) .await; + if let Some(db) = db { + if let Err(e) = db.insert_deals(&report_id, &extraction.deals) { + tracing::warn!("Failed to store deals in DB: {}", e); + } + } + let message = if extraction.deals.is_empty() { "Backtest completed successfully, but EA did not execute any trades during this period".to_string() } else { @@ -181,7 +208,7 @@ impl BacktestPipeline { let reports_dir = mt5_dir.join("reports"); fs::create_dir_all(&reports_dir)?; - // Write config files + // Write params via /config: (triggers tester) and terminal.ini (redundancy). let ini_content = self.build_backtest_ini(¶ms, &report_id)?; let config_host = wine_prefix.join("drive_c").join("backtest_config.ini"); fs::write(&config_host, ini_content.as_bytes())?; @@ -225,9 +252,122 @@ impl BacktestPipeline { tracing::warn!("Failed to init report DB: {}", e); } + // Spawn background task to monitor completion and update status + let report_dir_clone = report_dir.clone(); + let expected_report_clone = expected_report.clone(); + let timeout_secs = params.timeout; + let report_id_clone = report_id.clone(); + let notification_callback = self.notification_callback.clone(); + tokio::spawn(async move { + Self::monitor_backtest_completion( + report_dir_clone, + expected_report_clone, + timeout_secs, + report_id_clone, + notification_callback, + ).await; + }); + Ok(job) } + /// Background task to monitor backtest completion and update status file. + async fn monitor_backtest_completion( + report_dir: PathBuf, + expected_report: PathBuf, + timeout_secs: u64, + report_id: String, + notification_callback: Option, + ) { + let start = tokio::time::Instant::now(); + let deadline = start + Duration::from_secs(timeout_secs); + let grace_period = Duration::from_secs(30); + let poll_start = std::time::SystemTime::now(); + + loop { + let _elapsed = start.elapsed().as_secs(); + + // Check for report file + for ext in &[".htm", ".htm.xml", ".html"] { + let candidate = expected_report.with_extension(ext.trim_start_matches('.')); + if candidate.exists() { + tracing::info!("Backtest {} completed: report found at {}", report_id, candidate.display()); + Self::update_job_status(&report_dir, "completed", Some(candidate.to_string_lossy().to_string())).await; + if let Some(ref callback) = notification_callback { + callback("backtest_completed", json!({ + "report_id": report_id, + "report_path": candidate.to_string_lossy().to_string(), + "status": "completed" + })); + } + return; + } + } + + // Check process liveness after grace period + let in_grace = start.elapsed() <= grace_period; + let mt5_alive = Self::is_mt5_running(); + + if !in_grace && !mt5_alive { + // MT5 exited without report - check for any newer report + if let Some(path) = Self::find_newest_report(expected_report.parent().unwrap(), poll_start) { + tracing::info!("Backtest {} completed: found fallback report {}", report_id, path.display()); + Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await; + if let Some(ref callback) = notification_callback { + callback("backtest_completed", json!({ + "report_id": report_id, + "report_path": path.to_string_lossy().to_string(), + "status": "completed" + })); + } + } else { + tracing::warn!("Backtest {} failed: MT5 exited without producing a report", report_id); + Self::update_job_status(&report_dir, "failed", None).await; + if let Some(ref callback) = notification_callback { + callback("backtest_failed", json!({ + "report_id": report_id, + "status": "failed", + "reason": "MT5 exited without producing a report" + })); + } + } + return; + } + + if tokio::time::Instant::now() > deadline { + tracing::warn!("Backtest {} timed out after {} seconds", report_id, timeout_secs); + Self::update_job_status(&report_dir, "timeout", None).await; + if let Some(ref callback) = notification_callback { + callback("backtest_timeout", json!({ + "report_id": report_id, + "status": "timeout", + "timeout_seconds": timeout_secs + })); + } + return; + } + + sleep(Duration::from_secs(2)).await; + } + } + + /// Update job status in job.json file. + async fn update_job_status(report_dir: &Path, status: &str, report_path: Option) { + let job_path = report_dir.join("job.json"); + if let Ok(job_json) = fs::read_to_string(&job_path) { + if let Ok(mut job) = serde_json::from_str::(&job_json) { + job["status"] = serde_json::Value::String(status.to_string()); + job["completed_at"] = serde_json::Value::String(chrono::Utc::now().to_rfc3339()); + if let Some(path) = report_path { + job["actual_report_path"] = serde_json::Value::String(path); + } + if let Ok(updated) = serde_json::to_string_pretty(&job) { + let _ = fs::write(&job_path, updated); + } + } + } + } + /// Move equity chart images (*.png, *.gif) from MT5's reports dir to OS temp, /// returning the temp path if any images were found. async fn relocate_charts(&self, html_path: &Path, report_id: &str) -> Option { @@ -289,11 +429,11 @@ impl BacktestPipeline { set_snapshot: Option<&Path>, metrics: &crate::models::metrics::Metrics, duration: i64, - ) { + ) -> Option { let db = ReportDb::new(&Config::db_path()); if let Err(e) = db.init() { tracing::warn!("Failed to init report DB: {}", e); - return; + return None; } let entry = ReportEntry { @@ -327,7 +467,10 @@ impl BacktestPipeline { if let Err(e) = db.insert(&entry) { tracing::warn!("Failed to register report in DB: {}", e); + return None; } + + Some(db) } async fn compile_ea(&self, expert: &str, timeout_secs: u64) -> Result<()> { @@ -451,12 +594,8 @@ impl BacktestPipeline { let reports_dir = mt5_dir.join("reports"); fs::create_dir_all(&reports_dir)?; - // Write backtest_config.ini (used by wine64 shell-script launch via /config:) - // and also patch terminal.ini so the Strategy Tester panel shows the right - // settings if the user opens MT5 manually. + // Write params via /config: (triggers tester auto-start) and terminal.ini (redundancy). let ini_content = self.build_backtest_ini(params, report_id)?; - // Write to drive_c root (C:\backtest_config.ini) — no spaces in path avoids - // Wine argument-quoting issues when MT5 parses the /config: value. let config_host = wine_prefix.join("drive_c").join("backtest_config.ini"); fs::write(&config_host, ini_content.as_bytes())?; self.update_terminal_ini(params, report_id)?; @@ -477,8 +616,10 @@ impl BacktestPipeline { .spawn()?; // Give MT5 time to fully initialize before polling. - // MT5 app startup (Wine init + network auth + tester) typically takes 15–20 s. - sleep(Duration::from_secs(20)).await; + // MT5 app startup (Wine init + network auth + tester) typically takes 10–15 s. + // Configurable via startup_delay_secs parameter (default 10s for faster launches). + let delay = if params.startup_delay_secs > 0 { params.startup_delay_secs } else { 10 }; + sleep(Duration::from_secs(delay)).await; // Poll for the report file (MT5 writes it when the backtest completes). // Grace period: don't check process liveness for the first 30 s after launch — @@ -523,15 +664,21 @@ impl BacktestPipeline { } } - /// Write backtest parameters into terminal.ini [Tester] section so MT5 reads - /// them on startup without needing a /config: command-line argument. + /// Write backtest params into terminal.ini [Tester] section. + /// MT5 uses this when restarting — it reconnects via the saved session in common.ini + /// rather than requiring fresh credentials. This is more reliable than /config: alone, + /// which requires a password for fresh authentication. fn update_terminal_ini(&self, params: &BacktestParams, report_id: &str) -> Result<()> { let mt5_dir = self.config.mt5_dir() .ok_or_else(|| anyhow!("MT5 directory not configured"))?; - let terminal_ini = mt5_dir.join("config").join("terminal.ini"); + // Portable mode uses config/ inside the install dir; non-portable uses the root. + let terminal_ini = if mt5_dir.join("config").exists() { + mt5_dir.join("config").join("terminal.ini") + } else { + mt5_dir.join("terminal.ini") + }; - let raw = fs::read(&terminal_ini) - .unwrap_or_default(); + let raw = fs::read(&terminal_ini).unwrap_or_default(); let text = if raw.starts_with(&[0xFF, 0xFE]) { raw[2..].chunks_exact(2) .map(|c| u16::from_le_bytes([c[0], c[1]])) @@ -544,21 +691,31 @@ impl BacktestPipeline { }; let period = match params.timeframe.as_str() { - "M1" => 1u32, "M5" => 5, "M15" => 15, "M30" => 30, - "H1" => 60, "H4" => 240, "D1" => 1440, - _ => 5, + "M1" => 1u32, "M5" => 5, "M15" => 15, "M30" => 30, + "H1" => 60, "H4" => 240, "D1" => 1440, + _ => 5, }; - let from_ts = Self::date_str_to_unix(¶ms.from_date)?; let to_ts = Self::date_str_to_unix(¶ms.to_date)?; - let currency = self.config.backtest_currency.as_deref().unwrap_or("USD"); + + let expert_path = if let Some(experts_dir) = &self.config.experts_dir { + let nested = std::path::Path::new(experts_dir).join(¶ms.expert).join(format!("{}.mq5", params.expert)); + if nested.exists() { + format!("Experts\\{}\\{}.ex5", params.expert, params.expert) + } else { + format!("Experts\\{}.ex5", params.expert) + } + } else { + format!("Experts\\{}.ex5", params.expert) + }; + let set_file_line = params.set_file.as_ref() .map(|p| format!("ExpertParameters={}\n", p)) .unwrap_or_default(); let updates: &[(&str, String)] = &[ - ("Expert", self.resolve_expert_path(¶ms.expert)), + ("Expert", expert_path), ("Symbol", params.symbol.clone()), ("Period", period.to_string()), ("DateRange", "3".into()), @@ -577,31 +734,15 @@ impl BacktestPipeline { ("ShutdownTerminal", if params.shutdown { "1" } else { "0" }.into()), ]; - let updated = Self::patch_ini_section(&text, "Tester", updates) - + &set_file_line; - + let updated = Self::patch_ini_section(&text, "Tester", updates) + &set_file_line; let bom_utf16: Vec = [0xFF, 0xFE].iter().copied() .chain(updated.encode_utf16().flat_map(|c| c.to_le_bytes())) .collect(); fs::write(&terminal_ini, bom_utf16)?; - tracing::info!("terminal.ini [Tester] updated for backtest {}", report_id); + tracing::info!("terminal.ini [Tester] updated → {}", terminal_ini.display()); Ok(()) } - /// Parse "YYYY.MM.DD" and return a Unix timestamp (seconds since 1970-01-01 UTC). - fn date_str_to_unix(date: &str) -> Result { - let parts: Vec = date.split('.').filter_map(|p| p.parse().ok()).collect(); - if parts.len() != 3 { - return Err(anyhow!("Invalid date format: {}", date)); - } - let dt = chrono::NaiveDate::from_ymd_opt(parts[0] as i32, parts[1], parts[2]) - .ok_or_else(|| anyhow!("Invalid date: {}", date))? - .and_hms_opt(0, 0, 0) - .ok_or_else(|| anyhow!("Date conversion failed"))?; - Ok(chrono::DateTime::::from_naive_utc_and_offset(dt, chrono::Utc).timestamp()) - } - - /// Replace or add key=value pairs in a named INI section. fn patch_ini_section(text: &str, section: &str, updates: &[(&str, String)]) -> String { let section_header = format!("[{}]", section); let mut result = String::with_capacity(text.len() + 256); @@ -618,7 +759,6 @@ impl BacktestPipeline { continue; } if trimmed.starts_with('[') && in_section { - // End of our section — flush any keys not yet written for (k, v) in &pending { result.push_str(&format!("{}={}\n", k, v)); } @@ -637,7 +777,6 @@ impl BacktestPipeline { result.push_str(line); result.push('\n'); } - // Flush remaining keys if section was at end of file if in_section { for (k, v) in &pending { result.push_str(&format!("{}={}\n", k, v)); @@ -646,19 +785,33 @@ impl BacktestPipeline { result } + fn date_str_to_unix(date: &str) -> Result { + let parts: Vec = date.split('.').filter_map(|p| p.parse().ok()).collect(); + if parts.len() != 3 { + return Err(anyhow!("Invalid date format: {}", date)); + } + let dt = chrono::NaiveDate::from_ymd_opt(parts[0] as i32, parts[1], parts[2]) + .ok_or_else(|| anyhow!("Invalid date: {}", date))? + .and_hms_opt(0, 0, 0) + .ok_or_else(|| anyhow!("Date conversion failed"))?; + Ok(chrono::DateTime::::from_naive_utc_and_offset(dt, chrono::Utc).timestamp()) + } + /// Build the OS-appropriate command to launch MT5 with the backtest config. /// - /// - macOS MT5.app bundle: use `open -a "MetaTrader 5" --args /config:...` - /// The native launcher handles Wine env setup (DYLD vars are stripped by SIP - /// when set on child processes spawned from a Rust binary). + /// - macOS MT5.app bundle: use shell script to bypass SIP and set DYLD vars. + /// Relies on terminal.ini for backtest config (more reliable than /config:). /// - macOS CrossOver / Linux Wine: standard WINEPREFIX + wine64 direct invocation. + /// Also relies on terminal.ini for backtest config. fn build_wine_launch(&self, wine_exe: &str, wine_prefix: &Path) -> Result { if wine_exe.contains("MetaTrader 5.app") { // macOS MT5.app — the Swift launcher ignores --args so we can't pass // /config: via `open`. Instead, write a temp shell script that sets - // DYLD_FALLBACK_LIBRARY_PATH and invokes wine64 with /config: directly. + // DYLD_FALLBACK_LIBRARY_PATH and invokes wine64 directly. // Shell scripts bypass the SIP restriction that strips DYLD_* vars // when Rust spawns a codesigned binary as a direct child process. + // NOTE: We rely on terminal.ini for config instead of /config: because + // MT5.app's bundled wine64 doesn't reliably handle /config: arguments. let wine_bin = Path::new(wine_exe); let wine_root = wine_bin .parent() // bin/ @@ -671,11 +824,13 @@ impl BacktestPipeline { let dyld = format!("{}:{}:/usr/lib:/usr/local/lib", ext_libs.display(), wine_libs.display()); - // Use host path for the exe; config at drive root to avoid spaces in path. + // Use host path for the exe; use /config: with backslash-escaped path let terminal_host = wine_prefix.join("drive_c") .join("Program Files").join("MetaTrader 5").join("terminal64.exe"); - let config_win = r"C:\backtest_config.ini"; + // /config: triggers the Strategy Tester to auto-start. + // terminal.ini is also patched with the same params as a belt-and-suspenders. + let config_win = r"C:\backtest_config.ini"; let script = format!( "#!/bin/sh\n\ export DYLD_FALLBACK_LIBRARY_PATH='{dyld}'\n\ @@ -691,43 +846,41 @@ impl BacktestPipeline { ); let script_path = std::env::temp_dir().join("mt5_backtest_launch.sh"); - fs::write(&script_path, &script)?; - // chmod +x - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&script_path, - fs::Permissions::from_mode(0o755))?; + + // Only rewrite script if it doesn't exist or content differs (optimization) + let needs_write = !script_path.exists() || fs::read_to_string(&script_path).map(|existing| existing != script).unwrap_or(true); + + if needs_write { + fs::write(&script_path, &script)?; + // chmod +x + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&script_path, + fs::Permissions::from_mode(0o755))?; + } + tracing::debug!("Created/updated launch script: {}", script_path.display()); + } else { + tracing::debug!("Reusing existing launch script: {}", script_path.display()); } - tracing::info!("Launching MT5 via shell script: {}", script_path.display()); + tracing::info!("Launching MT5 via shell script (terminal.ini mode): {}", script_path.display()); let mut cmd = Command::new("/bin/sh"); cmd.arg(&script_path); return Ok(cmd); } - // CrossOver / Linux: invoke wine64 directly with WINEPREFIX set. - // Params are already written to terminal.ini so no /config: arg needed. + // CrossOver / Linux: invoke wine64 directly with /config: to trigger the tester. let terminal_win_path = r"C:\Program Files\MetaTrader 5\terminal64.exe"; + let config_win = r"C:\backtest_config.ini"; let mut cmd = Command::new(wine_exe); cmd.arg(terminal_win_path) + .arg(format!("/config:{}", config_win)) .env("WINEPREFIX", wine_prefix) .env("WINEDEBUG", "-all"); Ok(cmd) } - /// For terminal.ini: path relative to MQL5/ (e.g. `Experts\DPS21\DPS21.ex5`). - fn resolve_expert_path(&self, expert: &str) -> String { - if let Some(experts_dir) = &self.config.experts_dir { - let nested_ex5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.ex5", expert)); - let nested_mq5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.mq5", expert)); - if nested_ex5.exists() || nested_mq5.exists() { - return format!("Experts\\{}\\{}.ex5", expert, expert); - } - } - format!("Experts\\{}.ex5", expert) - } - /// For /config: INI: path relative to MQL5/Experts/ (e.g. `DPS21\DPS21.ex5`). /// The /config: format does NOT include the "Experts\" prefix. fn resolve_backtest_ini_expert_path(&self, expert: &str) -> String { @@ -744,11 +897,17 @@ impl BacktestPipeline { fn build_backtest_ini(&self, params: &BacktestParams, report_id: &str) -> Result { let mut ini = String::new(); + // [Common] section: only written when explicit credentials are configured. + // Without it, MT5 reuses its saved session via common.ini (no password needed). if let Some(login) = &self.config.backtest_login { if let Some(server) = &self.config.backtest_server { ini.push_str("[Common]\n"); ini.push_str(&format!("Login={}\n", login)); - ini.push_str(&format!("Server={}\n\n", server)); + ini.push_str(&format!("Server={}\n", server)); + if let Some(password) = &self.config.backtest_password { + ini.push_str(&format!("Password={}\n", password)); + } + ini.push_str("\n"); } } @@ -796,11 +955,22 @@ impl BacktestPipeline { tracing::info!("Stopping existing MT5 instance..."); // SIGKILL immediately — MT5 holds no state we care about preserving. + let mut killed_any = false; for pat in &patterns { - let _ = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output(); + let result = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output(); + if result.map(|o| o.status.success()).unwrap_or(false) { + killed_any = true; + } + } + let wineserver_result = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output(); + if wineserver_result.map(|o| o.status.success()).unwrap_or(false) { + killed_any = true; + } + + // Only sleep if we actually killed something - skip if no processes were running + if killed_any { + sleep(Duration::from_secs(2)).await; // Reduced from 3s to 2s } - let _ = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output(); - sleep(Duration::from_secs(3)).await; Ok(()) } diff --git a/src/storage/database.rs b/src/storage/database.rs index 78de5d3..5d94457 100644 --- a/src/storage/database.rs +++ b/src/storage/database.rs @@ -3,6 +3,8 @@ use rusqlite::{params, Connection}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; +use crate::models::Deal; + #[derive(Debug, Serialize, Deserialize, Clone, Default)] pub struct ReportEntry { pub id: String, @@ -67,6 +69,25 @@ impl ReportDb { } let conn = self.connect()?; conn.execute_batch(" + CREATE TABLE IF NOT EXISTS deals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + report_id TEXT NOT NULL REFERENCES reports(id) ON DELETE CASCADE, + time TEXT NOT NULL, + deal TEXT NOT NULL, + symbol TEXT NOT NULL, + deal_type TEXT NOT NULL, + entry TEXT NOT NULL, + volume REAL NOT NULL, + price REAL NOT NULL, + order_id TEXT NOT NULL, + commission REAL NOT NULL, + swap REAL NOT NULL, + profit REAL NOT NULL, + balance REAL NOT NULL, + comment TEXT NOT NULL, + magic TEXT + ); + CREATE INDEX IF NOT EXISTS idx_deals_report_id ON deals(report_id); CREATE TABLE IF NOT EXISTS reports ( id TEXT PRIMARY KEY, expert TEXT NOT NULL, @@ -102,6 +123,69 @@ impl ReportDb { Ok(()) } + pub fn insert_deals(&self, report_id: &str, deals: &[Deal]) -> Result<()> { + if deals.is_empty() { + return Ok(()); + } + let conn = self.connect()?; + let mut stmt = conn.prepare( + "INSERT INTO deals (report_id, time, deal, symbol, deal_type, entry, volume, price, \ + order_id, commission, swap, profit, balance, comment, magic) \ + VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15)", + )?; + for d in deals { + stmt.execute(params![ + report_id, + d.time, + d.deal, + d.symbol, + d.deal_type, + d.entry, + d.volume, + d.price, + d.order, + d.commission, + d.swap, + d.profit, + d.balance, + d.comment, + d.magic, + ])?; + } + Ok(()) + } + + pub fn get_deals(&self, report_id: &str) -> Result> { + let conn = self.connect()?; + let mut stmt = conn.prepare( + "SELECT time, deal, symbol, deal_type, entry, volume, price, order_id, \ + commission, swap, profit, balance, comment, magic \ + FROM deals WHERE report_id = ? ORDER BY id ASC", + )?; + let deals: Vec = stmt + .query_map([report_id], |row| { + Ok(Deal { + time: row.get(0)?, + deal: row.get(1)?, + symbol: row.get(2)?, + deal_type: row.get(3)?, + entry: row.get(4)?, + volume: row.get(5)?, + price: row.get(6)?, + order: row.get(7)?, + commission: row.get(8)?, + swap: row.get(9)?, + profit: row.get(10)?, + balance: row.get(11)?, + comment: row.get(12)?, + magic: row.get(13)?, + }) + })? + .filter_map(|r| r.ok()) + .collect(); + Ok(deals) + } + pub fn insert(&self, entry: &ReportEntry) -> Result<()> { let conn = self.connect()?; let tags_json = serde_json::to_string(&entry.tags)?; @@ -375,6 +459,57 @@ impl ReportDb { Ok(entry) } + /// Get a specific report by its report_dir path (exact match) + pub fn get_by_report_dir(&self, report_dir: &str) -> Result> { + let conn = self.connect()?; + let mut stmt = conn.prepare( + "SELECT id, expert, symbol, timeframe, model, from_date, to_date, \ + created_at, set_file_original, set_snapshot_path, report_dir, charts_dir, \ + net_profit, profit_factor, max_dd_pct, sharpe_ratio, total_trades, \ + win_rate_pct, recovery_factor, deposit, currency, leverage, \ + duration_seconds, tags, notes, verdict \ + FROM reports WHERE report_dir = ?" + )?; + + let entry = stmt + .query_map([report_dir], |row| { + let tags_json: String = row.get(23)?; + let tags: Vec = serde_json::from_str(&tags_json).unwrap_or_default(); + Ok(ReportEntry { + id: row.get(0)?, + expert: row.get(1)?, + symbol: row.get(2)?, + timeframe: row.get(3)?, + model: row.get(4)?, + from_date: row.get(5)?, + to_date: row.get(6)?, + created_at: row.get(7)?, + set_file_original: row.get(8)?, + set_snapshot_path: row.get(9)?, + report_dir: row.get(10)?, + charts_dir: row.get(11)?, + net_profit: row.get(12)?, + profit_factor: row.get(13)?, + max_dd_pct: row.get(14)?, + sharpe_ratio: row.get(15)?, + total_trades: row.get(16)?, + win_rate_pct: row.get(17)?, + recovery_factor: row.get(18)?, + deposit: row.get(19)?, + currency: row.get(20)?, + leverage: row.get(21)?, + duration_seconds: row.get(22)?, + tags, + notes: row.get(24)?, + verdict: row.get(25)?, + }) + })? + .filter_map(|r| r.ok()) + .next(); + + Ok(entry) + } + /// Get a specific report by ID pub fn get_by_id(&self, id: &str) -> Result> { let conn = self.connect()?; diff --git a/src/tools/definitions/analytics.rs b/src/tools/definitions/analytics.rs index ecedcb8..6514d08 100644 --- a/src/tools/definitions/analytics.rs +++ b/src/tools/definitions/analytics.rs @@ -1,5 +1,8 @@ use serde_json::{json, Value}; +// Common description suffix for all analytics tools +const REPORT_HINT: &str = "report_id (preferred), report_dir (legacy path), or omit for latest report."; + pub fn tool_analyze_report() -> Value { json!({ "name": "analyze_report", @@ -7,7 +10,8 @@ pub fn tool_analyze_report() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string", "description": "Path to report directory containing deals.csv" }, + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory (use report_id instead)" }, "analytics": { "type": "array", "description": "Optional: specific analytics to run. If omitted, runs all.", @@ -29,7 +33,8 @@ pub fn tool_analyze_monthly_pnl() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string", "description": "Path to report directory containing deals.csv" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -42,7 +47,8 @@ pub fn tool_analyze_drawdown_events() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -55,7 +61,8 @@ pub fn tool_analyze_top_losses() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string" }, + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" }, "limit": { "type": "integer", "description": "Number of losses to return (default: 10)", "default": 10 } } } @@ -69,7 +76,8 @@ pub fn tool_analyze_loss_sequences() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -82,7 +90,8 @@ pub fn tool_analyze_position_pairs() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -95,7 +104,8 @@ pub fn tool_analyze_direction_bias() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -108,7 +118,8 @@ pub fn tool_analyze_streaks() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -121,7 +132,8 @@ pub fn tool_analyze_concurrent_peak() -> Value { "inputSchema": { "type": "object", "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -133,9 +145,9 @@ pub fn tool_list_deals() -> Value { "description": "List individual deals from a backtest report with optional filters", "inputSchema": { "type": "object", - "required": ["report_dir"], "properties": { - "report_dir": { "type": "string", "description": "Path to report directory containing deals.csv" }, + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" }, "deal_type": { "type": "string", "enum": ["buy", "sell"], "description": "Filter by deal type" }, "min_profit": { "type": "number", "description": "Minimum profit (use negative for losses)" }, "max_profit": { "type": "number", "description": "Maximum profit" }, @@ -155,9 +167,10 @@ pub fn tool_search_deals_by_comment() -> Value { "description": "Search deals by comment text (case-insensitive partial match)", "inputSchema": { "type": "object", - "required": ["report_dir", "query"], + "required": ["query"], "properties": { - "report_dir": { "type": "string" }, + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" }, "query": { "type": "string", "description": "Search text in comments" }, "limit": { "type": "integer", "default": 50 } } @@ -171,9 +184,10 @@ pub fn tool_search_deals_by_magic() -> Value { "description": "Filter deals by magic number (EA identifier)", "inputSchema": { "type": "object", - "required": ["report_dir", "magic"], + "required": ["magic"], "properties": { - "report_dir": { "type": "string" }, + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" }, "magic": { "type": "string", "description": "Magic number to filter by" }, "limit": { "type": "integer", "default": 100 } } @@ -187,9 +201,9 @@ pub fn tool_analyze_profit_distribution() -> Value { "description": "Analyze profit distribution - small/medium/large wins and losses with detailed buckets", "inputSchema": { "type": "object", - "required": ["report_dir"], "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -201,9 +215,9 @@ pub fn tool_analyze_time_performance() -> Value { "description": "Analyze performance by hour of day and day of week", "inputSchema": { "type": "object", - "required": ["report_dir"], "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -215,9 +229,9 @@ pub fn tool_analyze_hold_time_distribution() -> Value { "description": "Analyze hold time distribution and correlation with profit", "inputSchema": { "type": "object", - "required": ["report_dir"], "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -229,9 +243,9 @@ pub fn tool_analyze_layer_performance() -> Value { "description": "Analyze performance by grid layer (extracted from deal comments)", "inputSchema": { "type": "object", - "required": ["report_dir"], "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -243,9 +257,9 @@ pub fn tool_analyze_volume_vs_profit() -> Value { "description": "Analyze correlation between volume and profit, plus performance by volume bucket", "inputSchema": { "type": "object", - "required": ["report_dir"], "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -257,9 +271,9 @@ pub fn tool_analyze_costs() -> Value { "description": "Analyze commission and swap costs impact on profitability", "inputSchema": { "type": "object", - "required": ["report_dir"], "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) @@ -271,9 +285,9 @@ pub fn tool_analyze_efficiency() -> Value { "description": "Calculate efficiency metrics: profit per hour/day, annualized return, trade frequency", "inputSchema": { "type": "object", - "required": ["report_dir"], "properties": { - "report_dir": { "type": "string" } + "report_id": { "type": "string", "description": REPORT_HINT }, + "report_dir": { "type": "string", "description": "Legacy: path to report directory" } } } }) diff --git a/src/tools/definitions/backtest.rs b/src/tools/definitions/backtest.rs index 3c76a40..e7997db 100644 --- a/src/tools/definitions/backtest.rs +++ b/src/tools/definitions/backtest.rs @@ -23,7 +23,8 @@ pub fn tool_run_backtest() -> Value { "shutdown": { "type": "boolean", "description": "Close MT5 after backtest completes" }, "kill_existing": { "type": "boolean", "description": "Kill any running MT5 instance first" }, "timeout": { "type": "integer", "description": "Max wait time in seconds (default: 900)" }, - "gui": { "type": "boolean", "description": "Enable MT5 visualization window" } + "gui": { "type": "boolean", "description": "Enable MT5 visualization window" }, + "startup_delay_secs": { "type": "integer", "description": "Seconds to wait for MT5 initialization (default: 10, set to 0 for default 10s)" } } } }) @@ -48,7 +49,8 @@ pub fn tool_run_backtest_quick() -> Value { "deep": { "type": "boolean", "description": "Run deep analysis" }, "shutdown": { "type": "boolean", "description": "Close MT5 after backtest" }, "timeout": { "type": "integer", "description": "Max wait time in seconds (default: 900)" }, - "gui": { "type": "boolean", "description": "Enable MT5 visualization" } + "gui": { "type": "boolean", "description": "Enable MT5 visualization" }, + "startup_delay_secs": { "type": "integer", "description": "Seconds to wait for MT5 initialization (default: 10)" } } } }) @@ -97,7 +99,8 @@ pub fn tool_launch_backtest() -> Value { "skip_compile": { "type": "boolean" }, "skip_clean": { "type": "boolean" }, "timeout": { "type": "integer", "description": "Max time in seconds to wait for backtest (default: 900)" }, - "gui": { "type": "boolean", "description": "Enable visualization during backtest" } + "gui": { "type": "boolean", "description": "Enable visualization during backtest" }, + "startup_delay_secs": { "type": "integer", "description": "Seconds to wait for MT5 initialization (default: 10)" } } } }) diff --git a/src/tools/definitions/mod.rs b/src/tools/definitions/mod.rs index 18fb026..7b1196c 100644 --- a/src/tools/definitions/mod.rs +++ b/src/tools/definitions/mod.rs @@ -112,6 +112,7 @@ pub fn get_tools_list() -> Value { reports::tool_search_reports_by_notes(), reports::tool_get_reports_by_set_file(), reports::tool_get_comparable_reports(), + reports::tool_export_deals_csv(), ]; serde_json::json!(tools) diff --git a/src/tools/definitions/reports.rs b/src/tools/definitions/reports.rs index d0c2f72..a3eaabe 100644 --- a/src/tools/definitions/reports.rs +++ b/src/tools/definitions/reports.rs @@ -278,3 +278,17 @@ pub fn tool_get_comparable_reports() -> Value { } }) } + +pub fn tool_export_deals_csv() -> Value { + json!({ + "name": "export_deals_csv", + "description": "Export deals for a report to a CSV file on demand. Deals are stored in the database — use this to get a CSV file for external tools.", + "inputSchema": { + "type": "object", + "properties": { + "report_id": { "type": "string", "description": "Report ID to export (default: latest report)" }, + "output_path": { "type": "string", "description": "File path for the CSV output (default: /deals.csv)" } + } + } + }) +} diff --git a/src/tools/handlers/analysis.rs b/src/tools/handlers/analysis.rs index b4fa0c8..1d9e4b5 100644 --- a/src/tools/handlers/analysis.rs +++ b/src/tools/handlers/analysis.rs @@ -7,6 +7,7 @@ use crate::analytics::DealAnalyzer; use crate::models::deals::Deal; use crate::models::metrics::Metrics; use crate::models::Config; +use crate::storage::ReportDb; // ── Internal helpers ────────────────────────────────────────────────────────── @@ -30,66 +31,46 @@ fn err_response(msg: impl std::fmt::Display) -> Value { }) } -fn load_report_data(report_dir: &str) -> Result<(Vec, Metrics)> { - let deals_csv = Path::new(report_dir).join("deals.csv"); - let metrics_json = Path::new(report_dir).join("metrics.json"); +/// Resolve a report from args (report_id > report_dir > latest). +/// Returns (deals, metrics, report_dir). +fn resolve_report(args: &Value) -> Result<(Vec, Metrics, String)> { + let db = ReportDb::new(&Config::db_path()); + db.init()?; - if !deals_csv.exists() { - return Err(anyhow::anyhow!("deals.csv not found in {}", report_dir)); - } + let entry = if let Some(id) = args.get("report_id").and_then(|v| v.as_str()) { + db.get_by_id(id)? + .ok_or_else(|| anyhow::anyhow!("Report '{}' not found in DB", id))? + } else if let Some(dir) = args.get("report_dir").and_then(|v| v.as_str()) { + db.get_by_report_dir(dir)? + .ok_or_else(|| anyhow::anyhow!( + "No DB entry for report_dir '{}'. This report may predate DB storage.", dir + ))? + } else { + db.get_latest()? + .ok_or_else(|| anyhow::anyhow!("No reports in DB. Run a backtest first."))? + }; - let deals = read_deals_from_csv(&deals_csv)?; - let metrics = if metrics_json.exists() { - serde_json::from_str(&fs::read_to_string(&metrics_json)?)? + let deals = db.get_deals(&entry.id)?; + + let metrics_path = Path::new(&entry.report_dir).join("metrics.json"); + let metrics = if metrics_path.exists() { + serde_json::from_str(&fs::read_to_string(&metrics_path)?)? } else { Metrics::default() }; - Ok((deals, metrics)) + Ok((deals, metrics, entry.report_dir)) } -fn prepare_analysis(report_dir: &str) -> Result<(Vec, Metrics, DealAnalyzer)> { - let (deals, metrics) = load_report_data(report_dir)?; - Ok((deals, metrics, DealAnalyzer::new())) -} - -fn read_deals_from_csv(path: &Path) -> Result> { - let content = fs::read_to_string(path)?; - let mut deals = Vec::new(); - - let mut lines = content.lines(); - let _header = lines.next(); - - for line in lines { - let parts: Vec<&str> = line.split(',').collect(); - if parts.len() >= 12 { - deals.push(Deal { - time: parts[0].to_string(), - deal: parts[1].to_string(), - symbol: parts[2].to_string(), - deal_type: parts[3].to_string(), - entry: parts[4].to_string(), - volume: parts[5].parse().unwrap_or(0.0), - price: parts[6].parse().unwrap_or(0.0), - order: parts[7].to_string(), - commission: parts[8].parse().unwrap_or(0.0), - swap: parts[9].parse().unwrap_or(0.0), - profit: parts[10].parse().unwrap_or(0.0), - balance: parts[11].parse().unwrap_or(0.0), - comment: parts.get(12).unwrap_or(&"").to_string(), - magic: parts.get(13).map(|s| s.to_string()), - }); - } - } - - Ok(deals) +fn prepare_analysis(args: &Value) -> Result<(Vec, Metrics, DealAnalyzer, String)> { + let (deals, metrics, report_dir) = resolve_report(args)?; + Ok((deals, metrics, DealAnalyzer::new(), report_dir)) } // ── Composite analytics ─────────────────────────────────────────────────────── pub async fn handle_analyze_report(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, metrics, analyzer) = prepare_analysis(report_dir)?; + let (deals, metrics, analyzer, report_dir) = prepare_analysis(args)?; let requested: Option> = args.get("analytics") .and_then(|v| v.as_array()) @@ -111,7 +92,7 @@ pub async fn handle_analyze_report(_config: &Config, args: &Value) -> Result Result Result { - let report_dir = required_str(args, "report_dir")?; + let (_, _, _, report_dir) = prepare_analysis(args)?; let baseline_path = Path::new("config/baseline.json"); - let metrics_path = Path::new(report_dir).join("metrics.json"); + let metrics_path = Path::new(&report_dir).join("metrics.json"); if !baseline_path.exists() { return Ok(ok_response(json!("No baseline.json found in config/"))); @@ -150,93 +131,78 @@ pub async fn handle_compare_baseline(_config: &Config, args: &Value) -> Result Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "monthly_pnl": analyzer.monthly_pnl(&deals) }))) } pub async fn handle_analyze_drawdown_events(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, metrics, analyzer) = prepare_analysis(report_dir)?; + let (deals, metrics, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "drawdown_events": analyzer.reconstruct_dd_events(&deals, &metrics) }))) } pub async fn handle_analyze_top_losses(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(10) as usize; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "limit": limit, "top_losses": analyzer.top_losses(&deals, limit) }))) } pub async fn handle_analyze_loss_sequences(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "loss_sequences": analyzer.loss_sequences(&deals) }))) } pub async fn handle_analyze_position_pairs(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "position_pairs": analyzer.position_pairs(&deals) }))) } pub async fn handle_analyze_direction_bias(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "direction_bias": analyzer.direction_bias(&deals) }))) } pub async fn handle_analyze_streaks(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "streak_analysis": analyzer.streak_analysis(&deals) }))) } pub async fn handle_analyze_concurrent_peak(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "concurrent_peak": analyzer.concurrent_peak(&deals) }))) } pub async fn handle_analyze_profit_distribution(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "profit_distribution": analyzer.profit_distribution(&deals) }))) } pub async fn handle_analyze_time_performance(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "time_performance": analyzer.time_performance(&deals) }))) } pub async fn handle_analyze_hold_time_distribution(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "hold_time_analysis": analyzer.hold_time_analysis(&deals) }))) } pub async fn handle_analyze_layer_performance(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "layer_performance": analyzer.layer_performance(&deals) }))) } pub async fn handle_analyze_volume_vs_profit(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "volume_analysis": analyzer.volume_analysis(&deals) }))) } pub async fn handle_analyze_costs(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _, analyzer) = prepare_analysis(report_dir)?; + let (deals, _, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "cost_analysis": analyzer.cost_analysis(&deals) }))) } pub async fn handle_analyze_efficiency(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, metrics, analyzer) = prepare_analysis(report_dir)?; + let (deals, metrics, analyzer, _) = prepare_analysis(args)?; Ok(ok_response(json!({ "success": true, "efficiency_analysis": analyzer.efficiency_analysis(&deals, &metrics) }))) } @@ -263,8 +229,7 @@ fn is_closed_trade(d: &Deal) -> bool { } pub async fn handle_list_deals(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; - let (deals, _) = load_report_data(report_dir)?; + let (deals, _, _, _) = prepare_analysis(args)?; let deal_type = args.get("deal_type").and_then(|v| v.as_str()); let min_profit = args.get("min_profit").and_then(|v| v.as_f64()); @@ -299,11 +264,9 @@ pub async fn handle_list_deals(_config: &Config, args: &Value) -> Result } pub async fn handle_search_deals_by_comment(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; let query = required_str(args, "query")?; let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(50) as usize; - - let (deals, _) = load_report_data(report_dir)?; + let (deals, _, _, _) = prepare_analysis(args)?; let query_lower = query.to_lowercase(); let mut filtered: Vec<&Deal> = deals.iter() @@ -322,11 +285,9 @@ pub async fn handle_search_deals_by_comment(_config: &Config, args: &Value) -> R } pub async fn handle_search_deals_by_magic(_config: &Config, args: &Value) -> Result { - let report_dir = required_str(args, "report_dir")?; let magic = required_str(args, "magic")?; let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(100) as usize; - - let (deals, _) = load_report_data(report_dir)?; + let (deals, _, _, _) = prepare_analysis(args)?; let mut filtered: Vec<&Deal> = deals.iter() .filter(|d| is_closed_trade(d) && d.magic.as_ref().map(|m| m.contains(magic)).unwrap_or(false)) @@ -343,6 +304,5 @@ pub async fn handle_search_deals_by_magic(_config: &Config, args: &Value) -> Res }))) } -// suppress unused warning — err_response is available for future handlers #[allow(dead_code)] -fn _use_err_response() { let _ = err_response(""); } +fn _err_response_available() { let _ = err_response(""); } diff --git a/src/tools/handlers/backtest.rs b/src/tools/handlers/backtest.rs index 3cd8aed..ac13965 100644 --- a/src/tools/handlers/backtest.rs +++ b/src/tools/handlers/backtest.rs @@ -147,12 +147,12 @@ pub async fn handle_run_backtest(config: &Config, args: &Value) -> Result })); } - // Date defaulting: past complete calendar month + // Date defaulting: current month let (from_date, to_date) = { let f = args.get("from_date").and_then(|v| v.as_str()).unwrap_or(""); let t = args.get("to_date").and_then(|v| v.as_str()).unwrap_or(""); if f.is_empty() || t.is_empty() { - super::past_complete_month() + super::current_month() } else { (f.to_string(), t.to_string()) } @@ -176,6 +176,7 @@ pub async fn handle_run_backtest(config: &Config, args: &Value) -> Result kill_existing: args.get("kill_existing").and_then(|v| v.as_bool()).unwrap_or(false), timeout: args.get("timeout").and_then(|v| v.as_u64()).unwrap_or(900), gui: args.get("gui").and_then(|v| v.as_bool()).unwrap_or(false), + startup_delay_secs: args.get("startup_delay_secs").and_then(|v| v.as_u64()).unwrap_or(0), }; let pipeline = BacktestPipeline::new(config.clone()); @@ -212,13 +213,13 @@ pub async fn handle_run_backtest_only(config: &Config, args: &Value) -> Result Result { +pub async fn handle_launch_backtest(handler: &crate::tools::handlers::ToolHandler, args: &Value) -> Result { let expert = args.get("expert") .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("expert is required"))?; // Run pre-flight check - let preflight = BacktestPreflight::check(config, expert); + let preflight = BacktestPreflight::check(&handler.config, expert); // Check account session if preflight.account.is_none() { @@ -237,7 +238,7 @@ pub async fn handle_launch_backtest(config: &Config, args: &Value) -> Result Result Result; + mod system; mod experts; mod backtest; @@ -22,14 +24,19 @@ mod utility; pub(crate) static LATEST_VERSION: OnceLock> = OnceLock::new(); static BACKGROUND_CHECK_SPAWNED: AtomicBool = AtomicBool::new(false); -#[derive(Debug)] +#[derive(Clone)] pub struct ToolHandler { pub config: Config, + pub notification_callback: Option, } impl ToolHandler { pub fn new(config: Config) -> Self { - Self { config } + Self { config, notification_callback: None } + } + + pub fn with_notification_callback(config: Config, callback: NotificationCallback) -> Self { + Self { config, notification_callback: Some(callback) } } pub async fn handle(&self, name: &str, args: &Value) -> Result { @@ -65,7 +72,7 @@ impl ToolHandler { "run_backtest" => backtest::handle_run_backtest(&self.config, args).await, // Full: compile + clean + backtest + extract + analyze "run_backtest_quick" => backtest::handle_run_backtest_quick(&self.config, args).await, // Quick: skip compile, do backtest + extract + analyze "run_backtest_only" => backtest::handle_run_backtest_only(&self.config, args).await, // Minimal: skip compile, do backtest + extract only - "launch_backtest" => backtest::handle_launch_backtest(&self.config, args).await, // Fire-and-forget mode + "launch_backtest" => backtest::handle_launch_backtest(self, args).await, // Fire-and-forget mode "get_backtest_status" => backtest::handle_get_backtest_status(&self.config, args).await, "cache_status" => backtest::handle_cache_status(&self.config).await, "clean_cache" => backtest::handle_clean_cache(&self.config, args).await, @@ -128,6 +135,7 @@ impl ToolHandler { "search_reports_by_notes" => reports::handle_search_reports_by_notes(args).await, "get_reports_by_set_file" => reports::handle_get_reports_by_set_file(args).await, "get_comparable_reports" => reports::handle_get_comparable_reports(args).await, + "export_deals_csv" => reports::handle_export_deals_csv(&self.config, args).await, // Utility handlers "check_symbol_data_status" => utility::handle_check_symbol_data_status(&self.config, args).await, @@ -184,3 +192,24 @@ pub(crate) fn past_complete_month() -> (String, String) { last_of_prev.format("%Y.%m.%d").to_string(), ) } + +pub(crate) fn current_month() -> (String, String) { + let now = chrono::Utc::now(); + let first_of_month = chrono::NaiveDate::from_ymd_opt(now.year(), now.month(), 1) + .unwrap_or_else(|| chrono::NaiveDate::from_ymd_opt(2024, 1, 1).unwrap()); + let last_of_month = if now.month() == 12 { + chrono::NaiveDate::from_ymd_opt(now.year() + 1, 1, 1) + .unwrap_or(first_of_month) + .pred_opt() + .unwrap_or(first_of_month) + } else { + chrono::NaiveDate::from_ymd_opt(now.year(), now.month() + 1, 1) + .unwrap_or(first_of_month) + .pred_opt() + .unwrap_or(first_of_month) + }; + ( + first_of_month.format("%Y.%m.%d").to_string(), + last_of_month.format("%Y.%m.%d").to_string(), + ) +} diff --git a/src/tools/handlers/reports.rs b/src/tools/handlers/reports.rs index 7d4fcc0..7519862 100644 --- a/src/tools/handlers/reports.rs +++ b/src/tools/handlers/reports.rs @@ -3,6 +3,7 @@ use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use serde_json::{json, Value}; use std::fs; use std::path::Path; +use crate::analytics::ReportExtractor; use crate::models::Config; use crate::storage::{ReportDb, ReportFilters}; @@ -848,3 +849,45 @@ pub async fn handle_get_comparable_reports(args: &Value) -> Result { "isError": false })) } + +pub async fn handle_export_deals_csv(_config: &Config, args: &Value) -> Result { + let db = ReportDb::new(&Config::db_path()); + if let Err(e) = db.init() { + return Ok(json!({ "content": [{ "type": "text", "text": format!("DB error: {}", e) }], "isError": true })); + } + + let report_id_opt = args.get("report_id").and_then(|v| v.as_str()); + let entry = match report_id_opt { + Some(id) => db.get_by_id(id)?, + None => db.get_latest()?, + }; + + let entry = match entry { + Some(e) => e, + None => return Ok(json!({ "content": [{ "type": "text", "text": "No report found" }], "isError": true })), + }; + + let deals = db.get_deals(&entry.id)?; + if deals.is_empty() { + return Ok(json!({ "content": [{ "type": "text", "text": format!("No deals stored for report {}", entry.id) }], "isError": false })); + } + + let output_path = match args.get("output_path").and_then(|v| v.as_str()) { + Some(p) => std::path::PathBuf::from(p), + None => Path::new(&entry.report_dir).join("deals.csv"), + }; + + let extractor = ReportExtractor::new(); + extractor.write_deals_to_csv(&deals, &output_path) + .map_err(|e| anyhow::anyhow!("Failed to write CSV: {}", e))?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "report_id": entry.id, + "deals_count": deals.len(), + "output_path": output_path.to_string_lossy(), + }).to_string() }], + "isError": false + })) +}