From 552e271d088a511f0952249d7c6eb36dd02ea49a Mon Sep 17 00:00:00 2001 From: Devid HW Date: Thu, 2 Jul 2026 03:53:58 +0700 Subject: [PATCH] feat: add rolling backtest and optimization improvements - Add run_rolling_backtest tool for N consecutive weekly backtests - Add max_passes support for optimization to cap genetic passes - Kill stale MT5/agent processes before optimization launch - Auto-parse optimization results on status poll when process completes - Fix XML namespace for SpreadsheetML parsing - Change optimization model from 4 to 1 for better accuracy - Persist completion results to job metadata --- src/optimization/optimizer.rs | 113 ++++++--- src/optimization/parser.rs | 23 +- src/tools/definitions/backtest.rs | 28 +++ src/tools/definitions/mod.rs | 1 + src/tools/definitions/optimization.rs | 3 +- src/tools/handlers/backtest.rs | 346 +++++++++++++++++++++++++- src/tools/handlers/mod.rs | 1 + src/tools/handlers/optimization.rs | 31 ++- 8 files changed, 504 insertions(+), 42 deletions(-) diff --git a/src/optimization/optimizer.rs b/src/optimization/optimizer.rs index ad11ca0..7bf0354 100644 --- a/src/optimization/optimizer.rs +++ b/src/optimization/optimizer.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use crate::models::Config; +use crate::optimization::OptimizationParser; /// Read a file that may be UTF-16LE (with BOM) or UTF-8, returning a UTF-8 String. /// MT5 .set and .ini files are typically UTF-16LE with BOM (0xFF 0xFE). @@ -36,6 +37,7 @@ pub struct OptimizationParams { pub deposit: u32, pub leverage: u32, pub currency: String, + pub max_passes: Option, } impl Default for OptimizationParams { @@ -49,6 +51,7 @@ impl Default for OptimizationParams { deposit: 10000, leverage: 500, currency: "USD".to_string(), + max_passes: None, } } } @@ -91,6 +94,12 @@ impl OptimizationRunner { return Err(anyhow!("Set file not found: {}", params.set_file)); } + // Kill any existing MT5/agent processes to avoid stale zombies + for pat in &["terminal64\\.exe", "metatester64\\.exe"] { + let _ = Command::new("pkill").args(["-KILL", "-f", pat]).output(); + } + std::thread::sleep(std::time::Duration::from_secs(3)); + // Generate job ID and log file let timestamp = Utc::now().format("%Y%m%d_%H%M%S").to_string(); let job_id = format!("opt_{}", timestamp); @@ -136,13 +145,25 @@ impl OptimizationRunner { } else { format!("Experts\\{}.ex5", params.expert) }; - let tester_section = format!( + let set_param = { + let base = if !params.set_file.is_empty() && params.set_file != format!("{}.set", params.expert) { + // Extract just the filename from the set file path + std::path::Path::new(¶ms.set_file).file_name() + .and_then(|n| n.to_str()) + .unwrap_or(&format!("{}.set", params.expert)) + .to_string() + } else { + format!("{}.set", params.expert) + }; + base + }; + let mut tester_section = format!( "[Tester]\n\ Expert={}\n\ - ExpertParameters={}.set\n\ + ExpertParameters={}\n\ Symbol={}\n\ Period=M1\n\ - Model=4\n\ + Model=1\n\ FromDate={}\n\ ToDate={}\n\ ForwardMode=0\n\ @@ -152,13 +173,16 @@ impl OptimizationRunner { Leverage={}\n\ Execution=10\n\ Optimization=2\n\ - Visual=0\n\ - Report=reports\\opt_report.htm\n\ - ReplaceReport=1\n\ - ShutdownTerminal=1", - expert_path, params.expert, params.symbol, + Visual=0\n\ + Report=..\\..\\mt5mcp_opt_report.htm\n\ + ReplaceReport=1\n\ + ShutdownTerminal=1", + expert_path, set_param, params.symbol, params.from_date, params.to_date, params.deposit, params.currency, params.leverage, ); + if let Some(mp) = params.max_passes { + tester_section.push_str(&format!("\nMaxPass={}", mp)); + } let updated_ini = Self::patch_ini_section(&mt5_ini_text, "Tester", &tester_section); // Strip any stale [Agents] sections from previous runs (no local agent processes) let cleaned = Self::strip_ini_section(&updated_ini, "Agents"); @@ -185,10 +209,10 @@ impl OptimizationRunner { } opt_ini.push_str("[Tester]\n"); opt_ini.push_str(&format!("Expert={}.ex5\n", params.expert)); - opt_ini.push_str(&format!("ExpertParameters={}.set\n", params.expert)); + opt_ini.push_str(&format!("ExpertParameters={}\n", set_param)); opt_ini.push_str(&format!("Symbol={}\n", params.symbol)); opt_ini.push_str("Period=M1\n"); - opt_ini.push_str("Model=4\n"); + opt_ini.push_str("Model=1\n"); opt_ini.push_str("Optimization=2\n"); opt_ini.push_str(&format!("FromDate={}\n", params.from_date)); opt_ini.push_str(&format!("ToDate={}\n", params.to_date)); @@ -199,9 +223,12 @@ impl OptimizationRunner { opt_ini.push_str(&format!("Leverage={}\n", params.leverage)); opt_ini.push_str("Execution=10\n"); opt_ini.push_str("Visual=0\n"); - opt_ini.push_str("Report=reports\\opt_report.htm\n"); + opt_ini.push_str("Report=..\\..\\mt5mcp_opt_report.htm\n"); opt_ini.push_str("ReplaceReport=1\n"); opt_ini.push_str("ShutdownTerminal=1\n"); + if let Some(mp) = params.max_passes { + opt_ini.push_str(&format!("MaxPass={}\n", mp)); + } fs::write(&opt_config_host, opt_ini.as_bytes())?; // Build launch script (macOS-compatible with /config: to trigger tester mode) @@ -346,6 +373,7 @@ impl OptimizationRunner { let meta_path = jobs_dir.join(format!("{}.json", job_id)); let started_at = Utc::now().to_rfc3339(); + let report_path = wine_prefix.join("drive_c").join("mt5mcp_opt_report"); let metadata = serde_json::json!({ "job_id": job_id, @@ -358,6 +386,7 @@ impl OptimizationRunner { "combinations": combinations, "log_file": log_file.to_string_lossy(), "wine_prefix": wine_prefix.to_string_lossy(), + "report_path": report_path.to_string_lossy(), "started_at": started_at, }); @@ -445,37 +474,51 @@ impl OptimizationRunner { let meta: serde_json::Value = serde_json::from_str(&fs::read_to_string(&meta_path)?)?; let pid = meta.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - - // Check if process is still running let is_running = self.is_process_running(pid); - // Check for completion marker in log - let log_file = meta.get("log_file").and_then(|v| v.as_str()).unwrap_or(""); - let is_complete = if !log_file.is_empty() && Path::new(log_file).exists() { - fs::read_to_string(log_file) - .map(|content| content.contains("Optimization complete")) - .unwrap_or(false) - } else { - false - }; - - let status = if is_complete { - "completed" - } else if is_running { - "running" - } else { - "stopped" - }; - - Ok(serde_json::json!({ - "status": status, + let mut result = serde_json::json!({ + "status": if is_running { "running" } else { "stopped" }, "job_id": job_id, "pid": pid, "expert": meta.get("expert"), "symbol": meta.get("symbol"), + "from_date": meta.get("from_date"), + "to_date": meta.get("to_date"), "started_at": meta.get("started_at"), - "log_file": log_file, - })) + }); + + // If not running, try to parse the optimization report + if !is_running { + let parser = OptimizationParser::new(); + match parser.parse_job(job_id) { + Ok(passes) if !passes.is_empty() => { + let mut sorted_by_pf = passes.clone(); + sorted_by_pf.sort_by(|a, b| b.profit_factor.partial_cmp(&a.profit_factor).unwrap()); + let top10: Vec<_> = sorted_by_pf.into_iter().take(10).collect(); + + let best_pf = parser.find_best_pass(&passes, "profit_factor"); + let best_profit = parser.find_best_pass(&passes, "profit"); + + let m = result.as_object_mut() + .ok_or_else(|| anyhow!("result is not object"))?; + m.insert("status".into(), serde_json::Value::String("completed".into())); + m.insert("total_passes".into(), serde_json::json!(passes.len())); + m.insert("top_10".into(), serde_json::to_value(&top10).unwrap_or_default()); + m.insert("best_pf".into(), serde_json::to_value(best_pf).unwrap_or_default()); + m.insert("best_profit".into(), serde_json::to_value(best_profit).unwrap_or_default()); + } + _ => { + let m = result.as_object_mut() + .ok_or_else(|| anyhow!("result is not object"))?; + m.insert("status".into(), serde_json::Value::String("stopped".into())); + m.insert("message".into(), serde_json::Value::String( + "Optimization stopped but no report found — may have crashed or was killed early".into() + )); + } + } + } + + Ok(result) } fn is_process_running(&self, pid: u32) -> bool { diff --git a/src/optimization/parser.rs b/src/optimization/parser.rs index 4d6a360..ab4943d 100644 --- a/src/optimization/parser.rs +++ b/src/optimization/parser.rs @@ -31,6 +31,23 @@ impl OptimizationParser { } let meta: serde_json::Value = serde_json::from_str(&fs::read_to_string(&meta_path)?)?; + + // Check if report_path is stored in metadata + if let Some(report_base) = meta.get("report_path").and_then(|v| v.as_str()) { + let base_path = Path::new(report_base); + for ext in &[".htm", ".htm.xml", ".html"] { + let candidate = base_path.with_extension(ext.trim_start_matches('.')); + if candidate.exists() { + return self.parse_file(&candidate); + } + } + return Err(anyhow!( + "Optimization report not found at {}.*. Is MT5 optimization still running?", + base_path.display() + )); + } + + // Fallback: derive from wine_prefix (legacy) let wine_prefix = meta.get("wine_prefix") .and_then(|v| v.as_str()) .ok_or_else(|| anyhow!("wine_prefix not in job metadata"))?; @@ -169,13 +186,13 @@ impl OptimizationParser { // Find all rows in Worksheet/Table for node in doc.descendants() { - if node.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Row")) || + if node.has_tag_name(("urn:schemas-microsoft-com:office:spreadsheet", "Row")) || node.has_tag_name("Row") { let cells: Vec = node.children() .filter(|n: &roxmltree::Node<'_, '_>| { - n.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Cell")) || + n.has_tag_name(("urn:schemas-microsoft-com:office:spreadsheet", "Cell")) || n.has_tag_name("Cell") || - n.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Data")) || + n.has_tag_name(("urn:schemas-microsoft-com:office:spreadsheet", "Data")) || n.has_tag_name("Data") }) .map(|n| n.text().unwrap_or("").trim().to_string().replace(',', "")) diff --git a/src/tools/definitions/backtest.rs b/src/tools/definitions/backtest.rs index 3116bfe..5206429 100644 --- a/src/tools/definitions/backtest.rs +++ b/src/tools/definitions/backtest.rs @@ -147,6 +147,34 @@ pub fn tool_cache_status() -> Value { }) } +pub fn tool_run_rolling_backtest() -> Value { + json!({ + "name": "run_rolling_backtest", + "description": "Rolling backtest: run N consecutive weekly backtests sequentially and return aggregated results. Compiles once, then each week runs with skip_compile. Use this to test EA stability across multiple weeks.", + "inputSchema": { + "type": "object", + "required": ["expert"], + "properties": { + "expert": { "type": "string", "description": "EA name without path or extension" }, + "symbol": { "type": "string", "description": "Trading symbol (default: from config or first available)" }, + "from_date": { "type": "string", "description": "Start date YYYY.MM.DD (default: auto-calculate N weeks back)" }, + "to_date": { "type": "string", "description": "End date YYYY.MM.DD (default: auto-calculate to last Sunday)" }, + "timeframe": { "type": "string", "enum": ["M1", "M5", "M15", "M30", "H1", "H4", "D1"], "description": "Chart timeframe (default: M5)" }, + "deposit": { "type": "integer", "description": "Initial deposit (default: 10000)" }, + "model": { "type": "integer", "enum": [0, 1, 2], "description": "Tick model: 0=Every tick, 1=OHLC, 2=Open prices" }, + "set_file": { "type": "string", "description": "Path to .set parameter file for EA inputs" }, + "weeks": { "type": "integer", "description": "Number of weekly backtests to run (default: 4)" }, + "skip_compile": { "type": "boolean", "description": "Skip initial compilation (default: false — compiles on first week)" }, + "shutdown": { "type": "boolean", "description": "Close MT5 after backtest completes (default: true)" }, + "kill_existing": { "type": "boolean", "description": "Kill any running MT5 instance first (default: true)" }, + "timeout": { "type": "integer", "description": "Max wait time per week in seconds (default: 900)" }, + "gui": { "type": "boolean", "description": "Enable MT5 visualization window" }, + "startup_delay_secs": { "type": "integer", "description": "Seconds to wait for MT5 initialization (default: 10)" } + } + } + }) +} + pub fn tool_clean_cache() -> Value { json!({ "name": "clean_cache", diff --git a/src/tools/definitions/mod.rs b/src/tools/definitions/mod.rs index 8f35578..9c93377 100644 --- a/src/tools/definitions/mod.rs +++ b/src/tools/definitions/mod.rs @@ -19,6 +19,7 @@ pub fn get_tools_list() -> Value { backtest::tool_launch_backtest(), // Fire-and-forget: compile + clean + launch MT5 backtest::tool_get_backtest_status(), // Poll for completion backtest::tool_get_tester_log(), // Live journal reading mid-backtest or after + backtest::tool_run_rolling_backtest(), // Rolling backtest: multiple sequential weeks backtest::tool_cache_status(), backtest::tool_clean_cache(), // Optimization diff --git a/src/tools/definitions/optimization.rs b/src/tools/definitions/optimization.rs index 0c3856d..91acb2e 100644 --- a/src/tools/definitions/optimization.rs +++ b/src/tools/definitions/optimization.rs @@ -13,7 +13,8 @@ pub fn tool_run_optimization() -> Value { "symbol": { "type": "string", "description": "Trading symbol (default: XAUUSD)" }, "from_date": { "type": "string", "description": "Start date YYYY.MM.DD" }, "to_date": { "type": "string", "description": "End date YYYY.MM.DD" }, - "deposit": { "type": "integer", "description": "Initial deposit (default: 10000)" } + "deposit": { "type": "integer", "description": "Initial deposit (default: 10000)" }, + "max_passes": { "type": "integer", "description": "Cap on genetic optimization passes (MT5 default: ~10496 for 10 params). Use e.g. 5000 to run ~50% fewer passes." } } } }) diff --git a/src/tools/handlers/backtest.rs b/src/tools/handlers/backtest.rs index b953733..a25aa84 100644 --- a/src/tools/handlers/backtest.rs +++ b/src/tools/handlers/backtest.rs @@ -1,8 +1,10 @@ use anyhow::Result; +use chrono::Datelike; use serde_json::{json, Value}; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; +use tokio::time::Duration; use crate::models::Config; use crate::models::report::BacktestJob; use crate::pipeline::backtest::{BacktestParams, BacktestPipeline}; @@ -375,6 +377,348 @@ pub async fn handle_launch_backtest(handler: &crate::tools::handlers::ToolHandle })) } +pub async fn handle_run_rolling_backtest(config: &Config, args: &Value) -> Result { + let expert = args.get("expert") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("expert is required"))?; + + let weeks_count = args.get("weeks").and_then(|v| v.as_u64()).unwrap_or(4) as i64; + if weeks_count < 1 || weeks_count > 52 { + return Ok(json!({ + "content": [{ "type": "text", "text": "weeks must be between 1 and 52".to_string() }], + "isError": true + })); + } + + // Calculate weekly date ranges + let from_arg = args.get("from_date").and_then(|v| v.as_str()).unwrap_or(""); + let to_arg = args.get("to_date").and_then(|v| v.as_str()).unwrap_or(""); + + let weeks: Vec<(String, String, String)> = if !from_arg.is_empty() && !to_arg.is_empty() { + let start = chrono::NaiveDate::parse_from_str(from_arg, "%Y.%m.%d") + .map_err(|e| anyhow::anyhow!("Invalid from_date '{}': {}", from_arg, e))?; + let end = chrono::NaiveDate::parse_from_str(to_arg, "%Y.%m.%d") + .map_err(|e| anyhow::anyhow!("Invalid to_date '{}': {}", to_arg, e))?; + if end <= start { + return Ok(json!({ + "content": [{ "type": "text", "text": "to_date must be after from_date".to_string() }], + "isError": true + })); + } + let mut weeks = Vec::new(); + let mut current = start; + let mut week_idx = 0; + while current < end && weeks.len() < weeks_count as usize { + week_idx += 1; + let week_end = if current + chrono::Duration::days(7) >= end { + end + } else { + let days_to_sun = 6 - current.weekday().num_days_from_monday(); + let week_end_raw = current + chrono::Duration::days(days_to_sun as i64); + std::cmp::min(week_end_raw, end) + }; + let label = format!("Week {}", week_idx); + weeks.push((label, current.format("%Y.%m.%d").to_string(), week_end.format("%Y.%m.%d").to_string())); + current = week_end + chrono::Duration::days(1); + } + weeks + } else { + let now = chrono::Utc::now().date_naive(); + let days_from_sun = now.weekday().num_days_from_sunday(); + let last_sun = now - chrono::Duration::days(days_from_sun as i64); + let mut weeks = Vec::new(); + for i in 0..weeks_count { + let i = weeks_count - 1 - i; + let week_end = last_sun - chrono::Duration::days(i * 7); + let week_start = week_end - chrono::Duration::days(6); + let label = format!( + "{} - {}", + week_start.format("%b %d"), + week_end.format("%b %d") + ); + weeks.push((label, week_start.format("%Y.%m.%d").to_string(), week_end.format("%Y.%m.%d").to_string())); + } + weeks + }; + + let symbol = args.get("symbol").and_then(|v| v.as_str()).unwrap_or("").to_string(); + let timeframe = args.get("timeframe").and_then(|v| v.as_str()).unwrap_or("M5").to_string(); + let deposit = args.get("deposit").and_then(|v| v.as_u64()).unwrap_or(10000) as u32; + let model = args.get("model").and_then(|v| v.as_u64()).unwrap_or(0) as u8; + let leverage: u32 = args.get("leverage").and_then(|v| v.as_u64()).unwrap_or(500) as u32; + let set_file = args.get("set_file").and_then(|v| v.as_str()).map(|s| s.to_string()); + let deep_analyze = args.get("deep").and_then(|v| v.as_bool()).unwrap_or(false); + let shutdown = args.get("shutdown").and_then(|v| v.as_bool()).unwrap_or(true); + let kill_existing = args.get("kill_existing").and_then(|v| v.as_bool()).unwrap_or(true); + let timeout = args.get("timeout").and_then(|v| v.as_u64()).unwrap_or(900); + let gui = args.get("gui").and_then(|v| v.as_bool()).unwrap_or(false); + let startup_delay_secs = args.get("startup_delay_secs").and_then(|v| v.as_u64()).unwrap_or(0); + let skip_compile_first = args.get("skip_compile").and_then(|v| v.as_bool()).unwrap_or(false); + + // Create a rolling job report dir for status tracking + let report_id = format!( + "ROLLING_{}_{}_{}", + expert, + weeks.first().map(|w| &w.1).unwrap_or(&"unknown".to_string()), + weeks.last().map(|w| &w.2).unwrap_or(&"unknown".to_string()), + ); + let report_dir = config.reports_dir().join(&report_id); + fs::create_dir_all(&report_dir)?; + + let job_path = report_dir.join("job.json"); + let job = BacktestJob { + report_id: report_id.clone(), + report_dir: report_dir.to_string_lossy().to_string(), + expert: expert.to_string(), + symbol: symbol.clone(), + timeframe: timeframe.clone(), + mt5_pid: None, + expected_report_path: String::new(), + timeout_seconds: timeout * weeks.len() as u64, + launched_at: chrono::Utc::now().to_rfc3339(), + status: Some("launched".to_string()), + }; + fs::write(&job_path, serde_json::to_string_pretty(&job)?)?; + + // Save the weekly schedule for status polling + let weeks_path = report_dir.join("weeks.json"); + fs::write(&weeks_path, serde_json::to_string_pretty(&json!({ + "weeks": weeks.iter().map(|(l, f, t)| json!({"label": l, "from_date": f, "to_date": t})).collect::>() + }))?)?; + + // Spawn background task to run all weeks sequentially + let config_clone = config.clone(); + let report_dir_clone = report_dir.clone(); + let expert_clone = expert.to_string(); + let symbol_clone = symbol; + let timeframe_clone = timeframe; + let set_file_clone = set_file; + let weeks_clone = weeks.clone(); + + tokio::spawn(async move { + if let Err(e) = run_rolling_weeks_sequential( + config_clone, + report_dir_clone, + expert_clone, + symbol_clone, + timeframe_clone, + deposit, + model, + leverage, + set_file_clone, + deep_analyze, + shutdown, + kill_existing, + timeout, + gui, + startup_delay_secs, + skip_compile_first, + weeks_clone, + ).await { + tracing::error!("Rolling backtest failed: {}", e); + } + }); + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "message": format!("Rolling backtest launched with {} weeks. Use get_backtest_status to poll for completion.", weeks.len()), + "report_id": report_id, + "report_dir": report_dir.to_string_lossy(), + "expert": expert, + "weeks": weeks.iter().map(|(l, f, t)| json!({"label": l, "from_date": f, "to_date": t})).collect::>(), + "poll_hint": "Call get_backtest_status with report_dir to check progress" + }).to_string() }], + "isError": false + })) +} + +async fn run_rolling_weeks_sequential( + config: Config, + report_dir: PathBuf, + expert: String, + symbol: String, + timeframe: String, + deposit: u32, + model: u8, + leverage: u32, + set_file: Option, + deep_analyze: bool, + _shutdown: bool, + _kill_existing: bool, + timeout: u64, + gui: bool, + startup_delay_secs: u64, + skip_compile_first: bool, + weeks: Vec<(String, String, String)>, +) -> Result<()> { + let pipeline = if cfg!(debug_assertions) { + BacktestPipeline::new(config.clone()) + } else { + BacktestPipeline::new(config.clone()) + }; + let progress_log = report_dir.join("progress.log"); + + // Clear stale results + let results_path = report_dir.join("rolling_results.json"); + let _ = fs::remove_file(&results_path); + + let mut week_results = Vec::new(); + let mut total_net_profit: f64 = 0.0; + let mut max_drawdown: f64 = 0.0; + let mut total_trades: u64 = 0; + + // Kill MT5/wineserver before starting + for pat in &["MetaTrader 5.app", "terminal64.exe", "wineserver"] { + let _ = Command::new("pkill").args(["-KILL", "-f", pat]).output(); + } + tokio::time::sleep(Duration::from_secs(5)).await; + + for (idx, (label, from_date, to_date)) in weeks.iter().enumerate() { + let skip_compile = if idx == 0 { skip_compile_first } else { true }; + + fs::write(&progress_log, format!("WEEK {}: {} ({} -> {})\n", idx + 1, label, from_date, to_date)) + .unwrap_or(()); + + tracing::info!("Rolling week {}/{}: {} ({} -> {})", idx + 1, weeks.len(), label, from_date, to_date); + + // Clean slate before each week + for pat in &["MetaTrader 5.app", "terminal64.exe", "wineserver"] { + let _ = Command::new("pkill").args(["-KILL", "-f", pat]).output(); + } + tokio::time::sleep(Duration::from_secs(3)).await; + + // Fire-and-forget launch (handles journal fallback if HTML not produced) + let params = BacktestParams { + expert: expert.clone(), + symbol: symbol.clone(), + from_date: from_date.clone(), + to_date: to_date.clone(), + timeframe: timeframe.clone(), + deposit, + model, + leverage, + set_file: set_file.clone(), + skip_compile, + skip_clean: false, + skip_analyze: false, + deep_analyze, + shutdown: true, + kill_existing: false, + timeout, + gui, + startup_delay_secs, + inactivity_kill_secs: None, + }; + + let job = match pipeline.launch_backtest(params).await { + Ok(j) => j, + Err(e) => { + tracing::error!("Rolling week {}/{} launch failed: {}", idx + 1, weeks.len(), e); + week_results.push(json!({ + "label": label, "from_date": from_date, "to_date": to_date, + "success": false, "error": format!("launch failed: {}", e) + })); + continue; + } + }; + + // Poll for completion (up to timeout) + let week_report_dir = Path::new(&job.report_dir); + let poll_start = std::time::Instant::now(); + let max_wait = Duration::from_secs(timeout); + loop { + tokio::time::sleep(Duration::from_secs(5)).await; + + // Check for metrics.json (written after extraction, including journal fallback) + let metrics_path = week_report_dir.join("metrics.json"); + if metrics_path.exists() { break; } + + let jp = week_report_dir.join("job.json"); + if let Ok(content) = fs::read_to_string(&jp) { + if let Ok(j) = serde_json::from_str::(&content) { + if let Some(ref s) = j.status { + if s == "completed" || s == "completed_no_html" { break; } + if s == "failed" || s == "timeout" || s == "timeout_inactive" { + tracing::warn!("Rolling week {}/{} background status: {}", idx + 1, weeks.len(), s); + break; + } + } + } + } + + if poll_start.elapsed() > max_wait { + tracing::warn!("Rolling week {}/{} timed out after {}s", idx + 1, weeks.len(), timeout); + break; + } + } + + // Wait a moment for extraction to finish + tokio::time::sleep(Duration::from_secs(2)).await; + + // Read results + let metrics_path = week_report_dir.join("metrics.json"); + if let Ok(content) = fs::read_to_string(&metrics_path) { + if let Ok(metrics) = serde_json::from_str::(&content) { + let np = metrics.get("net_profit").and_then(|v| v.as_f64()).unwrap_or(0.0); + let dd = metrics.get("max_dd_pct").and_then(|v| v.as_f64()).unwrap_or(0.0); + let trades = metrics.get("total_trades").and_then(|v| v.as_u64()).unwrap_or(0); + let pf = metrics.get("profit_factor").and_then(|v| v.as_f64()).unwrap_or(0.0); + + total_net_profit += np; + if dd > max_drawdown { max_drawdown = dd; } + total_trades += trades; + + tracing::info!("Rolling week {}/{} done: profit={:.2}, dd={:.1}%", idx + 1, weeks.len(), np, dd); + + week_results.push(json!({ + "label": label, "from_date": from_date, "to_date": to_date, + "success": true, "net_profit": np, "max_dd_pct": dd, + "total_trades": trades, "profit_factor": pf, + "report_dir": job.report_dir + })); + continue; + } + } + + // No metrics found + week_results.push(json!({ + "label": label, "from_date": from_date, "to_date": to_date, + "success": false, "error": "No metrics extracted" + })); + } + + // Write final results + let summary = json!({ + "success": true, + "weeks_run": week_results.len(), + "summary": { + "total_net_profit": total_net_profit, + "max_drawdown_pct": max_drawdown, + "total_trades": total_trades, + }, + "weekly_results": week_results + }); + if let Ok(json_str) = serde_json::to_string_pretty(&summary) { + let _ = fs::write(&results_path, &json_str); + } + + // Update job status + let job_path = report_dir.join("job.json"); + if let Ok(content) = fs::read_to_string(&job_path) { + if let Ok(mut job) = serde_json::from_str::(&content) { + job.status = Some("completed".to_string()); + if let Ok(json_str) = serde_json::to_string_pretty(&job) { + let _ = fs::write(&job_path, json_str); + } + } + } + + fs::write(&progress_log, "DONE\n").unwrap_or(()); + tracing::info!("Rolling backtest completed: {} weeks, profit={:.2}, max_dd={:.1}%", week_results.len(), total_net_profit, max_drawdown); + Ok(()) +} + pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Result { let report_dir = args.get("report_dir") .and_then(|v| v.as_str()) diff --git a/src/tools/handlers/mod.rs b/src/tools/handlers/mod.rs index 3dad05c..30ef834 100644 --- a/src/tools/handlers/mod.rs +++ b/src/tools/handlers/mod.rs @@ -75,6 +75,7 @@ impl ToolHandler { "launch_backtest" => backtest::handle_launch_backtest(self, args).await, // Fire-and-forget mode "get_backtest_status" => backtest::handle_get_backtest_status(&self.config, args).await, "get_tester_log" => backtest::handle_get_tester_log(&self.config, args).await, + "run_rolling_backtest" => backtest::handle_run_rolling_backtest(&self.config, args).await, "cache_status" => backtest::handle_cache_status(&self.config).await, "clean_cache" => backtest::handle_clean_cache(&self.config, args).await, diff --git a/src/tools/handlers/optimization.rs b/src/tools/handlers/optimization.rs index 3975a7c..21b9933 100644 --- a/src/tools/handlers/optimization.rs +++ b/src/tools/handlers/optimization.rs @@ -1,5 +1,7 @@ use anyhow::Result; use serde_json::{json, Value}; +use std::fs; +use std::path::Path; use crate::models::Config; use crate::optimization::{OptimizationParams, OptimizationParser, OptimizationRunner}; @@ -29,6 +31,7 @@ pub async fn handle_run_optimization(config: &Config, args: &Value) -> Result Re let runner = OptimizationRunner::new(config.clone()); let status = runner.get_job_status(job_id)?; + // Store completion results back to metadata for persistence + if status.get("status").and_then(|v| v.as_str()) == Some("completed") { + let jobs_dir = Path::new(".mt5mcp_jobs"); + let meta_path = jobs_dir.join(format!("{}.json", job_id)); + if let Ok(meta_str) = fs::read_to_string(&meta_path) { + if let Ok(mut meta) = serde_json::from_str::(&meta_str) { + if let Some(obj) = meta.as_object_mut() { + obj.insert("status".into(), serde_json::Value::String("completed".into())); + obj.insert("completed_at".into(), serde_json::Value::String(chrono::Utc::now().to_rfc3339())); + if let Some(top) = status.get("top_10") { + obj.insert("top_10".into(), top.clone()); + } + if let Some(best) = status.get("best_pf") { + obj.insert("best_pf".into(), best.clone()); + } + if let Some(total) = status.get("total_passes") { + obj.insert("total_passes".into(), total.clone()); + } + let _ = fs::write(&meta_path, serde_json::to_string_pretty(&meta).unwrap_or_default()); + } + } + } + } + Ok(json!({ "content": [{ "type": "text", "text": status.to_string() }], "isError": false @@ -65,7 +92,7 @@ pub async fn handle_get_optimization_results(_config: &Config, args: &Value) -> let job_id = args.get("job_id") .and_then(|v| v.as_str()); - let file = args.get("file") + let file = args.get("report_file") .and_then(|v| v.as_str()); let parser = OptimizationParser::new(); @@ -79,7 +106,7 @@ pub async fn handle_get_optimization_results(_config: &Config, args: &Value) -> }; let sort_by = args.get("sort").and_then(|v| v.as_str()).unwrap_or("profit"); - let top_n = args.get("top").and_then(|v| v.as_u64()).unwrap_or(30) as usize; + let top_n = args.get("top_n").and_then(|v| v.as_u64()).unwrap_or(30) as usize; let best = parser.find_best_pass(&passes, sort_by);