Devid HW
ecf5606b7a
- fix: inactivity watchdog now waits 30s for HTML report then kills terminal64.exe unconditionally — ShutdownTerminal=1 does not cause MT5 to exit on Wine/macOS; relying on natural exit caused all runs to fall back to journal extraction - fix: tester agent log deduplication — each deal is written twice in the log; use HashSet to skip already-seen deal numbers - fix: position tracker for entry direction inference — infer "in"/"out" from per-symbol signed lot accumulation instead of guessing - fix: is_closed_trade() no longer gates on profit!=0.0 — journal deals have profit=0.0 legitimately; the old gate hid all 140 deals from list_deals - fix: log file selection priority — prefer Agent-127.0.0.1 over Agent-0.0.0.0 (startup-only log), tiebreak by file size - feat: launch_backtest default shutdown=true; inactivity_kill_secs default 120s exposed as tool parameter - feat: report DB stores deals in SQLite; analytics resolve by report_id / report_dir / latest Verified: 1-month DPS21/XAUUSD.cent/M5 backtest produces full HTML report with all metrics (win_rate=70%, profit_factor=0.77, sharpe=-3.61, max_dd=6.93%) and all 17 analytics tools returning real data. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1618 lines
70 KiB
Rust
1618 lines
70 KiB
Rust
use anyhow::{anyhow, Result};
|
||
use chrono;
|
||
use serde_json::json;
|
||
use std::fs;
|
||
use std::path::{Path, PathBuf};
|
||
use std::process::Command;
|
||
use std::sync::Arc;
|
||
use tokio::time::{sleep, Duration};
|
||
|
||
use crate::analytics::{DealAnalyzer, ReportExtractor};
|
||
use crate::compile::MqlCompiler;
|
||
use crate::models::config::Config;
|
||
use crate::models::report::{PipelineMetadata, FilePaths, BacktestJob};
|
||
use crate::storage::{ReportDb, ReportEntry};
|
||
|
||
type NotificationCallback = Arc<dyn Fn(&str, serde_json::Value) + Send + Sync>;
|
||
|
||
pub struct BacktestPipeline {
|
||
config: Config,
|
||
compiler: MqlCompiler,
|
||
extractor: ReportExtractor,
|
||
analyzer: DealAnalyzer,
|
||
notification_callback: Option<NotificationCallback>,
|
||
}
|
||
|
||
pub struct BacktestParams {
|
||
pub expert: String,
|
||
pub symbol: String,
|
||
pub from_date: String,
|
||
pub to_date: String,
|
||
pub timeframe: String,
|
||
pub deposit: u32,
|
||
pub model: u8,
|
||
pub leverage: u32,
|
||
pub set_file: Option<String>,
|
||
pub skip_compile: bool,
|
||
pub skip_clean: bool,
|
||
pub skip_analyze: bool,
|
||
#[allow(dead_code)]
|
||
pub deep_analyze: bool,
|
||
pub shutdown: bool,
|
||
#[allow(dead_code)]
|
||
pub kill_existing: bool,
|
||
pub timeout: u64,
|
||
pub gui: bool,
|
||
pub startup_delay_secs: u64,
|
||
/// Kill MT5 if tester agent log hasn't grown for this many seconds.
|
||
/// 0 = disabled. Useful to abort EAs that stop trading mid-backtest.
|
||
pub inactivity_kill_secs: Option<u64>,
|
||
}
|
||
|
||
pub struct PipelineResult {
|
||
pub success: bool,
|
||
pub report_dir: PathBuf,
|
||
pub duration_seconds: i64,
|
||
pub message: String,
|
||
}
|
||
|
||
impl BacktestPipeline {
|
||
pub fn new(config: Config) -> Self {
|
||
let compiler = MqlCompiler::new(config.clone());
|
||
let extractor = ReportExtractor::new();
|
||
let analyzer = DealAnalyzer::new();
|
||
|
||
Self {
|
||
config,
|
||
compiler,
|
||
extractor,
|
||
analyzer,
|
||
notification_callback: None,
|
||
}
|
||
}
|
||
|
||
pub fn with_notification_callback(config: Config, callback: NotificationCallback) -> Self {
|
||
let compiler = MqlCompiler::new(config.clone());
|
||
let extractor = ReportExtractor::new();
|
||
let analyzer = DealAnalyzer::new();
|
||
|
||
Self {
|
||
config,
|
||
compiler,
|
||
extractor,
|
||
analyzer,
|
||
notification_callback: Some(callback),
|
||
}
|
||
}
|
||
|
||
pub async fn run(&self, params: BacktestParams) -> Result<PipelineResult> {
|
||
let start_time = chrono::Utc::now();
|
||
let report_id = self.generate_report_id(¶ms);
|
||
let report_dir = self.config.reports_dir().join(&report_id);
|
||
|
||
fs::create_dir_all(&report_dir)?;
|
||
|
||
let progress_log = report_dir.join("progress.log");
|
||
self.log_progress(&progress_log, "START").await;
|
||
|
||
if !params.skip_compile {
|
||
self.log_progress(&progress_log, "COMPILE").await;
|
||
self.compile_ea(¶ms.expert, params.timeout).await?;
|
||
}
|
||
|
||
if !params.skip_clean {
|
||
self.log_progress(&progress_log, "CLEAN").await;
|
||
self.clean_cache(¶ms.expert).await?;
|
||
}
|
||
|
||
self.log_progress(&progress_log, "BACKTEST").await;
|
||
let report_path = self.run_backtest(¶ms, &report_id).await?;
|
||
|
||
self.log_progress(&progress_log, "EXTRACT").await;
|
||
let extraction = self.extractor.extract(
|
||
&report_path.to_string_lossy(),
|
||
&report_dir.to_string_lossy(),
|
||
)?;
|
||
|
||
// Handle case where EA didn't trade - no deals generated
|
||
if extraction.deals.is_empty() {
|
||
tracing::warn!("Backtest completed but no deals were generated - EA did not trade during this period");
|
||
let warning_path = report_dir.join("NO_TRADES_WARNING.txt");
|
||
let _ = fs::write(&warning_path, "Warning: No deals were generated during this backtest.\nThe EA did not execute any trades during the specified date range.\n");
|
||
}
|
||
|
||
// Move equity chart images to OS temp dir, then delete the HTML report.
|
||
let charts_dir = self.relocate_charts(&report_path, &report_id).await;
|
||
let _ = fs::remove_file(&report_path);
|
||
|
||
// Snapshot the set file alongside the extracted data.
|
||
let set_snapshot = self.snapshot_set_file(¶ms, &report_dir).await;
|
||
|
||
if !params.skip_analyze {
|
||
self.log_progress(&progress_log, "ANALYZE").await;
|
||
let analysis = self.analyzer.analyze(&extraction.deals, &extraction.metrics);
|
||
|
||
let analysis_path = report_dir.join("analysis.json");
|
||
fs::write(&analysis_path, serde_json::to_string_pretty(&analysis)?)?;
|
||
}
|
||
|
||
self.log_progress(&progress_log, "DONE").await;
|
||
|
||
let duration = (chrono::Utc::now() - start_time).num_seconds();
|
||
self.save_metadata(¶ms, &report_dir, duration, extraction.deals.is_empty()).await?;
|
||
|
||
// Register in the SQLite report registry and store deals.
|
||
let db = self.register_in_db(
|
||
&report_id,
|
||
¶ms,
|
||
&report_dir,
|
||
charts_dir.as_deref(),
|
||
set_snapshot.as_deref(),
|
||
&extraction.metrics,
|
||
duration,
|
||
)
|
||
.await;
|
||
|
||
if let Some(db) = db {
|
||
if let Err(e) = db.insert_deals(&report_id, &extraction.deals) {
|
||
tracing::warn!("Failed to store deals in DB: {}", e);
|
||
}
|
||
}
|
||
|
||
let message = if extraction.deals.is_empty() {
|
||
"Backtest completed successfully, but EA did not execute any trades during this period".to_string()
|
||
} else {
|
||
"Backtest completed successfully".to_string()
|
||
};
|
||
|
||
Ok(PipelineResult {
|
||
success: true,
|
||
report_dir,
|
||
duration_seconds: duration,
|
||
message,
|
||
})
|
||
}
|
||
|
||
/// Launch backtest in fire-and-forget mode: compile, clean, launch MT5, return immediately.
|
||
/// Returns a BacktestJob that can be used with get_backtest_status to poll for completion.
|
||
pub async fn launch_backtest(&self, params: BacktestParams) -> Result<BacktestJob> {
|
||
let _start_time = chrono::Utc::now();
|
||
let report_id = self.generate_report_id(¶ms);
|
||
let report_dir = self.config.reports_dir().join(&report_id);
|
||
|
||
fs::create_dir_all(&report_dir)?;
|
||
|
||
let progress_log = report_dir.join("progress.log");
|
||
self.log_progress(&progress_log, "START").await;
|
||
|
||
if !params.skip_compile {
|
||
self.log_progress(&progress_log, "COMPILE").await;
|
||
self.compile_ea(¶ms.expert, params.timeout).await?;
|
||
}
|
||
|
||
if !params.skip_clean {
|
||
self.log_progress(&progress_log, "CLEAN").await;
|
||
self.clean_cache(¶ms.expert).await?;
|
||
}
|
||
|
||
self.log_progress(&progress_log, "BACKTEST").await;
|
||
|
||
// Get MT5 paths
|
||
let mt5_dir = self.config.mt5_dir()
|
||
.ok_or_else(|| anyhow!("MT5 directory not configured"))?;
|
||
let wine_exe = self.config.wine_executable.as_ref()
|
||
.ok_or_else(|| anyhow!("wine_executable not configured"))?;
|
||
let wine_prefix = mt5_dir
|
||
.parent()
|
||
.and_then(|p| p.parent())
|
||
.and_then(|p| p.parent())
|
||
.map(|p| p.to_path_buf())
|
||
.ok_or_else(|| anyhow!("Could not determine Wine prefix from terminal_dir"))?;
|
||
let reports_dir = mt5_dir.join("reports");
|
||
fs::create_dir_all(&reports_dir)?;
|
||
|
||
// Kill first — MT5 writes terminal.ini on exit, which would clobber
|
||
// the backtest params we're about to write.
|
||
self.kill_mt5().await?;
|
||
|
||
// Write params *after* MT5 is dead so nothing can overwrite them.
|
||
let ini_content = self.build_backtest_ini(¶ms, &report_id)?;
|
||
let config_host = wine_prefix.join("drive_c").join("backtest_config.ini");
|
||
fs::write(&config_host, ini_content.as_bytes())?;
|
||
self.update_terminal_ini(¶ms, &report_id)?;
|
||
|
||
// Launch MT5 (fire and forget)
|
||
let mut cmd = self.build_wine_launch(wine_exe, &wine_prefix)?;
|
||
let child = cmd.stdin(std::process::Stdio::null())
|
||
.stdout(std::process::Stdio::null())
|
||
.stderr(std::process::Stdio::null())
|
||
.spawn()?;
|
||
|
||
let pid = child.id();
|
||
tracing::info!("MT5 launched with PID {:?} for backtest {}", pid, report_id);
|
||
|
||
// Create and save the job tracking file
|
||
let expected_report = reports_dir.join(format!("{}.htm", report_id));
|
||
let job = BacktestJob::new(
|
||
report_id.clone(),
|
||
report_dir.to_string_lossy().to_string(),
|
||
params.expert.clone(),
|
||
params.symbol.clone(),
|
||
params.timeframe.clone(),
|
||
expected_report.to_string_lossy().to_string(),
|
||
params.timeout,
|
||
);
|
||
|
||
// Save job info for polling
|
||
let job_path = report_dir.join("job.json");
|
||
fs::write(&job_path, serde_json::to_string_pretty(&job)?)?;
|
||
|
||
// Save initial metadata
|
||
self.save_metadata(¶ms, &report_dir, 0, false).await?;
|
||
|
||
// Register in DB as "running"
|
||
let db = ReportDb::new(&Config::db_path());
|
||
if let Err(e) = db.init() {
|
||
tracing::warn!("Failed to init report DB: {}", e);
|
||
}
|
||
|
||
// Spawn background task to monitor completion and update status
|
||
let report_dir_clone = report_dir.clone();
|
||
let expected_report_clone = expected_report.clone();
|
||
let timeout_secs = params.timeout;
|
||
let report_id_clone = report_id.clone();
|
||
let notification_callback = self.notification_callback.clone();
|
||
let config_clone = self.config.clone();
|
||
let params_clone = BacktestParams {
|
||
expert: params.expert.clone(),
|
||
symbol: params.symbol.clone(),
|
||
from_date: params.from_date.clone(),
|
||
to_date: params.to_date.clone(),
|
||
timeframe: params.timeframe.clone(),
|
||
deposit: params.deposit,
|
||
model: params.model,
|
||
leverage: params.leverage,
|
||
set_file: params.set_file.clone(),
|
||
skip_compile: params.skip_compile,
|
||
skip_clean: params.skip_clean,
|
||
skip_analyze: params.skip_analyze,
|
||
deep_analyze: params.deep_analyze,
|
||
shutdown: params.shutdown,
|
||
kill_existing: params.kill_existing,
|
||
timeout: params.timeout,
|
||
gui: params.gui,
|
||
startup_delay_secs: params.startup_delay_secs,
|
||
inactivity_kill_secs: params.inactivity_kill_secs,
|
||
};
|
||
tokio::spawn(async move {
|
||
Self::monitor_backtest_completion(
|
||
report_dir_clone,
|
||
expected_report_clone,
|
||
timeout_secs,
|
||
report_id_clone,
|
||
notification_callback,
|
||
config_clone,
|
||
params_clone,
|
||
).await;
|
||
});
|
||
|
||
Ok(job)
|
||
}
|
||
|
||
/// Extract deals from a completed report and store them in the DB.
|
||
/// Returns true if extraction and DB registration succeeded.
|
||
async fn extract_and_store(
|
||
report_path: &Path,
|
||
report_dir: &Path,
|
||
report_id: &str,
|
||
config: &Config,
|
||
params: &BacktestParams,
|
||
) -> bool {
|
||
let extractor = ReportExtractor::new();
|
||
let start_time = chrono::Utc::now();
|
||
match extractor.extract(
|
||
&report_path.to_string_lossy(),
|
||
&report_dir.to_string_lossy(),
|
||
) {
|
||
Ok(extraction) => {
|
||
let duration = (chrono::Utc::now() - start_time).num_seconds();
|
||
let db = ReportDb::new(&Config::db_path());
|
||
if db.init().is_err() {
|
||
tracing::warn!("launch_backtest: failed to init DB for {}", report_id);
|
||
return false;
|
||
}
|
||
let entry = ReportEntry {
|
||
id: report_id.to_string(),
|
||
expert: params.expert.clone(),
|
||
symbol: params.symbol.clone(),
|
||
timeframe: params.timeframe.clone(),
|
||
model: params.model as i64,
|
||
from_date: params.from_date.clone(),
|
||
to_date: params.to_date.clone(),
|
||
created_at: chrono::Utc::now().to_rfc3339(),
|
||
set_file_original: params.set_file.clone(),
|
||
set_snapshot_path: None,
|
||
report_dir: report_dir.to_string_lossy().to_string(),
|
||
charts_dir: None,
|
||
net_profit: Some(extraction.metrics.net_profit),
|
||
profit_factor: Some(extraction.metrics.profit_factor),
|
||
max_dd_pct: Some(extraction.metrics.max_dd_pct),
|
||
sharpe_ratio: Some(extraction.metrics.sharpe_ratio),
|
||
total_trades: Some(extraction.metrics.total_trades as i64),
|
||
win_rate_pct: Some(extraction.metrics.win_rate_pct),
|
||
recovery_factor: Some(extraction.metrics.recovery_factor),
|
||
deposit: Some(params.deposit as f64),
|
||
currency: config.backtest_currency.clone(),
|
||
leverage: Some(params.leverage as i64),
|
||
duration_seconds: Some(duration),
|
||
tags: Vec::new(),
|
||
notes: None,
|
||
verdict: None,
|
||
};
|
||
if let Err(e) = db.insert(&entry) {
|
||
tracing::warn!("launch_backtest: failed to register report in DB: {}", e);
|
||
return false;
|
||
}
|
||
if let Err(e) = db.insert_deals(report_id, &extraction.deals) {
|
||
tracing::warn!("launch_backtest: failed to store deals in DB: {}", e);
|
||
}
|
||
tracing::info!("launch_backtest: extracted {} deals for {}", extraction.deals.len(), report_id);
|
||
true
|
||
}
|
||
Err(e) => {
|
||
tracing::warn!("launch_backtest: extraction failed for {}: {}", report_id, e);
|
||
false
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Background task to monitor backtest completion and update status file.
|
||
async fn monitor_backtest_completion(
|
||
report_dir: PathBuf,
|
||
expected_report: PathBuf,
|
||
timeout_secs: u64,
|
||
report_id: String,
|
||
notification_callback: Option<NotificationCallback>,
|
||
config: Config,
|
||
params: BacktestParams,
|
||
) {
|
||
let start = tokio::time::Instant::now();
|
||
let deadline = start + Duration::from_secs(timeout_secs);
|
||
let grace_period = Duration::from_secs(30);
|
||
let poll_start = std::time::SystemTime::now();
|
||
|
||
// Inactivity watchdog: kill MT5 if tester agent log hasn't grown for this long.
|
||
// Catches EAs that stall (no ticks processed) or flat periods with zero trades.
|
||
let inactivity_threshold = Duration::from_secs(
|
||
params.inactivity_kill_secs.unwrap_or(0)
|
||
);
|
||
let mut last_log_size: u64 = 0;
|
||
let mut last_log_activity = tokio::time::Instant::now();
|
||
let inactivity_enabled = inactivity_threshold.as_secs() > 0;
|
||
|
||
loop {
|
||
let _elapsed = start.elapsed().as_secs();
|
||
|
||
// Check for report file (exact name first)
|
||
for ext in &["htm", "htm.xml", "html"] {
|
||
let candidate = if *ext == "htm" {
|
||
expected_report.clone()
|
||
} else {
|
||
// Build alternate extension path without touching expected_report extension
|
||
let stem = expected_report.file_stem()
|
||
.map(|s| s.to_string_lossy().to_string())
|
||
.unwrap_or_default();
|
||
expected_report.with_file_name(format!("{}.{}", stem, ext))
|
||
};
|
||
if candidate.exists() {
|
||
tracing::info!("Backtest {} completed: report found at {}", report_id, candidate.display());
|
||
let extracted = Self::extract_and_store(&candidate, &report_dir, &report_id, &config, ¶ms).await;
|
||
if extracted {
|
||
let _ = fs::remove_file(&candidate);
|
||
} else {
|
||
tracing::warn!("Backtest {}: extraction failed, keeping report file at {}", report_id, candidate.display());
|
||
}
|
||
// When ShutdownTerminal=0 (shutdown=false), MT5 stays running after the
|
||
// test so the report can be written reliably. Kill it ourselves now that
|
||
// extraction is done so we don't leave zombie Wine processes.
|
||
if !params.shutdown && Self::is_mt5_running() {
|
||
tracing::info!("Backtest {}: killing MT5 after report extraction (shutdown=false)", report_id);
|
||
let _ = std::process::Command::new("pkill")
|
||
.args(["-TERM", "-f", "terminal64\\.exe"])
|
||
.output();
|
||
}
|
||
Self::update_job_status(&report_dir, "completed", Some(candidate.to_string_lossy().to_string())).await;
|
||
if let Some(ref callback) = notification_callback {
|
||
callback("backtest_completed", json!({
|
||
"report_id": report_id,
|
||
"report_path": candidate.to_string_lossy().to_string(),
|
||
"status": "completed"
|
||
}));
|
||
}
|
||
return;
|
||
}
|
||
}
|
||
|
||
// Inactivity watchdog: check if tester agent log is growing.
|
||
// The agent log is written incrementally as the tester processes ticks.
|
||
// If it stops growing for `inactivity_threshold` seconds → the test is done
|
||
// (or EA is stuck). Either way, we wait 30 s for MT5 to write the HTML
|
||
// report, then kill it unconditionally.
|
||
//
|
||
// NOTE: ShutdownTerminal=1 is supposed to make MT5 exit after the test,
|
||
// but on Wine/macOS this is unreliable — terminal64.exe often stays alive
|
||
// indefinitely. So we always kill after the HTML-wait window rather than
|
||
// depending on natural exit. The 30 s window is long enough for MT5 to flush
|
||
// the report file; if it isn't there by then, it won't appear.
|
||
if inactivity_enabled && Self::is_mt5_running() {
|
||
if let Some(log_path) = Self::find_active_tester_agent_log(&config) {
|
||
let current_size = fs::metadata(&log_path)
|
||
.map(|m| m.len())
|
||
.unwrap_or(0);
|
||
if current_size > last_log_size {
|
||
last_log_size = current_size;
|
||
last_log_activity = tokio::time::Instant::now();
|
||
} else if last_log_activity.elapsed() >= inactivity_threshold && last_log_size > 0 {
|
||
tracing::info!(
|
||
"Backtest {}: tester log inactive for {}s — waiting 30s for HTML report, then killing MT5",
|
||
report_id, inactivity_threshold.as_secs()
|
||
);
|
||
// Poll for the HTML during the grace window (30 s, 1 s intervals).
|
||
let reports_parent = expected_report.parent();
|
||
let mut html_found: Option<std::path::PathBuf> = None;
|
||
for _wait in 0u32..30 {
|
||
// Check exact expected path first.
|
||
for ext in &["htm", "htm.xml", "html"] {
|
||
let candidate = if *ext == "htm" {
|
||
expected_report.clone()
|
||
} else {
|
||
let stem = expected_report.file_stem()
|
||
.map(|s| s.to_string_lossy().to_string())
|
||
.unwrap_or_default();
|
||
expected_report.with_file_name(format!("{}.{}", stem, ext))
|
||
};
|
||
if candidate.exists() {
|
||
html_found = Some(candidate);
|
||
break;
|
||
}
|
||
}
|
||
if html_found.is_some() { break; }
|
||
// Also scan for any newly created report.
|
||
if let Some(parent) = reports_parent {
|
||
if let Some(path) = Self::find_newest_report(parent, poll_start) {
|
||
html_found = Some(path);
|
||
break;
|
||
}
|
||
}
|
||
sleep(Duration::from_secs(1)).await;
|
||
}
|
||
|
||
// Kill MT5 now — it either wrote the report or it won't.
|
||
tracing::info!("Backtest {}: killing MT5 after inactivity+HTML-wait window", report_id);
|
||
let _ = std::process::Command::new("pkill")
|
||
.args(["-TERM", "-f", "terminal64\\.exe"])
|
||
.output();
|
||
sleep(Duration::from_secs(2)).await;
|
||
let _ = std::process::Command::new("pkill")
|
||
.args(["-KILL", "-f", "terminal64\\.exe"])
|
||
.output();
|
||
|
||
if let Some(path) = html_found {
|
||
tracing::info!("Backtest {}: HTML report found during wait: {}", report_id, path.display());
|
||
let extracted = Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await;
|
||
if extracted { let _ = fs::remove_file(&path); }
|
||
Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await;
|
||
if let Some(ref callback) = notification_callback {
|
||
callback("backtest_completed", json!({
|
||
"report_id": report_id,
|
||
"status": "completed"
|
||
}));
|
||
}
|
||
return;
|
||
}
|
||
|
||
// No HTML — fall back to journal extraction.
|
||
sleep(Duration::from_secs(1)).await;
|
||
if let Some(log) = Self::find_active_tester_agent_log(&config) {
|
||
if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await {
|
||
Self::update_job_status(&report_dir, "completed_no_html", None).await;
|
||
if let Some(ref callback) = notification_callback {
|
||
callback("backtest_completed", json!({
|
||
"report_id": report_id,
|
||
"status": "completed_no_html",
|
||
"reason": "extracted from journal after inactivity kill (no HTML produced)"
|
||
}));
|
||
}
|
||
return;
|
||
}
|
||
}
|
||
Self::update_job_status(&report_dir, "timeout_inactive", None).await;
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
// Check process liveness after grace period
|
||
let in_grace = start.elapsed() <= grace_period;
|
||
let mt5_alive = Self::is_mt5_running();
|
||
|
||
if !in_grace && !mt5_alive {
|
||
// MT5 exited. Poll for the .htm report for up to 10 s (check every 1 s).
|
||
// Wine on macOS can take several seconds to flush the file after the
|
||
// process exits — a single fixed wait often misses the window.
|
||
let reports_parent = match expected_report.parent() {
|
||
Some(p) => p,
|
||
None => {
|
||
tracing::error!("Backtest {}: expected_report path has no parent", report_id);
|
||
Self::update_job_status(&report_dir, "failed", None).await;
|
||
return;
|
||
}
|
||
};
|
||
let mut found_report: Option<std::path::PathBuf> = None;
|
||
for attempt in 1u32..=10 {
|
||
sleep(Duration::from_secs(1)).await;
|
||
if let Some(path) = Self::find_newest_report(reports_parent, poll_start) {
|
||
tracing::info!(
|
||
"Backtest {}: found report after {}s — {}",
|
||
report_id, attempt, path.display()
|
||
);
|
||
found_report = Some(path);
|
||
break;
|
||
}
|
||
tracing::debug!("Backtest {}: no report yet ({}s elapsed after MT5 exit)", report_id, attempt);
|
||
}
|
||
if let Some(path) = found_report {
|
||
tracing::info!("Backtest {} completed: found report {}", report_id, path.display());
|
||
let extracted = Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await;
|
||
if extracted {
|
||
let _ = fs::remove_file(&path);
|
||
} else {
|
||
tracing::warn!("Backtest {}: extraction failed, keeping report at {}", report_id, path.display());
|
||
}
|
||
Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await;
|
||
if let Some(ref callback) = notification_callback {
|
||
callback("backtest_completed", json!({
|
||
"report_id": report_id,
|
||
"report_path": path.to_string_lossy().to_string(),
|
||
"status": "completed"
|
||
}));
|
||
}
|
||
return;
|
||
}
|
||
|
||
// No HTML report found — fallback to journal extraction.
|
||
tracing::warn!("Backtest {}: no HTML report found, trying journal extraction", report_id);
|
||
if let Some(log) = Self::find_active_tester_agent_log(&config) {
|
||
if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await {
|
||
Self::update_job_status(&report_dir, "completed_no_html", None).await;
|
||
if let Some(ref callback) = notification_callback {
|
||
callback("backtest_completed", json!({
|
||
"report_id": report_id,
|
||
"status": "completed_no_html",
|
||
"reason": "extracted from tester journal (HTML report not produced)"
|
||
}));
|
||
}
|
||
return;
|
||
}
|
||
}
|
||
|
||
tracing::warn!("Backtest {} failed: MT5 exited without producing a report", report_id);
|
||
Self::update_job_status(&report_dir, "failed", None).await;
|
||
if let Some(ref callback) = notification_callback {
|
||
callback("backtest_failed", json!({
|
||
"report_id": report_id,
|
||
"status": "failed",
|
||
"reason": "MT5 exited without producing a report or recoverable journal"
|
||
}));
|
||
}
|
||
return;
|
||
}
|
||
|
||
if tokio::time::Instant::now() > deadline {
|
||
tracing::warn!("Backtest {} timed out after {} seconds", report_id, timeout_secs);
|
||
// Last-chance journal extraction on timeout
|
||
if let Some(log) = Self::find_active_tester_agent_log(&config) {
|
||
if Self::extract_from_journal(&log, &report_dir, &report_id, &config, ¶ms).await {
|
||
Self::update_job_status(&report_dir, "completed_no_html", None).await;
|
||
return;
|
||
}
|
||
}
|
||
Self::update_job_status(&report_dir, "timeout", None).await;
|
||
if let Some(ref callback) = notification_callback {
|
||
callback("backtest_timeout", json!({
|
||
"report_id": report_id,
|
||
"status": "timeout",
|
||
"timeout_seconds": timeout_secs
|
||
}));
|
||
}
|
||
return;
|
||
}
|
||
|
||
sleep(Duration::from_secs(2)).await;
|
||
}
|
||
}
|
||
|
||
/// Find the best tester agent log file for today.
|
||
///
|
||
/// Selection priority:
|
||
/// 1. Local agents (127.0.0.1) preferred over external/cloud agents (0.0.0.0)
|
||
/// — 0.0.0.0 logs only contain startup info, never actual deal lines.
|
||
/// 2. Among equal-priority agents, pick the **largest** file (most content).
|
||
pub fn find_active_tester_agent_log(config: &Config) -> Option<PathBuf> {
|
||
let mt5_dir = config.mt5_dir()?;
|
||
let tester_dir = mt5_dir.join("Tester");
|
||
let today = chrono::Utc::now().format("%Y%m%d").to_string();
|
||
|
||
// (priority, size, path) — higher priority = more preferred
|
||
// priority 1 = local (127.0.0.1), priority 0 = other (0.0.0.0 / any)
|
||
let mut best: Option<(u8, u64, PathBuf)> = None;
|
||
|
||
if let Ok(agents) = fs::read_dir(&tester_dir) {
|
||
for agent in agents.filter_map(|e| e.ok()) {
|
||
let agent_name = agent.file_name();
|
||
let agent_str = agent_name.to_string_lossy();
|
||
|
||
// Only consider Agent-* directories
|
||
if !agent_str.starts_with("Agent-") {
|
||
continue;
|
||
}
|
||
|
||
let logs_dir = agent.path().join("logs");
|
||
let candidate = logs_dir.join(format!("{}.log", today));
|
||
if !candidate.exists() {
|
||
continue;
|
||
}
|
||
|
||
let meta = match fs::metadata(&candidate) {
|
||
Ok(m) => m,
|
||
Err(_) => continue,
|
||
};
|
||
let size = meta.len();
|
||
|
||
// Prefer local agents (127.0.0.1) — they log actual deal execution
|
||
let priority: u8 = if agent_str.contains("127.0.0.1") { 1 } else { 0 };
|
||
|
||
let is_better = match &best {
|
||
None => true,
|
||
Some((bp, bs, _)) => priority > *bp || (priority == *bp && size > *bs),
|
||
};
|
||
|
||
if is_better {
|
||
best = Some((priority, size, candidate));
|
||
}
|
||
}
|
||
}
|
||
best.map(|(_, _, p)| p)
|
||
}
|
||
|
||
/// Read the most recent tester agent log and return its lines (UTF-16 or UTF-8).
|
||
pub fn read_tester_agent_log(log_path: &Path) -> Option<Vec<String>> {
|
||
let bytes = fs::read(log_path).ok()?;
|
||
let text = if bytes.len() >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE {
|
||
// UTF-16 LE with BOM
|
||
let words: Vec<u16> = bytes[2..].chunks_exact(2)
|
||
.map(|c| u16::from_le_bytes([c[0], c[1]]))
|
||
.collect();
|
||
String::from_utf16_lossy(&words).to_string()
|
||
} else {
|
||
String::from_utf8_lossy(&bytes).to_string()
|
||
};
|
||
Some(text.lines().map(|l| l.to_string()).collect())
|
||
}
|
||
|
||
/// Parse deal entries from a tester agent log.
|
||
/// Returns (deals_parsed, final_balance_pips, sim_progress_line).
|
||
///
|
||
/// Entry direction (in/out) is inferred via a per-symbol position tracker:
|
||
/// - No open position → "in"
|
||
/// - Same direction as existing position → "in" (grid/martingale add)
|
||
/// - Opposite direction → "out" (closing)
|
||
/// Profit/balance are unavailable in the journal; they remain 0.0.
|
||
pub fn parse_journal_deals(lines: &[String]) -> (Vec<crate::models::deals::Deal>, f64, String) {
|
||
use regex::Regex;
|
||
use std::collections::HashMap;
|
||
|
||
// Format: "... YYYY.MM.DD HH:MM:SS deal #N buy/sell VOLUME SYM at PRICE done ..."
|
||
let deal_re = Regex::new(
|
||
r"(\d{4}\.\d{2}\.\d{2} \d{2}:\d{2}:\d{2})\s+deal #(\d+) (buy|sell) ([\d.]+) (\S+) at ([\d.]+) done"
|
||
).unwrap();
|
||
let balance_re = Regex::new(r"final balance ([\d.]+) pips").unwrap();
|
||
let progress_re = Regex::new(r"Test passed in (.+)").unwrap();
|
||
|
||
let mut deals = Vec::new();
|
||
let mut final_balance = 0.0f64;
|
||
let mut progress_str = String::new();
|
||
|
||
// signed lots per symbol: positive = net long, negative = net short
|
||
let mut position: HashMap<String, f64> = HashMap::new();
|
||
// MT5 tester logs each deal TWICE (dual-agent logging) — deduplicate by deal number
|
||
let mut seen_deals: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||
|
||
for line in lines {
|
||
if let Some(cap) = deal_re.captures(line) {
|
||
let sim_time = cap[1].to_string();
|
||
let deal_num = cap[2].to_string();
|
||
let direction = cap[3].to_string(); // "buy" | "sell"
|
||
let volume: f64 = cap[4].parse().unwrap_or(0.0);
|
||
let symbol = cap[5].to_string();
|
||
let price: f64 = cap[6].parse().unwrap_or(0.0);
|
||
|
||
// Skip duplicate deal entries (MT5 writes each deal twice)
|
||
if !seen_deals.insert(deal_num.clone()) {
|
||
continue;
|
||
}
|
||
|
||
let signed = if direction == "buy" { volume } else { -volume };
|
||
let current = position.get(&symbol).copied().unwrap_or(0.0);
|
||
|
||
// Determine entry type by comparing new direction against open position
|
||
let entry_type = if current.abs() < 1e-9 {
|
||
// flat → opening a new position
|
||
"in"
|
||
} else if (current > 0.0 && direction == "buy")
|
||
|| (current < 0.0 && direction == "sell")
|
||
{
|
||
// same direction as existing → adding (grid/martingale)
|
||
"in"
|
||
} else {
|
||
// opposite direction → closing / partial close
|
||
"out"
|
||
};
|
||
|
||
// Update tracked position
|
||
let new_pos = current + signed;
|
||
if new_pos.abs() < 1e-9 {
|
||
position.remove(&symbol);
|
||
} else {
|
||
position.insert(symbol.clone(), new_pos);
|
||
}
|
||
|
||
deals.push(crate::models::deals::Deal {
|
||
time: sim_time,
|
||
deal: deal_num,
|
||
symbol,
|
||
deal_type: direction,
|
||
entry: entry_type.to_string(),
|
||
volume,
|
||
price,
|
||
order: String::new(),
|
||
commission: 0.0,
|
||
swap: 0.0,
|
||
profit: 0.0, // not available in journal
|
||
balance: 0.0, // not available in journal
|
||
comment: String::new(),
|
||
magic: None,
|
||
});
|
||
}
|
||
if let Some(cap) = balance_re.captures(line) {
|
||
final_balance = cap[1].parse().unwrap_or(0.0);
|
||
}
|
||
if let Some(cap) = progress_re.captures(line) {
|
||
progress_str = cap[1].to_string();
|
||
}
|
||
}
|
||
|
||
(deals, final_balance, progress_str)
|
||
}
|
||
|
||
/// Fallback: extract deals from the tester agent journal log when no HTML report exists.
|
||
/// Stores partial deal data (no per-deal P&L) and records the final balance only.
|
||
async fn extract_from_journal(
|
||
log_path: &Path,
|
||
report_dir: &Path,
|
||
report_id: &str,
|
||
config: &Config,
|
||
params: &BacktestParams,
|
||
) -> bool {
|
||
let lines = match Self::read_tester_agent_log(log_path) {
|
||
Some(l) => l,
|
||
None => {
|
||
tracing::warn!("Journal extraction: could not read log {}", log_path.display());
|
||
return false;
|
||
}
|
||
};
|
||
|
||
let (deals, final_balance_pips, progress) = Self::parse_journal_deals(&lines);
|
||
if deals.is_empty() {
|
||
tracing::warn!("Journal extraction: no deals found in {}", log_path.display());
|
||
return false;
|
||
}
|
||
|
||
tracing::info!(
|
||
"Journal extraction: {} deals, final balance {} pips, {}",
|
||
deals.len(), final_balance_pips, progress
|
||
);
|
||
|
||
// Save journal summary to report_dir
|
||
let summary_path = report_dir.join("journal_extraction.json");
|
||
let summary = json!({
|
||
"source": "tester_agent_log",
|
||
"log_path": log_path.to_string_lossy(),
|
||
"total_deals": deals.len(),
|
||
"final_balance_pips": final_balance_pips,
|
||
"progress": progress,
|
||
"note": "No HTML report was produced. Deals extracted from tester agent log. profit/balance fields are 0 (not available in log format)."
|
||
});
|
||
let _ = fs::write(&summary_path, serde_json::to_string_pretty(&summary).unwrap_or_default());
|
||
|
||
// Register in DB with partial metrics
|
||
let db = crate::storage::ReportDb::new(&Config::db_path());
|
||
if db.init().is_err() {
|
||
return false;
|
||
}
|
||
let entry = crate::storage::ReportEntry {
|
||
id: report_id.to_string(),
|
||
expert: params.expert.clone(),
|
||
symbol: params.symbol.clone(),
|
||
timeframe: params.timeframe.clone(),
|
||
model: params.model as i64,
|
||
from_date: params.from_date.clone(),
|
||
to_date: params.to_date.clone(),
|
||
created_at: chrono::Utc::now().to_rfc3339(),
|
||
set_file_original: params.set_file.clone(),
|
||
set_snapshot_path: None,
|
||
report_dir: report_dir.to_string_lossy().to_string(),
|
||
charts_dir: None,
|
||
net_profit: Some(final_balance_pips - params.deposit as f64),
|
||
profit_factor: None,
|
||
max_dd_pct: None,
|
||
sharpe_ratio: None,
|
||
total_trades: Some(deals.len() as i64 / 2), // open+close pairs
|
||
win_rate_pct: None,
|
||
recovery_factor: None,
|
||
deposit: Some(params.deposit as f64),
|
||
currency: config.backtest_currency.clone(),
|
||
leverage: Some(params.leverage as i64),
|
||
duration_seconds: None,
|
||
tags: vec!["journal-only".to_string()],
|
||
notes: Some(format!("Extracted from journal: {} deals, final balance {} pips. No HTML report.", deals.len(), final_balance_pips)),
|
||
verdict: None,
|
||
};
|
||
if db.insert(&entry).is_err() {
|
||
return false;
|
||
}
|
||
if let Err(e) = db.insert_deals(report_id, &deals) {
|
||
tracing::warn!("Journal extraction: failed to store deals: {}", e);
|
||
}
|
||
true
|
||
}
|
||
|
||
/// Update job status in job.json file.
|
||
async fn update_job_status(report_dir: &Path, status: &str, report_path: Option<String>) {
|
||
let job_path = report_dir.join("job.json");
|
||
if let Ok(job_json) = fs::read_to_string(&job_path) {
|
||
if let Ok(mut job) = serde_json::from_str::<serde_json::Value>(&job_json) {
|
||
job["status"] = serde_json::Value::String(status.to_string());
|
||
job["completed_at"] = serde_json::Value::String(chrono::Utc::now().to_rfc3339());
|
||
if let Some(path) = report_path {
|
||
job["actual_report_path"] = serde_json::Value::String(path);
|
||
}
|
||
if let Ok(updated) = serde_json::to_string_pretty(&job) {
|
||
let _ = fs::write(&job_path, updated);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Move equity chart images (*.png, *.gif) from MT5's reports dir to OS temp,
|
||
/// returning the temp path if any images were found.
|
||
async fn relocate_charts(&self, html_path: &Path, report_id: &str) -> Option<PathBuf> {
|
||
let reports_dir = html_path.parent()?;
|
||
let charts_dir = Config::charts_temp_dir(report_id);
|
||
let image_exts = ["png", "gif", "jpg", "jpeg"];
|
||
|
||
let entries = fs::read_dir(reports_dir).ok()?;
|
||
let mut found = false;
|
||
|
||
for entry in entries.filter_map(|e| e.ok()) {
|
||
let path = entry.path();
|
||
let name = path.file_name()?.to_string_lossy().to_string();
|
||
|
||
let is_chart = name.starts_with(report_id)
|
||
&& path
|
||
.extension()
|
||
.and_then(|e| e.to_str())
|
||
.map(|e| image_exts.contains(&e))
|
||
.unwrap_or(false);
|
||
|
||
if is_chart {
|
||
if !found {
|
||
if fs::create_dir_all(&charts_dir).is_err() {
|
||
return None;
|
||
}
|
||
}
|
||
let dest = charts_dir.join(entry.file_name());
|
||
let _ = fs::rename(&path, &dest);
|
||
found = true;
|
||
}
|
||
}
|
||
|
||
if found { Some(charts_dir) } else { None }
|
||
}
|
||
|
||
/// Copy the set file into the report dir as set_snapshot.set.
|
||
async fn snapshot_set_file(
|
||
&self,
|
||
params: &BacktestParams,
|
||
report_dir: &Path,
|
||
) -> Option<PathBuf> {
|
||
let set_src = params.set_file.as_ref()?;
|
||
let src_path = Path::new(set_src);
|
||
if !src_path.exists() {
|
||
return None;
|
||
}
|
||
let dest = report_dir.join("set_snapshot.set");
|
||
fs::copy(src_path, &dest).ok()?;
|
||
Some(dest)
|
||
}
|
||
|
||
async fn register_in_db(
|
||
&self,
|
||
report_id: &str,
|
||
params: &BacktestParams,
|
||
report_dir: &Path,
|
||
charts_dir: Option<&Path>,
|
||
set_snapshot: Option<&Path>,
|
||
metrics: &crate::models::metrics::Metrics,
|
||
duration: i64,
|
||
) -> Option<ReportDb> {
|
||
let db = ReportDb::new(&Config::db_path());
|
||
if let Err(e) = db.init() {
|
||
tracing::warn!("Failed to init report DB: {}", e);
|
||
return None;
|
||
}
|
||
|
||
let entry = ReportEntry {
|
||
id: report_id.to_string(),
|
||
expert: params.expert.clone(),
|
||
symbol: params.symbol.clone(),
|
||
timeframe: params.timeframe.clone(),
|
||
model: params.model as i64,
|
||
from_date: params.from_date.clone(),
|
||
to_date: params.to_date.clone(),
|
||
created_at: chrono::Utc::now().to_rfc3339(),
|
||
set_file_original: params.set_file.clone(),
|
||
set_snapshot_path: set_snapshot.map(|p| p.to_string_lossy().to_string()),
|
||
report_dir: report_dir.to_string_lossy().to_string(),
|
||
charts_dir: charts_dir.map(|p| p.to_string_lossy().to_string()),
|
||
net_profit: Some(metrics.net_profit),
|
||
profit_factor: Some(metrics.profit_factor),
|
||
max_dd_pct: Some(metrics.max_dd_pct),
|
||
sharpe_ratio: Some(metrics.sharpe_ratio),
|
||
total_trades: Some(metrics.total_trades as i64),
|
||
win_rate_pct: Some(metrics.win_rate_pct),
|
||
recovery_factor: Some(metrics.recovery_factor),
|
||
deposit: Some(params.deposit as f64),
|
||
currency: self.config.backtest_currency.clone(),
|
||
leverage: Some(params.leverage as i64),
|
||
duration_seconds: Some(duration),
|
||
tags: Vec::new(),
|
||
notes: None,
|
||
verdict: None,
|
||
};
|
||
|
||
if let Err(e) = db.insert(&entry) {
|
||
tracing::warn!("Failed to register report in DB: {}", e);
|
||
return None;
|
||
}
|
||
|
||
Some(db)
|
||
}
|
||
|
||
async fn compile_ea(&self, expert: &str, timeout_secs: u64) -> Result<()> {
|
||
let mut search_paths = vec![
|
||
PathBuf::from(&self.config.get("project_dir")).join("src/experts").join(format!("{}.mq5", expert)),
|
||
PathBuf::from(&self.config.get("project_dir")).join("src").join(format!("{}.mq5", expert)),
|
||
PathBuf::from(&self.config.get("project_dir")).join(format!("{}.mq5", expert)),
|
||
PathBuf::from("src/experts").join(format!("{}.mq5", expert)),
|
||
PathBuf::from("src").join(format!("{}.mq5", expert)),
|
||
PathBuf::from(format!("{}.mq5", expert)),
|
||
];
|
||
// Also search in MT5 Experts dir: Experts/{expert}/{expert}.mq5 and Experts/{expert}.mq5
|
||
if let Some(experts_dir) = &self.config.experts_dir {
|
||
search_paths.push(PathBuf::from(experts_dir).join(expert).join(format!("{}.mq5", expert)));
|
||
search_paths.push(PathBuf::from(experts_dir).join(format!("{}.mq5", expert)));
|
||
}
|
||
|
||
let source_path = search_paths
|
||
.into_iter()
|
||
.find(|p| p.exists())
|
||
.ok_or_else(|| anyhow!("Cannot find {}.mq5 — searched project_dir and MT5 Experts dir", expert))?;
|
||
|
||
let timeout = std::time::Duration::from_secs(timeout_secs.min(300)); // Max 5 min for compile
|
||
let result = self.compiler.compile_with_timeout(&source_path.to_string_lossy(), timeout).await?;
|
||
|
||
if !result.success {
|
||
return Err(anyhow!(
|
||
"Compilation failed: {}",
|
||
result.errors.join("; ")
|
||
));
|
||
}
|
||
|
||
Ok(())
|
||
}
|
||
|
||
async fn clean_cache(&self, expert: &str) -> Result<()> {
|
||
if let Some(cache_dir) = &self.config.tester_cache_dir {
|
||
let cache_path = Path::new(cache_dir);
|
||
if cache_path.exists() {
|
||
for entry in walkdir::WalkDir::new(cache_path) {
|
||
if let Ok(entry) = entry {
|
||
let path = entry.path();
|
||
if path.extension().map(|e| e == "tst").unwrap_or(false) {
|
||
let _ = fs::remove_file(path);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if let Some(tester_dir) = &self.config.tester_profiles_dir {
|
||
let cached_set = Path::new(tester_dir).join(format!("{}.set", expert));
|
||
if cached_set.exists() {
|
||
let _ = fs::remove_file(&cached_set);
|
||
}
|
||
}
|
||
|
||
self.reset_terminal_ini().await?;
|
||
|
||
Ok(())
|
||
}
|
||
|
||
async fn reset_terminal_ini(&self) -> Result<()> {
|
||
let mt5_dir = self.config.mt5_dir()
|
||
.ok_or_else(|| anyhow!("MT5 directory not configured"))?;
|
||
|
||
let terminal_ini = mt5_dir.join("config").join("terminal.ini");
|
||
if !terminal_ini.exists() {
|
||
return Ok(());
|
||
}
|
||
|
||
let content = fs::read(&terminal_ini)?;
|
||
|
||
let (text, encoding) = if content.starts_with(&[0xFF, 0xFE]) || content.starts_with(&[0xFE, 0xFF]) {
|
||
let text = String::from_utf16_lossy(
|
||
content.chunks_exact(2)
|
||
.map(|c| u16::from_le_bytes([c[0], c[1]]))
|
||
.collect::<Vec<_>>()
|
||
.as_slice()
|
||
);
|
||
(text, "utf-16")
|
||
} else {
|
||
(String::from_utf8_lossy(&content).to_string(), "utf-8")
|
||
};
|
||
|
||
let updated = text
|
||
.replace("OptMode=-1", "OptMode=0")
|
||
.replace("LastOptimization=1", "");
|
||
|
||
let output = if encoding == "utf-16" {
|
||
let utf16: Vec<u16> = updated.encode_utf16().collect();
|
||
let bytes: Vec<u8> = utf16.iter()
|
||
.flat_map(|&c| c.to_le_bytes())
|
||
.collect();
|
||
bytes
|
||
} else {
|
||
updated.into_bytes()
|
||
};
|
||
|
||
fs::write(&terminal_ini, output)?;
|
||
|
||
Ok(())
|
||
}
|
||
|
||
async fn run_backtest(&self, params: &BacktestParams, report_id: &str) -> Result<PathBuf> {
|
||
let mt5_dir = self.config.mt5_dir()
|
||
.ok_or_else(|| anyhow!("MT5 directory not configured"))?;
|
||
|
||
let wine_exe = self.config.wine_executable.as_ref()
|
||
.ok_or_else(|| anyhow!("wine_executable not configured"))?;
|
||
|
||
// mt5_dir = {prefix}/drive_c/Program Files/MetaTrader 5
|
||
// WINEPREFIX = three levels up
|
||
let wine_prefix = mt5_dir
|
||
.parent() // .../drive_c/Program Files
|
||
.and_then(|p| p.parent()) // .../drive_c
|
||
.and_then(|p| p.parent()) // .../<prefix>
|
||
.map(|p| p.to_path_buf())
|
||
.ok_or_else(|| anyhow!("Could not determine Wine prefix from terminal_dir"))?;
|
||
|
||
let reports_dir = mt5_dir.join("reports");
|
||
fs::create_dir_all(&reports_dir)?;
|
||
|
||
// Kill first — MT5 writes terminal.ini on exit, which would clobber
|
||
// the backtest params we're about to write.
|
||
self.kill_mt5().await?;
|
||
|
||
// Write params *after* MT5 is dead so nothing can overwrite them.
|
||
let ini_content = self.build_backtest_ini(params, report_id)?;
|
||
let config_host = wine_prefix.join("drive_c").join("backtest_config.ini");
|
||
fs::write(&config_host, ini_content.as_bytes())?;
|
||
self.update_terminal_ini(params, report_id)?;
|
||
|
||
// Record launch time before sleeping so find_newest_report doesn't miss
|
||
// reports written during the startup wait.
|
||
let poll_start = std::time::SystemTime::now();
|
||
let launch_instant = tokio::time::Instant::now();
|
||
|
||
// Build the launch command, adapting for the Wine runtime in use.
|
||
let mut cmd = self.build_wine_launch(wine_exe, &wine_prefix)?;
|
||
cmd.stdin(std::process::Stdio::null())
|
||
.stdout(std::process::Stdio::null())
|
||
.stderr(std::process::Stdio::null())
|
||
.spawn()?;
|
||
|
||
// Give MT5 time to fully initialize before polling.
|
||
// MT5 app startup (Wine init + network auth + tester) typically takes 10–15 s.
|
||
// Configurable via startup_delay_secs parameter (default 10s for faster launches).
|
||
let delay = if params.startup_delay_secs > 0 { params.startup_delay_secs } else { 10 };
|
||
sleep(Duration::from_secs(delay)).await;
|
||
|
||
// Poll for the report file (MT5 writes it when the backtest completes).
|
||
// Grace period: don't check process liveness for the first 30 s after launch —
|
||
// MT5 may still be appearing in the process list while wineserver re-initializes.
|
||
let grace_period = Duration::from_secs(30);
|
||
let deadline = launch_instant + Duration::from_secs(params.timeout);
|
||
|
||
loop {
|
||
let elapsed = launch_instant.elapsed().as_secs();
|
||
|
||
// 1. Check for the exact expected report filename.
|
||
for ext in &[".htm", ".htm.xml", ".html"] {
|
||
let candidate = reports_dir.join(format!("{}{}", report_id, ext));
|
||
tracing::debug!("poll t+{}s: checking {}", elapsed, candidate.display());
|
||
if candidate.exists() {
|
||
tracing::info!("poll t+{}s: found exact report {}", elapsed, candidate.display());
|
||
return Ok(candidate);
|
||
}
|
||
}
|
||
|
||
// 2. Only check process liveness after the grace period — this prevents
|
||
// a false "not running" when the new instance is still starting up.
|
||
let in_grace = launch_instant.elapsed() <= grace_period;
|
||
let mt5_alive = Self::is_mt5_running();
|
||
tracing::info!("poll t+{}s: in_grace={} mt5_alive={}", elapsed, in_grace, mt5_alive);
|
||
|
||
if !in_grace && !mt5_alive {
|
||
// MT5 writes the .htm report file right before exiting. There is a
|
||
// short window where the process is gone but the file hasn't been
|
||
// flushed to the directory. Wait 3 s to let Wine/macOS finish the
|
||
// write before we scan — this prevents false "no report" failures.
|
||
sleep(Duration::from_secs(3)).await;
|
||
if let Some(path) = Self::find_newest_report(&reports_dir, poll_start) {
|
||
tracing::info!("poll: MT5 exited, found report {}", path.display());
|
||
return Ok(path);
|
||
}
|
||
return Err(anyhow!(
|
||
"MT5 exited without producing a report. \
|
||
The backtest may have been stopped mid-way or failed to start."
|
||
));
|
||
}
|
||
|
||
if tokio::time::Instant::now() > deadline {
|
||
return Err(anyhow!("Timeout: no report after {} seconds", params.timeout));
|
||
}
|
||
sleep(Duration::from_secs(2)).await;
|
||
}
|
||
}
|
||
|
||
/// Write backtest params into terminal.ini [Tester] section.
|
||
/// MT5 uses this when restarting — it reconnects via the saved session in common.ini
|
||
/// rather than requiring fresh credentials. This is more reliable than /config: alone,
|
||
/// which requires a password for fresh authentication.
|
||
fn update_terminal_ini(&self, params: &BacktestParams, report_id: &str) -> Result<()> {
|
||
let mt5_dir = self.config.mt5_dir()
|
||
.ok_or_else(|| anyhow!("MT5 directory not configured"))?;
|
||
// Portable mode uses config/ inside the install dir; non-portable uses the root.
|
||
let terminal_ini = if mt5_dir.join("config").exists() {
|
||
mt5_dir.join("config").join("terminal.ini")
|
||
} else {
|
||
mt5_dir.join("terminal.ini")
|
||
};
|
||
|
||
let raw = fs::read(&terminal_ini).unwrap_or_default();
|
||
let text = if raw.starts_with(&[0xFF, 0xFE]) {
|
||
raw[2..].chunks_exact(2)
|
||
.map(|c| u16::from_le_bytes([c[0], c[1]]))
|
||
.collect::<Vec<_>>()
|
||
.iter()
|
||
.map(|&c| char::from_u32(c as u32).unwrap_or('?'))
|
||
.collect::<String>()
|
||
} else {
|
||
String::from_utf8_lossy(&raw).into_owned()
|
||
};
|
||
|
||
let period = match params.timeframe.as_str() {
|
||
"M1" => 1u32, "M5" => 5, "M15" => 15, "M30" => 30,
|
||
"H1" => 60, "H4" => 240, "D1" => 1440,
|
||
_ => 5,
|
||
};
|
||
let from_ts = Self::date_str_to_unix(¶ms.from_date)?;
|
||
let to_ts = Self::date_str_to_unix(¶ms.to_date)?;
|
||
let currency = self.config.backtest_currency.as_deref().unwrap_or("USD");
|
||
|
||
let expert_path = if let Some(experts_dir) = &self.config.experts_dir {
|
||
let nested = std::path::Path::new(experts_dir).join(¶ms.expert).join(format!("{}.mq5", params.expert));
|
||
if nested.exists() {
|
||
format!("Experts\\{}\\{}.ex5", params.expert, params.expert)
|
||
} else {
|
||
format!("Experts\\{}.ex5", params.expert)
|
||
}
|
||
} else {
|
||
format!("Experts\\{}.ex5", params.expert)
|
||
};
|
||
|
||
let set_file_line = params.set_file.as_ref()
|
||
.map(|p| format!("ExpertParameters={}\n", Self::ini_safe(p)))
|
||
.unwrap_or_default();
|
||
|
||
let updates: &[(&str, String)] = &[
|
||
("Expert", Self::ini_safe(&expert_path)),
|
||
("Symbol", Self::ini_safe(¶ms.symbol)),
|
||
("Period", period.to_string()),
|
||
("DateRange", "3".into()),
|
||
("DateFrom", from_ts.to_string()),
|
||
("DateTo", to_ts.to_string()),
|
||
("Visualization", "0".into()),
|
||
("Execution", "10".into()),
|
||
("Currency", currency.into()),
|
||
("Leverage", params.leverage.to_string()),
|
||
("Deposit", format!("{:.2}", params.deposit)),
|
||
("TicksMode", params.model.to_string()),
|
||
("PipsCalculation", "1".into()),
|
||
("OptMode", "0".into()),
|
||
("Report", format!("reports\\{}.htm", report_id)),
|
||
("ReplaceReport", "1".into()),
|
||
("ShutdownTerminal", if params.shutdown { "1" } else { "0" }.into()),
|
||
];
|
||
|
||
let updated = Self::patch_ini_section(&text, "Tester", updates) + &set_file_line;
|
||
let bom_utf16: Vec<u8> = [0xFF, 0xFE].iter().copied()
|
||
.chain(updated.encode_utf16().flat_map(|c| c.to_le_bytes()))
|
||
.collect();
|
||
fs::write(&terminal_ini, bom_utf16)?;
|
||
tracing::info!("terminal.ini [Tester] updated → {}", terminal_ini.display());
|
||
Ok(())
|
||
}
|
||
|
||
/// Strip CR/LF from a user-supplied INI value to prevent newline injection.
|
||
fn ini_safe(value: &str) -> String {
|
||
value.replace(['\n', '\r'], "")
|
||
}
|
||
|
||
fn patch_ini_section(text: &str, section: &str, updates: &[(&str, String)]) -> String {
|
||
let section_header = format!("[{}]", section);
|
||
let mut result = String::with_capacity(text.len() + 256);
|
||
let mut in_section = false;
|
||
let mut pending: std::collections::HashMap<&str, &String> =
|
||
updates.iter().map(|(k, v)| (*k, v)).collect();
|
||
|
||
for line in text.lines() {
|
||
let trimmed = line.trim();
|
||
if trimmed == section_header {
|
||
in_section = true;
|
||
result.push_str(line);
|
||
result.push('\n');
|
||
continue;
|
||
}
|
||
if trimmed.starts_with('[') && in_section {
|
||
for (k, v) in &pending {
|
||
result.push_str(&format!("{}={}\n", k, v));
|
||
}
|
||
pending.clear();
|
||
in_section = false;
|
||
}
|
||
if in_section {
|
||
if let Some((key, _)) = trimmed.split_once('=') {
|
||
let key = key.trim();
|
||
if let Some(val) = pending.remove(key) {
|
||
result.push_str(&format!("{}={}\n", key, val));
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
result.push_str(line);
|
||
result.push('\n');
|
||
}
|
||
if in_section {
|
||
for (k, v) in &pending {
|
||
result.push_str(&format!("{}={}\n", k, v));
|
||
}
|
||
}
|
||
result
|
||
}
|
||
|
||
fn date_str_to_unix(date: &str) -> Result<i64> {
|
||
let parts: Vec<u32> = date.split('.').filter_map(|p| p.parse().ok()).collect();
|
||
if parts.len() != 3 {
|
||
return Err(anyhow!("Invalid date format: {}", date));
|
||
}
|
||
let dt = chrono::NaiveDate::from_ymd_opt(parts[0] as i32, parts[1], parts[2])
|
||
.ok_or_else(|| anyhow!("Invalid date: {}", date))?
|
||
.and_hms_opt(0, 0, 0)
|
||
.ok_or_else(|| anyhow!("Date conversion failed"))?;
|
||
Ok(chrono::DateTime::<chrono::Utc>::from_naive_utc_and_offset(dt, chrono::Utc).timestamp())
|
||
}
|
||
|
||
/// Build the OS-appropriate command to launch MT5 with the backtest config.
|
||
///
|
||
/// - macOS MT5.app bundle: use shell script to bypass SIP and set DYLD vars.
|
||
/// Relies on terminal.ini for backtest config (more reliable than /config:).
|
||
/// - macOS CrossOver / Linux Wine: standard WINEPREFIX + wine64 direct invocation.
|
||
/// Also relies on terminal.ini for backtest config.
|
||
fn build_wine_launch(&self, wine_exe: &str, wine_prefix: &Path) -> Result<Command> {
|
||
if wine_exe.contains("MetaTrader 5.app") {
|
||
// macOS MT5.app — the Swift launcher ignores --args so we can't pass
|
||
// /config: via `open`. Instead, write a temp shell script that sets
|
||
// DYLD_FALLBACK_LIBRARY_PATH and invokes wine64 directly.
|
||
// Shell scripts bypass the SIP restriction that strips DYLD_* vars
|
||
// when Rust spawns a codesigned binary as a direct child process.
|
||
// NOTE: We rely on terminal.ini for config instead of /config: because
|
||
// MT5.app's bundled wine64 doesn't reliably handle /config: arguments.
|
||
let wine_bin = Path::new(wine_exe);
|
||
let wine_root = wine_bin
|
||
.parent() // bin/
|
||
.and_then(|p| p.parent()) // wine/
|
||
.map(|p| p.to_path_buf())
|
||
.ok_or_else(|| anyhow!("Cannot derive Wine root from wine_exe"))?;
|
||
|
||
let ext_libs = wine_root.join("lib").join("external");
|
||
let wine_libs = wine_root.join("lib");
|
||
let dyld = format!("{}:{}:/usr/lib:/usr/local/lib",
|
||
ext_libs.display(), wine_libs.display());
|
||
|
||
// Use host path for the exe; use /config: with backslash-escaped path
|
||
let terminal_host = wine_prefix.join("drive_c")
|
||
.join("Program Files").join("MetaTrader 5").join("terminal64.exe");
|
||
|
||
// /config: triggers the Strategy Tester to auto-start.
|
||
// terminal.ini is also patched with the same params as a belt-and-suspenders.
|
||
let config_win = r"C:\backtest_config.ini";
|
||
let script = format!(
|
||
"#!/bin/sh\n\
|
||
export DYLD_FALLBACK_LIBRARY_PATH='{dyld}'\n\
|
||
export WINEPREFIX='{prefix}'\n\
|
||
export WINEDEBUG='-all'\n\
|
||
nohup '{wine}' '{terminal}' '/config:{config}' \
|
||
>/dev/null 2>&1 &\n",
|
||
dyld = dyld,
|
||
prefix = wine_prefix.display(),
|
||
wine = wine_exe,
|
||
terminal = terminal_host.display(),
|
||
config = config_win,
|
||
);
|
||
|
||
let script_path = std::env::temp_dir().join("mt5_backtest_launch.sh");
|
||
|
||
// Always rewrite: script content changes per backtest (DYLD paths are
|
||
// dynamic) and we must ensure +x permissions are set every time.
|
||
fs::write(&script_path, &script)?;
|
||
#[cfg(unix)]
|
||
{
|
||
use std::os::unix::fs::PermissionsExt;
|
||
fs::set_permissions(&script_path, fs::Permissions::from_mode(0o755))?;
|
||
}
|
||
tracing::debug!("Wrote launch script: {}", script_path.display());
|
||
|
||
tracing::info!("Launching MT5 via shell script (terminal.ini mode): {}", script_path.display());
|
||
let mut cmd = Command::new("/bin/sh");
|
||
cmd.arg(&script_path);
|
||
return Ok(cmd);
|
||
}
|
||
|
||
// CrossOver / Linux: invoke wine64 directly with /config: to trigger the tester.
|
||
let terminal_win_path = r"C:\Program Files\MetaTrader 5\terminal64.exe";
|
||
let config_win = r"C:\backtest_config.ini";
|
||
let mut cmd = Command::new(wine_exe);
|
||
cmd.arg(terminal_win_path)
|
||
.arg(format!("/config:{}", config_win))
|
||
.env("WINEPREFIX", wine_prefix)
|
||
.env("WINEDEBUG", "-all");
|
||
Ok(cmd)
|
||
}
|
||
|
||
/// For /config: INI: path relative to MQL5/Experts/ (e.g. `DPS21\DPS21.ex5`).
|
||
/// The /config: format does NOT include the "Experts\" prefix.
|
||
fn resolve_backtest_ini_expert_path(&self, expert: &str) -> String {
|
||
if let Some(experts_dir) = &self.config.experts_dir {
|
||
let nested_ex5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.ex5", expert));
|
||
let nested_mq5 = PathBuf::from(experts_dir).join(expert).join(format!("{}.mq5", expert));
|
||
if nested_ex5.exists() || nested_mq5.exists() {
|
||
return format!("{}\\{}.ex5", expert, expert);
|
||
}
|
||
}
|
||
format!("{}.ex5", expert)
|
||
}
|
||
|
||
fn build_backtest_ini(&self, params: &BacktestParams, report_id: &str) -> Result<String> {
|
||
let mut ini = String::new();
|
||
|
||
// [Common] section: only written when explicit credentials are configured.
|
||
// Without it, MT5 reuses its saved session via common.ini (no password needed).
|
||
if let Some(login) = &self.config.backtest_login {
|
||
if let Some(server) = &self.config.backtest_server {
|
||
ini.push_str("[Common]\n");
|
||
ini.push_str(&format!("Login={}\n", login));
|
||
ini.push_str(&format!("Server={}\n", server));
|
||
if let Some(password) = &self.config.backtest_password {
|
||
ini.push_str(&format!("Password={}\n", password));
|
||
}
|
||
ini.push_str("\n");
|
||
}
|
||
}
|
||
|
||
ini.push_str("[Tester]\n");
|
||
// Expert path is relative to MQL5/Experts/ in the /config: format (no "Experts\" prefix).
|
||
ini.push_str(&format!("Expert={}\n", Self::ini_safe(&self.resolve_backtest_ini_expert_path(¶ms.expert))));
|
||
ini.push_str(&format!("Symbol={}\n", Self::ini_safe(¶ms.symbol)));
|
||
ini.push_str(&format!("Period={}\n", Self::ini_safe(¶ms.timeframe)));
|
||
ini.push_str("Optimization=0\n");
|
||
ini.push_str(&format!("Model={}\n", params.model));
|
||
ini.push_str(&format!("FromDate={}\n", params.from_date));
|
||
ini.push_str(&format!("ToDate={}\n", params.to_date));
|
||
ini.push_str("ForwardMode=0\n");
|
||
ini.push_str(&format!("Deposit={}\n", params.deposit));
|
||
ini.push_str(&format!("Currency={}\n", self.config.backtest_currency.as_ref().unwrap_or(&"USD".to_string())));
|
||
ini.push_str("ProfitInPips=1\n");
|
||
ini.push_str(&format!("Leverage={}\n", params.leverage));
|
||
ini.push_str("Execution=10\n");
|
||
ini.push_str(&format!("Visual={}\n", if params.gui { "1" } else { "0" }));
|
||
ini.push_str(&format!("Report=reports\\{}.htm\n", report_id));
|
||
ini.push_str("ReplaceReport=1\n");
|
||
ini.push_str(&format!("ShutdownTerminal={}\n", if params.shutdown { "1" } else { "0" }));
|
||
|
||
if let Some(set_file) = ¶ms.set_file {
|
||
ini.push_str(&format!("ExpertParameters={}\n", Self::ini_safe(set_file)));
|
||
}
|
||
|
||
Ok(ini)
|
||
}
|
||
|
||
async fn kill_mt5(&self) -> Result<()> {
|
||
let patterns = Self::mt5_process_patterns();
|
||
|
||
let running = patterns.iter().any(|pat| {
|
||
Command::new("pgrep")
|
||
.args(["-f", pat.as_str()])
|
||
.output()
|
||
.map(|o| o.status.success())
|
||
.unwrap_or(false)
|
||
});
|
||
|
||
if !running {
|
||
return Ok(());
|
||
}
|
||
|
||
tracing::info!("Stopping existing MT5 instance...");
|
||
// SIGKILL immediately — MT5 holds no state we care about preserving.
|
||
for pat in &patterns {
|
||
let _ = Command::new("pkill").args(["-KILL", "-f", pat.as_str()]).output();
|
||
}
|
||
// Also kill wineserver so the Wine prefix is fully reset before relaunch.
|
||
// If wineserver is still alive when the new MT5 spawns, the new Wine
|
||
// instance may attach to the dying server and never enter tester mode.
|
||
let _ = Command::new("pkill").args(["-KILL", "-f", "wineserver"]).output();
|
||
|
||
// Poll until wineserver is actually gone (max 10 s) rather than sleeping
|
||
// a fixed amount. On macOS wineserver cleanup varies from 1 s to 6+ s.
|
||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||
loop {
|
||
sleep(Duration::from_millis(500)).await;
|
||
let ws_alive = Command::new("pgrep")
|
||
.args(["-f", "wineserver"])
|
||
.output()
|
||
.map(|o| o.status.success())
|
||
.unwrap_or(false);
|
||
let mt5_alive = patterns.iter().any(|pat| {
|
||
Command::new("pgrep")
|
||
.args(["-f", pat.as_str()])
|
||
.output()
|
||
.map(|o| o.status.success())
|
||
.unwrap_or(false)
|
||
});
|
||
if !ws_alive && !mt5_alive {
|
||
tracing::info!("MT5 and wineserver fully exited");
|
||
break;
|
||
}
|
||
if tokio::time::Instant::now() >= deadline {
|
||
tracing::warn!("wineserver still alive after 10 s — proceeding anyway");
|
||
break;
|
||
}
|
||
}
|
||
// Brief extra pause to let the kernel release sockets and shared memory.
|
||
sleep(Duration::from_millis(500)).await;
|
||
|
||
Ok(())
|
||
}
|
||
|
||
fn is_mt5_running() -> bool {
|
||
Self::mt5_process_patterns().iter().any(|pat| {
|
||
Command::new("pgrep")
|
||
.args(["-f", pat.as_str()])
|
||
.output()
|
||
.map(|o| o.status.success())
|
||
.unwrap_or(false)
|
||
})
|
||
}
|
||
|
||
/// Scan `dir` for the newest .htm/.htm.xml/.html file written after `since`.
|
||
fn find_newest_report(dir: &Path, since: std::time::SystemTime) -> Option<PathBuf> {
|
||
let entries = fs::read_dir(dir).ok()?;
|
||
let mut candidates: Vec<(std::time::SystemTime, PathBuf)> = entries
|
||
.filter_map(|e| e.ok())
|
||
.filter(|e| {
|
||
let ext = e.path().extension()
|
||
.and_then(|x| x.to_str())
|
||
.unwrap_or("")
|
||
.to_lowercase();
|
||
matches!(ext.as_str(), "htm" | "xml" | "html")
|
||
})
|
||
.filter_map(|e| {
|
||
let mtime = e.metadata().ok()?.modified().ok()?;
|
||
if mtime >= since { Some((mtime, e.path())) } else { None }
|
||
})
|
||
.collect();
|
||
|
||
candidates.sort_by_key(|(t, _)| *t);
|
||
candidates.into_iter().last().map(|(_, p)| p)
|
||
}
|
||
|
||
fn mt5_process_patterns() -> Vec<String> {
|
||
if cfg!(target_os = "macos") {
|
||
// macOS: official MT5.app bundle (contains its own Wine runtime)
|
||
// Also match Wine-hosted terminal64.exe for CrossOver installs
|
||
vec![
|
||
"MetaTrader 5\\.app".to_string(),
|
||
"terminal64\\.exe".to_string(),
|
||
]
|
||
} else {
|
||
// Linux: MT5 always runs as a Wine process
|
||
vec![
|
||
"terminal64\\.exe".to_string(),
|
||
"metatrader".to_string(),
|
||
]
|
||
}
|
||
}
|
||
|
||
async fn log_progress(&self, log_path: &Path, stage: &str) {
|
||
let timestamp = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ");
|
||
let line = format!("{} {}\n", stage, timestamp);
|
||
let _ = fs::write(log_path, line);
|
||
}
|
||
|
||
async fn save_metadata(&self, params: &BacktestParams, report_dir: &Path, duration: i64, no_trades: bool) -> Result<()> {
|
||
let metadata = PipelineMetadata {
|
||
expert: params.expert.clone(),
|
||
symbol: params.symbol.clone(),
|
||
timeframe: params.timeframe.clone(),
|
||
from_date: params.from_date.clone(),
|
||
to_date: params.to_date.clone(),
|
||
deposit: params.deposit as f64,
|
||
currency: self.config.backtest_currency.clone().unwrap_or_else(|| "USD".to_string()),
|
||
model: params.model as i32,
|
||
leverage: params.leverage as i32,
|
||
set_file: params.set_file.clone(),
|
||
report_dir: report_dir.to_string_lossy().to_string(),
|
||
duration_seconds: duration,
|
||
files: FilePaths {
|
||
metrics: report_dir.join("metrics.json").to_string_lossy().to_string(),
|
||
analysis: report_dir.join("analysis.json").to_string_lossy().to_string(),
|
||
},
|
||
no_trades,
|
||
};
|
||
|
||
let json = serde_json::to_string_pretty(&metadata)?;
|
||
fs::write(report_dir.join("pipeline_metadata.json"), json)?;
|
||
|
||
Ok(())
|
||
}
|
||
|
||
fn generate_report_id(&self, params: &BacktestParams) -> String {
|
||
let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");
|
||
format!(
|
||
"{}_{}_{}_{}_{}",
|
||
timestamp, params.expert, params.symbol, params.timeframe, params.model
|
||
)
|
||
}
|
||
}
|