From 4bc5ca22e9efbaa7ed256e5bea2b0e68b4a24553 Mon Sep 17 00:00:00 2001 From: Devid HW Date: Thu, 23 Apr 2026 01:03:48 +0700 Subject: [PATCH] fix: remove stale deals.csv references, add extract+store to launch_backtest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - FilePaths struct: remove deals_csv/deals_json fields (deals go to DB only) - save_metadata: stop writing stale CSV/JSON paths to pipeline_metadata.json - get_backtest_status: remove deals.csv artifact check (was unused in logic) - get_backtest_crash_info: check DB deal count and metrics.json instead of deals.csv - export_report: remove unused _deals_path variable - launch_backtest: monitor_backtest_completion now calls extract_and_store when report found — extracts metrics+deals, registers report in DB, deletes htm file Co-Authored-By: Claude Sonnet 4.6 --- src/models/report.rs | 4 -- src/pipeline/backtest.rs | 95 +++++++++++++++++++++++++++++++- src/tools/definitions/utility.rs | 2 +- src/tools/handlers/backtest.rs | 2 - src/tools/handlers/utility.rs | 52 ++++++++++------- 5 files changed, 126 insertions(+), 29 deletions(-) diff --git a/src/models/report.rs b/src/models/report.rs index 79a12fb..669533b 100644 --- a/src/models/report.rs +++ b/src/models/report.rs @@ -14,8 +14,6 @@ pub struct Report { pub from_date: String, pub to_date: String, pub metrics_file: PathBuf, - pub deals_csv: PathBuf, - pub deals_json: PathBuf, pub analysis_file: Option, } @@ -42,8 +40,6 @@ pub struct PipelineMetadata { pub struct FilePaths { pub metrics: String, pub analysis: String, - pub deals_csv: String, - pub deals_json: String, } #[allow(dead_code)] diff --git a/src/pipeline/backtest.rs b/src/pipeline/backtest.rs index 2e77cf8..2ed42cb 100644 --- a/src/pipeline/backtest.rs +++ b/src/pipeline/backtest.rs @@ -258,6 +258,27 @@ impl BacktestPipeline { let timeout_secs = params.timeout; let report_id_clone = report_id.clone(); let notification_callback = self.notification_callback.clone(); + let config_clone = self.config.clone(); + let params_clone = BacktestParams { + expert: params.expert.clone(), + symbol: params.symbol.clone(), + from_date: params.from_date.clone(), + to_date: params.to_date.clone(), + timeframe: params.timeframe.clone(), + deposit: params.deposit, + model: params.model, + leverage: params.leverage, + set_file: params.set_file.clone(), + skip_compile: params.skip_compile, + skip_clean: params.skip_clean, + skip_analyze: params.skip_analyze, + deep_analyze: params.deep_analyze, + shutdown: params.shutdown, + kill_existing: params.kill_existing, + timeout: params.timeout, + gui: params.gui, + startup_delay_secs: params.startup_delay_secs, + }; tokio::spawn(async move { Self::monitor_backtest_completion( report_dir_clone, @@ -265,12 +286,78 @@ impl BacktestPipeline { timeout_secs, report_id_clone, notification_callback, + config_clone, + params_clone, ).await; }); Ok(job) } + /// Extract deals from a completed report and store them in the DB. + async fn extract_and_store( + report_path: &Path, + report_dir: &Path, + report_id: &str, + config: &Config, + params: &BacktestParams, + ) { + let extractor = ReportExtractor::new(); + let start_time = chrono::Utc::now(); + match extractor.extract( + &report_path.to_string_lossy(), + &report_dir.to_string_lossy(), + ) { + Ok(extraction) => { + let duration = (chrono::Utc::now() - start_time).num_seconds(); + let db = ReportDb::new(&Config::db_path()); + if db.init().is_err() { + tracing::warn!("launch_backtest: failed to init DB for {}", report_id); + return; + } + let entry = ReportEntry { + id: report_id.to_string(), + expert: params.expert.clone(), + symbol: params.symbol.clone(), + timeframe: params.timeframe.clone(), + model: params.model as i64, + from_date: params.from_date.clone(), + to_date: params.to_date.clone(), + created_at: chrono::Utc::now().to_rfc3339(), + set_file_original: params.set_file.clone(), + set_snapshot_path: None, + report_dir: report_dir.to_string_lossy().to_string(), + charts_dir: None, + net_profit: Some(extraction.metrics.net_profit), + profit_factor: Some(extraction.metrics.profit_factor), + max_dd_pct: Some(extraction.metrics.max_dd_pct), + sharpe_ratio: Some(extraction.metrics.sharpe_ratio), + total_trades: Some(extraction.metrics.total_trades as i64), + win_rate_pct: Some(extraction.metrics.win_rate_pct), + recovery_factor: Some(extraction.metrics.recovery_factor), + deposit: Some(params.deposit as f64), + currency: config.backtest_currency.clone(), + leverage: Some(params.leverage as i64), + duration_seconds: Some(duration), + tags: Vec::new(), + notes: None, + verdict: None, + }; + if let Err(e) = db.insert(&entry) { + tracing::warn!("launch_backtest: failed to register report in DB: {}", e); + return; + } + if let Err(e) = db.insert_deals(report_id, &extraction.deals) { + tracing::warn!("launch_backtest: failed to store deals in DB: {}", e); + } + tracing::info!("launch_backtest: extracted {} deals for {}", extraction.deals.len(), report_id); + } + Err(e) => { + tracing::warn!("launch_backtest: extraction failed for {}: {}", report_id, e); + } + } + } + /// Background task to monitor backtest completion and update status file. async fn monitor_backtest_completion( report_dir: PathBuf, @@ -278,6 +365,8 @@ impl BacktestPipeline { timeout_secs: u64, report_id: String, notification_callback: Option, + config: Config, + params: BacktestParams, ) { let start = tokio::time::Instant::now(); let deadline = start + Duration::from_secs(timeout_secs); @@ -292,6 +381,8 @@ impl BacktestPipeline { let candidate = expected_report.with_extension(ext.trim_start_matches('.')); if candidate.exists() { tracing::info!("Backtest {} completed: report found at {}", report_id, candidate.display()); + Self::extract_and_store(&candidate, &report_dir, &report_id, &config, ¶ms).await; + let _ = fs::remove_file(&candidate); Self::update_job_status(&report_dir, "completed", Some(candidate.to_string_lossy().to_string())).await; if let Some(ref callback) = notification_callback { callback("backtest_completed", json!({ @@ -312,6 +403,8 @@ impl BacktestPipeline { // MT5 exited without report - check for any newer report if let Some(path) = Self::find_newest_report(expected_report.parent().unwrap(), poll_start) { tracing::info!("Backtest {} completed: found fallback report {}", report_id, path.display()); + Self::extract_and_store(&path, &report_dir, &report_id, &config, ¶ms).await; + let _ = fs::remove_file(&path); Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await; if let Some(ref callback) = notification_callback { callback("backtest_completed", json!({ @@ -1047,8 +1140,6 @@ impl BacktestPipeline { files: FilePaths { metrics: report_dir.join("metrics.json").to_string_lossy().to_string(), analysis: report_dir.join("analysis.json").to_string_lossy().to_string(), - deals_csv: report_dir.join("deals.csv").to_string_lossy().to_string(), - deals_json: report_dir.join("deals.json").to_string_lossy().to_string(), }, no_trades, }; diff --git a/src/tools/definitions/utility.rs b/src/tools/definitions/utility.rs index e1882af..3e64c28 100644 --- a/src/tools/definitions/utility.rs +++ b/src/tools/definitions/utility.rs @@ -289,7 +289,7 @@ pub fn tool_get_wine_prefix_info() -> Value { pub fn tool_get_backtest_crash_info() -> Value { json!({ "name": "get_backtest_crash_info", - "description": "Investigate backtest crashes/failures. Checks for incomplete markers, missing deals.csv, error logs. Can scan recent reports.", + "description": "Investigate backtest crashes/failures. Checks for incomplete markers, missing metrics.json, DB deal count, error logs. Can scan recent reports.", "inputSchema": { "type": "object", "properties": { diff --git a/src/tools/handlers/backtest.rs b/src/tools/handlers/backtest.rs index ac13965..e4390bb 100644 --- a/src/tools/handlers/backtest.rs +++ b/src/tools/handlers/backtest.rs @@ -355,7 +355,6 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul // Check for completed artifacts let metrics_exists = report_path.join("metrics.json").exists(); - let deals_exists = report_path.join("deals.csv").exists(); let is_complete = stage == "DONE" || (report_found && metrics_exists); // Calculate elapsed time if job exists @@ -400,7 +399,6 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul "mt5_running": mt5_running, "report_found": report_found, "metrics_extracted": metrics_exists, - "deals_extracted": deals_exists, "elapsed_seconds": elapsed_seconds, "message": message, "job": job.map(|j| { diff --git a/src/tools/handlers/utility.rs b/src/tools/handlers/utility.rs index 79fc62a..7af49ce 100644 --- a/src/tools/handlers/utility.rs +++ b/src/tools/handlers/utility.rs @@ -3,6 +3,7 @@ use serde_json::{json, Value}; use std::fs; use std::path::{Path, PathBuf}; use crate::models::Config; +use crate::storage::ReportDb; /// Validate that `user_path` resolves to a location within `allowed_base`. /// Returns the canonicalized absolute path on success. @@ -785,7 +786,6 @@ pub async fn handle_export_report(_config: &Config, args: &Value) -> Result Re } } - // Check if deals.csv is missing or empty - let deals_csv = path.join("deals.csv"); - if !deals_csv.exists() { - result["crashes_found"].as_array_mut().unwrap().push(json!({ - "report_dir": dir, - "type": "missing_deals", - "reason": "deals.csv not found - backtest likely failed", - })); - } else if let Ok(meta) = deals_csv.metadata() { - if meta.len() < 100 { - result["crashes_found"].as_array_mut().unwrap().push(json!({ - "report_dir": dir, - "type": "empty_deals", - "reason": "deals.csv is nearly empty - no trades were made", - })); + // Check deal count in DB + let db = ReportDb::new(&Config::db_path()); + if db.init().is_ok() { + match db.get_by_report_dir(dir) { + Ok(Some(entry)) => { + let deal_count = db.get_deals(&entry.id) + .map(|d| d.len()) + .unwrap_or(0); + if deal_count == 0 { + result["crashes_found"].as_array_mut().unwrap().push(json!({ + "report_dir": dir, + "type": "empty_deals", + "reason": "No deals stored in DB - EA did not trade or extraction failed", + })); + } + } + Ok(None) | Err(_) => { + // Not in DB at all means extraction never completed + let metrics_exists = path.join("metrics.json").exists(); + if !metrics_exists { + result["crashes_found"].as_array_mut().unwrap().push(json!({ + "report_dir": dir, + "type": "missing_deals", + "reason": "Report not found in DB and no metrics.json - backtest likely crashed before extraction", + })); + } + } } } } @@ -1676,10 +1688,10 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re if let Ok(modified) = meta.modified() { if modified >= cutoff { // Check for failure indicators - let has_deals = path.join("deals.csv").exists(); + let has_metrics = path.join("metrics.json").exists(); let has_incomplete = path.join(".incomplete").exists(); - - if !has_deals || has_incomplete { + + if !has_metrics || has_incomplete { failures += 1; } } @@ -1701,7 +1713,7 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re if types.contains(&"missing_deals".to_string()) { result["common_patterns"].as_array_mut().unwrap().push( - json!("Missing deals.csv suggests MT5 crashed during backtest") + json!("Report not in DB and no metrics.json suggests MT5 crashed before extraction completed") ); } if types.contains(&"incomplete".to_string()) {