fix: remove stale deals.csv references, add extract+store to launch_backtest

- FilePaths struct: remove deals_csv/deals_json fields (deals go to DB only)
- save_metadata: stop writing stale CSV/JSON paths to pipeline_metadata.json
- get_backtest_status: remove deals.csv artifact check (was unused in logic)
- get_backtest_crash_info: check DB deal count and metrics.json instead of deals.csv
- export_report: remove unused _deals_path variable
- launch_backtest: monitor_backtest_completion now calls extract_and_store when
  report found — extracts metrics+deals, registers report in DB, deletes htm file

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/models/report.rs

@@ -14,8 +14,6 @@ pub struct Report {
     pub from_date: String,
     pub to_date: String,
     pub metrics_file: PathBuf,
-    pub deals_csv: PathBuf,
-    pub deals_json: PathBuf,
     pub analysis_file: Option<PathBuf>,
 }
 
@@ -42,8 +40,6 @@ pub struct PipelineMetadata {
 pub struct FilePaths {
     pub metrics: String,
     pub analysis: String,
-    pub deals_csv: String,
-    pub deals_json: String,
 }
 
 #[allow(dead_code)]

src/pipeline/backtest.rs

@@ -258,6 +258,27 @@ impl BacktestPipeline {
         let timeout_secs = params.timeout;
         let report_id_clone = report_id.clone();
         let notification_callback = self.notification_callback.clone();
+        let config_clone = self.config.clone();
+        let params_clone = BacktestParams {
+            expert: params.expert.clone(),
+            symbol: params.symbol.clone(),
+            from_date: params.from_date.clone(),
+            to_date: params.to_date.clone(),
+            timeframe: params.timeframe.clone(),
+            deposit: params.deposit,
+            model: params.model,
+            leverage: params.leverage,
+            set_file: params.set_file.clone(),
+            skip_compile: params.skip_compile,
+            skip_clean: params.skip_clean,
+            skip_analyze: params.skip_analyze,
+            deep_analyze: params.deep_analyze,
+            shutdown: params.shutdown,
+            kill_existing: params.kill_existing,
+            timeout: params.timeout,
+            gui: params.gui,
+            startup_delay_secs: params.startup_delay_secs,
+        };
         tokio::spawn(async move {
             Self::monitor_backtest_completion(
                 report_dir_clone,
@@ -265,12 +286,78 @@ impl BacktestPipeline {
                 timeout_secs,
                 report_id_clone,
                 notification_callback,
+                config_clone,
+                params_clone,
             ).await;
         });
 
         Ok(job)
     }
 
+    /// Extract deals from a completed report and store them in the DB.
+    async fn extract_and_store(
+        report_path: &Path,
+        report_dir: &Path,
+        report_id: &str,
+        config: &Config,
+        params: &BacktestParams,
+    ) {
+        let extractor = ReportExtractor::new();
+        let start_time = chrono::Utc::now();
+        match extractor.extract(
+            &report_path.to_string_lossy(),
+            &report_dir.to_string_lossy(),
+        ) {
+            Ok(extraction) => {
+                let duration = (chrono::Utc::now() - start_time).num_seconds();
+                let db = ReportDb::new(&Config::db_path());
+                if db.init().is_err() {
+                    tracing::warn!("launch_backtest: failed to init DB for {}", report_id);
+                    return;
+                }
+                let entry = ReportEntry {
+                    id: report_id.to_string(),
+                    expert: params.expert.clone(),
+                    symbol: params.symbol.clone(),
+                    timeframe: params.timeframe.clone(),
+                    model: params.model as i64,
+                    from_date: params.from_date.clone(),
+                    to_date: params.to_date.clone(),
+                    created_at: chrono::Utc::now().to_rfc3339(),
+                    set_file_original: params.set_file.clone(),
+                    set_snapshot_path: None,
+                    report_dir: report_dir.to_string_lossy().to_string(),
+                    charts_dir: None,
+                    net_profit: Some(extraction.metrics.net_profit),
+                    profit_factor: Some(extraction.metrics.profit_factor),
+                    max_dd_pct: Some(extraction.metrics.max_dd_pct),
+                    sharpe_ratio: Some(extraction.metrics.sharpe_ratio),
+                    total_trades: Some(extraction.metrics.total_trades as i64),
+                    win_rate_pct: Some(extraction.metrics.win_rate_pct),
+                    recovery_factor: Some(extraction.metrics.recovery_factor),
+                    deposit: Some(params.deposit as f64),
+                    currency: config.backtest_currency.clone(),
+                    leverage: Some(params.leverage as i64),
+                    duration_seconds: Some(duration),
+                    tags: Vec::new(),
+                    notes: None,
+                    verdict: None,
+                };
+                if let Err(e) = db.insert(&entry) {
+                    tracing::warn!("launch_backtest: failed to register report in DB: {}", e);
+                    return;
+                }
+                if let Err(e) = db.insert_deals(report_id, &extraction.deals) {
+                    tracing::warn!("launch_backtest: failed to store deals in DB: {}", e);
+                }
+                tracing::info!("launch_backtest: extracted {} deals for {}", extraction.deals.len(), report_id);
+            }
+            Err(e) => {
+                tracing::warn!("launch_backtest: extraction failed for {}: {}", report_id, e);
+            }
+        }
+    }
+
     /// Background task to monitor backtest completion and update status file.
     async fn monitor_backtest_completion(
         report_dir: PathBuf,
@@ -278,6 +365,8 @@ impl BacktestPipeline {
         timeout_secs: u64,
         report_id: String,
         notification_callback: Option<NotificationCallback>,
+        config: Config,
+        params: BacktestParams,
     ) {
         let start = tokio::time::Instant::now();
         let deadline = start + Duration::from_secs(timeout_secs);
@@ -292,6 +381,8 @@ impl BacktestPipeline {
                 let candidate = expected_report.with_extension(ext.trim_start_matches('.'));
                 if candidate.exists() {
                     tracing::info!("Backtest {} completed: report found at {}", report_id, candidate.display());
+                    Self::extract_and_store(&candidate, &report_dir, &report_id, &config, &params).await;
+                    let _ = fs::remove_file(&candidate);
                     Self::update_job_status(&report_dir, "completed", Some(candidate.to_string_lossy().to_string())).await;
                     if let Some(ref callback) = notification_callback {
                         callback("backtest_completed", json!({
@@ -312,6 +403,8 @@ impl BacktestPipeline {
                 // MT5 exited without report - check for any newer report
                 if let Some(path) = Self::find_newest_report(expected_report.parent().unwrap(), poll_start) {
                     tracing::info!("Backtest {} completed: found fallback report {}", report_id, path.display());
+                    Self::extract_and_store(&path, &report_dir, &report_id, &config, &params).await;
+                    let _ = fs::remove_file(&path);
                     Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await;
                     if let Some(ref callback) = notification_callback {
                         callback("backtest_completed", json!({
@@ -1047,8 +1140,6 @@ impl BacktestPipeline {
             files: FilePaths {
                 metrics: report_dir.join("metrics.json").to_string_lossy().to_string(),
                 analysis: report_dir.join("analysis.json").to_string_lossy().to_string(),
-                deals_csv: report_dir.join("deals.csv").to_string_lossy().to_string(),
-                deals_json: report_dir.join("deals.json").to_string_lossy().to_string(),
             },
             no_trades,
         };

src/tools/definitions/utility.rs

@@ -289,7 +289,7 @@ pub fn tool_get_wine_prefix_info() -> Value {
 pub fn tool_get_backtest_crash_info() -> Value {
     json!({
         "name": "get_backtest_crash_info",
-        "description": "Investigate backtest crashes/failures. Checks for incomplete markers, missing deals.csv, error logs. Can scan recent reports.",
+        "description": "Investigate backtest crashes/failures. Checks for incomplete markers, missing metrics.json, DB deal count, error logs. Can scan recent reports.",
         "inputSchema": {
             "type": "object",
             "properties": {

src/tools/handlers/backtest.rs

@@ -355,7 +355,6 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul
     
     // Check for completed artifacts
     let metrics_exists = report_path.join("metrics.json").exists();
-    let deals_exists = report_path.join("deals.csv").exists();
     let is_complete = stage == "DONE" || (report_found && metrics_exists);
     
     // Calculate elapsed time if job exists
@@ -400,7 +399,6 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul
             "mt5_running": mt5_running,
             "report_found": report_found,
             "metrics_extracted": metrics_exists,
-            "deals_extracted": deals_exists,
             "elapsed_seconds": elapsed_seconds,
             "message": message,
             "job": job.map(|j| {

src/tools/handlers/utility.rs

@@ -3,6 +3,7 @@ use serde_json::{json, Value};
 use std::fs;
 use std::path::{Path, PathBuf};
 use crate::models::Config;
+use crate::storage::ReportDb;
 
 /// Validate that `user_path` resolves to a location within `allowed_base`.
 /// Returns the canonicalized absolute path on success.
@@ -785,7 +786,6 @@ pub async fn handle_export_report(_config: &Config, args: &Value) -> Result<Valu
     
     let path = Path::new(report_dir);
     let metrics_path = path.join("metrics.json");
-    let _deals_path = path.join("deals.csv");
     
     // Read metrics
     let metrics: Value = if metrics_path.exists() {
@@ -1640,21 +1640,33 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
                 }
             }
             
-            // Check if deals.csv is missing or empty
+            // Check deal count in DB
-            let deals_csv = path.join("deals.csv");
+            let db = ReportDb::new(&Config::db_path());
-            if !deals_csv.exists() {
+            if db.init().is_ok() {
-                result["crashes_found"].as_array_mut().unwrap().push(json!({
+                match db.get_by_report_dir(dir) {
-                    "report_dir": dir,
+                    Ok(Some(entry)) => {
-                    "type": "missing_deals",
+                        let deal_count = db.get_deals(&entry.id)
-                    "reason": "deals.csv not found - backtest likely failed",
+                            .map(|d| d.len())
-                }));
+                            .unwrap_or(0);
-            } else if let Ok(meta) = deals_csv.metadata() {
+                        if deal_count == 0 {
-                if meta.len() < 100 {
+                            result["crashes_found"].as_array_mut().unwrap().push(json!({
-                    result["crashes_found"].as_array_mut().unwrap().push(json!({
+                                "report_dir": dir,
-                        "report_dir": dir,
+                                "type": "empty_deals",
-                        "type": "empty_deals",
+                                "reason": "No deals stored in DB - EA did not trade or extraction failed",
-                        "reason": "deals.csv is nearly empty - no trades were made",
+                            }));
-                    }));
+                        }
+                    }
+                    Ok(None) | Err(_) => {
+                        // Not in DB at all means extraction never completed
+                        let metrics_exists = path.join("metrics.json").exists();
+                        if !metrics_exists {
+                            result["crashes_found"].as_array_mut().unwrap().push(json!({
+                                "report_dir": dir,
+                                "type": "missing_deals",
+                                "reason": "Report not found in DB and no metrics.json - backtest likely crashed before extraction",
+                            }));
+                        }
+                    }
                 }
             }
         }
@@ -1676,10 +1688,10 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
                             if let Ok(modified) = meta.modified() {
                                 if modified >= cutoff {
                                     // Check for failure indicators
-                                    let has_deals = path.join("deals.csv").exists();
+                                    let has_metrics = path.join("metrics.json").exists();
                                     let has_incomplete = path.join(".incomplete").exists();
-                                    
+
-                                    if !has_deals || has_incomplete {
+                                    if !has_metrics || has_incomplete {
                                         failures += 1;
                                     }
                                 }
@@ -1701,7 +1713,7 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
         
         if types.contains(&"missing_deals".to_string()) {
             result["common_patterns"].as_array_mut().unwrap().push(
-                json!("Missing deals.csv suggests MT5 crashed during backtest")
+                json!("Report not in DB and no metrics.json suggests MT5 crashed before extraction completed")
             );
         }
         if types.contains(&"incomplete".to_string()) {