gavindiaz/mt5-quant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: remove stale deals.csv references, add extract+store to launch_backtest

Browse Source 
- FilePaths struct: remove deals_csv/deals_json fields (deals go to DB only)
- save_metadata: stop writing stale CSV/JSON paths to pipeline_metadata.json
- get_backtest_status: remove deals.csv artifact check (was unused in logic)
- get_backtest_crash_info: check DB deal count and metrics.json instead of deals.csv
- export_report: remove unused _deals_path variable
- launch_backtest: monitor_backtest_completion now calls extract_and_store when
  report found — extracts metrics+deals, registers report in DB, deletes htm file

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Devid HW
2026-04-23 01:03:48 +07:00
parent e44345a05f
commit 4bc5ca22e9
5 changed files with 126 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/report.rs
-4
View File
@@ -14,8 +14,6 @@ pub struct Report {
pub from_date: String,
pub to_date: String,
pub metrics_file: PathBuf,
pub deals_csv: PathBuf,
pub deals_json: PathBuf,
pub analysis_file: Option<PathBuf>,
}
@@ -42,8 +40,6 @@ pub struct PipelineMetadata {
pub struct FilePaths {
pub metrics: String,
pub analysis: String,
pub deals_csv: String,
pub deals_json: String,
}
#[allow(dead_code)]
src/pipeline/backtest.rs
+93 -2
View File
@@ -258,6 +258,27 @@ impl BacktestPipeline {
let timeout_secs = params.timeout;
let report_id_clone = report_id.clone();
let notification_callback = self.notification_callback.clone();
let config_clone = self.config.clone();
let params_clone = BacktestParams {
expert: params.expert.clone(),
symbol: params.symbol.clone(),
from_date: params.from_date.clone(),
to_date: params.to_date.clone(),
timeframe: params.timeframe.clone(),
deposit: params.deposit,
model: params.model,
leverage: params.leverage,
set_file: params.set_file.clone(),
skip_compile: params.skip_compile,
skip_clean: params.skip_clean,
skip_analyze: params.skip_analyze,
deep_analyze: params.deep_analyze,
shutdown: params.shutdown,
kill_existing: params.kill_existing,
timeout: params.timeout,
gui: params.gui,
startup_delay_secs: params.startup_delay_secs,
};
tokio::spawn(async move {
Self::monitor_backtest_completion(
report_dir_clone,
@@ -265,12 +286,78 @@ impl BacktestPipeline {
timeout_secs,
report_id_clone,
notification_callback,
config_clone,
params_clone,
).await;
});
Ok(job)
}
/// Extract deals from a completed report and store them in the DB.
async fn extract_and_store(
report_path: &Path,
report_dir: &Path,
report_id: &str,
config: &Config,
params: &BacktestParams,
) {
let extractor = ReportExtractor::new();
let start_time = chrono::Utc::now();
match extractor.extract(
&report_path.to_string_lossy(),
&report_dir.to_string_lossy(),
) {
Ok(extraction) => {
let duration = (chrono::Utc::now() - start_time).num_seconds();
let db = ReportDb::new(&Config::db_path());
if db.init().is_err() {
tracing::warn!("launch_backtest: failed to init DB for {}", report_id);
return;
}
let entry = ReportEntry {
id: report_id.to_string(),
expert: params.expert.clone(),
symbol: params.symbol.clone(),
timeframe: params.timeframe.clone(),
model: params.model as i64,
from_date: params.from_date.clone(),
to_date: params.to_date.clone(),
created_at: chrono::Utc::now().to_rfc3339(),
set_file_original: params.set_file.clone(),
set_snapshot_path: None,
report_dir: report_dir.to_string_lossy().to_string(),
charts_dir: None,
net_profit: Some(extraction.metrics.net_profit),
profit_factor: Some(extraction.metrics.profit_factor),
max_dd_pct: Some(extraction.metrics.max_dd_pct),
sharpe_ratio: Some(extraction.metrics.sharpe_ratio),
total_trades: Some(extraction.metrics.total_trades as i64),
win_rate_pct: Some(extraction.metrics.win_rate_pct),
recovery_factor: Some(extraction.metrics.recovery_factor),
deposit: Some(params.deposit as f64),
currency: config.backtest_currency.clone(),
leverage: Some(params.leverage as i64),
duration_seconds: Some(duration),
tags: Vec::new(),
notes: None,
verdict: None,
};
if let Err(e) = db.insert(&entry) {
tracing::warn!("launch_backtest: failed to register report in DB: {}", e);
return;
}
if let Err(e) = db.insert_deals(report_id, &extraction.deals) {
tracing::warn!("launch_backtest: failed to store deals in DB: {}", e);
}
tracing::info!("launch_backtest: extracted {} deals for {}", extraction.deals.len(), report_id);
}
Err(e) => {
tracing::warn!("launch_backtest: extraction failed for {}: {}", report_id, e);
}
}
}
/// Background task to monitor backtest completion and update status file.
async fn monitor_backtest_completion(
report_dir: PathBuf,
@@ -278,6 +365,8 @@ impl BacktestPipeline {
timeout_secs: u64,
report_id: String,
notification_callback: Option<NotificationCallback>,
config: Config,
params: BacktestParams,
) {
let start = tokio::time::Instant::now();
let deadline = start + Duration::from_secs(timeout_secs);
@@ -292,6 +381,8 @@ impl BacktestPipeline {
let candidate = expected_report.with_extension(ext.trim_start_matches('.'));
if candidate.exists() {
tracing::info!("Backtest {} completed: report found at {}", report_id, candidate.display());
Self::extract_and_store(&candidate, &report_dir, &report_id, &config, &params).await;
let _ = fs::remove_file(&candidate);
Self::update_job_status(&report_dir, "completed", Some(candidate.to_string_lossy().to_string())).await;
if let Some(ref callback) = notification_callback {
callback("backtest_completed", json!({
@@ -312,6 +403,8 @@ impl BacktestPipeline {
// MT5 exited without report - check for any newer report
if let Some(path) = Self::find_newest_report(expected_report.parent().unwrap(), poll_start) {
tracing::info!("Backtest {} completed: found fallback report {}", report_id, path.display());
Self::extract_and_store(&path, &report_dir, &report_id, &config, &params).await;
let _ = fs::remove_file(&path);
Self::update_job_status(&report_dir, "completed", Some(path.to_string_lossy().to_string())).await;
if let Some(ref callback) = notification_callback {
callback("backtest_completed", json!({
@@ -1047,8 +1140,6 @@ impl BacktestPipeline {
files: FilePaths {
metrics: report_dir.join("metrics.json").to_string_lossy().to_string(),
analysis: report_dir.join("analysis.json").to_string_lossy().to_string(),
deals_csv: report_dir.join("deals.csv").to_string_lossy().to_string(),
deals_json: report_dir.join("deals.json").to_string_lossy().to_string(),
},
no_trades,
};
src/tools/definitions/utility.rs
+1 -1
View File
@@ -289,7 +289,7 @@ pub fn tool_get_wine_prefix_info() -> Value {
pub fn tool_get_backtest_crash_info() -> Value {
json!({
"name": "get_backtest_crash_info",
"description": "Investigate backtest crashes/failures. Checks for incomplete markers, missing deals.csv, error logs. Can scan recent reports.",
"description": "Investigate backtest crashes/failures. Checks for incomplete markers, missing metrics.json, DB deal count, error logs. Can scan recent reports.",
"inputSchema": {
"type": "object",
"properties": {
src/tools/handlers/backtest.rs
-2
View File
@@ -355,7 +355,6 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul
// Check for completed artifacts
let metrics_exists = report_path.join("metrics.json").exists();
let deals_exists = report_path.join("deals.csv").exists();
let is_complete = stage == "DONE" || (report_found && metrics_exists);
// Calculate elapsed time if job exists
@@ -400,7 +399,6 @@ pub async fn handle_get_backtest_status(_config: &Config, args: &Value) -> Resul
"mt5_running": mt5_running,
"report_found": report_found,
"metrics_extracted": metrics_exists,
"deals_extracted": deals_exists,
"elapsed_seconds": elapsed_seconds,
"message": message,
"job": job.map(|j| {
src/tools/handlers/utility.rs
+32 -20
View File
@@ -3,6 +3,7 @@ use serde_json::{json, Value};
use std::fs;
use std::path::{Path, PathBuf};
use crate::models::Config;
use crate::storage::ReportDb;
/// Validate that `user_path` resolves to a location within `allowed_base`.
/// Returns the canonicalized absolute path on success.
@@ -785,7 +786,6 @@ pub async fn handle_export_report(_config: &Config, args: &Value) -> Result<Valu
let path = Path::new(report_dir);
let metrics_path = path.join("metrics.json");
let _deals_path = path.join("deals.csv");
// Read metrics
let metrics: Value = if metrics_path.exists() {
@@ -1640,21 +1640,33 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
}
}
// Check if deals.csv is missing or empty
let deals_csv = path.join("deals.csv");
if !deals_csv.exists() {
result["crashes_found"].as_array_mut().unwrap().push(json!({
"report_dir": dir,
"type": "missing_deals",
"reason": "deals.csv not found - backtest likely failed",
}));
} else if let Ok(meta) = deals_csv.metadata() {
if meta.len() < 100 {
result["crashes_found"].as_array_mut().unwrap().push(json!({
"report_dir": dir,
"type": "empty_deals",
"reason": "deals.csv is nearly empty - no trades were made",
}));
// Check deal count in DB
let db = ReportDb::new(&Config::db_path());
if db.init().is_ok() {
match db.get_by_report_dir(dir) {
Ok(Some(entry)) => {
let deal_count = db.get_deals(&entry.id)
.map(|d| d.len())
.unwrap_or(0);
if deal_count == 0 {
result["crashes_found"].as_array_mut().unwrap().push(json!({
"report_dir": dir,
"type": "empty_deals",
"reason": "No deals stored in DB - EA did not trade or extraction failed",
}));
}
}
Ok(None) | Err(_) => {
// Not in DB at all means extraction never completed
let metrics_exists = path.join("metrics.json").exists();
if !metrics_exists {
result["crashes_found"].as_array_mut().unwrap().push(json!({
"report_dir": dir,
"type": "missing_deals",
"reason": "Report not found in DB and no metrics.json - backtest likely crashed before extraction",
}));
}
}
}
}
}
@@ -1676,10 +1688,10 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
if let Ok(modified) = meta.modified() {
if modified >= cutoff {
// Check for failure indicators
let has_deals = path.join("deals.csv").exists();
let has_metrics = path.join("metrics.json").exists();
let has_incomplete = path.join(".incomplete").exists();
if !has_deals || has_incomplete {
if !has_metrics || has_incomplete {
failures += 1;
}
}
@@ -1701,7 +1713,7 @@ pub async fn handle_get_backtest_crash_info(config: &Config, args: &Value) -> Re
if types.contains(&"missing_deals".to_string()) {
result["common_patterns"].as_array_mut().unwrap().push(
json!("Missing deals.csv suggests MT5 crashed during backtest")
json!("Report not in DB and no metrics.json suggests MT5 crashed before extraction completed")
);
}
if types.contains(&"incomplete".to_string()) {