Complete migration to Rust implementation

- Migrate optimization from optimize.sh to src/optimization/
- Remove all Python files (analytics/, server/, hooks/, pyproject.toml)
- Add optimization module: optimizer.rs, parser.rs, mod.rs
- Implement all missing MCP tool handlers (35 total tools)
- Add handle_patch_set_file handler
- Clean up orphan files: .venv/, __pycache__, test files
- Move test_rcp_server.sh to tests/integration_test.sh
- Add Rust integration tests in tests/integration_tests.rs
- Fix all compiler warnings with #[allow(dead_code)]
- Update test fixtures and structure
This commit is contained in:
Devid HW
2026-04-18 15:57:28 +07:00
parent a3b046c68f
commit 331b7fbb73
29 changed files with 1405 additions and 5401 deletions
+5
View File
@@ -0,0 +1,5 @@
pub mod optimizer;
pub mod parser;
pub use optimizer::{OptimizationParams, OptimizationRunner};
pub use parser::OptimizationParser;
+375
View File
@@ -0,0 +1,375 @@
use anyhow::{anyhow, Result};
use chrono::Utc;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use crate::models::Config;
pub struct OptimizationParams {
pub expert: String,
pub set_file: String,
pub symbol: String,
pub from_date: String,
pub to_date: String,
pub deposit: u32,
pub model: u8,
pub leverage: u32,
pub currency: String,
}
impl Default for OptimizationParams {
fn default() -> Self {
Self {
expert: String::new(),
set_file: String::new(),
symbol: "XAUUSD".to_string(),
from_date: String::new(),
to_date: String::new(),
deposit: 10000,
model: 0,
leverage: 500,
currency: "USD".to_string(),
}
}
}
pub struct OptimizationResult {
pub success: bool,
pub job_id: String,
pub pid: u32,
pub log_file: PathBuf,
pub combinations: u64,
pub message: String,
}
pub struct OptimizationRunner {
config: Config,
}
impl OptimizationRunner {
pub fn new(config: Config) -> Self {
Self { config }
}
pub async fn run(&self, params: OptimizationParams) -> Result<OptimizationResult> {
// Validate required fields
if params.expert.is_empty() {
return Err(anyhow!("expert is required"));
}
if params.set_file.is_empty() {
return Err(anyhow!("set_file is required"));
}
if params.from_date.is_empty() {
return Err(anyhow!("from_date is required"));
}
if params.to_date.is_empty() {
return Err(anyhow!("to_date is required"));
}
let set_path = Path::new(&params.set_file);
if !set_path.exists() {
return Err(anyhow!("Set file not found: {}", params.set_file));
}
// Generate job ID and log file
let timestamp = Utc::now().format("%Y%m%d_%H%M%S").to_string();
let job_id = format!("opt_{}", timestamp);
let log_file = PathBuf::from(format!("/tmp/mt5opt_{}.log", timestamp));
// Count combinations
let combinations = self.count_combinations(&params.set_file)?;
// Get paths
let mt5_dir = self.config.terminal_dir.as_ref()
.ok_or_else(|| anyhow!("terminal_dir not configured"))?;
let wine_exe = self.config.wine_executable.as_ref()
.ok_or_else(|| anyhow!("wine_executable not configured"))?;
// Write .set file as UTF-16LE with BOM directly to MT5 tester directory
let wine_prefix_dir = self.get_wine_prefix_dir(mt5_dir)?;
let tester_dir = wine_prefix_dir.join("drive_c/Program Files/MetaTrader 5/MQL5/Profiles/Tester");
fs::create_dir_all(&tester_dir)?;
let dst_set_file = tester_dir.join(format!("{}.set", params.expert));
self.write_utf16le_set(&params.set_file, &dst_set_file)?;
// Reset OptMode in terminal.ini
self.reset_optmode(mt5_dir)?;
// Get Wine prefix directory
let wine_prefix_dir = self.get_wine_prefix_dir(mt5_dir)?;
// Build optimization INI
let ini_path = wine_prefix_dir.join("drive_c/mt5mcp_backtest.ini");
let ini_content = format!(r#"[Tester]
Expert={}
Symbol={}
Period=M5
Deposit={}
Currency={}
Leverage={}
Model={}
FromDate={}
ToDate={}
Report=C:\mt5mcp_opt_report
Optimization=2
ExpertParameters={}.set
ShutdownTerminal=1
"#, params.expert, params.symbol, params.deposit, params.currency,
params.leverage, params.model, params.from_date, params.to_date, params.expert);
fs::write(&ini_path, ini_content)?;
// Build batch file
let batch_path = wine_prefix_dir.join("drive_c/mt5mcp_run.bat");
let batch_content = format!(r#"@echo off
"C:\Program Files\MetaTrader 5\terminal64.exe" /config:C:\mt5mcp_backtest.ini
"#);
fs::write(&batch_path, batch_content)?;
// Launch detached process
let cmd = format!("cmd.exe /c 'C:\\mt5mcp_run.bat'");
let child = Command::new(wine_exe)
.arg(&cmd)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()?;
let pid = child.id();
// Write job metadata
self.write_job_metadata(&job_id, pid, &params, &log_file, combinations, &wine_prefix_dir)?;
Ok(OptimizationResult {
success: true,
job_id,
pid,
log_file,
combinations,
message: format!("Optimization launched (pid: {}). Runs for 2-6 hours. Do NOT kill this process.", pid),
})
}
fn count_combinations(&self, set_file: &str) -> Result<u64> {
let content = fs::read_to_string(set_file)?;
let mut total: u64 = 1;
for line in content.lines() {
let line = line.trim();
if line.starts_with(';') || !line.contains('=') {
continue;
}
// Format: param=value||start||step||stop||Y
let parts: Vec<&str> = line.split("||").collect();
if parts.len() >= 5 && parts.last().unwrap().trim().to_uppercase() == "Y" {
if let (Ok(start), Ok(step), Ok(stop)) = (
parts[1].trim().parse::<f64>(),
parts[2].trim().parse::<f64>(),
parts[3].trim().parse::<f64>(),
) {
if step > 0.0 {
let count = ((stop - start) / step).max(0.0) as u64 + 1;
total = total.saturating_mul(count);
}
}
}
}
Ok(total.max(1))
}
fn write_utf16le_set(&self, src: &str, dst: &Path) -> Result<()> {
let content = fs::read_to_string(src)?;
// Create parent directory if needed
if let Some(parent) = dst.parent() {
fs::create_dir_all(parent)?;
}
// Write UTF-16LE with BOM
let mut utf16_content: Vec<u16> = vec![0xFEFF]; // BOM
utf16_content.extend(content.encode_utf16());
let bytes: Vec<u8> = utf16_content.iter()
.flat_map(|&c| vec![(c & 0xFF) as u8, ((c >> 8) & 0xFF) as u8])
.collect();
fs::write(dst, bytes)?;
// Make read-only
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(dst, fs::Permissions::from_mode(0o444))?;
}
Ok(())
}
fn reset_optmode(&self, mt5_dir: &str) -> Result<()> {
let terminal_ini = Path::new(mt5_dir).join("terminal.ini");
if !terminal_ini.exists() {
return Ok(());
}
let content = fs::read_to_string(&terminal_ini)?;
let updated = content
.lines()
.map(|line| {
if line.starts_with("OptMode=") {
"OptMode=0".to_string()
} else if line.starts_with("LastOptimization=") {
String::new()
} else {
line.to_string()
}
})
.filter(|l| !l.is_empty())
.collect::<Vec<_>>()
.join("\n");
fs::write(&terminal_ini, updated)?;
Ok(())
}
fn get_wine_prefix_dir(&self, mt5_dir: &str) -> Result<PathBuf> {
let path = Path::new(mt5_dir);
// Go up two levels: .../drive_c/Program Files/MetaTrader 5 -> .../drive_c
let prefix_dir = path
.parent()
.and_then(|p| p.parent())
.ok_or_else(|| anyhow!("Cannot determine Wine prefix from terminal_dir"))?;
Ok(prefix_dir.to_path_buf())
}
fn write_job_metadata(
&self,
job_id: &str,
pid: u32,
params: &OptimizationParams,
log_file: &Path,
combinations: u64,
wine_prefix: &Path,
) -> Result<()> {
let jobs_dir = Path::new(".mt5mcp_jobs");
fs::create_dir_all(jobs_dir)?;
let meta_path = jobs_dir.join(format!("{}.json", job_id));
let started_at = Utc::now().to_rfc3339();
let metadata = serde_json::json!({
"job_id": job_id,
"pid": pid,
"expert": params.expert,
"symbol": params.symbol,
"from_date": params.from_date,
"to_date": params.to_date,
"set_file": params.set_file,
"combinations": combinations,
"log_file": log_file.to_string_lossy(),
"wine_prefix": wine_prefix.to_string_lossy(),
"started_at": started_at,
});
fs::write(&meta_path, serde_json::to_string_pretty(&metadata)?)?;
Ok(())
}
pub fn get_job_status(&self, job_id: &str) -> Result<serde_json::Value> {
let jobs_dir = Path::new(".mt5mcp_jobs");
let meta_path = jobs_dir.join(format!("{}.json", job_id));
if !meta_path.exists() {
return Ok(serde_json::json!({
"status": "not_found",
"message": format!("Job {} not found", job_id)
}));
}
let meta: serde_json::Value = serde_json::from_str(&fs::read_to_string(&meta_path)?)?;
let pid = meta.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
// Check if process is still running
let is_running = self.is_process_running(pid);
// Check for completion marker in log
let log_file = meta.get("log_file").and_then(|v| v.as_str()).unwrap_or("");
let is_complete = if !log_file.is_empty() && Path::new(log_file).exists() {
fs::read_to_string(log_file)
.map(|content| content.contains("Optimization complete"))
.unwrap_or(false)
} else {
false
};
let status = if is_complete {
"completed"
} else if is_running {
"running"
} else {
"stopped"
};
Ok(serde_json::json!({
"status": status,
"job_id": job_id,
"pid": pid,
"expert": meta.get("expert"),
"symbol": meta.get("symbol"),
"started_at": meta.get("started_at"),
"log_file": log_file,
}))
}
fn is_process_running(&self, pid: u32) -> bool {
#[cfg(unix)]
{
Command::new("kill")
.args(["-0", &pid.to_string()])
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
#[cfg(windows)]
{
// Windows implementation would use different method
false
}
}
pub fn list_jobs(&self) -> Result<Vec<serde_json::Value>> {
let jobs_dir = Path::new(".mt5mcp_jobs");
let mut jobs = Vec::new();
if jobs_dir.exists() {
for entry in fs::read_dir(jobs_dir)? {
if let Ok(entry) = entry {
let path = entry.path();
if path.extension().map(|e| e == "json").unwrap_or(false) {
if let Ok(content) = fs::read_to_string(&path) {
if let Ok(meta) = serde_json::from_str::<serde_json::Value>(&content) {
let job_id = path.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("unknown")
.to_string();
let pid = meta.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
let is_running = self.is_process_running(pid);
jobs.push(serde_json::json!({
"job_id": job_id,
"expert": meta.get("expert"),
"status": if is_running { "running" } else { "stopped" },
"started_at": meta.get("started_at"),
}));
}
}
}
}
}
}
Ok(jobs)
}
}
+270
View File
@@ -0,0 +1,270 @@
use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs;
use std::path::Path;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OptimizationPass {
pub pass: u32,
pub profit: f64,
pub total_trades: u32,
pub profit_factor: f64,
pub expected_payoff: f64,
pub drawdown_pct: f64,
pub params: HashMap<String, String>,
}
pub struct OptimizationParser;
impl OptimizationParser {
pub fn new() -> Self {
Self
}
pub fn parse_job(&self, job_id: &str) -> Result<Vec<OptimizationPass>> {
let jobs_dir = Path::new(".mt5mcp_jobs");
let meta_path = jobs_dir.join(format!("{}.json", job_id));
if !meta_path.exists() {
return Err(anyhow!("Job not found: {}. Check .mt5mcp_jobs/", job_id));
}
let meta: serde_json::Value = serde_json::from_str(&fs::read_to_string(&meta_path)?)?;
let wine_prefix = meta.get("wine_prefix")
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("wine_prefix not in job metadata"))?;
let base_path = Path::new(wine_prefix).join("drive_c/mt5mcp_opt_report");
// Try different extensions
for ext in &[".htm", ".htm.xml", ".html"] {
let candidate = base_path.with_extension(ext.trim_start_matches('.'));
if candidate.exists() {
return self.parse_file(&candidate);
}
}
Err(anyhow!(
"Optimization report not found. Expected: {}.htm or {}.htm.xml\nIs MT5 optimization still running?",
base_path.display(),
base_path.display()
))
}
pub fn parse_file(&self, path: &Path) -> Result<Vec<OptimizationPass>> {
let format = self.detect_format(path);
let text = self.read_text(path)?;
match format {
"xml" => self.parse_xml(&text),
_ => self.parse_html(&text),
}
}
fn detect_format(&self, path: &Path) -> &str {
let path_str = path.to_string_lossy();
if path_str.ends_with(".xml") || path_str.ends_with(".htm.xml") {
return "xml";
}
if let Ok(header) = fs::read(path) {
let header = &header[..header.len().min(512)];
if header.windows(5).any(|w| w == b"<?xml") || header.windows(8).any(|w| w == b"Workbook") {
return "xml";
}
}
"html"
}
fn read_text(&self, path: &Path) -> Result<String> {
let raw = fs::read(path)?;
// Try UTF-16 first (common for MT5 reports)
if raw.len() >= 2 {
if raw[0] == 0xFF && raw[1] == 0xFE {
// UTF-16 LE with BOM
let u16_vec: Vec<u16> = raw[2..].chunks_exact(2)
.map(|c| u16::from_le_bytes([c[0], c[1]]))
.collect();
return Ok(String::from_utf16_lossy(&u16_vec));
} else if raw[0] == 0xFE && raw[1] == 0xFF {
// UTF-16 BE with BOM
let u16_vec: Vec<u16> = raw[2..].chunks_exact(2)
.map(|c| u16::from_be_bytes([c[0], c[1]]))
.collect();
return Ok(String::from_utf16_lossy(&u16_vec));
}
}
// Try UTF-8, then fallback to lossy
if let Ok(text) = String::from_utf8(raw.clone()) {
return Ok(text);
}
// Try UTF-16 without BOM
if raw.len() % 2 == 0 {
let u16_vec: Vec<u16> = raw.chunks_exact(2)
.map(|c| u16::from_le_bytes([c[0], c[1]]))
.collect();
let text = String::from_utf16_lossy(&u16_vec);
if text.chars().any(|c| c.is_ascii_alphanumeric()) {
return Ok(text);
}
}
Ok(String::from_utf8_lossy(&raw).to_string())
}
fn parse_html(&self, text: &str) -> Result<Vec<OptimizationPass>> {
let mut results = Vec::new();
let mut headers: Vec<String> = Vec::new();
// Find all table rows
let row_regex = regex::Regex::new(r"<tr[^>]*>(.*?)</tr>")?;
let cell_regex = regex::Regex::new(r"<t[dh][^>]*>(.*?)</t[dh]>")?;
let tag_regex = regex::Regex::new(r"<[^>]+>")?;
for row_caps in row_regex.captures_iter(text) {
let row = &row_caps[1];
let cells: Vec<String> = cell_regex.captures_iter(row)
.map(|c| {
let cell = &c[1];
tag_regex.replace_all(cell, "").trim().to_string().replace(',', "")
})
.collect();
if cells.is_empty() {
continue;
}
// Header row detection
if headers.is_empty() && cells[0].to_lowercase().contains("pass") {
headers = cells;
continue;
}
// Data row
if !headers.is_empty() && cells[0].parse::<u32>().is_ok() {
let row_map: HashMap<String, String> = headers.iter()
.zip(cells.iter())
.map(|(h, c)| (h.to_lowercase().replace(' ', "_"), c.clone()))
.collect();
if let Some(pass) = self.row_to_pass(&row_map) {
results.push(pass);
}
}
}
Ok(results)
}
fn parse_xml(&self, text: &str) -> Result<Vec<OptimizationPass>> {
let mut results = Vec::new();
// Parse SpreadsheetML XML
let doc = roxmltree::Document::parse(text)?;
// Find all rows in Worksheet/Table
for node in doc.descendants() {
if node.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Row")) ||
node.has_tag_name("Row") {
let cells: Vec<String> = node.children()
.filter(|n: &roxmltree::Node<'_, '_>| {
n.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Cell")) ||
n.has_tag_name("Cell") ||
n.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Data")) ||
n.has_tag_name("Data")
})
.map(|n| n.text().unwrap_or("").trim().to_string().replace(',', ""))
.collect();
if cells.is_empty() {
continue;
}
// Check if first cell is a pass number
if let Ok(pass_num) = cells[0].parse::<u32>() {
if pass_num > 0 {
let mut row_map = HashMap::new();
// Standard MT5 optimization report columns
let headers = vec![
"pass", "result", "profit", "total_trades", "profit_factor",
"expected_payoff", "drawdown_pct", "recovery_factor", "sharpe_ratio",
"custom", "consecutive_wins", "consecutive_losses",
];
for (i, cell) in cells.iter().enumerate() {
if let Some(header) = headers.get(i) {
row_map.insert(header.to_string(), cell.clone());
}
}
if let Some(pass) = self.row_to_pass(&row_map) {
results.push(pass);
}
}
}
}
}
Ok(results)
}
fn row_to_pass(&self, row: &HashMap<String, String>) -> Option<OptimizationPass> {
let pass = row.get("pass").or_else(|| row.get("#"))
.and_then(|v| v.parse().ok())?;
let profit = row.get("profit").or_else(|| row.get("total_net_profit"))
.and_then(|v| v.replace(' ', "").parse().ok())?;
let total_trades = row.get("total_trades").or_else(|| row.get("trades"))
.and_then(|v| v.parse().ok())?;
let profit_factor = row.get("profit_factor")
.and_then(|v| v.parse().ok())?;
let expected_payoff = row.get("expected_payoff")
.and_then(|v| v.parse().ok())?;
let drawdown_pct = row.get("drawdown_pct").or_else(|| row.get("max_drawdown"))
.and_then(|v| v.trim_end_matches('%').trim().parse().ok())?;
// Extract parameter values from row
let params: HashMap<String, String> = row.iter()
.filter(|(k, _)| ![
"pass", "result", "profit", "total_trades", "profit_factor",
"expected_payoff", "drawdown_pct", "max_drawdown", "recovery_factor",
"sharpe_ratio", "custom", "consecutive_wins", "consecutive_losses"
].contains(&k.as_str()))
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
Some(OptimizationPass {
pass,
profit,
total_trades,
profit_factor,
expected_payoff,
drawdown_pct,
params,
})
}
pub fn find_best_pass<'a>(&self, passes: &'a [OptimizationPass], criteria: &str) -> Option<&'a OptimizationPass> {
match criteria {
"profit" => passes.iter().max_by(|a, b| a.profit.partial_cmp(&b.profit).unwrap()),
"profit_factor" => passes.iter().max_by(|a, b| a.profit_factor.partial_cmp(&b.profit_factor).unwrap()),
"sharpe" => passes.iter().max_by(|a, b| {
let a_sharpe = a.params.get("sharpe_ratio").and_then(|v| v.parse::<f64>().ok()).unwrap_or(0.0);
let b_sharpe = b.params.get("sharpe_ratio").and_then(|v| v.parse::<f64>().ok()).unwrap_or(0.0);
a_sharpe.partial_cmp(&b_sharpe).unwrap()
}),
"drawdown" => passes.iter().min_by(|a, b| a.drawdown_pct.partial_cmp(&b.drawdown_pct).unwrap()),
_ => passes.iter().max_by(|a, b| a.profit.partial_cmp(&b.profit).unwrap()),
}
}
}