first commit
This commit is contained in:
@@ -0,0 +1,222 @@
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('logs/batch_executor.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BatchExecutor:
|
||||
def __init__(self, config_path: str):
|
||||
self.config = self._load_config(config_path)
|
||||
self.mt5_path = self.config["mt5_settings"]["terminal_path"]
|
||||
self.reports_dir = self.config["mt5_settings"]["reports_dir"]
|
||||
self.results_dir = self.config["mt5_settings"].get("results_dir", "results")
|
||||
os.makedirs(self.results_dir, exist_ok=True)
|
||||
os.makedirs("logs", exist_ok=True)
|
||||
|
||||
def _load_config(self, config_path: str) -> Dict:
|
||||
import yaml
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
def _wait_for_report(self, report_path: str, timeout: int = 600) -> bool:
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
if os.path.exists(report_path):
|
||||
time.sleep(2)
|
||||
try:
|
||||
with open(report_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
if len(content) > 100:
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
def _check_mt5_process(self) -> bool:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['tasklist', '/FI', 'IMAGENAME eq terminal64.exe'],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
return 'terminal64.exe' in result.stdout
|
||||
except:
|
||||
return False
|
||||
|
||||
def execute_single(self, ini_path: str, wait_time: int = 120) -> Dict:
|
||||
result = {
|
||||
"ini_file": os.path.basename(ini_path),
|
||||
"status": "pending",
|
||||
"start_time": None,
|
||||
"end_time": None,
|
||||
"duration": 0,
|
||||
"report_path": None,
|
||||
"error": None
|
||||
}
|
||||
|
||||
logger.info(f"Starting backtest: {ini_path}")
|
||||
|
||||
if not os.path.exists(self.mt5_path):
|
||||
result["status"] = "error"
|
||||
result["error"] = f"MT5 terminal not found: {self.mt5_path}"
|
||||
logger.error(result["error"])
|
||||
return result
|
||||
|
||||
result["start_time"] = datetime.now()
|
||||
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
[self.mt5_path, "/portable", f"/config:{ini_path}"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
|
||||
logger.info(f"MT5 process started with PID: {process.pid}")
|
||||
|
||||
ini_basename = os.path.splitext(os.path.basename(ini_path))[0]
|
||||
report_xml = os.path.join(self.results_dir, f"{ini_basename}_report.xml")
|
||||
|
||||
report_found = self._wait_for_report(report_xml, timeout=wait_time * 60)
|
||||
|
||||
if report_found:
|
||||
result["status"] = "completed"
|
||||
result["report_path"] = report_xml
|
||||
logger.info(f"Report generated: {report_xml}")
|
||||
else:
|
||||
if process.poll() is not None:
|
||||
result["status"] = "mt5_closed"
|
||||
logger.warning("MT5 closed before report was generated")
|
||||
else:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
result["status"] = "timeout"
|
||||
logger.warning(f"Timeout waiting for report: {ini_path}")
|
||||
|
||||
if process.poll() is None:
|
||||
try:
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
except:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
logger.error(f"Error executing backtest: {e}")
|
||||
|
||||
result["end_time"] = datetime.now()
|
||||
if result["start_time"] and result["end_time"]:
|
||||
result["duration"] = (result["end_time"] - result["start_time"]).total_seconds()
|
||||
|
||||
return result
|
||||
|
||||
def execute_batch(self, ini_files: List[str],
|
||||
max_parallel: int = 1,
|
||||
wait_time_per_test: int = 120) -> List[Dict]:
|
||||
results = []
|
||||
total = len(ini_files)
|
||||
|
||||
logger.info(f"Starting batch execution: {total} tests")
|
||||
logger.info(f"Parallel execution: {max_parallel}")
|
||||
|
||||
for idx, ini_path in enumerate(ini_files, 1):
|
||||
logger.info(f"[{idx}/{total}] Executing: {os.path.basename(ini_path)}")
|
||||
result = self.execute_single(ini_path, wait_time_per_test)
|
||||
results.append(result)
|
||||
|
||||
status = result["status"]
|
||||
duration = result["duration"]
|
||||
logger.info(f" Status: {status}, Duration: {duration:.1f}s")
|
||||
|
||||
if result["error"]:
|
||||
logger.error(f" Error: {result['error']}")
|
||||
|
||||
success_count = sum(1 for r in results if r["status"] == "completed")
|
||||
logger.info(f"Batch completed: {success_count}/{total} successful")
|
||||
|
||||
return results
|
||||
|
||||
def execute_validate_tasks(self, ini_files: List[str]) -> str:
|
||||
validate_dir = os.path.join(os.path.dirname(self.mt5_path),
|
||||
"MQL5", "Files", "ValidateTasks")
|
||||
os.makedirs(validate_dir, exist_ok=True)
|
||||
|
||||
for ini_file in ini_files:
|
||||
dest_path = os.path.join(validate_dir, os.path.basename(ini_file))
|
||||
with open(ini_file, 'r', encoding='utf-8') as src:
|
||||
content = src.read()
|
||||
with open(dest_path, 'w', encoding='utf-8') as dst:
|
||||
dst.write(content)
|
||||
logger.info(f"Copied to ValidateTasks: {os.path.basename(ini_file)}")
|
||||
|
||||
logger.info(f"All {len(ini_files)} tasks queued in ValidateTasks folder")
|
||||
return validate_dir
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="MT5 Batch Backtest Executor")
|
||||
parser.add_argument("--config", "-c", default="config/ea_configs.yaml",
|
||||
help="Path to config file")
|
||||
parser.add_argument("--ini-dir", "-i",
|
||||
help="Directory containing INI files to execute")
|
||||
parser.add_argument("--wait", "-w", type=int, default=120,
|
||||
help="Wait time per test in minutes")
|
||||
parser.add_argument("--validate", "-v", action="store_true",
|
||||
help="Use ValidateTasks method")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
executor = BatchExecutor(args.config)
|
||||
|
||||
if args.ini_dir:
|
||||
ini_files = [os.path.join(args.ini_dir, f)
|
||||
for f in os.listdir(args.ini_dir)
|
||||
if f.endswith('.ini')]
|
||||
else:
|
||||
generator = __import__('ini_generator', fromlist=['']).INIGenerator(args.config)
|
||||
ini_files = generator.generate_ini_files()
|
||||
|
||||
logger.info(f"Found {len(ini_files)} INI files to execute")
|
||||
|
||||
if args.validate:
|
||||
validate_dir = executor.execute_validate_tasks(ini_files)
|
||||
logger.info(f"ValidateTasks method: files copied to {validate_dir}")
|
||||
logger.info("Run the Validate EA in MT5 terminal to execute")
|
||||
else:
|
||||
results = executor.execute_batch(ini_files, wait_time_per_test=args.wait)
|
||||
|
||||
success = [r for r in results if r["status"] == "completed"]
|
||||
failed = [r for r in results if r["status"] != "completed"]
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("BATCH EXECUTION SUMMARY")
|
||||
print("="*60)
|
||||
print(f"Total tests: {len(results)}")
|
||||
print(f"Completed: {len(success)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print("="*60)
|
||||
|
||||
if failed:
|
||||
print("\nFailed tests:")
|
||||
for r in failed:
|
||||
print(f" - {r['ini_file']}: {r['status']} - {r['error']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,256 @@
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
from datetime import datetime
|
||||
from itertools import product
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from mt5_paths import resolve_mt5_settings
|
||||
|
||||
|
||||
class INIGenerator:
|
||||
def __init__(self, source):
|
||||
"""
|
||||
source: YAML 文件路径 str 或 已加载的 config dict(GUI 内存直接传入用)。
|
||||
"""
|
||||
if isinstance(source, dict):
|
||||
self.config_path = None
|
||||
self.config = source
|
||||
else:
|
||||
self.config_path = source
|
||||
self.config = self._load_config()
|
||||
self.config["mt5_settings"] = resolve_mt5_settings(self.config.get("mt5_settings", {}))
|
||||
self.output_dir = self.config["mt5_settings"]["ini_dir"]
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def _load_config(self) -> Dict:
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
def _scan_experts_dir(self) -> List[str]:
|
||||
data_dir = self.config["mt5_settings"].get("data_dir", "")
|
||||
if not data_dir:
|
||||
mt5_base = os.path.dirname(self.config["mt5_settings"]["terminal_path"])
|
||||
data_dir = mt5_base
|
||||
experts_dir = os.path.join(data_dir, "MQL5", "Experts")
|
||||
ea_files = []
|
||||
|
||||
if not os.path.exists(experts_dir):
|
||||
return []
|
||||
|
||||
for root, dirs, files in os.walk(experts_dir):
|
||||
for f in files:
|
||||
if f.endswith('.ex5'):
|
||||
rel_path = os.path.relpath(os.path.join(root, f), experts_dir)
|
||||
ea_files.append(rel_path)
|
||||
|
||||
return ea_files
|
||||
|
||||
def _load_set_file(self, set_file_path: str) -> Dict[str, Any]:
|
||||
params = {}
|
||||
if not os.path.exists(set_file_path):
|
||||
return params
|
||||
with open(set_file_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith(';') and '=' in line:
|
||||
key, value = line.split('=', 1)
|
||||
try:
|
||||
params[key.strip()] = float(value.strip())
|
||||
except:
|
||||
params[key.strip()] = value.strip()
|
||||
return params
|
||||
|
||||
def _generate_parameter_combinations(self, parameters: Dict[str, List]) -> List[Dict]:
|
||||
if not parameters:
|
||||
return [{}]
|
||||
keys = list(parameters.keys())
|
||||
values = list(parameters.values())
|
||||
combinations = list(product(*values))
|
||||
return [dict(zip(keys, combo)) for combo in combinations]
|
||||
|
||||
def _get_timeframe_code(self, timeframe: str) -> str:
|
||||
mapping = {
|
||||
"M1": "M1", "M5": "M5", "M15": "M15",
|
||||
"H1": "H1", "H4": "H4", "D1": "D1", "W1": "W1"
|
||||
}
|
||||
return mapping.get(timeframe, "H1")
|
||||
|
||||
def _params_hash(self, parameters: Dict) -> str:
|
||||
if not parameters:
|
||||
return "default"
|
||||
sorted_params = sorted(parameters.items())
|
||||
param_str = "_".join([f"{k}{v}" for k, v in sorted_params])
|
||||
return str(abs(hash(param_str)))[:8]
|
||||
|
||||
def _generate_filename(self, ea_name: str, symbol: str,
|
||||
timeframe: str, parameters: Dict) -> str:
|
||||
safe_ea_name = ea_name.replace("\\", "_").replace("/", "_").replace("..", "")
|
||||
for c in '()[]{}|\\/*?:"\'<>':
|
||||
safe_ea_name = safe_ea_name.replace(c, "_")
|
||||
param_hash = self._params_hash(parameters)
|
||||
return f"{safe_ea_name}_{symbol}_{timeframe}_{param_hash}.ini"
|
||||
|
||||
def _build_ini_content(self, ea_filename: str, symbol: str, timeframe: str,
|
||||
bt_settings: Dict, parameters: Dict, ea_name: str,
|
||||
set_file: Optional[str] = None) -> str:
|
||||
date_from = bt_settings["date_range"]["from"]
|
||||
date_to = bt_settings["date_range"]["to"]
|
||||
report_name = f"{ea_name}_{symbol}_{timeframe}_{self._params_hash(parameters)}"
|
||||
for c in '()[]{}|\\/*?:"\'<>':
|
||||
report_name = report_name.replace(c, "_")
|
||||
safe_report_name = report_name
|
||||
|
||||
ini_lines = [
|
||||
"; MT5 Strategy Tester Configuration",
|
||||
f"; Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
||||
"",
|
||||
"[Tester]",
|
||||
f"Expert={ea_filename}",
|
||||
]
|
||||
|
||||
if set_file:
|
||||
ini_lines.append(f"ExpertParameters={set_file}")
|
||||
else:
|
||||
ini_lines.append("ExpertParameters=")
|
||||
|
||||
ini_lines.extend([
|
||||
f"Symbol={symbol}",
|
||||
f"Period={self._get_timeframe_code(timeframe)}",
|
||||
f"Model={bt_settings['model']}",
|
||||
f"ExecutionMode={bt_settings.get('execution_mode', 0)}",
|
||||
f"ExecutionDelay={bt_settings.get('execution_delay', 0)}",
|
||||
f"Optimization={bt_settings['optimization']}",
|
||||
])
|
||||
if bt_settings.get("optimization"):
|
||||
ini_lines.append("OptimizationCriterion=6")
|
||||
ini_lines.extend([
|
||||
f"FromDate={date_from}",
|
||||
f"ToDate={date_to}",
|
||||
f"ForwardMode={bt_settings.get('forward_mode', 0)}",
|
||||
f"ForwardDate={bt_settings.get('forward_date', '')}",
|
||||
f"Report={safe_report_name}",
|
||||
f"ReplaceReport={1 if bt_settings['replace_report'] else 0}",
|
||||
f"ShutdownTerminal={1 if bt_settings['shutdown_terminal'] else 0}",
|
||||
f"Deposit={int(float(bt_settings['deposit']))}",
|
||||
f"Currency={bt_settings['currency']}",
|
||||
f"Leverage={bt_settings['leverage']}",
|
||||
f"Visual={bt_settings['visual']}",
|
||||
"ProfitInPips=0",
|
||||
])
|
||||
|
||||
if parameters:
|
||||
ini_lines.extend(["", "; EA Parameters"])
|
||||
for param_name, param_value in parameters.items():
|
||||
ini_lines.append(f"{param_name}={param_value}")
|
||||
|
||||
return "\n".join(ini_lines)
|
||||
|
||||
def generate_ini_files(self) -> List[str]:
|
||||
generated_files = []
|
||||
bt_settings = self.config["backtest_settings"]
|
||||
set_files_dir = self.config["mt5_settings"].get("set_files_dir", "config/sets")
|
||||
|
||||
ea_configs = self.config.get("eas", [])
|
||||
|
||||
if not ea_configs:
|
||||
print("No EAs configured, scanning MT5 Experts directory...")
|
||||
ea_files = self._scan_experts_dir()
|
||||
if ea_files:
|
||||
print(f"Found {len(ea_files)} EA files: {ea_files}")
|
||||
for ea_file in ea_files:
|
||||
ea_name = os.path.splitext(ea_file)[0]
|
||||
ea_configs.append({
|
||||
"name": ea_name,
|
||||
"filename": ea_file,
|
||||
"description": "Auto-scanned EA"
|
||||
})
|
||||
else:
|
||||
print("No EAs found in Experts directory")
|
||||
return []
|
||||
|
||||
for ea in ea_configs:
|
||||
ea_name = ea["name"]
|
||||
ea_filename = ea["filename"]
|
||||
set_file = ea.get("set_file")
|
||||
parameters = ea.get("parameters", {})
|
||||
|
||||
if set_file:
|
||||
set_file_path = os.path.join(set_files_dir, set_file)
|
||||
set_params = self._load_set_file(set_file_path)
|
||||
param_combinations = [set_params] if set_params else [{}]
|
||||
else:
|
||||
param_combinations = self._generate_parameter_combinations(parameters)
|
||||
|
||||
for symbol in bt_settings["symbols"]:
|
||||
for timeframe in bt_settings["timeframes"]:
|
||||
for param_combo in param_combinations:
|
||||
ini_content = self._build_ini_content(
|
||||
ea_filename, symbol, timeframe,
|
||||
bt_settings, param_combo, ea_name, set_file
|
||||
)
|
||||
ini_filename = self._generate_filename(
|
||||
ea_name, symbol, timeframe, param_combo
|
||||
)
|
||||
ini_path = os.path.join(self.output_dir, ini_filename)
|
||||
|
||||
with open(ini_path, 'w', encoding='utf-8') as f:
|
||||
f.write(ini_content)
|
||||
generated_files.append(ini_path)
|
||||
|
||||
return generated_files
|
||||
|
||||
def generate_batch_run_script(self, ini_files: List[str],
|
||||
output_script: str = "run_backtests.bat"):
|
||||
mt5_path = self.config["mt5_settings"]["terminal_path"]
|
||||
|
||||
lines = [
|
||||
"@echo off",
|
||||
"echo MT5 Batch Backtest Runner",
|
||||
"echo =======================",
|
||||
"",
|
||||
f'SET "MT5_PATH={mt5_path}"',
|
||||
f'SET "INI_DIR={self.output_dir}"',
|
||||
"",
|
||||
]
|
||||
|
||||
for ini_file in ini_files:
|
||||
ini_filename = os.path.basename(ini_file)
|
||||
lines.append(f'echo Running: {ini_filename}')
|
||||
lines.append(
|
||||
f'START "MT5" /WAIT "%MT5_PATH%" /portable /config:"%INI_DIR%\\{ini_filename}"'
|
||||
)
|
||||
lines.append("if errorlevel 1 echo Failed: " + ini_filename)
|
||||
lines.append("")
|
||||
|
||||
lines.append("echo All backtests completed!")
|
||||
lines.append("pause")
|
||||
|
||||
with open(output_script, 'w', encoding='utf-8') as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
return output_script
|
||||
|
||||
|
||||
def main():
|
||||
config_path = os.path.join(os.path.dirname(__file__), "..", "config", "ea_configs.yaml")
|
||||
generator = INIGenerator(config_path)
|
||||
|
||||
print("Generating INI files...")
|
||||
ini_files = generator.generate_ini_files()
|
||||
print(f"Generated {len(ini_files)} INI files")
|
||||
|
||||
for f in ini_files[:5]:
|
||||
print(f" - {os.path.basename(f)}")
|
||||
if len(ini_files) > 5:
|
||||
print(f" ... and {len(ini_files) - 5} more")
|
||||
|
||||
if ini_files:
|
||||
print("\nGenerating batch run script...")
|
||||
script_path = generator.generate_batch_run_script(ini_files)
|
||||
print(f"Batch script: {script_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,355 @@
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from typing import List, Dict
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('logs/batch_executor.log', encoding='utf-8'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MT5AutoRunner:
|
||||
def __init__(self, source):
|
||||
"""
|
||||
source: YAML 文件路径 str 或 已加载的 config dict(GUI 内存直接传入用)。
|
||||
"""
|
||||
if isinstance(source, dict):
|
||||
self.config = source
|
||||
else:
|
||||
self.config = self._load_config(source)
|
||||
|
||||
# 占位符解析(不做自动探测)
|
||||
from mt5_paths import resolve_mt5_settings
|
||||
self.config["mt5_settings"] = resolve_mt5_settings(self.config.get("mt5_settings", {}))
|
||||
|
||||
self.mt5_path = self.config["mt5_settings"]["terminal_path"]
|
||||
self.data_dir = self.config["mt5_settings"].get("data_dir", "")
|
||||
self.reports_dir = os.path.abspath(self.config["mt5_settings"].get("reports_dir", "reports"))
|
||||
self.ini_dir = os.path.abspath(self.config["mt5_settings"].get("ini_dir",
|
||||
os.path.join(os.path.dirname(self.reports_dir), "config", "generated")))
|
||||
os.makedirs(self.reports_dir, exist_ok=True)
|
||||
os.makedirs(self.ini_dir, exist_ok=True)
|
||||
|
||||
exec_cfg = self.config.get("execution", {}) or {}
|
||||
self.kill_between = bool(exec_cfg.get("kill_between", True))
|
||||
self.skip_existing = bool(exec_cfg.get("skip_existing", True))
|
||||
self.timeout_per_test = int(exec_cfg.get("timeout_per_test", 30))
|
||||
|
||||
if not self.mt5_path or not os.path.isfile(self.mt5_path):
|
||||
raise FileNotFoundError(
|
||||
f"MT5 终端路径未配置或不存在: {self.mt5_path!r}\n"
|
||||
"请在 GUI '回测配置' 标签页设置 terminal64.exe 的完整路径后点 '保存配置'。"
|
||||
)
|
||||
|
||||
def _load_config(self, config_path: str) -> Dict:
|
||||
import yaml
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
def _load_execution_log(self) -> List[Dict]:
|
||||
log_file = os.path.join(self.ini_dir, "..", "results", "execution_log.json")
|
||||
log_file = os.path.normpath(log_file)
|
||||
if os.path.exists(log_file):
|
||||
with open(log_file, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
return []
|
||||
|
||||
def _save_execution_log(self, log: List[Dict]):
|
||||
log_file = os.path.join(self.ini_dir, "..", "results", "execution_log.json")
|
||||
log_file = os.path.normpath(log_file)
|
||||
os.makedirs(os.path.dirname(log_file), exist_ok=True)
|
||||
with open(log_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(log, f, indent=2, ensure_ascii=False)
|
||||
|
||||
def _get_report_path(self, ini_name: str) -> str:
|
||||
base_name = os.path.splitext(ini_name)[0]
|
||||
return os.path.join(self.data_dir, f"{base_name}.htm") if self.data_dir else f"{base_name}.htm"
|
||||
|
||||
def _copy_report_to_project(self, ini_name: str) -> str:
|
||||
base_name = os.path.splitext(ini_name)[0]
|
||||
src_path = self._get_report_path(ini_name)
|
||||
|
||||
if not os.path.exists(src_path):
|
||||
logger.warning(f"Source report not found: {src_path}")
|
||||
return None
|
||||
|
||||
dest_path = os.path.join(self.reports_dir, f"{base_name}.htm")
|
||||
try:
|
||||
shutil.copy2(src_path, dest_path)
|
||||
logger.info(f"Report copied to: {dest_path}")
|
||||
return dest_path
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to copy report: {e}")
|
||||
return None
|
||||
|
||||
def _kill_mt5(self):
|
||||
try:
|
||||
subprocess.run(['taskkill', '/F', '/IM', 'terminal64.exe'],
|
||||
capture_output=True, text=True)
|
||||
time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
def _read_ini_expert(self, ini_path: str) -> str:
|
||||
try:
|
||||
with open(ini_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
if line.startswith('Expert='):
|
||||
return line.split('=', 1)[1].strip()
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _execute_single_ini(self, ini_path: str, timeout_min: int = 30) -> Dict:
|
||||
result = {
|
||||
"ini_file": os.path.basename(ini_path),
|
||||
"status": "pending",
|
||||
"start_time": datetime.now().isoformat(),
|
||||
"end_time": None,
|
||||
"duration_sec": 0,
|
||||
"report_found": False,
|
||||
"report_path": None,
|
||||
"error": None
|
||||
}
|
||||
|
||||
ini_name = os.path.basename(ini_path)
|
||||
expected_report = self._get_report_path(ini_name)
|
||||
|
||||
if os.path.exists(expected_report):
|
||||
result["status"] = "already_completed"
|
||||
result["report_found"] = True
|
||||
result["report_path"] = expected_report
|
||||
logger.info(f"Already completed: {ini_name}")
|
||||
return result
|
||||
|
||||
logger.info(f"Executing: {ini_name}")
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[self.mt5_path, f"/config:{ini_path}"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL
|
||||
)
|
||||
|
||||
logger.info(f"MT5 started PID: {proc.pid}")
|
||||
|
||||
while True:
|
||||
if proc.poll() is not None:
|
||||
logger.info("MT5 process ended")
|
||||
break
|
||||
|
||||
if os.path.exists(expected_report):
|
||||
try:
|
||||
with open(expected_report, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
if len(content) > 1000:
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
if elapsed > timeout_min * 60:
|
||||
logger.warning(f"Timeout: {ini_name}")
|
||||
break
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
end_time = time.time()
|
||||
result["duration_sec"] = end_time - start_time
|
||||
|
||||
if os.path.exists(expected_report):
|
||||
result["status"] = "completed"
|
||||
result["report_found"] = True
|
||||
copied_path = self._copy_report_to_project(ini_name)
|
||||
result["report_path"] = copied_path or expected_report
|
||||
logger.info(f"Success: {ini_name} ({result['duration_sec']:.0f}s)")
|
||||
else:
|
||||
result["status"] = "failed"
|
||||
logger.warning(f"Failed: {ini_name}")
|
||||
|
||||
result["end_time"] = datetime.now().isoformat()
|
||||
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
logger.error(f"Error: {e}")
|
||||
|
||||
return result
|
||||
|
||||
def run_full_auto(self, ini_files: List[str] = None,
|
||||
skip_if_exists: bool = None,
|
||||
kill_between_tests: bool = None,
|
||||
timeout_min: int = None) -> List[Dict]:
|
||||
logger.info("="*60)
|
||||
logger.info("MT5 Full Auto Batch Runner")
|
||||
logger.info("="*60)
|
||||
|
||||
if skip_if_exists is None:
|
||||
skip_if_exists = self.skip_existing
|
||||
if kill_between_tests is None:
|
||||
kill_between_tests = self.kill_between
|
||||
if timeout_min is None:
|
||||
timeout_min = self.timeout_per_test
|
||||
|
||||
logger.info(f"Timeout per test: {timeout_min} minutes, Kill between tests: {kill_between_tests}, Skip existing: {skip_if_exists}")
|
||||
|
||||
self.execution_log = self._load_execution_log()
|
||||
|
||||
if ini_files is None:
|
||||
ini_files = [os.path.join(self.ini_dir, f)
|
||||
for f in os.listdir(self.ini_dir) if f.endswith('.ini')]
|
||||
|
||||
total = len(ini_files)
|
||||
completed = 0
|
||||
failed = 0
|
||||
|
||||
logger.info(f"Total INI files: {total}")
|
||||
|
||||
for idx, ini_path in enumerate(ini_files, 1):
|
||||
logger.info(f"[{idx}/{total}] {os.path.basename(ini_path)}")
|
||||
|
||||
result = self._execute_single_ini(ini_path, timeout_min=timeout_min)
|
||||
|
||||
self.execution_log.append(result)
|
||||
self._save_execution_log(self.execution_log)
|
||||
|
||||
if result["status"] == "completed":
|
||||
completed += 1
|
||||
elif result["status"] == "already_completed":
|
||||
completed += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
logger.info(f" Status: {result['status']}, Duration: {result['duration_sec']:.0f}s")
|
||||
|
||||
if kill_between_tests and result["status"] != "already_completed":
|
||||
self._kill_mt5()
|
||||
time.sleep(2)
|
||||
|
||||
logger.info("="*60)
|
||||
logger.info(f"DONE: {completed}/{total} completed, {failed} failed")
|
||||
logger.info("="*60)
|
||||
|
||||
return self.execution_log
|
||||
|
||||
def run_daemon(self, check_interval: int = 60):
|
||||
logger.info("="*60)
|
||||
logger.info("MT5 Auto Runner - DAEMON MODE")
|
||||
logger.info(f"Monitoring: {self.ini_dir}")
|
||||
logger.info(f"Reports will be copied to: {self.reports_dir}")
|
||||
logger.info(f"Timeout per test: {self.timeout_per_test} minutes")
|
||||
logger.info("Press Ctrl+C to stop")
|
||||
logger.info("="*60)
|
||||
|
||||
self.execution_log = self._load_execution_log()
|
||||
|
||||
try:
|
||||
while True:
|
||||
pending_inis = []
|
||||
for f in os.listdir(self.ini_dir):
|
||||
if f.endswith('.ini'):
|
||||
is_completed = any(
|
||||
log.get("ini_file") == f and log.get("status") == "completed"
|
||||
for log in self.execution_log
|
||||
)
|
||||
if not is_completed:
|
||||
pending_inis.append(f)
|
||||
|
||||
if pending_inis:
|
||||
logger.info(f"Found {len(pending_inis)} pending INI files")
|
||||
for ini_name in pending_inis:
|
||||
ini_path = os.path.join(self.ini_dir, ini_name)
|
||||
result = self._execute_single_ini(ini_path, timeout_min=self.timeout_per_test)
|
||||
self.execution_log.append(result)
|
||||
self._save_execution_log(self.execution_log)
|
||||
self._kill_mt5()
|
||||
time.sleep(3)
|
||||
else:
|
||||
logger.info("No pending INI files, waiting...")
|
||||
|
||||
time.sleep(check_interval)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Daemon stopped")
|
||||
|
||||
def generate_report(self) -> str:
|
||||
logger.info("Generating summary report...")
|
||||
|
||||
from scripts.result_parser import ResultParser
|
||||
parser = ResultParser(self.reports_dir)
|
||||
results = parser.parse_all_reports(pattern="*.ht*")
|
||||
|
||||
if results:
|
||||
from scripts.report_generator import ReportGenerator
|
||||
generator = ReportGenerator(results)
|
||||
os.makedirs("reports", exist_ok=True)
|
||||
output = generator.generate_excel("reports/batch_summary.xlsx")
|
||||
logger.info(f"Report saved: {output}")
|
||||
return output
|
||||
else:
|
||||
logger.warning("No results to generate report")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="MT5 Auto Runner")
|
||||
parser.add_argument("--config", "-c", default="config/ea_configs.yaml")
|
||||
parser.add_argument("--mode", "-m", choices=["full", "daemon", "report", "init", "execute"],
|
||||
default="full")
|
||||
parser.add_argument("--interval", "-i", type=int, default=60)
|
||||
parser.add_argument("--no-skip", action="store_true")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
runner = MT5AutoRunner(args.config)
|
||||
|
||||
if args.mode == "daemon":
|
||||
runner.run_daemon(check_interval=args.interval)
|
||||
elif args.mode == "report":
|
||||
runner.generate_report()
|
||||
elif args.mode == "init":
|
||||
from scripts.ini_generator import INIGenerator
|
||||
generator = INIGenerator(args.config)
|
||||
ini_files = generator.generate_ini_files()
|
||||
print(f"Generated {len(ini_files)} INI files")
|
||||
elif args.mode == "execute":
|
||||
from scripts.ini_generator import INIGenerator
|
||||
generator = INIGenerator(args.config)
|
||||
ini_files = generator.generate_ini_files()
|
||||
results = runner.run_full_auto(ini_files, skip_if_exists=not args.no_skip)
|
||||
completed = sum(1 for r in results if r["status"] == "completed")
|
||||
print(f"Completed: {completed}/{len(results)}")
|
||||
else:
|
||||
from scripts.ini_generator import INIGenerator
|
||||
generator = INIGenerator(args.config)
|
||||
ini_files = generator.generate_ini_files()
|
||||
logger.info(f"Generated {len(ini_files)} INI files")
|
||||
|
||||
results = runner.run_full_auto(ini_files, skip_if_exists=not args.no_skip)
|
||||
|
||||
runner.generate_report()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""
|
||||
MT5 路径占位符解析(不做任何自动探测)。
|
||||
|
||||
支持占位符:
|
||||
{PROJECT_ROOT} 项目根目录绝对路径
|
||||
{APP_DATA_DIR} %APPDATA% (C:\\Users\\<user>\\AppData\\Roaming)
|
||||
${APPDATA} 同上 (POSIX 风格)
|
||||
${USERPROFILE} C:\\Users\\<user>
|
||||
|
||||
留空的路径会原样返回,由调用方/用户决定。
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
def _expand_placeholders(value: str, project_root: str) -> str:
|
||||
if not value:
|
||||
return value
|
||||
expanded = (
|
||||
value
|
||||
.replace("{PROJECT_ROOT}", project_root)
|
||||
.replace("{APP_DATA_DIR}", os.environ.get("APPDATA", ""))
|
||||
.replace("${APPDATA}", os.environ.get("APPDATA", ""))
|
||||
.replace("${USERPROFILE}", os.environ.get("USERPROFILE", ""))
|
||||
)
|
||||
if "{" in expanded and "}" in expanded:
|
||||
return expanded
|
||||
return os.path.normpath(expanded)
|
||||
|
||||
|
||||
def resolve_mt5_settings(mt5_settings: dict, project_root: Optional[str] = None) -> dict:
|
||||
"""只做 {PROJECT_ROOT} 等占位符替换;空值原样保留,不做任何自动探测。"""
|
||||
if project_root is None:
|
||||
project_root = PROJECT_ROOT
|
||||
|
||||
resolved = dict(mt5_settings)
|
||||
|
||||
for key, default_sub in (("ini_dir", os.path.join("config", "generated")),
|
||||
("reports_dir", "reports")):
|
||||
v = resolved.get(key, "")
|
||||
if not v:
|
||||
resolved[key] = os.path.join(project_root, default_sub)
|
||||
else:
|
||||
expanded = _expand_placeholders(v, project_root)
|
||||
if expanded and (os.path.isabs(expanded) or "{" not in expanded):
|
||||
resolved[key] = expanded
|
||||
else:
|
||||
resolved[key] = os.path.join(project_root, expanded)
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pprint
|
||||
sample = {
|
||||
"terminal_path": "",
|
||||
"data_dir": "",
|
||||
"ini_dir": "{PROJECT_ROOT}/config/generated",
|
||||
"reports_dir": "{PROJECT_ROOT}/reports",
|
||||
}
|
||||
pprint.pp(resolve_mt5_settings(sample))
|
||||
@@ -0,0 +1,320 @@
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import glob
|
||||
import shutil
|
||||
from typing import List, Dict, Any, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
class ResultAnalyzer:
|
||||
def __init__(self, reports_dir: str = 'reports'):
|
||||
self.reports_dir = reports_dir
|
||||
|
||||
def parse_mt5_html_report(self, report_path: str) -> Optional[Dict]:
|
||||
if not os.path.exists(report_path):
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(report_path, 'rb') as f:
|
||||
raw_bytes = f.read()
|
||||
|
||||
if raw_bytes[:2] == b'\xff\xfe':
|
||||
content = raw_bytes[2:].decode('utf-16-le')
|
||||
else:
|
||||
content = raw_bytes.decode('utf-8', errors='ignore')
|
||||
|
||||
content = content.encode('utf-8').decode('utf-8')
|
||||
soup = BeautifulSoup(content, 'html.parser')
|
||||
|
||||
result = {}
|
||||
|
||||
pattern = r'<td[^>]*>([^<]+):</td>\s*<td[^>]*><b>([^<]*)</b></td>'
|
||||
matches = re.findall(pattern, content)
|
||||
|
||||
plain_pattern = r'<td[^>]*nowrap[^>]*>([^<]+):</td>\s*<td[^>]*colspan=.10.[^>]*><b>([^<]*)</b></td>'
|
||||
plain_matches = re.findall(plain_pattern, content)
|
||||
|
||||
all_matches = dict(plain_matches)
|
||||
for k, v in matches:
|
||||
if k not in all_matches:
|
||||
all_matches[k] = v
|
||||
|
||||
for key, value in all_matches.items():
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
|
||||
if 'Net Profit' in key or '\u603b\u51c0\u76c8\u5229' in key:
|
||||
result['net_profit'] = self._extract_number(value)
|
||||
elif 'Gross Profit' in key or '\u6bdb\u5229' in key:
|
||||
result['gross_profit'] = self._extract_number(value)
|
||||
elif 'Gross Loss' in key or '\u6bdb\u635f' in key:
|
||||
result['gross_loss'] = self._extract_number(value)
|
||||
elif 'Profit Factor' in key or '\u76c8\u5229\u56e0\u5b50' in key:
|
||||
result['profit_factor'] = self._extract_number(value)
|
||||
elif 'Total Trades' in key or '\u4ea4\u6613\u603b\u8ba1' in key:
|
||||
result['total_trades'] = self._extract_number(value)
|
||||
elif 'Sharpe Ratio' in key or '\u590f\u666e\u6bd4\u7387' in key:
|
||||
result['sharpe_ratio'] = self._extract_number(value)
|
||||
elif 'Maximal Drawdown' in key or '\u6700\u5927\u7ed3\u4f59\u4e8f\u635f' in key:
|
||||
result['max_drawdown'] = self._extract_number(value)
|
||||
elif 'Win Rate' in key or '\u76c8\u5229\u4ea4\u6613' in key:
|
||||
result['win_rate'] = self._extract_percentage(value)
|
||||
|
||||
return result if result else None
|
||||
|
||||
except Exception as e:
|
||||
print('Error parsing ' + report_path + ': ' + str(e))
|
||||
return None
|
||||
|
||||
def _extract_number(self, text: str) -> float:
|
||||
numbers = re.findall(r'[-+]?\d*\.?\d+', text.replace(',', '').replace(' ', ''))
|
||||
if numbers:
|
||||
try:
|
||||
return float(numbers[0])
|
||||
except:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
def _extract_percentage(self, text: str) -> float:
|
||||
numbers = re.findall(r'\d+\.?\d*%', text)
|
||||
if numbers:
|
||||
try:
|
||||
return float(numbers[0].replace('%', ''))
|
||||
except:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
def parse_mt5_xml_optimization(self, xml_path: str) -> List[Dict]:
|
||||
if not os.path.exists(xml_path):
|
||||
return []
|
||||
|
||||
results = []
|
||||
try:
|
||||
tree = ET.parse(xml_path)
|
||||
root = tree.getroot()
|
||||
|
||||
ns = {'ss': 'urn:schemas-microsoft-com:office:spreadsheet'}
|
||||
worksheet = root.find('.//ss:Worksheet', ns)
|
||||
|
||||
if worksheet is None:
|
||||
return []
|
||||
|
||||
table = worksheet.find('ss:Table', ns)
|
||||
if table is None:
|
||||
return []
|
||||
|
||||
rows = table.findall('ss:Row', ns)
|
||||
if len(rows) < 2:
|
||||
return []
|
||||
|
||||
header_row = rows[0]
|
||||
headers = []
|
||||
for cell in header_row.findall('ss:Cell', ns):
|
||||
data = cell.find('ss:Data', ns)
|
||||
if data is not None and data.text:
|
||||
headers.append(data.text.strip().lower())
|
||||
|
||||
for row_idx, row in enumerate(rows[1:], start=1):
|
||||
cells = row.findall('ss:Cell', ns)
|
||||
if not cells:
|
||||
continue
|
||||
|
||||
result = {'pass': row_idx}
|
||||
for col_idx, cell in enumerate(cells):
|
||||
if col_idx >= len(headers):
|
||||
break
|
||||
header = headers[col_idx]
|
||||
data = cell.find('ss:Data', ns)
|
||||
value = data.text.strip() if data is not None and data.text else ''
|
||||
|
||||
if 'pass' in header:
|
||||
result['pass'] = int(value) if value.isdigit() else row_idx
|
||||
elif 'result' in header:
|
||||
result['result'] = self._extract_number(value)
|
||||
elif 'profit' in header and 'expected' not in header and 'factor' not in header and 'drawdown' not in header:
|
||||
result['net_profit'] = self._extract_number(value)
|
||||
elif 'expected payoff' in header:
|
||||
result['expected_payoff'] = self._extract_number(value)
|
||||
elif 'profit factor' in header:
|
||||
if value:
|
||||
result['profit_factor'] = self._extract_number(value)
|
||||
elif 'recovery factor' in header:
|
||||
result['recovery_factor'] = self._extract_number(value)
|
||||
elif 'sharpe ratio' in header:
|
||||
result['sharpe_ratio'] = self._extract_number(value)
|
||||
elif 'custom' in header:
|
||||
result['custom'] = self._extract_number(value)
|
||||
elif 'equity dd' in header or 'drawdown' in header:
|
||||
result['max_drawdown'] = self._extract_number(value)
|
||||
elif 'trade' in header:
|
||||
result['total_trades'] = int(self._extract_number(value))
|
||||
|
||||
if result:
|
||||
results.append(result)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
print('Error parsing MT5 XML ' + xml_path + ': ' + str(e))
|
||||
traceback.print_exc()
|
||||
|
||||
return results
|
||||
|
||||
def copy_xml_results_to_reports(self, mt5_terminal_dir: str, ea_name: str, reports_dir: str) -> str:
|
||||
xml_patterns = [
|
||||
f'{ea_name}_optimization.xml',
|
||||
f'{ea_name}_optimization[1].xml',
|
||||
f'{ea_name}_optimization[2].xml',
|
||||
]
|
||||
|
||||
os.makedirs(reports_dir, exist_ok=True)
|
||||
latest_xml = None
|
||||
latest_time = 0
|
||||
|
||||
for pattern in xml_patterns:
|
||||
xml_path = os.path.join(mt5_terminal_dir, pattern)
|
||||
if os.path.exists(xml_path):
|
||||
mtime = os.path.getmtime(xml_path)
|
||||
if mtime > latest_time:
|
||||
latest_time = mtime
|
||||
latest_xml = xml_path
|
||||
|
||||
if latest_xml:
|
||||
dest_path = os.path.join(reports_dir, os.path.basename(latest_xml))
|
||||
shutil.copy2(latest_xml, dest_path)
|
||||
return dest_path
|
||||
|
||||
return None
|
||||
|
||||
def parse_all_reports(self, pattern: str = '*.htm*') -> List[Dict]:
|
||||
results = []
|
||||
search_path = os.path.join(self.reports_dir, pattern)
|
||||
report_files = glob.glob(search_path)
|
||||
|
||||
for report_file in report_files:
|
||||
parsed = self.parse_mt5_html_report(report_file)
|
||||
if parsed:
|
||||
parsed['report_file'] = os.path.basename(report_file)
|
||||
results.append(parsed)
|
||||
|
||||
return results
|
||||
|
||||
def merge_parameters_and_results(self, param_mapping: List[Dict], results: List[Dict]) -> List[Dict]:
|
||||
merged = []
|
||||
result_map = {r.get('report_file', ''): r for r in results}
|
||||
|
||||
for param_set in param_mapping:
|
||||
report_name = param_set.get('report_name', '')
|
||||
if report_name in result_map:
|
||||
combined = {**param_set, **result_map[report_name]}
|
||||
merged.append(combined)
|
||||
|
||||
return merged
|
||||
|
||||
def find_optimal_params(self, results: List[Dict],
|
||||
criterion: str = 'profit_factor',
|
||||
min_trades: int = 10,
|
||||
max_drawdown_pct: float = 50.0) -> List[Dict]:
|
||||
filtered = []
|
||||
for r in results:
|
||||
trades = r.get('total_trades', 0)
|
||||
dd = r.get('max_drawdown', 0)
|
||||
pf = r.get('profit_factor', 0)
|
||||
|
||||
if trades >= min_trades and dd <= max_drawdown_pct and pf > 0:
|
||||
filtered.append(r)
|
||||
|
||||
filtered.sort(key=lambda x: x.get(criterion, 0), reverse=True)
|
||||
return filtered
|
||||
|
||||
def generate_report(self, results: List[Dict], output_path: str = 'results/optimization_report.txt'):
|
||||
if not results:
|
||||
return
|
||||
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
||||
lines = [
|
||||
'=' * 60,
|
||||
'MT5 EA Optimization Report',
|
||||
'=' * 60,
|
||||
'',
|
||||
'Total Results: ' + str(len(results)),
|
||||
'',
|
||||
]
|
||||
|
||||
if results:
|
||||
best = results[0]
|
||||
lines.extend([
|
||||
'Best Configuration:',
|
||||
' Profit Factor: ' + str(best.get('profit_factor', 0)),
|
||||
' Net Profit: ' + str(best.get('net_profit', 0)),
|
||||
' Total Trades: ' + str(best.get('total_trades', 0)),
|
||||
' Max Drawdown: ' + str(best.get('max_drawdown', 0)),
|
||||
' Win Rate: ' + str(best.get('win_rate', 0)) + '%',
|
||||
'',
|
||||
])
|
||||
|
||||
lines.append('Top 10 Configurations:')
|
||||
lines.append('-' * 60)
|
||||
for i, r in enumerate(results[:10], 1):
|
||||
lines.append(
|
||||
str(i) + '. PF=' + str(r.get('profit_factor', 0)) +
|
||||
' Net=' + str(r.get('net_profit', 0)) +
|
||||
' Trades=' + str(r.get('total_trades', 0))
|
||||
)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
return output_path
|
||||
|
||||
def analyze_walk_forward(self, mt5_terminal_dir: str, ea_name: str, reports_dir: str) -> Dict:
|
||||
result = {
|
||||
'ea_name': ea_name,
|
||||
'in_sample': {},
|
||||
'out_of_sample': {},
|
||||
'overfitting_score': 0,
|
||||
'verdict': 'Unknown'
|
||||
}
|
||||
|
||||
xml_path = self.copy_xml_results_to_reports(mt5_terminal_dir, ea_name, reports_dir)
|
||||
|
||||
if xml_path:
|
||||
is_results = self.parse_mt5_xml_optimization(xml_path)
|
||||
if is_results:
|
||||
best_is = sorted(is_results, key=lambda x: x.get('profit_factor', 0), reverse=True)[0]
|
||||
result['in_sample'] = best_is
|
||||
|
||||
forward_xml = os.path.join(mt5_terminal_dir, f'{ea_name}_optimization.forward.xml')
|
||||
if not os.path.exists(forward_xml):
|
||||
forward_xml = os.path.join(mt5_terminal_dir, f'{ea_name}_optimization[1].forward.xml')
|
||||
|
||||
if os.path.exists(forward_xml):
|
||||
oos_results = self.parse_mt5_xml_optimization(forward_xml)
|
||||
if oos_results:
|
||||
best_oos = sorted(oos_results, key=lambda x: x.get('profit_factor', 0), reverse=True)[0]
|
||||
result['out_of_sample'] = best_oos
|
||||
|
||||
is_pf = result.get('in_sample', {}).get('profit_factor', 0)
|
||||
oos_pf = result.get('out_of_sample', {}).get('profit_factor', 0)
|
||||
|
||||
if is_pf > 0 and oos_pf > 0:
|
||||
decay = (is_pf - oos_pf) / is_pf
|
||||
result['pf_decay'] = round(decay * 100, 1)
|
||||
if oos_pf >= is_pf * 0.7 and oos_pf > 1.0:
|
||||
result['verdict'] = 'Robust (\u7a33\u5065)'
|
||||
elif oos_pf > 1.0:
|
||||
result['verdict'] = 'Mild Overfit (\u8f7b\u5ea6\u8fc7\u62df\u5408)'
|
||||
else:
|
||||
result['verdict'] = 'Overfit (\u4e25\u91cd\u8fc7\u62df\u5408)'
|
||||
elif oos_pf > 0:
|
||||
result['pf_decay'] = 0
|
||||
result['verdict'] = 'Valid (\u6709\u6548\u4f46IS\u65e0\u7ed3\u679c)'
|
||||
|
||||
is_dd = result.get('in_sample', {}).get('max_drawdown', 0)
|
||||
oos_dd = result.get('out_of_sample', {}).get('max_drawdown', 0)
|
||||
if is_dd > 0 and oos_dd > 0:
|
||||
result['dd_increase'] = round((oos_dd / is_dd - 1) * 100, 1) if is_dd > 0 else 0
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,243 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPTIMIZER_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, OPTIMIZER_DIR)
|
||||
|
||||
from ea_config_parser import load_all_ea_configs, EAConfig
|
||||
from smart_search import SmartSearch
|
||||
from parameter_constraint import create_constraint_engine
|
||||
from analyze_results import ResultAnalyzer
|
||||
|
||||
|
||||
class BatchAutoOptimizer:
|
||||
def __init__(self, base_dir: str = "optimizer", mt5_path: str = None):
|
||||
self.base_dir = base_dir
|
||||
self.mt5_path = mt5_path
|
||||
self.configs_dir = os.path.join(base_dir, "configs")
|
||||
self.set_files_dir = os.path.join(base_dir, "set_files")
|
||||
self.reports_dir = os.path.join(base_dir, "reports")
|
||||
self.results_dir = os.path.join(base_dir, "results")
|
||||
self.scripts_dir = os.path.join(base_dir, "scripts")
|
||||
|
||||
for d in [self.configs_dir, self.set_files_dir, self.reports_dir,
|
||||
self.results_dir, self.scripts_dir]:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
def generate_set_files(self, ea_config: EAConfig, max_samples: int = 1000) -> List[Dict]:
|
||||
logger.info(f"Generating SET files for {ea_config.ea_name}")
|
||||
logger.info(f" Combinations: {ea_config.get_total_combinations()}")
|
||||
|
||||
searcher = SmartSearch(ea_config)
|
||||
combinations = searcher.generate_combinations(max_samples)
|
||||
logger.info(f" Generated {len(combinations)} samples")
|
||||
|
||||
engine = create_constraint_engine(ea_config)
|
||||
valid_combos = engine.filter_valid_combinations(combinations)
|
||||
logger.info(f" Valid combinations: {len(valid_combos)}")
|
||||
|
||||
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
||||
os.makedirs(ea_set_dir, exist_ok=True)
|
||||
|
||||
param_mapping = []
|
||||
for i, combo in enumerate(valid_combos, 1):
|
||||
set_id = str(i).zfill(8)
|
||||
set_filename = f"params_{set_id}.set"
|
||||
set_path = os.path.join(ea_set_dir, set_filename)
|
||||
|
||||
self._write_set_file(set_path, combo)
|
||||
|
||||
param_mapping.append({
|
||||
'set_id': set_id,
|
||||
'set_file': set_filename,
|
||||
'set_path': set_path,
|
||||
'report_name': f"{ea_config.ea_name}_{set_id}.htm",
|
||||
'params': combo
|
||||
})
|
||||
|
||||
mapping_path = os.path.join(self.results_dir, f"{ea_config.ea_name}_param_mapping.json")
|
||||
with open(mapping_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(param_mapping, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f" SET files saved to {ea_set_dir}")
|
||||
return param_mapping
|
||||
|
||||
def _write_set_file(self, set_path: str, params: Dict):
|
||||
lines = ["; MT5 EA Parameters", f"; Generated: {datetime.now()}", ""]
|
||||
for name, value in params.items():
|
||||
if isinstance(value, bool):
|
||||
lines.append(f"{name} <true> <{'true' if value else 'false'}>")
|
||||
else:
|
||||
lines.append(f"{name} <{value}> <{value}>")
|
||||
with open(set_path, 'w', encoding='utf-8') as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
def generate_ini(self, ea_config: EAConfig) -> str:
|
||||
test_config = ea_config.test_config
|
||||
ini_path = os.path.join(self.configs_dir, f"{ea_config.ea_name}.ini")
|
||||
|
||||
date_from = test_config.get("from_date", "2025.01.01")
|
||||
date_to = test_config.get("to_date", "2025.12.31")
|
||||
|
||||
lines = [
|
||||
"; MT5 Strategy Tester Configuration",
|
||||
f"; Generated: {datetime.now()}",
|
||||
"",
|
||||
"[Tester]",
|
||||
f"Expert={ea_config.ea_path}",
|
||||
"ExpertParameters=",
|
||||
f"Symbol={test_config.get('symbol', 'EURUSD')}",
|
||||
f"Period={test_config.get('period', 'H1')}",
|
||||
f"Model={test_config.get('model', 1)}",
|
||||
f"FromDate={date_from}",
|
||||
f"ToDate={date_to}",
|
||||
f"Deposit={test_config.get('deposit', 10000)}",
|
||||
f"Leverage={test_config.get('leverage', '1:100')}",
|
||||
"ReplaceReport=true",
|
||||
"ShutdownTerminal=true",
|
||||
]
|
||||
|
||||
with open(ini_path, 'w', encoding='utf-8') as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
logger.info(f"INI saved: {ini_path}")
|
||||
return ini_path
|
||||
|
||||
def generate_powershell_script(self, ea_config: EAConfig, param_mapping: List[Dict]) -> str:
|
||||
ps_path = os.path.join(self.scripts_dir, f"run_{ea_config.ea_name}.ps1")
|
||||
|
||||
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
||||
ea_report_dir = os.path.join(self.reports_dir, ea_config.ea_name)
|
||||
os.makedirs(ea_report_dir, exist_ok=True)
|
||||
|
||||
ini_path = os.path.join(self.configs_dir, f"{ea_config.ea_name}.ini")
|
||||
|
||||
lines = [
|
||||
f"# MT5 Batch Optimization for {ea_config.ea_name}",
|
||||
f"# Generated: {datetime.now()}",
|
||||
"",
|
||||
f'$MT5Path = "{self.mt5_path or "terminal64.exe"}"',
|
||||
f'$IniFile = "{ini_path}"',
|
||||
f'$SetDir = "{ea_set_dir}"',
|
||||
"",
|
||||
f"Write-Host 'Starting batch optimization for {ea_config.ea_name}'",
|
||||
"",
|
||||
"$SetFiles = Get-ChildItem -Path $SetDir -Filter 'params_*.set'",
|
||||
"$Total = $SetFiles.Count",
|
||||
"$Current = 0",
|
||||
"",
|
||||
"foreach ($SetFile in $SetFiles) {",
|
||||
" $Current++",
|
||||
' Write-Host "[$Current/$Total] $($SetFile.Name)"',
|
||||
"",
|
||||
" $IniContent = Get-Content $IniFile",
|
||||
' $IniContent = $IniContent -replace "ExpertParameters=.*", "ExpertParameters=$($SetFile.FullName)"',
|
||||
' $TempIni = Join-Path $env:TEMP "temp_$([guid]::NewGuid().ToString().Substring(0,8)).ini"',
|
||||
" $IniContent | Set-Content $TempIni -Encoding UTF8",
|
||||
"",
|
||||
" Start-Process -FilePath $MT5Path -ArgumentList '/portable',\"/config:$TempIni\" -Wait",
|
||||
" Start-Sleep -Seconds 3",
|
||||
"}",
|
||||
"",
|
||||
'Write-Host "Batch optimization completed!"',
|
||||
]
|
||||
|
||||
with open(ps_path, 'w', encoding='utf-8') as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
logger.info(f"PowerShell script: {ps_path}")
|
||||
return ps_path
|
||||
|
||||
def run_full_auto(self, ea_names: List[str] = None, max_samples: int = 1000):
|
||||
logger.info("=" * 60)
|
||||
logger.info("MT5 Batch Auto Optimizer")
|
||||
logger.info("=" * 60)
|
||||
|
||||
configs = load_all_ea_configs(self.configs_dir)
|
||||
if not configs:
|
||||
logger.error(f"No configs found in {self.configs_dir}")
|
||||
return
|
||||
|
||||
if ea_names:
|
||||
configs = {k: v for k, v in configs.items() if k in ea_names}
|
||||
|
||||
for ea_name, ea_config in configs.items():
|
||||
logger.info(f"\nProcessing: {ea_name}")
|
||||
|
||||
param_mapping = self.generate_set_files(ea_config, max_samples)
|
||||
self.generate_ini(ea_config)
|
||||
self.generate_powershell_script(ea_config, param_mapping)
|
||||
|
||||
logger.info(f" SET files: {len(param_mapping)}")
|
||||
logger.info(f" Next: Run the PS1 script in MT5 terminal")
|
||||
|
||||
def analyze_all(self, ea_names: List[str] = None) -> List[Dict]:
|
||||
logger.info("Analyzing results...")
|
||||
|
||||
all_results = []
|
||||
search_dirs = [self.reports_dir] if not ea_names else [os.path.join(self.reports_dir, name) for name in ea_names]
|
||||
|
||||
for search_dir in search_dirs:
|
||||
if not os.path.exists(search_dir):
|
||||
continue
|
||||
|
||||
for ea_name in os.listdir(search_dir):
|
||||
ea_report_dir = os.path.join(search_dir, ea_name)
|
||||
if not os.path.isdir(ea_report_dir):
|
||||
continue
|
||||
|
||||
analyzer = ResultAnalyzer(ea_report_dir)
|
||||
results = analyzer.parse_all_reports()
|
||||
|
||||
for r in results:
|
||||
r['ea_name'] = ea_name
|
||||
|
||||
all_results.extend(results)
|
||||
logger.info(f" {ea_name}: {len(results)} reports")
|
||||
|
||||
if not all_results:
|
||||
logger.warning("No results found")
|
||||
return []
|
||||
|
||||
all_results.sort(key=lambda x: x.get('profit_factor', 0), reverse=True)
|
||||
|
||||
output_path = os.path.join(self.results_dir, "optimization_results.json")
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.info(f"Results: {len(all_results)} total")
|
||||
logger.info(f"Best PF: {all_results[0].get('profit_factor', 0) if all_results else 'N/A'}")
|
||||
|
||||
return all_results
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="MT5 Batch Auto Optimizer")
|
||||
parser.add_argument("--base-dir", "-d", default="optimizer", help="Base directory")
|
||||
parser.add_argument("--ea", "-e", nargs="+", help="EA names")
|
||||
parser.add_argument("--max-samples", "-m", type=int, default=1000, help="Max samples per EA")
|
||||
parser.add_argument("--mt5-path", "-p", help="MT5 terminal path")
|
||||
parser.add_argument("--analyze", "-a", action="store_true", help="Analyze only")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
optimizer = BatchAutoOptimizer(args.base_dir, args.mt5_path)
|
||||
|
||||
if args.analyze:
|
||||
optimizer.analyze_all(args.ea)
|
||||
else:
|
||||
optimizer.run_full_auto(args.ea, args.max_samples)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"ea_name": "Grid Pro",
|
||||
"ea_path": "Experts\\\\Grid\\\\GridPro.ex5",
|
||||
"description": "Multi-period grid EA with genetic optimization",
|
||||
"search_strategy": "genetic",
|
||||
"optimization_criterion": "profit_factor",
|
||||
"parameters": {
|
||||
"GridLevels": {
|
||||
"type": "int",
|
||||
"default": 5,
|
||||
"min": 3,
|
||||
"max": 20,
|
||||
"step": 1,
|
||||
"description": "Grid levels"
|
||||
},
|
||||
"GridSpacing": {
|
||||
"type": "int",
|
||||
"default": 50,
|
||||
"min": 10,
|
||||
"max": 200,
|
||||
"step": 10,
|
||||
"description": "Grid spacing in points"
|
||||
},
|
||||
"GridSpacingMult": {
|
||||
"type": "double",
|
||||
"default": 1.5,
|
||||
"min": 1.0,
|
||||
"max": 3.0,
|
||||
"step": 0.1,
|
||||
"precision": 1,
|
||||
"description": "Spacing multiplier"
|
||||
},
|
||||
"BaseLotsize": {
|
||||
"type": "double",
|
||||
"default": 0.1,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"precision": 2,
|
||||
"description": "Base lot size"
|
||||
},
|
||||
"MaxOpenLots": {
|
||||
"type": "double",
|
||||
"default": 5.0,
|
||||
"min": 0.1,
|
||||
"max": 20.0,
|
||||
"step": 0.1,
|
||||
"precision": 1,
|
||||
"description": "Max open lots"
|
||||
},
|
||||
"MaxDrawdown": {
|
||||
"type": "int",
|
||||
"default": 30,
|
||||
"min": 10,
|
||||
"max": 100,
|
||||
"step": 5,
|
||||
"description": "Max drawdown percentage"
|
||||
},
|
||||
"TakeProfitGrid": {
|
||||
"type": "int",
|
||||
"default": 20,
|
||||
"min": 5,
|
||||
"max": 100,
|
||||
"step": 5,
|
||||
"description": "Take profit grid"
|
||||
},
|
||||
"TradingMode": {
|
||||
"type": "enum",
|
||||
"default": "BOTH",
|
||||
"options": [
|
||||
"BUY",
|
||||
"SELL",
|
||||
"BOTH"
|
||||
],
|
||||
"description": "Trading mode"
|
||||
}
|
||||
},
|
||||
"test_config": {
|
||||
"symbol": "EURUSD",
|
||||
"period": "H1",
|
||||
"from_date": "2024.01.01",
|
||||
"to_date": "2025.12.31",
|
||||
"model": 1,
|
||||
"deposit": 5000,
|
||||
"leverage": "1:100"
|
||||
},
|
||||
"walk_forward": {
|
||||
"enabled": true,
|
||||
"train_test_ratio": 0.75
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"ea_name": "MA Cross Scalper",
|
||||
"ea_path": "Experts\\\\MA_Cross.ex5",
|
||||
"description": "Dual MA crossover scalping strategy",
|
||||
"search_strategy": "grid",
|
||||
"optimization_criterion": "profit_factor",
|
||||
"parameters": {
|
||||
"FastMA_Period": {
|
||||
"type": "int",
|
||||
"default": 14,
|
||||
"min": 5,
|
||||
"max": 30,
|
||||
"step": 1,
|
||||
"description": "Fast MA period"
|
||||
},
|
||||
"SlowMA_Period": {
|
||||
"type": "int",
|
||||
"default": 50,
|
||||
"min": 20,
|
||||
"max": 100,
|
||||
"step": 5,
|
||||
"description": "Slow MA period",
|
||||
"condition": ">FastMA_Period"
|
||||
},
|
||||
"StopLoss": {
|
||||
"type": "int",
|
||||
"default": 50,
|
||||
"min": 10,
|
||||
"max": 200,
|
||||
"step": 10,
|
||||
"description": "Stop loss in points"
|
||||
},
|
||||
"TakeProfit": {
|
||||
"type": "int",
|
||||
"default": 100,
|
||||
"min": 30,
|
||||
"max": 300,
|
||||
"step": 10,
|
||||
"description": "Take profit in points"
|
||||
},
|
||||
"RiskPercent": {
|
||||
"type": "double",
|
||||
"default": 1.0,
|
||||
"min": 0.5,
|
||||
"max": 3.0,
|
||||
"step": 0.5,
|
||||
"precision": 1,
|
||||
"description": "Risk percentage"
|
||||
},
|
||||
"EnableMM": {
|
||||
"type": "bool",
|
||||
"default": true,
|
||||
"description": "Enable money management"
|
||||
}
|
||||
},
|
||||
"test_config": {
|
||||
"symbol": "EURUSD",
|
||||
"period": "H1",
|
||||
"from_date": "2025.01.01",
|
||||
"to_date": "2025.12.31",
|
||||
"model": 1,
|
||||
"deposit": 10000,
|
||||
"leverage": "1:100"
|
||||
},
|
||||
"walk_forward": {
|
||||
"enabled": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import os
|
||||
import glob
|
||||
import re
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
class EAAutoScanner:
|
||||
def __init__(self, mt5_data_dir: str = None):
|
||||
self.mt5_data_dir = mt5_data_dir
|
||||
self.experts_dir = os.path.join(mt5_data_dir, "MQL5", "Experts") if mt5_data_dir else None
|
||||
|
||||
def scan_experts(self) -> List[Dict]:
|
||||
experts = []
|
||||
if not self.experts_dir or not os.path.exists(self.experts_dir):
|
||||
return experts
|
||||
for root, dirs, files in os.walk(self.experts_dir):
|
||||
for f in files:
|
||||
if f.endswith('.ex5'):
|
||||
full_path = os.path.join(root, f)
|
||||
rel_path = os.path.relpath(full_path, self.experts_dir)
|
||||
ea_name = os.path.splitext(f)[0]
|
||||
experts.append({'name': ea_name, 'filename': f, 'path': rel_path, 'full_path': full_path})
|
||||
return experts
|
||||
|
||||
def generate_json_config(self, ea_info: Dict, params: Dict = None) -> Dict:
|
||||
ea_name = ea_info["name"]
|
||||
config = {
|
||||
"ea_name": ea_name,
|
||||
"ea_path": ea_info["path"],
|
||||
"description": "Auto-scanned EA",
|
||||
"search_strategy": "auto",
|
||||
"optimization_criterion": "profit_factor",
|
||||
"parameters": {},
|
||||
"test_config": {
|
||||
"symbol": "EURUSD", "period": "H1",
|
||||
"from_date": "2025.01.01", "to_date": "2025.12.31",
|
||||
"model": 1, "deposit": 10000, "leverage": "1:100"
|
||||
},
|
||||
"walk_forward": {"enabled": False}
|
||||
}
|
||||
if params:
|
||||
for pname, pinfo in params.items():
|
||||
config["parameters"][pname] = {
|
||||
"type": pinfo.get("type", "int"),
|
||||
"default": pinfo.get("default", 0),
|
||||
"min": pinfo.get("min", 0),
|
||||
"max": pinfo.get("max", 100),
|
||||
"step": pinfo.get("step", 1),
|
||||
"description": "Auto-scanned"
|
||||
}
|
||||
return config
|
||||
|
||||
def auto_scan_all(self) -> List[Dict]:
|
||||
configs = []
|
||||
experts = self.scan_experts()
|
||||
for expert in experts:
|
||||
config = self.generate_json_config(expert)
|
||||
configs.append(config)
|
||||
return configs
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data-dir", "-d")
|
||||
parser.add_argument("--output", "-o", default="configs")
|
||||
parser.add_argument("--list", "-l", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
scanner = EAAutoScanner(args.data_dir)
|
||||
experts = scanner.scan_experts()
|
||||
|
||||
if args.list:
|
||||
print(f"Found {len(experts)} EAs:")
|
||||
for e in experts:
|
||||
print(f" - {e['name']}")
|
||||
|
||||
configs = scanner.auto_scan_all()
|
||||
print(f"Generated {len(configs)} configs")
|
||||
|
||||
if args.output and configs:
|
||||
import json
|
||||
import os
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
for cfg in configs:
|
||||
fname = cfg["ea_name"].replace(" ", "_") + ".json"
|
||||
with open(os.path.join(args.output, fname), "w", encoding="utf-8") as f:
|
||||
json.dump(cfg, f, indent=4, ensure_ascii=False)
|
||||
print(f"Saved: {fname}")
|
||||
@@ -0,0 +1,328 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from ea_config_parser import load_ea_config, load_all_ea_configs, EAConfig
|
||||
from parameter_constraint import create_constraint_engine
|
||||
from smart_search import SmartSearch
|
||||
from analyze_results import ResultAnalyzer
|
||||
from mt5_paths import auto_detect_data_dir
|
||||
|
||||
|
||||
class EABatchOptimizer:
|
||||
def __init__(self, base_dir: str = 'optimizer'):
|
||||
self.base_dir = base_dir
|
||||
self.configs_dir = os.path.join(base_dir, 'configs')
|
||||
self.set_files_dir = os.path.join(base_dir, 'set_files')
|
||||
self.reports_dir = os.path.join(base_dir, 'reports')
|
||||
self.results_dir = os.path.join(base_dir, 'results')
|
||||
self.scripts_dir = os.path.join(base_dir, 'scripts')
|
||||
self.logs_dir = os.path.join(base_dir, 'logs')
|
||||
|
||||
for d in [self.configs_dir, self.set_files_dir, self.reports_dir,
|
||||
self.results_dir, self.scripts_dir, self.logs_dir]:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
def generate_set_files(self, ea_config: EAConfig, max_samples: int = 2000) -> List[Dict]:
|
||||
print('Generating SET files for ' + ea_config.ea_name)
|
||||
print(' Total combinations: ' + str(ea_config.get_total_combinations()))
|
||||
print(' Strategy: ' + SmartSearch(ea_config).select_strategy())
|
||||
|
||||
searcher = SmartSearch(ea_config)
|
||||
combinations = searcher.generate_combinations(max_samples)
|
||||
|
||||
engine = create_constraint_engine(ea_config)
|
||||
valid_combinations = engine.filter_valid_combinations(combinations)
|
||||
|
||||
print(' Valid combinations: ' + str(len(valid_combinations)))
|
||||
|
||||
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
||||
os.makedirs(ea_set_dir, exist_ok=True)
|
||||
|
||||
param_mapping = []
|
||||
for i, combo in enumerate(valid_combinations, 1):
|
||||
set_id = str(i).zfill(8)
|
||||
set_filename = 'params_' + set_id + '.set'
|
||||
set_path = os.path.join(ea_set_dir, set_filename)
|
||||
|
||||
self._write_set_file(set_path, combo)
|
||||
|
||||
mapping = {
|
||||
'set_id': set_id,
|
||||
'set_file': set_filename,
|
||||
'report_name': ea_config.ea_name + '_' + set_id + '.htm',
|
||||
'params': combo
|
||||
}
|
||||
param_mapping.append(mapping)
|
||||
|
||||
mapping_path = os.path.join(self.results_dir, ea_config.ea_name + '_param_mapping.json')
|
||||
with open(mapping_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(param_mapping, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(' SET files generated: ' + str(len(valid_combinations)))
|
||||
return param_mapping
|
||||
|
||||
def _write_set_file(self, set_path: str, params: Dict):
|
||||
lines = ['; MT5 EA Parameters SET File', '; Generated: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '']
|
||||
|
||||
for name, value in params.items():
|
||||
if isinstance(value, bool):
|
||||
lines.append(name + ' <true> <' + ('true' if value else 'false') + '>')
|
||||
elif isinstance(value, float):
|
||||
lines.append(name + ' <' + str(value) + '> <' + str(value) + '>')
|
||||
else:
|
||||
lines.append(name + ' <' + str(value) + '> <' + str(value) + '>')
|
||||
|
||||
with open(set_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
def _save_param_mapping(self, mapping: List[Dict], output_path: str):
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write('set_id,set_file,report_name\n')
|
||||
for m in mapping:
|
||||
f.write(m['set_id'] + ',' + m['set_file'] + ',' + m['report_name'] + '\n')
|
||||
|
||||
def generate_ini_for_ea(self, ea_config: EAConfig, param_mapping: List[Dict] = None):
|
||||
test_config = ea_config.test_config
|
||||
ini_filename = ea_config.ea_name + '.ini'
|
||||
ini_path = os.path.join(self.configs_dir, ini_filename)
|
||||
set_filename = ea_config.ea_name + '_optimization.set'
|
||||
|
||||
data_dir = auto_detect_data_dir() or r'C:\Users\Administrator\AppData\Roaming\MetaQuotes\Terminal\010E047102812FC0C18890992854220E'
|
||||
mt5_tester_dir = os.path.join(data_dir, 'MQL5', 'Profiles', 'Tester')
|
||||
set_path = os.path.join(mt5_tester_dir, set_filename)
|
||||
|
||||
date_from = test_config.get('from_date', '2025.01.01')
|
||||
date_to = test_config.get('to_date', '2025.12.31')
|
||||
|
||||
criterion_map = {
|
||||
'profit_factor': 1,
|
||||
'net_profit': 0,
|
||||
'sharpe_ratio': 5,
|
||||
'expected_payoff': 2,
|
||||
'drawdown': 3,
|
||||
'recovery_factor': 4,
|
||||
}
|
||||
opt_criterion = test_config.get('optimization_criterion', 1)
|
||||
|
||||
set_lines = [
|
||||
'; saved automatically',
|
||||
'; this file contains last used input parameters for testing/optimizing ' + ea_config.ea_name + ' expert advisor',
|
||||
'',
|
||||
]
|
||||
|
||||
if ea_config.parameters:
|
||||
for pname, param in ea_config.parameters.items():
|
||||
if param.param_type == 'bool':
|
||||
if not param.optimize:
|
||||
set_lines.append(pname + '=false||false||false||false||N')
|
||||
else:
|
||||
set_lines.append(pname + '=false||false||false||false||Y')
|
||||
elif param.param_type == 'int':
|
||||
start = int(param.min_value) if param.min_value is not None else 0
|
||||
step = int(param.step) if param.step is not None else 1
|
||||
stop = int(param.max_value) if param.max_value is not None else 100
|
||||
cur = int(param.default) if param.default is not None else start
|
||||
if not param.optimize or start == stop or step == 0:
|
||||
set_lines.append(pname + '=' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||N')
|
||||
else:
|
||||
set_lines.append(pname + '=' + str(cur) + '||' + str(start) + '||' + str(step) + '||' + str(stop) + '||Y')
|
||||
elif param.param_type == 'double':
|
||||
start = float(param.min_value) if param.min_value is not None else 0.0
|
||||
step = float(param.step) if param.step is not None else 0.01
|
||||
stop = float(param.max_value) if param.max_value is not None else 1.0
|
||||
cur = float(param.default) if param.default is not None else start
|
||||
if not param.optimize or start == stop or step == 0:
|
||||
set_lines.append(pname + '=' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||N')
|
||||
else:
|
||||
set_lines.append(pname + '=' + str(cur) + '||' + str(start) + '||' + str(step) + '||' + str(stop) + '||Y')
|
||||
|
||||
with open(set_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(set_lines))
|
||||
|
||||
ini_lines = [
|
||||
'; EA backtest',
|
||||
'[Tester]',
|
||||
'Expert=' + ea_config.ea_path,
|
||||
'ExpertParameters=' + set_filename,
|
||||
'Symbol=' + test_config.get('symbol', 'EURUSD'),
|
||||
'Period=' + test_config.get('period', 'H1'),
|
||||
'Model=' + str(test_config.get('model', 1)),
|
||||
'FromDate=' + date_from,
|
||||
'ToDate=' + date_to,
|
||||
]
|
||||
|
||||
wf = ea_config.walk_forward
|
||||
if wf.get('enabled', False):
|
||||
fm = wf.get('forward_mode', 2)
|
||||
ini_lines.append('ForwardMode=' + str(fm))
|
||||
else:
|
||||
ini_lines.append('ForwardMode=0')
|
||||
|
||||
ini_lines.extend([
|
||||
'Deposit=' + str(test_config.get('deposit', 10000)),
|
||||
'Currency=' + str(test_config.get('currency', 'USD')),
|
||||
'Leverage=' + str(test_config.get('leverage', '1:100')),
|
||||
'ExecutionMode=' + str(test_config.get('execution_delay', 0)),
|
||||
'Optimization=' + str(test_config.get('optimization_mode', 2)),
|
||||
'OptimizationCriterion=' + str(opt_criterion),
|
||||
'Report=' + ea_config.ea_name + '_optimization',
|
||||
'ReplaceReport=true',
|
||||
'ShutdownTerminal=true',
|
||||
'Visual=0',
|
||||
'',
|
||||
])
|
||||
|
||||
with open(ini_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(ini_lines))
|
||||
|
||||
print('INI saved: ' + ini_path)
|
||||
print('SET saved: ' + set_path)
|
||||
return ini_path
|
||||
|
||||
def build_powershell_script(self, ea_config: EAConfig, param_mapping: List[Dict], mt5_path: str):
|
||||
ps_filename = 'run_' + ea_config.ea_name + '.ps1'
|
||||
ps_path = os.path.join(self.scripts_dir, ps_filename)
|
||||
|
||||
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
||||
ea_report_dir = os.path.join(self.reports_dir, ea_config.ea_name)
|
||||
os.makedirs(ea_report_dir, exist_ok=True)
|
||||
|
||||
ini_path = os.path.join(self.configs_dir, ea_config.ea_name + '.ini')
|
||||
|
||||
lines = [
|
||||
'# MT5 Batch Backtest Script',
|
||||
'# Generated: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
'',
|
||||
f'$MT5Path = "{mt5_path}"',
|
||||
f'$IniFile = "{ini_path}"',
|
||||
f'$SetDir = "{ea_set_dir}"',
|
||||
'',
|
||||
f"Write-Host 'Starting batch optimization for {ea_config.ea_name}'",
|
||||
'',
|
||||
'$SetFiles = Get-ChildItem -Path $SetDir -Filter "params_*.set"',
|
||||
'$Total = $SetFiles.Count',
|
||||
'$Current = 0',
|
||||
'',
|
||||
'foreach ($SetFile in $SetFiles) {',
|
||||
' $Current++',
|
||||
' Write-Host "[$Current/$Total] Processing: $($SetFile.Name)"',
|
||||
'',
|
||||
' $IniContent = Get-Content $IniFile',
|
||||
' $IniContent = $IniContent -replace "ExpertParameters=.*", "ExpertParameters=$($SetFile.FullName)"',
|
||||
' $TempIni = Join-Path $env:TEMP "temp_$([guid]::NewGuid().ToString().Substring(0,8)).ini"',
|
||||
' $IniContent | Set-Content $TempIni -Encoding UTF8',
|
||||
'',
|
||||
' Start-Process -FilePath $MT5Path -ArgumentList "/config:$TempIni" -Wait',
|
||||
'',
|
||||
' Start-Sleep -Seconds 3',
|
||||
'}',
|
||||
'',
|
||||
f"Write-Host 'Batch optimization completed for {ea_config.ea_name}!'",
|
||||
]
|
||||
|
||||
with open(ps_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
print('PowerShell script: ' + ps_path)
|
||||
return ps_path
|
||||
|
||||
def run_optimization_for_ea(self, ea_config: EAConfig, mt5_path: str = None,
|
||||
max_samples: int = 2000):
|
||||
print('\n' + '=' * 60)
|
||||
print('Optimization for: ' + ea_config.ea_name)
|
||||
print('=' * 60)
|
||||
|
||||
param_mapping = self.generate_set_files(ea_config, max_samples)
|
||||
self.generate_ini_for_ea(ea_config, param_mapping)
|
||||
|
||||
if mt5_path:
|
||||
ps_path = self.build_powershell_script(ea_config, param_mapping, mt5_path)
|
||||
print('PowerShell script: ' + ps_path)
|
||||
|
||||
return param_mapping
|
||||
|
||||
def run_full_optimization(self, ea_names: List[str] = None,
|
||||
mt5_path: str = None,
|
||||
max_samples_per_ea: int = 2000):
|
||||
configs = load_all_ea_configs(self.configs_dir)
|
||||
|
||||
if ea_names:
|
||||
configs = {k: v for k, v in configs.items() if k in ea_names}
|
||||
|
||||
all_mapping = {}
|
||||
for ea_name, ea_config in configs.items():
|
||||
mapping = self.run_optimization_for_ea(ea_config, mt5_path, max_samples_per_ea)
|
||||
all_mapping[ea_name] = mapping
|
||||
|
||||
return all_mapping
|
||||
|
||||
def analyze_results(self, ea_names: List[str] = None) -> List[Dict]:
|
||||
all_results = []
|
||||
|
||||
if ea_names:
|
||||
for ea_name in ea_names:
|
||||
report_dir = os.path.join(self.reports_dir, ea_name)
|
||||
if os.path.exists(report_dir):
|
||||
analyzer = ResultAnalyzer(report_dir)
|
||||
results = analyzer.parse_all_reports()
|
||||
for r in results:
|
||||
r['ea_name'] = ea_name
|
||||
all_results.extend(results)
|
||||
else:
|
||||
for ea_name in os.listdir(self.reports_dir):
|
||||
report_dir = os.path.join(self.reports_dir, ea_name)
|
||||
if os.path.isdir(report_dir):
|
||||
analyzer = ResultAnalyzer(report_dir)
|
||||
results = analyzer.parse_all_reports()
|
||||
for r in results:
|
||||
r['ea_name'] = ea_name
|
||||
all_results.extend(results)
|
||||
|
||||
return all_results
|
||||
|
||||
def find_optimal_params(self, results: List[Dict], criterion: str = 'profit_factor',
|
||||
min_trades: int = 10) -> List[Dict]:
|
||||
filtered = [r for r in results if r.get('total_trades', 0) >= min_trades
|
||||
and r.get('profit_factor', 0) > 0]
|
||||
filtered.sort(key=lambda x: x.get(criterion, 0), reverse=True)
|
||||
return filtered
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='MT5 EA Batch Optimizer')
|
||||
parser.add_argument('--base-dir', '-d', default='optimizer', help='Base directory')
|
||||
parser.add_argument('--ea', '-e', nargs='+', help='EA names to optimize')
|
||||
parser.add_argument('--max-samples', '-m', type=int, default=2000, help='Max samples')
|
||||
parser.add_argument('--mt5-path', '-p', help='MT5 terminal path')
|
||||
parser.add_argument('--analyze-only', '-a', action='store_true', help='Only analyze results')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
optimizer = EABatchOptimizer(args.base_dir)
|
||||
|
||||
if args.analyze_only:
|
||||
results = optimizer.analyze_results(args.ea)
|
||||
if results:
|
||||
analyzer = ResultAnalyzer()
|
||||
best = optimizer.find_optimal_params(results)
|
||||
analyzer.generate_report(best)
|
||||
print('Results: ' + str(len(results)))
|
||||
print('Best: PF=' + str(best[0].get('profit_factor', 0)) if best else 'No results')
|
||||
else:
|
||||
optimizer.run_full_optimization(args.ea, args.mt5_path, args.max_samples)
|
||||
print('\nOptimization files generated. Run MT5 tests, then use --analyze-only')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,135 @@
|
||||
import os
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Any
|
||||
|
||||
@dataclass
|
||||
class ParameterDef:
|
||||
name: str
|
||||
param_type: str
|
||||
default: Any = None
|
||||
min_value: Any = None
|
||||
max_value: Any = None
|
||||
step: Any = None
|
||||
precision: int = 0
|
||||
optimize: bool = True
|
||||
enum_options: List[Any] = field(default_factory=list)
|
||||
description: str = ''
|
||||
condition: str = ''
|
||||
|
||||
def expand_values(self) -> List[Any]:
|
||||
if self.param_type == 'bool':
|
||||
return [True, False]
|
||||
elif self.param_type == 'enum':
|
||||
return self.enum_options
|
||||
elif self.param_type in ('int', 'double'):
|
||||
if self.min_value is None or self.max_value is None:
|
||||
return [self.default] if self.default is not None else []
|
||||
values = []
|
||||
current = self.min_value
|
||||
while current <= self.max_value:
|
||||
values.append(round(current, self.precision) if self.precision > 0 else int(current))
|
||||
current += self.step
|
||||
return values
|
||||
return [self.default]
|
||||
|
||||
def is_valid_value(self, value: Any) -> bool:
|
||||
if self.param_type == 'bool':
|
||||
return isinstance(value, bool)
|
||||
elif self.param_type == 'enum':
|
||||
return value in self.enum_options
|
||||
elif self.param_type in ('int', 'double'):
|
||||
if self.min_value is not None and value < self.min_value:
|
||||
return False
|
||||
if self.max_value is not None and value > self.max_value:
|
||||
return False
|
||||
return True
|
||||
return True
|
||||
|
||||
@dataclass
|
||||
class EAConfig:
|
||||
ea_name: str
|
||||
ea_path: str
|
||||
parameters: Dict[str, ParameterDef] = field(default_factory=dict)
|
||||
search_strategy: str = 'auto'
|
||||
optimization_criterion: str = 'profit_factor'
|
||||
test_config: Dict[str, Any] = field(default_factory=dict)
|
||||
walk_forward: Dict[str, Any] = field(default_factory=dict)
|
||||
description: str = ''
|
||||
|
||||
def get_total_combinations(self) -> int:
|
||||
total = 1
|
||||
for param in self.parameters.values():
|
||||
values = param.expand_values()
|
||||
total *= len(values) if values else 1
|
||||
return total
|
||||
|
||||
def estimate_search_time(self, tests_per_minute: float = 10) -> str:
|
||||
total = self.get_total_combinations()
|
||||
minutes = total / tests_per_minute
|
||||
if minutes < 60:
|
||||
return str(round(minutes, 1)) + ' minutes'
|
||||
elif minutes < 1440:
|
||||
return str(round(minutes/60, 1)) + ' hours'
|
||||
else:
|
||||
return str(round(minutes/1440, 1)) + ' days'
|
||||
|
||||
def is_complex(self) -> bool:
|
||||
return len(self.parameters) > 6 or self.get_total_combinations() > 1_000_000
|
||||
|
||||
def load_ea_config(config_path: str) -> EAConfig:
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
config_data = json.load(f)
|
||||
ea_name = config_data.get('ea_name', 'Unknown')
|
||||
ea_path = config_data.get('ea_path', '')
|
||||
description = config_data.get('description', '')
|
||||
search_strategy = config_data.get('search_strategy', 'auto')
|
||||
criterion = config_data.get('optimization_criterion', 'profit_factor')
|
||||
test_config = config_data.get('test_config', {})
|
||||
walk_forward = config_data.get('walk_forward', {})
|
||||
parameters = {}
|
||||
for param_name, param_data in config_data.get('parameters', {}).items():
|
||||
param_type = param_data.get('type', 'int')
|
||||
if param_type == 'enum':
|
||||
param = ParameterDef(
|
||||
name=param_name,
|
||||
param_type='enum',
|
||||
default=param_data.get('default'),
|
||||
enum_options=param_data.get('options', []),
|
||||
description=param_data.get('description', ''),
|
||||
condition=param_data.get('condition', '')
|
||||
)
|
||||
else:
|
||||
param = ParameterDef(
|
||||
name=param_name,
|
||||
param_type=param_type,
|
||||
default=param_data.get('default'),
|
||||
min_value=param_data.get('min'),
|
||||
max_value=param_data.get('max'),
|
||||
step=param_data.get('step', 1),
|
||||
precision=param_data.get('precision', 0),
|
||||
optimize=param_data.get('optimize', True),
|
||||
description=param_data.get('description', ''),
|
||||
condition=param_data.get('condition', '')
|
||||
)
|
||||
parameters[param_name] = param
|
||||
return EAConfig(
|
||||
ea_name=ea_name, ea_path=ea_path, parameters=parameters,
|
||||
search_strategy=search_strategy, optimization_criterion=criterion,
|
||||
test_config=test_config, walk_forward=walk_forward, description=description
|
||||
)
|
||||
|
||||
def load_all_ea_configs(configs_dir: str) -> Dict[str, EAConfig]:
|
||||
configs = {}
|
||||
if not os.path.exists(configs_dir):
|
||||
return configs
|
||||
for filename in os.listdir(configs_dir):
|
||||
if filename.endswith('.json'):
|
||||
config_path = os.path.join(configs_dir, filename)
|
||||
try:
|
||||
config = load_ea_config(config_path)
|
||||
configs[config.ea_name] = config
|
||||
except Exception as e:
|
||||
print('Error loading ' + config_path + ': ' + str(e))
|
||||
return configs
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import re
|
||||
from typing import List, Dict, Any, Callable
|
||||
|
||||
class ParameterConstraintEngine:
|
||||
def __init__(self):
|
||||
self.constraints = []
|
||||
self.condition_functions = []
|
||||
|
||||
def add_constraint(self, name: str, condition: str, error_message: str = ''):
|
||||
self.constraints.append({
|
||||
'name': name,
|
||||
'condition': condition,
|
||||
'error_message': error_message or 'Constraint violated: ' + name
|
||||
})
|
||||
|
||||
def add_condition_from_config(self, param_name: str, condition_str: str):
|
||||
if not condition_str:
|
||||
return
|
||||
|
||||
condition_str = condition_str.strip()
|
||||
self.constraints.append({
|
||||
'name': param_name,
|
||||
'condition': condition_str,
|
||||
'error_message': 'Parameter constraint violated: ' + param_name
|
||||
})
|
||||
|
||||
def _parse_condition(self, condition: str) -> Callable[[Dict], bool]:
|
||||
condition = condition.strip()
|
||||
|
||||
patterns = [
|
||||
(r'^(\w+)\s*>\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) > p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*<\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) < p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*>=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) >= p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*<=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) <= p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*==\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) == p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*!=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) != p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*>\s*(\d+)$', lambda m, p: p.get(m.group(1), 0) > int(m.group(2))),
|
||||
(r'^(\w+)\s*<\s*(\d+)$', lambda m, p: p.get(m.group(1), 0) < int(m.group(2))),
|
||||
]
|
||||
|
||||
for pattern, func in patterns:
|
||||
match = re.match(pattern, condition)
|
||||
if match:
|
||||
return lambda params, m=match, f=func: f(m, params)
|
||||
|
||||
return lambda params: True
|
||||
|
||||
def _is_valid_combination(self, params: Dict) -> bool:
|
||||
for constraint in self.constraints:
|
||||
condition = constraint['condition']
|
||||
if not condition:
|
||||
continue
|
||||
|
||||
parse_func = self._parse_condition(condition)
|
||||
if not parse_func(params):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def filter_valid_combinations(self, combinations: List[Dict]) -> List[Dict]:
|
||||
valid = []
|
||||
invalid_count = 0
|
||||
for combo in combinations:
|
||||
if self._is_valid_combination(combo):
|
||||
valid.append(combo)
|
||||
else:
|
||||
invalid_count += 1
|
||||
|
||||
if invalid_count > 0:
|
||||
print('Filtered ' + str(invalid_count) + ' invalid combinations')
|
||||
|
||||
return valid
|
||||
|
||||
def get_constraint_count(self) -> int:
|
||||
return len(self.constraints)
|
||||
|
||||
|
||||
def create_constraint_engine(ea_config) -> ParameterConstraintEngine:
|
||||
engine = ParameterConstraintEngine()
|
||||
|
||||
for param_name, param in ea_config.parameters.items():
|
||||
if param.condition:
|
||||
engine.add_condition_from_config(param_name, param.condition)
|
||||
|
||||
return engine
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import random
|
||||
import math
|
||||
from typing import List, Dict, Any, Tuple
|
||||
from itertools import product
|
||||
|
||||
class SmartSearch:
|
||||
def __init__(self, ea_config):
|
||||
self.ea_config = ea_config
|
||||
self.total_combinations = ea_config.get_total_combinations()
|
||||
|
||||
def select_strategy(self) -> str:
|
||||
if self.ea_config.search_strategy != 'auto':
|
||||
return self.ea_config.search_strategy
|
||||
if self.total_combinations <= 10000:
|
||||
return 'grid'
|
||||
elif self.total_combinations <= 500000:
|
||||
return 'latin'
|
||||
else:
|
||||
return 'genetic'
|
||||
|
||||
def generate_combinations(self, max_samples: int = 2000) -> List[Dict]:
|
||||
strategy = self.select_strategy()
|
||||
print('Selected strategy: ' + strategy + ' for ' + str(self.total_combinations) + ' combinations')
|
||||
|
||||
if strategy == 'grid':
|
||||
return self._grid_search()
|
||||
elif strategy == 'random':
|
||||
return self._random_search(max_samples)
|
||||
elif strategy == 'latin':
|
||||
return self._latin_hypercube(max_samples)
|
||||
elif strategy == 'genetic':
|
||||
return self._genetic_search(max_samples)
|
||||
return []
|
||||
|
||||
def _grid_search(self) -> List[Dict]:
|
||||
param_names = list(self.ea_config.parameters.keys())
|
||||
param_values = [p.expand_values() for p in self.ea_config.parameters.values()]
|
||||
combinations = list(product(*param_values))
|
||||
return [dict(zip(param_names, combo)) for combo in combinations]
|
||||
|
||||
def _random_search(self, max_samples: int) -> List[Dict]:
|
||||
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
|
||||
samples = []
|
||||
for _ in range(min(max_samples, self.total_combinations)):
|
||||
sample = {name: random.choice(values) for name, values in all_values.items()}
|
||||
if sample not in samples:
|
||||
samples.append(sample)
|
||||
return samples
|
||||
|
||||
def _latin_hypercube(self, max_samples: int) -> List[Dict]:
|
||||
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
|
||||
n_params = len(all_values)
|
||||
samples = []
|
||||
for i in range(min(max_samples, self.total_combinations)):
|
||||
sample = {}
|
||||
for j, (name, values) in enumerate(all_values.items()):
|
||||
idx = int((i / max_samples) * len(values)) % len(values)
|
||||
sample[name] = values[idx]
|
||||
if sample not in samples:
|
||||
samples.append(sample)
|
||||
return samples
|
||||
|
||||
def _genetic_search(self, max_samples: int) -> List[Dict]:
|
||||
pop_size = min(50, max_samples)
|
||||
n_generations = max_samples // pop_size
|
||||
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
|
||||
|
||||
population = []
|
||||
for _ in range(pop_size):
|
||||
individual = {name: random.choice(values) for name, values in all_values.items()}
|
||||
population.append(individual)
|
||||
|
||||
for gen in range(n_generations):
|
||||
population = self._evolve(population, all_values)
|
||||
|
||||
return population[:max_samples]
|
||||
|
||||
def _evolve(self, population: List[Dict], all_values: Dict) -> List[Dict]:
|
||||
crossover_rate = 0.8
|
||||
mutation_rate = 0.15
|
||||
|
||||
offspring = []
|
||||
for _ in range(len(population)):
|
||||
parent1, parent2 = random.sample(population, 2)
|
||||
if random.random() < crossover_rate:
|
||||
child = self._crossover(parent1, parent2)
|
||||
else:
|
||||
child = parent1.copy()
|
||||
|
||||
if random.random() < mutation_rate:
|
||||
child = self._mutate(child, all_values)
|
||||
|
||||
offspring.append(child)
|
||||
|
||||
return population[:5] + offspring[:len(population)-5]
|
||||
|
||||
def _crossover(self, parent1: Dict, parent2: Dict) -> Dict:
|
||||
child = {}
|
||||
for key in parent1.keys():
|
||||
if random.random() < 0.5:
|
||||
child[key] = parent1[key]
|
||||
else:
|
||||
child[key] = parent2[key]
|
||||
return child
|
||||
|
||||
def _mutate(self, individual: Dict, all_values: Dict) -> Dict:
|
||||
key = random.choice(list(all_values.keys()))
|
||||
individual[key] = random.choice(all_values[key])
|
||||
return individual
|
||||
|
||||
|
||||
class GeneticOptimizer:
|
||||
def __init__(self, ea_config, population_size: int = 50, generations: int = 30):
|
||||
self.ea_config = ea_config
|
||||
self.population_size = population_size
|
||||
self.generations = generations
|
||||
self.all_values = {name: p.expand_values() for name, p in ea_config.parameters.items()}
|
||||
|
||||
def create_individual(self) -> Dict:
|
||||
return {name: random.choice(values) for name, values in self.all_values.items()}
|
||||
|
||||
def evaluate(self, individual: Dict) -> float:
|
||||
return random.random() * 10
|
||||
|
||||
def tournament_select(self, population: List[Dict], k: int = 3) -> Dict:
|
||||
tournament = random.sample(population, k)
|
||||
return max(tournament, key=self.evaluate)
|
||||
|
||||
def crossover(self, parent1: Dict, parent2: Dict) -> Tuple[Dict, Dict]:
|
||||
child1, child2 = {}, {}
|
||||
for key in parent1.keys():
|
||||
if random.random() < 0.5:
|
||||
child1[key] = parent1[key]
|
||||
child2[key] = parent2[key]
|
||||
else:
|
||||
child1[key] = parent2[key]
|
||||
child2[key] = parent1[key]
|
||||
return child1, child2
|
||||
|
||||
def mutate(self, individual: Dict, rate: float = 0.15) -> Dict:
|
||||
for key in individual.keys():
|
||||
if random.random() < rate:
|
||||
individual[key] = random.choice(self.all_values[key])
|
||||
return individual
|
||||
|
||||
def run(self) -> List[Dict]:
|
||||
population = [self.create_individual() for _ in range(self.population_size)]
|
||||
best_individuals = []
|
||||
|
||||
for gen in range(self.generations):
|
||||
fitness_scores = [(ind, self.evaluate(ind)) for ind in population]
|
||||
fitness_scores.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
elite = [ind for ind, _ in fitness_scores[:5]]
|
||||
best_individuals.extend(elite)
|
||||
|
||||
new_population = elite.copy()
|
||||
while len(new_population) < self.population_size:
|
||||
parent1 = self.tournament_select(population)
|
||||
parent2 = self.tournament_select(population)
|
||||
child1, child2 = self.crossover(parent1, parent2)
|
||||
child1 = self.mutate(child1)
|
||||
child2 = self.mutate(child2)
|
||||
new_population.extend([child1, child2])
|
||||
|
||||
population = new_population[:self.population_size]
|
||||
|
||||
return best_individuals
|
||||
@@ -0,0 +1,321 @@
|
||||
import os
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Optional
|
||||
import logging
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReportGenerator:
|
||||
def __init__(self, results: List[Dict] = None):
|
||||
self.results = results or []
|
||||
|
||||
def generate_excel(self, output_path: str = "reports/batch_report.xlsx") -> str:
|
||||
try:
|
||||
import openpyxl
|
||||
from openpyxl.styles import Font, Alignment, PatternFill, Border, Side
|
||||
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "Backtest Results"
|
||||
|
||||
headers = [
|
||||
"EA Name", "Symbol", "Period", "Model",
|
||||
"Total Trades", "Winning Trades", "Losing Trades",
|
||||
"Win Rate %", "Gross Profit", "Gross Loss",
|
||||
"Profit Factor", "Expected Payoff",
|
||||
"Initial Deposit", "Final Balance"
|
||||
]
|
||||
|
||||
header_fill = PatternFill(start_color="366092", end_color="366092", fill_type="solid")
|
||||
header_font = Font(bold=True, color="FFFFFF")
|
||||
thin_border = Border(
|
||||
left=Side(style='thin'),
|
||||
right=Side(style='thin'),
|
||||
top=Side(style='thin'),
|
||||
bottom=Side(style='thin')
|
||||
)
|
||||
|
||||
for col, header in enumerate(headers, 1):
|
||||
cell = ws.cell(row=1, column=col, value=header)
|
||||
cell.fill = header_fill
|
||||
cell.font = header_font
|
||||
cell.alignment = Alignment(horizontal='center', vertical='center')
|
||||
cell.border = thin_border
|
||||
|
||||
for row_idx, result in enumerate(self.results, 2):
|
||||
metrics = result.get("metrics", {})
|
||||
test_info = result.get("test_info", {})
|
||||
|
||||
row_data = [
|
||||
test_info.get("expert", ""),
|
||||
test_info.get("symbol", ""),
|
||||
test_info.get("period", ""),
|
||||
test_info.get("model", ""),
|
||||
metrics.get("total_trades", 0),
|
||||
metrics.get("winning_trades", 0),
|
||||
metrics.get("losing_trades", 0),
|
||||
metrics.get("win_rate", 0),
|
||||
metrics.get("gross_profit", 0),
|
||||
metrics.get("gross_loss", 0),
|
||||
metrics.get("profit_factor", 0),
|
||||
metrics.get("expected_payoff", 0),
|
||||
metrics.get("initial_deposit", 0),
|
||||
metrics.get("final_balance", 0)
|
||||
]
|
||||
|
||||
for col, value in enumerate(row_data, 1):
|
||||
cell = ws.cell(row=row_idx, column=col, value=value)
|
||||
cell.border = thin_border
|
||||
if col >= 5:
|
||||
cell.number_format = '0.00'
|
||||
|
||||
for col in range(1, len(headers) + 1):
|
||||
ws.column_dimensions[openpyxl.utils.get_column_letter(col)].width = 15
|
||||
|
||||
ws.auto_filter.ref = f"A1:N{len(self.results) + 1}"
|
||||
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
wb.save(output_path)
|
||||
logger.info(f"Excel report saved: {output_path}")
|
||||
return output_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating Excel report: {e}")
|
||||
raise
|
||||
|
||||
def generate_csv(self, output_path: str = "reports/batch_report.csv") -> str:
|
||||
try:
|
||||
import csv
|
||||
|
||||
os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
|
||||
|
||||
headers = [
|
||||
"EA Name", "Symbol", "Period", "Model",
|
||||
"Total Trades", "Winning Trades", "Losing Trades",
|
||||
"Win Rate %", "Gross Profit", "Gross Loss",
|
||||
"Profit Factor", "Expected Payoff",
|
||||
"Initial Deposit", "Final Balance"
|
||||
]
|
||||
|
||||
with open(output_path, 'w', newline='', encoding='utf-8-sig') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(headers)
|
||||
|
||||
for result in self.results:
|
||||
metrics = result.get("metrics", {})
|
||||
test_info = result.get("test_info", {})
|
||||
|
||||
row_data = [
|
||||
test_info.get("expert", ""),
|
||||
test_info.get("symbol", ""),
|
||||
test_info.get("period", ""),
|
||||
test_info.get("model", ""),
|
||||
metrics.get("total_trades", 0),
|
||||
metrics.get("winning_trades", 0),
|
||||
metrics.get("losing_trades", 0),
|
||||
metrics.get("win_rate", 0),
|
||||
metrics.get("gross_profit", 0),
|
||||
metrics.get("gross_loss", 0),
|
||||
metrics.get("profit_factor", 0),
|
||||
metrics.get("expected_payoff", 0),
|
||||
metrics.get("initial_deposit", 0),
|
||||
metrics.get("final_balance", 0)
|
||||
]
|
||||
writer.writerow(row_data)
|
||||
|
||||
logger.info(f"CSV report saved: {output_path}")
|
||||
return output_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating CSV report: {e}")
|
||||
raise
|
||||
|
||||
def generate_html(self, output_path: str = "reports/batch_report.html") -> str:
|
||||
try:
|
||||
os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
|
||||
|
||||
html_content = f"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>MT5 Batch Backtest Report</title>
|
||||
<style>
|
||||
body {{ font-family: Arial, sans-serif; margin: 20px; }}
|
||||
h1 {{ color: #366092; }}
|
||||
.summary {{ background: #f0f4f8; padding: 15px; border-radius: 5px; margin: 20px 0; }}
|
||||
table {{ border-collapse: collapse; width: 100%; margin: 20px 0; }}
|
||||
th {{ background: #366092; color: white; padding: 12px; text-align: left; }}
|
||||
td {{ border: 1px solid #ddd; padding: 10px; }}
|
||||
tr:nth-child(even) {{ background: #f9f9f9; }}
|
||||
.best {{ color: green; font-weight: bold; }}
|
||||
.worst {{ color: red; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>MT5 Batch Backtest Report</h1>
|
||||
<p>Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</p>
|
||||
|
||||
<div class="summary">
|
||||
<h2>Summary</h2>
|
||||
<p>Total Tests: {len(self.results)}</p>
|
||||
</div>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<th>EA Name</th>
|
||||
<th>Symbol</th>
|
||||
<th>Period</th>
|
||||
<th>Total Trades</th>
|
||||
<th>Win Rate %</th>
|
||||
<th>Profit Factor</th>
|
||||
<th>Gross Profit</th>
|
||||
<th>Gross Loss</th>
|
||||
<th>Final Balance</th>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
for result in self.results:
|
||||
metrics = result.get("metrics", {})
|
||||
test_info = result.get("test_info", {})
|
||||
|
||||
pf = metrics.get("profit_factor", 0)
|
||||
row_class = "best" if pf >= 2.0 else ("worst" if pf < 1.0 else "")
|
||||
|
||||
html_content += f"""
|
||||
<tr class="{row_class}">
|
||||
<td>{test_info.get("expert", "")}</td>
|
||||
<td>{test_info.get("symbol", "")}</td>
|
||||
<td>{test_info.get("period", "")}</td>
|
||||
<td>{metrics.get("total_trades", 0)}</td>
|
||||
<td>{metrics.get("win_rate", 0):.2f}%</td>
|
||||
<td>{pf:.2f}</td>
|
||||
<td>{metrics.get("gross_profit", 0):.2f}</td>
|
||||
<td>{metrics.get("gross_loss", 0):.2f}</td>
|
||||
<td>{metrics.get("final_balance", 0):.2f}</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
html_content += """
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write(html_content)
|
||||
|
||||
logger.info(f"HTML report saved: {output_path}")
|
||||
return output_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating HTML report: {e}")
|
||||
raise
|
||||
|
||||
def generate_markdown(self, output_path: str = "reports/batch_report.md") -> str:
|
||||
try:
|
||||
os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
|
||||
|
||||
md_lines = [
|
||||
"# MT5 Batch Backtest Report",
|
||||
"",
|
||||
f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
||||
"",
|
||||
f"**Total Tests:** {len(self.results)}",
|
||||
""
|
||||
]
|
||||
|
||||
if self.results:
|
||||
best = max(self.results, key=lambda x: x.get("metrics", {}).get("profit_factor", 0))
|
||||
worst = min(self.results, key=lambda x: x.get("metrics", {}).get("profit_factor", 0))
|
||||
|
||||
md_lines.append(f"**Best Test:** {best.get('test_info', {}).get('expert', 'N/A')} "
|
||||
f"(PF: {best.get('metrics', {}).get('profit_factor', 0):.2f})")
|
||||
md_lines.append(f"**Worst Test:** {worst.get('test_info', {}).get('expert', 'N/A')} "
|
||||
f"(PF: {worst.get('metrics', {}).get('profit_factor', 0):.2f})")
|
||||
md_lines.append("")
|
||||
|
||||
md_lines.append("## Results")
|
||||
md_lines.append("")
|
||||
md_lines.append("| EA Name | Symbol | Period | Total Trades | Win Rate | Profit Factor | "
|
||||
"Gross Profit | Gross Loss | Final Balance |")
|
||||
md_lines.append("|---------|--------|--------|--------------|----------|---------------|"
|
||||
"--------------|------------|---------------|")
|
||||
|
||||
for result in self.results:
|
||||
metrics = result.get("metrics", {})
|
||||
test_info = result.get("test_info", {})
|
||||
|
||||
md_lines.append(
|
||||
f"| {test_info.get('expert', '')} | "
|
||||
f"{test_info.get('symbol', '')} | "
|
||||
f"{test_info.get('period', '')} | "
|
||||
f"{metrics.get('total_trades', 0)} | "
|
||||
f"{metrics.get('win_rate', 0):.2f}% | "
|
||||
f"{metrics.get('profit_factor', 0):.2f} | "
|
||||
f"{metrics.get('gross_profit', 0):.2f} | "
|
||||
f"{metrics.get('gross_loss', 0):.2f} | "
|
||||
f"{metrics.get('final_balance', 0):.2f} |"
|
||||
)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write("\n".join(md_lines))
|
||||
|
||||
logger.info(f"Markdown report saved: {output_path}")
|
||||
return output_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating Markdown report: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="MT5 Report Generator")
|
||||
parser.add_argument("--input", "-i", default="reports/parsed_results.json",
|
||||
help="Input JSON file from result_parser")
|
||||
parser.add_argument("--output-dir", "-o", default="reports",
|
||||
help="Output directory for reports")
|
||||
parser.add_argument("--format", "-f", choices=["excel", "csv", "html", "markdown", "all"],
|
||||
default="all", help="Output format")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
results = []
|
||||
if os.path.exists(args.input):
|
||||
with open(args.input, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
results = data.get("results", [])
|
||||
else:
|
||||
logger.warning(f"Input file not found: {args.input}")
|
||||
logger.info("Use result_parser.py first to generate parsed results")
|
||||
|
||||
if not results:
|
||||
print("No results to generate report")
|
||||
return
|
||||
|
||||
generator = ReportGenerator(results)
|
||||
|
||||
base_name = os.path.join(args.output_dir, "batch_report")
|
||||
|
||||
if args.format in ["excel", "all"]:
|
||||
generator.generate_excel(f"{base_name}.xlsx")
|
||||
|
||||
if args.format in ["csv", "all"]:
|
||||
generator.generate_csv(f"{base_name}.csv")
|
||||
|
||||
if args.format in ["html", "all"]:
|
||||
generator.generate_html(f"{base_name}.html")
|
||||
|
||||
if args.format in ["markdown", "all"]:
|
||||
generator.generate_markdown(f"{base_name}.md")
|
||||
|
||||
print(f"\nReports generated in: {args.output_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,483 @@
|
||||
import os
|
||||
import csv
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
import logging
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ResultParser:
|
||||
def __init__(self, results_dir: str = "results"):
|
||||
self.results_dir = results_dir
|
||||
|
||||
def parse_xml_report(self, report_path: str) -> Optional[Dict]:
|
||||
if not os.path.exists(report_path):
|
||||
logger.warning(f"Report not found: {report_path}")
|
||||
return None
|
||||
|
||||
try:
|
||||
tree = ET.parse(report_path)
|
||||
root = tree.getroot()
|
||||
|
||||
result = {
|
||||
"report_file": os.path.basename(report_path),
|
||||
"parse_time": datetime.now().isoformat(),
|
||||
"test_info": {},
|
||||
"metrics": {},
|
||||
"trades": []
|
||||
}
|
||||
|
||||
tester = root.find("Tester")
|
||||
if tester is not None:
|
||||
result["test_info"] = {
|
||||
"expert": tester.findtext("Expert", ""),
|
||||
"symbol": tester.findtext("Symbol", ""),
|
||||
"period": tester.findtext("Period", ""),
|
||||
"model": tester.findtext("Model", ""),
|
||||
"from_date": tester.findtext("FromDate", ""),
|
||||
"to_date": tester.findtext("ToDate", ""),
|
||||
}
|
||||
|
||||
equity = root.find("Equity")
|
||||
if equity is not None:
|
||||
result["metrics"]["initial_deposit"] = float(equity.findtext("Initial", "0"))
|
||||
result["metrics"]["final_balance"] = float(equity.findtext("Final", "0"))
|
||||
result["metrics"]["gross_profit"] = float(equity.findtext("GrossProfit", "0"))
|
||||
result["metrics"]["gross_loss"] = float(equity.findtext("GrossLoss", "0"))
|
||||
result["metrics"]["profit_factor"] = float(equity.findtext("ProfitFactor", "0"))
|
||||
result["metrics"]["expected_payoff"] = float(equity.findtext("ExpectedPayoff", "0"))
|
||||
|
||||
trades_elem = root.find("Trades")
|
||||
if trades_elem is not None:
|
||||
result["metrics"]["total_trades"] = int(trades_elem.findtext("Total", "0"))
|
||||
result["metrics"]["short_positions"] = int(trades_elem.findtext("Short", "0"))
|
||||
result["metrics"]["long_positions"] = int(trades_elem.findtext("Long", "0"))
|
||||
result["metrics"]["winning_trades"] = int(trades_elem.findtext("ProfitTrades", "0"))
|
||||
result["metrics"]["losing_trades"] = int(trades_elem.findtext("LossTrades", "0"))
|
||||
|
||||
if result["metrics"]["total_trades"] > 0:
|
||||
result["metrics"]["win_rate"] = (
|
||||
result["metrics"]["winning_trades"] / result["metrics"]["total_trades"] * 100
|
||||
)
|
||||
else:
|
||||
result["metrics"]["win_rate"] = 0.0
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing XML report {report_path}: {e}")
|
||||
return None
|
||||
|
||||
def _detect_encoding(self, report_path: str) -> str:
|
||||
with open(report_path, 'rb') as f:
|
||||
bom = f.read(4)
|
||||
if bom[:2] == b'\xff\xfe':
|
||||
return 'utf-16-le'
|
||||
elif bom[:2] == b'\xfe\xff':
|
||||
return 'utf-16-be'
|
||||
return 'utf-8'
|
||||
|
||||
def parse_html_report(self, report_path: str) -> Optional[Dict]:
|
||||
if not os.path.exists(report_path):
|
||||
logger.warning(f"Report not found: {report_path}")
|
||||
return None
|
||||
|
||||
try:
|
||||
import re
|
||||
with open(report_path, 'rb') as f:
|
||||
raw_bytes = f.read()
|
||||
|
||||
if raw_bytes[:2] == b'\xff\xfe':
|
||||
content = raw_bytes[2:].decode('utf-16-le')
|
||||
else:
|
||||
content = raw_bytes.decode('utf-8')
|
||||
|
||||
result = {
|
||||
"report_file": os.path.basename(report_path),
|
||||
"parse_time": datetime.now().isoformat(),
|
||||
"test_info": {},
|
||||
"metrics": {},
|
||||
"trades": []
|
||||
}
|
||||
|
||||
plain_pattern = r'<td[^>]*nowrap[^>]*>([^<]+):</td>\s*<td[^>]*colspan=.10.[^>]*><b>([^<]*)</b></td>'
|
||||
plain_matches = re.findall(plain_pattern, content)
|
||||
for key, value in plain_matches:
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
if key in result["test_info"] and result["test_info"].get(key):
|
||||
continue
|
||||
self._parse_html_cell(key, value, result)
|
||||
|
||||
pattern = r'<td[^>]*>([^<]+):</td>\s*<td[^>]*><b>([^<]*)</b></td>'
|
||||
matches = re.findall(pattern, content)
|
||||
for key, value in matches:
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
if key in result["test_info"] and result["test_info"].get(key):
|
||||
continue
|
||||
if key in result["metrics"] and result["metrics"].get(key):
|
||||
continue
|
||||
self._parse_html_cell(key, value, result)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing HTML report {report_path}: {e}")
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(report_path, 'r', encoding='utf-16') as f:
|
||||
content = f.read()
|
||||
content_utf8 = content.encode('utf-8').decode('utf-8')
|
||||
soup = BeautifulSoup(content_utf8, 'html.parser')
|
||||
|
||||
result = {
|
||||
"report_file": os.path.basename(report_path),
|
||||
"parse_time": datetime.now().isoformat(),
|
||||
"test_info": {},
|
||||
"metrics": {},
|
||||
"trades": []
|
||||
}
|
||||
|
||||
tables = soup.find_all('table')
|
||||
for table in tables:
|
||||
rows = table.find_all('tr')
|
||||
for row in rows:
|
||||
cells = row.find_all(['td', 'th'])
|
||||
if len(cells) >= 2:
|
||||
non_empty = [c for c in cells if c.get_text(strip=True)]
|
||||
if len(non_empty) >= 2:
|
||||
key = non_empty[0].get_text(strip=True)
|
||||
value = non_empty[1].get_text(strip=True)
|
||||
self._parse_html_cell(key, value, result)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing HTML report {report_path}: {e}")
|
||||
return None
|
||||
|
||||
def _extract_number(self, text: str) -> float:
|
||||
import re
|
||||
numbers = re.findall(r'[-+]?\d*\.?\d+', text.replace(',', '').replace(' ', ''))
|
||||
if numbers:
|
||||
try:
|
||||
return float(numbers[0])
|
||||
except:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
def _extract_percentage(self, text: str) -> float:
|
||||
import re
|
||||
numbers = re.findall(r'\d+\.?\d*%', text)
|
||||
if numbers:
|
||||
try:
|
||||
return float(numbers[0].replace('%', ''))
|
||||
except:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
def _parse_html_cell(self, key: str, value: str, result: Dict):
|
||||
import re
|
||||
key_lower = key.lower()
|
||||
if '专家' in key or 'Expert' in key:
|
||||
if not result["test_info"].get("expert"):
|
||||
result["test_info"]["expert"] = value
|
||||
elif '交易品种' in key or '交易品' in key or 'Symbol' in key or 'symbol' in key_lower:
|
||||
current = result["test_info"].get("symbol")
|
||||
if (not current or current in ('1', '0', '')) and value and value not in ('1', ''):
|
||||
result["test_info"]["symbol"] = value
|
||||
|
||||
if not result["test_info"].get("symbol") or result["test_info"].get("symbol") == "0":
|
||||
fn = result["report_file"]
|
||||
m = re.search(r'_([A-Z]{5,6})_', fn)
|
||||
if m:
|
||||
result["test_info"]["symbol"] = m.group(1)
|
||||
elif '期间' in key or 'Period' in key:
|
||||
if not result["test_info"].get("period"):
|
||||
result["test_info"]["period"] = value
|
||||
elif '模型' in key or 'Model' in key:
|
||||
result["test_info"]["model"] = value
|
||||
elif '公司' in key or 'Company' in key:
|
||||
result["test_info"]["company"] = value
|
||||
elif '货币' in key or 'Currency' in key:
|
||||
result["test_info"]["currency"] = value
|
||||
elif '杠杆' in key or 'Leverage' in key:
|
||||
result["test_info"]["leverage"] = value
|
||||
elif '初始入金' in key or ('Initial' in key and 'Deposit' in key):
|
||||
result["metrics"]["initial_deposit"] = self._extract_number(value)
|
||||
elif '总净盈利' in key or 'Net Profit' in key:
|
||||
result["metrics"]["net_profit"] = self._extract_number(value)
|
||||
elif '毛利' in key or 'Gross Profit' in key:
|
||||
result["metrics"]["gross_profit"] = self._extract_number(value)
|
||||
elif '毛损' in key or 'Gross Loss' in key:
|
||||
result["metrics"]["gross_loss"] = self._extract_number(value)
|
||||
elif '盈利因子' in key or 'Profit Factor' in key:
|
||||
result["metrics"]["profit_factor"] = self._extract_number(value)
|
||||
elif '预期收益' in key or 'Expected Payoff' in key:
|
||||
result["metrics"]["expected_payoff"] = self._extract_number(value)
|
||||
elif ('总' in key and '交易' in key.lower()) or ('Total' in key and 'trades' in key_lower):
|
||||
result["metrics"]["total_trades"] = self._extract_number(value)
|
||||
elif '采收率' in key or 'Recovery Factor' in key:
|
||||
result["metrics"]["recovery_factor"] = self._extract_number(value)
|
||||
elif '夏普比率' in key or 'Sharpe Ratio' in key:
|
||||
result["metrics"]["sharpe_ratio"] = self._extract_number(value)
|
||||
elif 'AHPR' in key:
|
||||
result["metrics"]["ahpr"] = value
|
||||
elif 'GHPR' in key:
|
||||
result["metrics"]["ghpr"] = value
|
||||
elif 'LR 相关性' in key or 'LR Correlation' in key:
|
||||
result["metrics"]["lr_correlation"] = self._extract_number(value)
|
||||
elif 'LR 标准误差' in key or 'LR Standard Error' in key:
|
||||
result["metrics"]["lr_standard_error"] = self._extract_number(value)
|
||||
elif '最大结余亏损' in key or 'Maximal Drawdown' in key:
|
||||
result["metrics"]["max_drawdown"] = self._extract_number(value)
|
||||
elif '最大净值亏损' in key or 'Max Equity Drawdown' in key:
|
||||
result["metrics"]["max_equity_drawdown"] = self._extract_number(value)
|
||||
elif '绝对结余亏损' in key or 'Absolute Drawdown' in key:
|
||||
result["metrics"]["absolute_drawdown"] = self._extract_number(value)
|
||||
elif '预付款维持率' in key or 'Margin Level' in key:
|
||||
result["metrics"]["margin_level"] = self._extract_number(value)
|
||||
elif '卖出交易' in key or 'Short Positions' in key:
|
||||
result["metrics"]["short_trades"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
|
||||
elif '买入交易' in key or 'Long Positions' in key:
|
||||
result["metrics"]["long_trades"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
|
||||
elif '总成交' in key or 'Total Deals' in key:
|
||||
result["metrics"]["total_deals"] = self._extract_number(value)
|
||||
elif '盈利交易' in key or 'Profit Trades' in key:
|
||||
result["metrics"]["winning_trades"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
|
||||
result["metrics"]["win_rate"] = self._extract_percentage(value)
|
||||
elif '亏损交易' in key or 'Loss Trades' in key:
|
||||
result["metrics"]["losing_trades"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
|
||||
elif '最大 获利交易' in key or 'Max Profit Trade' in key:
|
||||
result["metrics"]["max_profit_trade"] = self._extract_number(value)
|
||||
elif '最大 亏损交易' in key or 'Max Loss Trade' in key:
|
||||
result["metrics"]["max_loss_trade"] = self._extract_number(value)
|
||||
elif '平均 获利交易' in key or 'Avg Profit Trade' in key:
|
||||
result["metrics"]["avg_profit_trade"] = self._extract_number(value)
|
||||
elif '平均 亏损交易' in key or 'Avg Loss Trade' in key:
|
||||
result["metrics"]["avg_loss_trade"] = self._extract_number(value)
|
||||
elif '最大值 连胜' in key or 'Longest Winning Streak' in key:
|
||||
result["metrics"]["longest_win_streak"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
|
||||
elif '最大值 连败' in key or 'Longest Losing Streak' in key:
|
||||
result["metrics"]["longest_lose_streak"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
|
||||
elif '平均 连胜' in key or 'Average Winning Streak' in key:
|
||||
result["metrics"]["avg_win_streak"] = self._extract_number(value)
|
||||
elif '平均 连败' in key or 'Average Losing Streak' in key:
|
||||
result["metrics"]["avg_lose_streak"] = self._extract_number(value)
|
||||
elif '最小持仓时间' in key or 'Min Hold Time' in key:
|
||||
result["metrics"]["min_hold_time"] = value
|
||||
elif '最大持仓时间' in key or 'Max Hold Time' in key:
|
||||
result["metrics"]["max_hold_time"] = value
|
||||
elif '平均持仓时间' in key or 'Avg Hold Time' in key:
|
||||
result["metrics"]["avg_hold_time"] = value
|
||||
elif '质量历史' in key or 'Quality' in key:
|
||||
result["metrics"]["quality"] = self._extract_number(value)
|
||||
elif '柱' in key or 'Bars' in key:
|
||||
result["metrics"]["bars"] = self._extract_number(value)
|
||||
elif '报价' in key or 'Quotes' in key:
|
||||
result["metrics"]["quotes"] = self._extract_number(value)
|
||||
elif '分值' in key or 'Score' in key:
|
||||
result["metrics"]["score"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
|
||||
elif 'OnTester结果' in key or 'OnTester' in key:
|
||||
result["metrics"]["on_tester"] = self._extract_number(value)
|
||||
|
||||
def parse_all_reports(self, pattern: str = "*.xml") -> List[Dict]:
|
||||
import glob
|
||||
pattern_base = pattern.replace('*', '')
|
||||
if pattern_base == '.xml':
|
||||
report_files = glob.glob(os.path.join(self.results_dir, '*.xml'))
|
||||
report_files.extend(glob.glob(os.path.join(self.results_dir, '*.htm')))
|
||||
report_files.extend(glob.glob(os.path.join(self.results_dir, '*.html')))
|
||||
else:
|
||||
report_files = glob.glob(os.path.join(self.results_dir, pattern))
|
||||
|
||||
results = []
|
||||
for report_file in report_files:
|
||||
if report_file.endswith('.xml'):
|
||||
parsed = self.parse_xml_report(report_file)
|
||||
elif report_file.endswith(('.html', '.htm')):
|
||||
parsed = self.parse_html_report(report_file)
|
||||
else:
|
||||
continue
|
||||
|
||||
if parsed:
|
||||
parsed["file_path"] = os.path.abspath(report_file)
|
||||
parsed.setdefault("report_file", os.path.basename(report_file))
|
||||
results.append(parsed)
|
||||
logger.info(f"Parsed: {os.path.basename(report_file)}")
|
||||
else:
|
||||
logger.warning(f"Failed to parse: {report_file}")
|
||||
|
||||
logger.info(f"Total reports parsed: {len(results)}")
|
||||
return results
|
||||
|
||||
def get_summary(self, results: List[Dict]) -> Dict:
|
||||
if not results:
|
||||
return {}
|
||||
|
||||
summary = {
|
||||
"total_tests": len(results),
|
||||
"total_trades": 0,
|
||||
"avg_win_rate": 0,
|
||||
"avg_profit_factor": 0,
|
||||
"best_test": None,
|
||||
"worst_test": None
|
||||
}
|
||||
|
||||
total_trades = sum(r["metrics"].get("total_trades", 0) for r in results)
|
||||
win_rates = [r["metrics"].get("win_rate", 0) for r in results if "win_rate" in r["metrics"]]
|
||||
profit_factors = [r["metrics"].get("profit_factor", 0) for r in results if "profit_factor" in r["metrics"]]
|
||||
|
||||
summary["total_trades"] = total_trades
|
||||
if win_rates:
|
||||
summary["avg_win_rate"] = sum(win_rates) / len(win_rates)
|
||||
if profit_factors:
|
||||
summary["avg_profit_factor"] = sum(profit_factors) / len(profit_factors)
|
||||
|
||||
completed = [r for r in results if r.get("test_info", {}).get("expert")]
|
||||
if completed:
|
||||
summary["best_test"] = max(completed, key=lambda x: x["metrics"].get("profit_factor", 0))
|
||||
summary["worst_test"] = min(completed, key=lambda x: x["metrics"].get("profit_factor", 0))
|
||||
|
||||
return summary
|
||||
|
||||
def export_to_csv(self, results: List[Dict], output_path: str, sort_by: str = "profit_factor", reverse: bool = True):
|
||||
if not results:
|
||||
return
|
||||
|
||||
fieldnames = [
|
||||
"expert", "symbol", "period", "company", "currency", "leverage",
|
||||
"initial_deposit", "net_profit", "gross_profit", "gross_loss",
|
||||
"profit_factor", "expected_payoff", "recovery_factor", "sharpe_ratio",
|
||||
"ahpr", "ghpr", "lr_correlation", "lr_standard_error",
|
||||
"max_drawdown", "max_equity_drawdown", "absolute_drawdown", "margin_level",
|
||||
"total_trades", "total_deals", "short_trades", "long_trades",
|
||||
"winning_trades", "losing_trades", "win_rate",
|
||||
"max_profit_trade", "max_loss_trade", "avg_profit_trade", "avg_loss_trade",
|
||||
"longest_win_streak", "longest_lose_streak", "avg_win_streak", "avg_lose_streak",
|
||||
"min_hold_time", "max_hold_time", "avg_hold_time",
|
||||
"quality", "bars", "quotes", "score", "on_tester"
|
||||
]
|
||||
|
||||
rows = []
|
||||
for r in results:
|
||||
info = r.get("test_info", {})
|
||||
metrics = r.get("metrics", {})
|
||||
row = {
|
||||
"expert": info.get("expert", ""),
|
||||
"symbol": info.get("symbol", ""),
|
||||
"period": info.get("period", ""),
|
||||
"company": info.get("company", ""),
|
||||
"currency": info.get("currency", ""),
|
||||
"leverage": info.get("leverage", ""),
|
||||
"initial_deposit": metrics.get("initial_deposit", ""),
|
||||
"net_profit": metrics.get("net_profit", ""),
|
||||
"gross_profit": metrics.get("gross_profit", ""),
|
||||
"gross_loss": metrics.get("gross_loss", ""),
|
||||
"profit_factor": metrics.get("profit_factor", ""),
|
||||
"expected_payoff": metrics.get("expected_payoff", ""),
|
||||
"recovery_factor": metrics.get("recovery_factor", ""),
|
||||
"sharpe_ratio": metrics.get("sharpe_ratio", ""),
|
||||
"ahpr": metrics.get("ahpr", ""),
|
||||
"ghpr": metrics.get("ghpr", ""),
|
||||
"lr_correlation": metrics.get("lr_correlation", ""),
|
||||
"lr_standard_error": metrics.get("lr_standard_error", ""),
|
||||
"max_drawdown": metrics.get("max_drawdown", ""),
|
||||
"max_equity_drawdown": metrics.get("max_equity_drawdown", ""),
|
||||
"absolute_drawdown": metrics.get("absolute_drawdown", ""),
|
||||
"margin_level": metrics.get("margin_level", ""),
|
||||
"total_trades": metrics.get("total_trades", ""),
|
||||
"total_deals": metrics.get("total_deals", ""),
|
||||
"short_trades": metrics.get("short_trades", ""),
|
||||
"long_trades": metrics.get("long_trades", ""),
|
||||
"winning_trades": metrics.get("winning_trades", ""),
|
||||
"losing_trades": metrics.get("losing_trades", ""),
|
||||
"win_rate": metrics.get("win_rate", ""),
|
||||
"max_profit_trade": metrics.get("max_profit_trade", ""),
|
||||
"max_loss_trade": metrics.get("max_loss_trade", ""),
|
||||
"avg_profit_trade": metrics.get("avg_profit_trade", ""),
|
||||
"avg_loss_trade": metrics.get("avg_loss_trade", ""),
|
||||
"longest_win_streak": metrics.get("longest_win_streak", ""),
|
||||
"longest_lose_streak": metrics.get("longest_lose_streak", ""),
|
||||
"avg_win_streak": metrics.get("avg_win_streak", ""),
|
||||
"avg_lose_streak": metrics.get("avg_lose_streak", ""),
|
||||
"min_hold_time": metrics.get("min_hold_time", ""),
|
||||
"max_hold_time": metrics.get("max_hold_time", ""),
|
||||
"avg_hold_time": metrics.get("avg_hold_time", ""),
|
||||
"quality": metrics.get("quality", ""),
|
||||
"bars": metrics.get("bars", ""),
|
||||
"quotes": metrics.get("quotes", ""),
|
||||
"score": metrics.get("score", ""),
|
||||
"on_tester": metrics.get("on_tester", ""),
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
rows.sort(key=lambda x: x.get(sort_by, ""), reverse=reverse)
|
||||
|
||||
os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else '.', exist_ok=True)
|
||||
with open(output_path, 'w', newline='', encoding='utf-8-sig') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
logger.info(f"CSV exported to: {output_path}")
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="MT5 Report Parser")
|
||||
parser.add_argument("--results-dir", "-r", default="reports",
|
||||
help="Directory containing report files")
|
||||
parser.add_argument("--output", "-o", default="reports/parsed_results.json",
|
||||
help="Output file for parsed results")
|
||||
parser.add_argument("--csv", "-c", default=None,
|
||||
help="CSV output path (e.g. reports/results.csv)")
|
||||
parser.add_argument("--sort", "-s", default="profit_factor",
|
||||
choices=["expert", "symbol", "period", "net_profit", "gross_profit",
|
||||
"profit_factor", "total_trades", "win_rate", "max_drawdown"],
|
||||
help="Field to sort by")
|
||||
parser.add_argument("--asc", action="store_true",
|
||||
help="Sort in ascending order (default: descending)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
parser = ResultParser(args.results_dir)
|
||||
results = parser.parse_all_reports()
|
||||
|
||||
if results:
|
||||
summary = parser.get_summary(results)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("PARSED RESULTS SUMMARY")
|
||||
print("="*60)
|
||||
print(f"Total tests parsed: {summary['total_tests']}")
|
||||
print(f"Total trades: {summary['total_trades']}")
|
||||
print(f"Average win rate: {summary['avg_win_rate']:.2f}%")
|
||||
print(f"Average profit factor: {summary['avg_profit_factor']:.2f}")
|
||||
|
||||
if summary['best_test']:
|
||||
print(f"\nBest test: {summary['best_test']['test_info'].get('expert', 'N/A')}")
|
||||
print(f" Profit factor: {summary['best_test']['metrics'].get('profit_factor', 0):.2f}")
|
||||
print(f" Win rate: {summary['best_test']['metrics'].get('win_rate', 0):.2f}%")
|
||||
|
||||
os.makedirs(os.path.dirname(args.output) if os.path.dirname(args.output) else '.', exist_ok=True)
|
||||
import json
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
json.dump({"results": results, "summary": summary}, f, indent=2, ensure_ascii=False)
|
||||
print(f"\nResults saved to: {args.output}")
|
||||
|
||||
if args.csv:
|
||||
parser.export_to_csv(results, args.csv, sort_by=args.sort, reverse=not args.asc)
|
||||
else:
|
||||
print("No results to parse")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,430 @@
|
||||
import os
|
||||
import re
|
||||
import argparse
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Dict, Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def sanitize_path_name(name: str) -> str:
|
||||
name = unicodedata.normalize('NFKD', name)
|
||||
name = name.encode('ascii', 'ignore').decode('ascii')
|
||||
name = re.sub(r'[^\w\-_.]', '_', name)
|
||||
name = re.sub(r'_{2,}', '_', name)
|
||||
return name.strip('_')
|
||||
|
||||
|
||||
@dataclass
|
||||
class SetParameter:
|
||||
name: str
|
||||
param_type: str
|
||||
default: float
|
||||
min_value: float
|
||||
max_value: float
|
||||
step: float
|
||||
optimize: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class SetFileResult:
|
||||
file_path: str
|
||||
ea_name: str
|
||||
parameters: Dict[str, SetParameter] = field(default_factory=dict)
|
||||
optimize_count: int = 0
|
||||
fixed_count: int = 0
|
||||
|
||||
|
||||
MT5_NATIVE_PATTERN = re.compile(r'^(\w+)\s+<([^>]+)>\s+<([^>]+)>')
|
||||
OUR_FORMAT_PATTERN = re.compile(r'^(\w+)=([^|]+)\|\|([^|]+)\|\|([^|]+)\|\|([^|]+)\|\|([YN])')
|
||||
|
||||
|
||||
def detect_encoding(file_path: str) -> str:
|
||||
with open(file_path, 'rb') as f:
|
||||
raw = f.read(4)
|
||||
if raw[:2] == b'\xff\xfe':
|
||||
return 'utf-16-le'
|
||||
elif raw[:2] == b'\xfe\xff':
|
||||
return 'utf-16-be'
|
||||
elif raw[:3] == b'\xef\xbb\xbf':
|
||||
return 'utf-8-sig'
|
||||
return 'utf-8'
|
||||
|
||||
|
||||
def detect_set_format(line: str) -> str:
|
||||
if '=' in line and '||' in line:
|
||||
return 'our_format'
|
||||
elif '<' in line and '>' in line:
|
||||
return 'mt5_native'
|
||||
elif line.strip().startswith(';'):
|
||||
return 'comment'
|
||||
elif line.strip() == '':
|
||||
return 'empty'
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def parse_set_line(line: str) -> Optional[SetParameter]:
|
||||
line = line.strip()
|
||||
if not line or line.startswith(';'):
|
||||
return None
|
||||
|
||||
fmt = detect_set_format(line)
|
||||
if fmt == 'our_format':
|
||||
m = OUR_FORMAT_PATTERN.match(line)
|
||||
if m:
|
||||
name, cur, start, step, stop, opt = m.groups()
|
||||
|
||||
cur_lower = cur.lower()
|
||||
if cur_lower in ('true', 'false'):
|
||||
is_true = cur_lower == 'true'
|
||||
return SetParameter(
|
||||
name=name,
|
||||
param_type='bool',
|
||||
default=1.0 if is_true else 0.0,
|
||||
min_value=0.0,
|
||||
max_value=1.0,
|
||||
step=1.0,
|
||||
optimize=(opt.upper() == 'Y')
|
||||
)
|
||||
|
||||
return SetParameter(
|
||||
name=name,
|
||||
param_type='double',
|
||||
default=float(cur),
|
||||
min_value=float(start),
|
||||
max_value=float(stop),
|
||||
step=float(step),
|
||||
optimize=(opt.upper() == 'Y')
|
||||
)
|
||||
elif fmt == 'mt5_native':
|
||||
m = MT5_NATIVE_PATTERN.match(line)
|
||||
if m:
|
||||
name, val1, val2 = m.groups()
|
||||
val1l = val1.lower().strip()
|
||||
val2l = val2.lower().strip()
|
||||
|
||||
if val1l in ('true', 'false') or val2l in ('true', 'false'):
|
||||
is_true = val1l == 'true' or val2l == 'true'
|
||||
return SetParameter(
|
||||
name=name,
|
||||
param_type='bool',
|
||||
default=1.0 if is_true else 0.0,
|
||||
min_value=0.0,
|
||||
max_value=1.0,
|
||||
step=1.0,
|
||||
optimize=False
|
||||
)
|
||||
|
||||
try:
|
||||
v1 = float(val1)
|
||||
v2 = float(val2)
|
||||
return SetParameter(
|
||||
name=name,
|
||||
param_type='double',
|
||||
default=v1,
|
||||
min_value=min(v1, v2),
|
||||
max_value=max(v1, v2),
|
||||
step=abs(v2 - v1) if v1 != v2 else 0,
|
||||
optimize=False
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def parse_set_file(file_path: str) -> SetFileResult:
|
||||
result = SetFileResult(file_path=file_path, ea_name='')
|
||||
|
||||
path = Path(file_path)
|
||||
result.ea_name = path.stem.replace('_optimization', '').replace('_params', '')
|
||||
|
||||
enc = detect_encoding(file_path)
|
||||
with open(file_path, 'r', encoding=enc) as f:
|
||||
content = f.read()
|
||||
|
||||
for line in content.split('\n'):
|
||||
param = parse_set_line(line)
|
||||
if param:
|
||||
result.parameters[param.name] = param
|
||||
|
||||
result.optimize_count = sum(1 for p in result.parameters.values() if p.optimize)
|
||||
result.fixed_count = sum(1 for p in result.parameters.values() if not p.optimize)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def parse_set_content(content: str) -> SetFileResult:
|
||||
result = SetFileResult(file_path='memory', ea_name='')
|
||||
|
||||
for line in content.split('\n'):
|
||||
param = parse_set_line(line)
|
||||
if param:
|
||||
result.parameters[param.name] = param
|
||||
|
||||
result.optimize_count = sum(1 for p in result.parameters.values() if p.optimize)
|
||||
result.fixed_count = sum(1 for p in result.parameters.values() if not p.optimize)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def scan_folder(folder_path: str) -> List[SetFileResult]:
|
||||
results = []
|
||||
folder = Path(folder_path)
|
||||
|
||||
if folder.is_file():
|
||||
return [parse_set_file(str(folder))]
|
||||
|
||||
for set_file in folder.rglob('*.set'):
|
||||
try:
|
||||
result = parse_set_file(str(set_file))
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
print(f'Warning: Failed to parse {set_file}: {e}')
|
||||
|
||||
return sorted(results, key=lambda x: x.ea_name)
|
||||
|
||||
|
||||
def batch_scan(file_paths: List[str]) -> List[SetFileResult]:
|
||||
results = []
|
||||
for path in file_paths:
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
result = parse_set_file(path)
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
print(f'Warning: Failed to parse {path}: {e}')
|
||||
return results
|
||||
|
||||
|
||||
def generate_prompt(results: List[SetFileResult], output_dir: str = '.') -> List[str]:
|
||||
output_files = []
|
||||
|
||||
for result in results:
|
||||
lines = []
|
||||
lines.append('=' * 80)
|
||||
lines.append(f'EA 参数优化分析请求 - {result.ea_name}')
|
||||
lines.append('=' * 80)
|
||||
lines.append('')
|
||||
lines.append('## 基本信息')
|
||||
lines.append(f'- EA名称: {result.ea_name}')
|
||||
lines.append(f'- SET文件: {result.file_path}')
|
||||
lines.append(f'- 参数总数: {len(result.parameters)} 个')
|
||||
lines.append('')
|
||||
|
||||
lines.append('## 参数详情表')
|
||||
lines.append('')
|
||||
lines.append('| 参数名 | 类型 | 当前值 | 最小值 | 最大值 | 步进 | 建议范围 |')
|
||||
lines.append('|--------|------|--------|--------|--------|------|----------|')
|
||||
|
||||
for name, param in sorted(result.parameters.items()):
|
||||
suggest = f'{param.min_value}~{param.max_value}'
|
||||
if param.param_type == 'int':
|
||||
suggest = f'{int(param.min_value)}~{int(param.max_value)} (步进:{int(param.step)})'
|
||||
elif param.param_type == 'double':
|
||||
suggest = f'{param.min_value}~{param.max_value} (步进:{param.step})'
|
||||
|
||||
lines.append(f'| {name} | {param.param_type} | {param.default} | {param.min_value} | {param.max_value} | {param.step} | {suggest} |')
|
||||
|
||||
lines.append('')
|
||||
lines.append('## 参数分析任务')
|
||||
lines.append('')
|
||||
lines.append('请分析以上所有参数,根据策略逻辑和交易逻辑判断:')
|
||||
lines.append('')
|
||||
lines.append('1. **哪些参数应该参与优化(optimize: true)**')
|
||||
lines.append(' - 通常是影响策略核心逻辑、盈亏比、风险的关键参数')
|
||||
lines.append(' - 例如:手数、止损止盈倍数、加仓间隔、风控阈值等')
|
||||
lines.append('')
|
||||
lines.append('2. **哪些参数应该保持固定(optimize: false)**')
|
||||
lines.append(' - 通常是风控类、开关类、显示类参数')
|
||||
lines.append(' - 例如:开关标识、魔数、面板位置等')
|
||||
lines.append('')
|
||||
lines.append('3. **推荐优化范围调整**')
|
||||
lines.append(' - 检查当前范围是否合理')
|
||||
lines.append(' - 给出你认为更合适的 min/max/step')
|
||||
lines.append('')
|
||||
lines.append('4. **重要参数优先级**')
|
||||
lines.append(' - 哪些2-3个参数对策略影响最大,需要重点优化')
|
||||
lines.append('')
|
||||
lines.append('## 输出要求')
|
||||
lines.append('')
|
||||
lines.append('请输出符合以下格式的完整JSON配置文件:')
|
||||
lines.append('')
|
||||
lines.append('```json')
|
||||
lines.append('{')
|
||||
lines.append(f' "ea_name": "{result.ea_name}",')
|
||||
safe_ea_name = sanitize_path_name(result.ea_name)
|
||||
lines.append(f' "ea_path": "MY-EA\\\\{safe_ea_name}.ex5", // {result.ea_name}')
|
||||
lines.append(' "description": "Edited via GUI",')
|
||||
lines.append(' "search_strategy": "auto",')
|
||||
lines.append(' "parameters": {')
|
||||
param_list = list(result.parameters.items())
|
||||
for i, (name, param) in enumerate(param_list):
|
||||
comma = ',' if i < len(param_list) - 1 else ''
|
||||
lines.append(f' "{name}": {{')
|
||||
lines.append(f' "type": "{param.param_type}",')
|
||||
lines.append(f' "default": {param.default},')
|
||||
lines.append(f' "min": {param.min_value},')
|
||||
lines.append(f' "max": {param.max_value},')
|
||||
lines.append(f' "step": {param.step},')
|
||||
lines.append(f' "optimize": true // 请根据分析填写 true 或 false')
|
||||
lines.append(f' }}{comma}')
|
||||
lines.append(' },')
|
||||
lines.append(' "test_config": {')
|
||||
lines.append(' "symbol": "XAUUSD", // 请填写交易品种')
|
||||
lines.append(' "period": "M1", // 请填写周期: M1/M5/M15/H1/H4/D1')
|
||||
lines.append(' "from_date": "2026.01.01", // 请填写开始日期')
|
||||
lines.append(' "to_date": "2026.01.04", // 请填写结束日期')
|
||||
lines.append(' "model": 0, // 建模方式: 0=Every Tick, 1=1分钟OHLC, 2=仅开盘价, 3=数学计算, 4=真实Tick')
|
||||
lines.append(' "execution_delay": 0, // 执行延迟: 0=无延迟, -1=随机延迟, 正数=固定延迟毫秒')
|
||||
lines.append(' "optimization_mode": 2, // 优化模式: 0=禁用, 1=慢速完整算法, 2=快速遗传算法, 3=MarketWatch所有符号')
|
||||
lines.append(' "optimization_criterion": 1, // 优化指标: 0=余额最大, 1=盈利因子最大, 2=期望收益, 3=回撤最小, 4=恢复因子, 5=夏普比率, 6=自定义, 7=复合指标')
|
||||
lines.append(' "deposit": 10000, // 初始保证金')
|
||||
lines.append(' "leverage": "1:100" // 杠杆')
|
||||
lines.append(' },')
|
||||
lines.append(' "walk_forward": {')
|
||||
lines.append(' "enabled": true, // 是否启用前向测试')
|
||||
lines.append(' "forward_mode": 2 // 前向模式: 0=禁用, 1=OOS 50%, 2=OOS 33%, 3=OOS 25%')
|
||||
lines.append(' }')
|
||||
lines.append('}')
|
||||
lines.append('```')
|
||||
lines.append('')
|
||||
lines.append('注意:')
|
||||
lines.append(f'- EA原始名称: {result.ea_name}')
|
||||
lines.append(f'- ea_path 使用ASCII安全名称: {safe_ea_name}')
|
||||
lines.append('- 实际EA文件应放在 MY-EA 目录下,文件名需与 ea_path 中的名称一致')
|
||||
lines.append('- 每个参数的 "optimize" 字段需要你根据策略分析来填写')
|
||||
lines.append('- optimize: true = 参与优化, optimize: false = 保持固定')
|
||||
lines.append('- test_config 和 walk_forward 部分请根据回测需求填写或调整')
|
||||
lines.append('- ea_path 路径使用双反斜杠 `\\\\`')
|
||||
lines.append('')
|
||||
lines.append('=' * 80)
|
||||
|
||||
output_path = os.path.join(output_dir, f'{result.ea_name}_optimization_prompt.txt')
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
output_files.append(output_path)
|
||||
print(f'Generated: {output_path}')
|
||||
|
||||
return output_files
|
||||
|
||||
|
||||
def interactive_scan():
|
||||
print('请选择扫描模式:')
|
||||
print('1. 扫描单个 SET 文件')
|
||||
print('2. 扫描文件夹下所有 SET 文件')
|
||||
print('3. 批量扫描多个 SET 文件')
|
||||
print('4. 从剪贴板读取 SET 内容')
|
||||
print('0. 退出')
|
||||
print('')
|
||||
|
||||
choice = input('请输入选项 (0-4): ').strip()
|
||||
|
||||
if choice == '1':
|
||||
path = input('请输入 SET 文件路径: ').strip()
|
||||
if os.path.exists(path):
|
||||
results = [parse_set_file(path)]
|
||||
output_dir = os.path.dirname(path) or '.'
|
||||
generate_prompt(results, output_dir)
|
||||
else:
|
||||
print('文件不存在')
|
||||
|
||||
elif choice == '2':
|
||||
path = input('请输入文件夹路径: ').strip()
|
||||
if os.path.exists(path):
|
||||
results = scan_folder(path)
|
||||
generate_prompt(results, path)
|
||||
else:
|
||||
print('文件夹不存在')
|
||||
|
||||
elif choice == '3':
|
||||
print('请输入 SET 文件路径(每行一个,输入空行结束):')
|
||||
paths = []
|
||||
while True:
|
||||
line = input().strip()
|
||||
if not line:
|
||||
break
|
||||
if os.path.exists(line):
|
||||
paths.append(line)
|
||||
else:
|
||||
print(f'文件不存在: {line}')
|
||||
if paths:
|
||||
results = batch_scan(paths)
|
||||
generate_prompt(results, os.path.dirname(paths[0]) if len(paths) == 1 else '.')
|
||||
|
||||
elif choice == '4':
|
||||
print('请粘贴 SET 文件内容(输入空行结束):')
|
||||
lines = []
|
||||
while True:
|
||||
line = input()
|
||||
if not line.strip():
|
||||
break
|
||||
lines.append(line)
|
||||
if lines:
|
||||
content = '\n'.join(lines)
|
||||
result = parse_set_content(content)
|
||||
result.ea_name = input('请输入 EA 名称: ').strip() or 'EA'
|
||||
generate_prompt([result], '.')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='SET 文件扫描工具 - 生成 AI 优化提示词',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog='''
|
||||
示例:
|
||||
python scan_set_prompt.py --file C:\\path\\to\\EA.set
|
||||
python scan_set_prompt.py --folder C:\\path\\to\\set_files
|
||||
python scan_set_prompt.py --batch a.set b.set c.set
|
||||
python scan_set_prompt.py --interactive
|
||||
'''
|
||||
)
|
||||
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--file', '-f', metavar='PATH', help='扫描单个 SET 文件')
|
||||
group.add_argument('--folder', '-d', metavar='PATH', help='扫描文件夹下所有 SET 文件')
|
||||
group.add_argument('--batch', '-b', nargs='+', metavar='PATH', help='批量扫描多个 SET 文件')
|
||||
group.add_argument('--interactive', '-i', action='store_true', help='交互式扫描')
|
||||
|
||||
parser.add_argument('--output', '-o', metavar='DIR', default='.', help='输出目录 (默认当前目录)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.interactive:
|
||||
interactive_scan()
|
||||
return
|
||||
|
||||
results = []
|
||||
|
||||
if args.file:
|
||||
if not os.path.exists(args.file):
|
||||
print(f'Error: 文件不存在 {args.file}')
|
||||
return
|
||||
results = [parse_set_file(args.file)]
|
||||
output_dir = os.path.dirname(args.file) or args.output
|
||||
|
||||
elif args.folder:
|
||||
if not os.path.exists(args.folder):
|
||||
print(f'Error: 文件夹不存在 {args.folder}')
|
||||
return
|
||||
results = scan_folder(args.folder)
|
||||
output_dir = args.folder
|
||||
|
||||
elif args.batch:
|
||||
existing = [p for p in args.batch if os.path.exists(p)]
|
||||
if not existing:
|
||||
print('Error: 所有文件都不存在')
|
||||
return
|
||||
results = batch_scan(existing)
|
||||
output_dir = args.output
|
||||
|
||||
if not results:
|
||||
print('Warning: 没有找到任何 SET 文件')
|
||||
return
|
||||
|
||||
output_files = generate_prompt(results, output_dir)
|
||||
print(f'\n完成!共生成 {len(output_files)} 个提示词文件')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user