first commit

This commit is contained in:
2026-07-06 03:32:12 +08:00
commit 160c68da39
54 changed files with 26556 additions and 0 deletions
View File
+222
View File
@@ -0,0 +1,222 @@
import os
import subprocess
import time
import logging
from datetime import datetime
from typing import List, Dict, Optional
from pathlib import Path
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('logs/batch_executor.log'),
logging.StreamHandler()
]
)
logger = logging.getLogger(__name__)
class BatchExecutor:
def __init__(self, config_path: str):
self.config = self._load_config(config_path)
self.mt5_path = self.config["mt5_settings"]["terminal_path"]
self.reports_dir = self.config["mt5_settings"]["reports_dir"]
self.results_dir = self.config["mt5_settings"].get("results_dir", "results")
os.makedirs(self.results_dir, exist_ok=True)
os.makedirs("logs", exist_ok=True)
def _load_config(self, config_path: str) -> Dict:
import yaml
with open(config_path, 'r', encoding='utf-8') as f:
return yaml.safe_load(f)
def _wait_for_report(self, report_path: str, timeout: int = 600) -> bool:
start_time = time.time()
while time.time() - start_time < timeout:
if os.path.exists(report_path):
time.sleep(2)
try:
with open(report_path, 'r', encoding='utf-8') as f:
content = f.read()
if len(content) > 100:
return True
except:
pass
time.sleep(5)
return False
def _check_mt5_process(self) -> bool:
try:
result = subprocess.run(
['tasklist', '/FI', 'IMAGENAME eq terminal64.exe'],
capture_output=True,
text=True
)
return 'terminal64.exe' in result.stdout
except:
return False
def execute_single(self, ini_path: str, wait_time: int = 120) -> Dict:
result = {
"ini_file": os.path.basename(ini_path),
"status": "pending",
"start_time": None,
"end_time": None,
"duration": 0,
"report_path": None,
"error": None
}
logger.info(f"Starting backtest: {ini_path}")
if not os.path.exists(self.mt5_path):
result["status"] = "error"
result["error"] = f"MT5 terminal not found: {self.mt5_path}"
logger.error(result["error"])
return result
result["start_time"] = datetime.now()
try:
process = subprocess.Popen(
[self.mt5_path, "/portable", f"/config:{ini_path}"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
logger.info(f"MT5 process started with PID: {process.pid}")
ini_basename = os.path.splitext(os.path.basename(ini_path))[0]
report_xml = os.path.join(self.results_dir, f"{ini_basename}_report.xml")
report_found = self._wait_for_report(report_xml, timeout=wait_time * 60)
if report_found:
result["status"] = "completed"
result["report_path"] = report_xml
logger.info(f"Report generated: {report_xml}")
else:
if process.poll() is not None:
result["status"] = "mt5_closed"
logger.warning("MT5 closed before report was generated")
else:
process.terminate()
process.wait(timeout=10)
result["status"] = "timeout"
logger.warning(f"Timeout waiting for report: {ini_path}")
if process.poll() is None:
try:
process.terminate()
process.wait(timeout=5)
except:
pass
except Exception as e:
result["status"] = "error"
result["error"] = str(e)
logger.error(f"Error executing backtest: {e}")
result["end_time"] = datetime.now()
if result["start_time"] and result["end_time"]:
result["duration"] = (result["end_time"] - result["start_time"]).total_seconds()
return result
def execute_batch(self, ini_files: List[str],
max_parallel: int = 1,
wait_time_per_test: int = 120) -> List[Dict]:
results = []
total = len(ini_files)
logger.info(f"Starting batch execution: {total} tests")
logger.info(f"Parallel execution: {max_parallel}")
for idx, ini_path in enumerate(ini_files, 1):
logger.info(f"[{idx}/{total}] Executing: {os.path.basename(ini_path)}")
result = self.execute_single(ini_path, wait_time_per_test)
results.append(result)
status = result["status"]
duration = result["duration"]
logger.info(f" Status: {status}, Duration: {duration:.1f}s")
if result["error"]:
logger.error(f" Error: {result['error']}")
success_count = sum(1 for r in results if r["status"] == "completed")
logger.info(f"Batch completed: {success_count}/{total} successful")
return results
def execute_validate_tasks(self, ini_files: List[str]) -> str:
validate_dir = os.path.join(os.path.dirname(self.mt5_path),
"MQL5", "Files", "ValidateTasks")
os.makedirs(validate_dir, exist_ok=True)
for ini_file in ini_files:
dest_path = os.path.join(validate_dir, os.path.basename(ini_file))
with open(ini_file, 'r', encoding='utf-8') as src:
content = src.read()
with open(dest_path, 'w', encoding='utf-8') as dst:
dst.write(content)
logger.info(f"Copied to ValidateTasks: {os.path.basename(ini_file)}")
logger.info(f"All {len(ini_files)} tasks queued in ValidateTasks folder")
return validate_dir
def main():
import argparse
parser = argparse.ArgumentParser(description="MT5 Batch Backtest Executor")
parser.add_argument("--config", "-c", default="config/ea_configs.yaml",
help="Path to config file")
parser.add_argument("--ini-dir", "-i",
help="Directory containing INI files to execute")
parser.add_argument("--wait", "-w", type=int, default=120,
help="Wait time per test in minutes")
parser.add_argument("--validate", "-v", action="store_true",
help="Use ValidateTasks method")
args = parser.parse_args()
executor = BatchExecutor(args.config)
if args.ini_dir:
ini_files = [os.path.join(args.ini_dir, f)
for f in os.listdir(args.ini_dir)
if f.endswith('.ini')]
else:
generator = __import__('ini_generator', fromlist=['']).INIGenerator(args.config)
ini_files = generator.generate_ini_files()
logger.info(f"Found {len(ini_files)} INI files to execute")
if args.validate:
validate_dir = executor.execute_validate_tasks(ini_files)
logger.info(f"ValidateTasks method: files copied to {validate_dir}")
logger.info("Run the Validate EA in MT5 terminal to execute")
else:
results = executor.execute_batch(ini_files, wait_time_per_test=args.wait)
success = [r for r in results if r["status"] == "completed"]
failed = [r for r in results if r["status"] != "completed"]
print("\n" + "="*60)
print("BATCH EXECUTION SUMMARY")
print("="*60)
print(f"Total tests: {len(results)}")
print(f"Completed: {len(success)}")
print(f"Failed: {len(failed)}")
print("="*60)
if failed:
print("\nFailed tests:")
for r in failed:
print(f" - {r['ini_file']}: {r['status']} - {r['error']}")
if __name__ == "__main__":
main()
+256
View File
@@ -0,0 +1,256 @@
import os
import sys
import yaml
from datetime import datetime
from itertools import product
from typing import Dict, List, Any, Optional
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from mt5_paths import resolve_mt5_settings
class INIGenerator:
def __init__(self, source):
"""
source: YAML 文件路径 str 或 已加载的 config dict(GUI 内存直接传入用)。
"""
if isinstance(source, dict):
self.config_path = None
self.config = source
else:
self.config_path = source
self.config = self._load_config()
self.config["mt5_settings"] = resolve_mt5_settings(self.config.get("mt5_settings", {}))
self.output_dir = self.config["mt5_settings"]["ini_dir"]
os.makedirs(self.output_dir, exist_ok=True)
def _load_config(self) -> Dict:
with open(self.config_path, 'r', encoding='utf-8') as f:
return yaml.safe_load(f)
def _scan_experts_dir(self) -> List[str]:
data_dir = self.config["mt5_settings"].get("data_dir", "")
if not data_dir:
mt5_base = os.path.dirname(self.config["mt5_settings"]["terminal_path"])
data_dir = mt5_base
experts_dir = os.path.join(data_dir, "MQL5", "Experts")
ea_files = []
if not os.path.exists(experts_dir):
return []
for root, dirs, files in os.walk(experts_dir):
for f in files:
if f.endswith('.ex5'):
rel_path = os.path.relpath(os.path.join(root, f), experts_dir)
ea_files.append(rel_path)
return ea_files
def _load_set_file(self, set_file_path: str) -> Dict[str, Any]:
params = {}
if not os.path.exists(set_file_path):
return params
with open(set_file_path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line and not line.startswith(';') and '=' in line:
key, value = line.split('=', 1)
try:
params[key.strip()] = float(value.strip())
except:
params[key.strip()] = value.strip()
return params
def _generate_parameter_combinations(self, parameters: Dict[str, List]) -> List[Dict]:
if not parameters:
return [{}]
keys = list(parameters.keys())
values = list(parameters.values())
combinations = list(product(*values))
return [dict(zip(keys, combo)) for combo in combinations]
def _get_timeframe_code(self, timeframe: str) -> str:
mapping = {
"M1": "M1", "M5": "M5", "M15": "M15",
"H1": "H1", "H4": "H4", "D1": "D1", "W1": "W1"
}
return mapping.get(timeframe, "H1")
def _params_hash(self, parameters: Dict) -> str:
if not parameters:
return "default"
sorted_params = sorted(parameters.items())
param_str = "_".join([f"{k}{v}" for k, v in sorted_params])
return str(abs(hash(param_str)))[:8]
def _generate_filename(self, ea_name: str, symbol: str,
timeframe: str, parameters: Dict) -> str:
safe_ea_name = ea_name.replace("\\", "_").replace("/", "_").replace("..", "")
for c in '()[]{}|\\/*?:"\'<>':
safe_ea_name = safe_ea_name.replace(c, "_")
param_hash = self._params_hash(parameters)
return f"{safe_ea_name}_{symbol}_{timeframe}_{param_hash}.ini"
def _build_ini_content(self, ea_filename: str, symbol: str, timeframe: str,
bt_settings: Dict, parameters: Dict, ea_name: str,
set_file: Optional[str] = None) -> str:
date_from = bt_settings["date_range"]["from"]
date_to = bt_settings["date_range"]["to"]
report_name = f"{ea_name}_{symbol}_{timeframe}_{self._params_hash(parameters)}"
for c in '()[]{}|\\/*?:"\'<>':
report_name = report_name.replace(c, "_")
safe_report_name = report_name
ini_lines = [
"; MT5 Strategy Tester Configuration",
f"; Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
"",
"[Tester]",
f"Expert={ea_filename}",
]
if set_file:
ini_lines.append(f"ExpertParameters={set_file}")
else:
ini_lines.append("ExpertParameters=")
ini_lines.extend([
f"Symbol={symbol}",
f"Period={self._get_timeframe_code(timeframe)}",
f"Model={bt_settings['model']}",
f"ExecutionMode={bt_settings.get('execution_mode', 0)}",
f"ExecutionDelay={bt_settings.get('execution_delay', 0)}",
f"Optimization={bt_settings['optimization']}",
])
if bt_settings.get("optimization"):
ini_lines.append("OptimizationCriterion=6")
ini_lines.extend([
f"FromDate={date_from}",
f"ToDate={date_to}",
f"ForwardMode={bt_settings.get('forward_mode', 0)}",
f"ForwardDate={bt_settings.get('forward_date', '')}",
f"Report={safe_report_name}",
f"ReplaceReport={1 if bt_settings['replace_report'] else 0}",
f"ShutdownTerminal={1 if bt_settings['shutdown_terminal'] else 0}",
f"Deposit={int(float(bt_settings['deposit']))}",
f"Currency={bt_settings['currency']}",
f"Leverage={bt_settings['leverage']}",
f"Visual={bt_settings['visual']}",
"ProfitInPips=0",
])
if parameters:
ini_lines.extend(["", "; EA Parameters"])
for param_name, param_value in parameters.items():
ini_lines.append(f"{param_name}={param_value}")
return "\n".join(ini_lines)
def generate_ini_files(self) -> List[str]:
generated_files = []
bt_settings = self.config["backtest_settings"]
set_files_dir = self.config["mt5_settings"].get("set_files_dir", "config/sets")
ea_configs = self.config.get("eas", [])
if not ea_configs:
print("No EAs configured, scanning MT5 Experts directory...")
ea_files = self._scan_experts_dir()
if ea_files:
print(f"Found {len(ea_files)} EA files: {ea_files}")
for ea_file in ea_files:
ea_name = os.path.splitext(ea_file)[0]
ea_configs.append({
"name": ea_name,
"filename": ea_file,
"description": "Auto-scanned EA"
})
else:
print("No EAs found in Experts directory")
return []
for ea in ea_configs:
ea_name = ea["name"]
ea_filename = ea["filename"]
set_file = ea.get("set_file")
parameters = ea.get("parameters", {})
if set_file:
set_file_path = os.path.join(set_files_dir, set_file)
set_params = self._load_set_file(set_file_path)
param_combinations = [set_params] if set_params else [{}]
else:
param_combinations = self._generate_parameter_combinations(parameters)
for symbol in bt_settings["symbols"]:
for timeframe in bt_settings["timeframes"]:
for param_combo in param_combinations:
ini_content = self._build_ini_content(
ea_filename, symbol, timeframe,
bt_settings, param_combo, ea_name, set_file
)
ini_filename = self._generate_filename(
ea_name, symbol, timeframe, param_combo
)
ini_path = os.path.join(self.output_dir, ini_filename)
with open(ini_path, 'w', encoding='utf-8') as f:
f.write(ini_content)
generated_files.append(ini_path)
return generated_files
def generate_batch_run_script(self, ini_files: List[str],
output_script: str = "run_backtests.bat"):
mt5_path = self.config["mt5_settings"]["terminal_path"]
lines = [
"@echo off",
"echo MT5 Batch Backtest Runner",
"echo =======================",
"",
f'SET "MT5_PATH={mt5_path}"',
f'SET "INI_DIR={self.output_dir}"',
"",
]
for ini_file in ini_files:
ini_filename = os.path.basename(ini_file)
lines.append(f'echo Running: {ini_filename}')
lines.append(
f'START "MT5" /WAIT "%MT5_PATH%" /portable /config:"%INI_DIR%\\{ini_filename}"'
)
lines.append("if errorlevel 1 echo Failed: " + ini_filename)
lines.append("")
lines.append("echo All backtests completed!")
lines.append("pause")
with open(output_script, 'w', encoding='utf-8') as f:
f.write("\n".join(lines))
return output_script
def main():
config_path = os.path.join(os.path.dirname(__file__), "..", "config", "ea_configs.yaml")
generator = INIGenerator(config_path)
print("Generating INI files...")
ini_files = generator.generate_ini_files()
print(f"Generated {len(ini_files)} INI files")
for f in ini_files[:5]:
print(f" - {os.path.basename(f)}")
if len(ini_files) > 5:
print(f" ... and {len(ini_files) - 5} more")
if ini_files:
print("\nGenerating batch run script...")
script_path = generator.generate_batch_run_script(ini_files)
print(f"Batch script: {script_path}")
if __name__ == "__main__":
main()
+355
View File
@@ -0,0 +1,355 @@
import os
import subprocess
import time
import logging
import json
import shutil
from datetime import datetime
from typing import List, Dict
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('logs/batch_executor.log', encoding='utf-8'),
logging.StreamHandler()
]
)
logger = logging.getLogger(__name__)
class MT5AutoRunner:
def __init__(self, source):
"""
source: YAML 文件路径 str 或 已加载的 config dict(GUI 内存直接传入用)。
"""
if isinstance(source, dict):
self.config = source
else:
self.config = self._load_config(source)
# 占位符解析(不做自动探测)
from mt5_paths import resolve_mt5_settings
self.config["mt5_settings"] = resolve_mt5_settings(self.config.get("mt5_settings", {}))
self.mt5_path = self.config["mt5_settings"]["terminal_path"]
self.data_dir = self.config["mt5_settings"].get("data_dir", "")
self.reports_dir = os.path.abspath(self.config["mt5_settings"].get("reports_dir", "reports"))
self.ini_dir = os.path.abspath(self.config["mt5_settings"].get("ini_dir",
os.path.join(os.path.dirname(self.reports_dir), "config", "generated")))
os.makedirs(self.reports_dir, exist_ok=True)
os.makedirs(self.ini_dir, exist_ok=True)
exec_cfg = self.config.get("execution", {}) or {}
self.kill_between = bool(exec_cfg.get("kill_between", True))
self.skip_existing = bool(exec_cfg.get("skip_existing", True))
self.timeout_per_test = int(exec_cfg.get("timeout_per_test", 30))
if not self.mt5_path or not os.path.isfile(self.mt5_path):
raise FileNotFoundError(
f"MT5 终端路径未配置或不存在: {self.mt5_path!r}\n"
"请在 GUI '回测配置' 标签页设置 terminal64.exe 的完整路径后点 '保存配置'"
)
def _load_config(self, config_path: str) -> Dict:
import yaml
with open(config_path, 'r', encoding='utf-8') as f:
return yaml.safe_load(f)
def _load_execution_log(self) -> List[Dict]:
log_file = os.path.join(self.ini_dir, "..", "results", "execution_log.json")
log_file = os.path.normpath(log_file)
if os.path.exists(log_file):
with open(log_file, 'r', encoding='utf-8') as f:
return json.load(f)
return []
def _save_execution_log(self, log: List[Dict]):
log_file = os.path.join(self.ini_dir, "..", "results", "execution_log.json")
log_file = os.path.normpath(log_file)
os.makedirs(os.path.dirname(log_file), exist_ok=True)
with open(log_file, 'w', encoding='utf-8') as f:
json.dump(log, f, indent=2, ensure_ascii=False)
def _get_report_path(self, ini_name: str) -> str:
base_name = os.path.splitext(ini_name)[0]
return os.path.join(self.data_dir, f"{base_name}.htm") if self.data_dir else f"{base_name}.htm"
def _copy_report_to_project(self, ini_name: str) -> str:
base_name = os.path.splitext(ini_name)[0]
src_path = self._get_report_path(ini_name)
if not os.path.exists(src_path):
logger.warning(f"Source report not found: {src_path}")
return None
dest_path = os.path.join(self.reports_dir, f"{base_name}.htm")
try:
shutil.copy2(src_path, dest_path)
logger.info(f"Report copied to: {dest_path}")
return dest_path
except Exception as e:
logger.error(f"Failed to copy report: {e}")
return None
def _kill_mt5(self):
try:
subprocess.run(['taskkill', '/F', '/IM', 'terminal64.exe'],
capture_output=True, text=True)
time.sleep(2)
except:
pass
def _read_ini_expert(self, ini_path: str) -> str:
try:
with open(ini_path, 'r', encoding='utf-8') as f:
for line in f:
if line.startswith('Expert='):
return line.split('=', 1)[1].strip()
except:
pass
return None
def _execute_single_ini(self, ini_path: str, timeout_min: int = 30) -> Dict:
result = {
"ini_file": os.path.basename(ini_path),
"status": "pending",
"start_time": datetime.now().isoformat(),
"end_time": None,
"duration_sec": 0,
"report_found": False,
"report_path": None,
"error": None
}
ini_name = os.path.basename(ini_path)
expected_report = self._get_report_path(ini_name)
if os.path.exists(expected_report):
result["status"] = "already_completed"
result["report_found"] = True
result["report_path"] = expected_report
logger.info(f"Already completed: {ini_name}")
return result
logger.info(f"Executing: {ini_name}")
try:
start_time = time.time()
proc = subprocess.Popen(
[self.mt5_path, f"/config:{ini_path}"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
logger.info(f"MT5 started PID: {proc.pid}")
while True:
if proc.poll() is not None:
logger.info("MT5 process ended")
break
if os.path.exists(expected_report):
try:
with open(expected_report, 'r', encoding='utf-8') as f:
content = f.read()
if len(content) > 1000:
break
except:
pass
elapsed = time.time() - start_time
if elapsed > timeout_min * 60:
logger.warning(f"Timeout: {ini_name}")
break
time.sleep(2)
end_time = time.time()
result["duration_sec"] = end_time - start_time
if os.path.exists(expected_report):
result["status"] = "completed"
result["report_found"] = True
copied_path = self._copy_report_to_project(ini_name)
result["report_path"] = copied_path or expected_report
logger.info(f"Success: {ini_name} ({result['duration_sec']:.0f}s)")
else:
result["status"] = "failed"
logger.warning(f"Failed: {ini_name}")
result["end_time"] = datetime.now().isoformat()
if proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=5)
except:
pass
except Exception as e:
result["status"] = "error"
result["error"] = str(e)
logger.error(f"Error: {e}")
return result
def run_full_auto(self, ini_files: List[str] = None,
skip_if_exists: bool = None,
kill_between_tests: bool = None,
timeout_min: int = None) -> List[Dict]:
logger.info("="*60)
logger.info("MT5 Full Auto Batch Runner")
logger.info("="*60)
if skip_if_exists is None:
skip_if_exists = self.skip_existing
if kill_between_tests is None:
kill_between_tests = self.kill_between
if timeout_min is None:
timeout_min = self.timeout_per_test
logger.info(f"Timeout per test: {timeout_min} minutes, Kill between tests: {kill_between_tests}, Skip existing: {skip_if_exists}")
self.execution_log = self._load_execution_log()
if ini_files is None:
ini_files = [os.path.join(self.ini_dir, f)
for f in os.listdir(self.ini_dir) if f.endswith('.ini')]
total = len(ini_files)
completed = 0
failed = 0
logger.info(f"Total INI files: {total}")
for idx, ini_path in enumerate(ini_files, 1):
logger.info(f"[{idx}/{total}] {os.path.basename(ini_path)}")
result = self._execute_single_ini(ini_path, timeout_min=timeout_min)
self.execution_log.append(result)
self._save_execution_log(self.execution_log)
if result["status"] == "completed":
completed += 1
elif result["status"] == "already_completed":
completed += 1
else:
failed += 1
logger.info(f" Status: {result['status']}, Duration: {result['duration_sec']:.0f}s")
if kill_between_tests and result["status"] != "already_completed":
self._kill_mt5()
time.sleep(2)
logger.info("="*60)
logger.info(f"DONE: {completed}/{total} completed, {failed} failed")
logger.info("="*60)
return self.execution_log
def run_daemon(self, check_interval: int = 60):
logger.info("="*60)
logger.info("MT5 Auto Runner - DAEMON MODE")
logger.info(f"Monitoring: {self.ini_dir}")
logger.info(f"Reports will be copied to: {self.reports_dir}")
logger.info(f"Timeout per test: {self.timeout_per_test} minutes")
logger.info("Press Ctrl+C to stop")
logger.info("="*60)
self.execution_log = self._load_execution_log()
try:
while True:
pending_inis = []
for f in os.listdir(self.ini_dir):
if f.endswith('.ini'):
is_completed = any(
log.get("ini_file") == f and log.get("status") == "completed"
for log in self.execution_log
)
if not is_completed:
pending_inis.append(f)
if pending_inis:
logger.info(f"Found {len(pending_inis)} pending INI files")
for ini_name in pending_inis:
ini_path = os.path.join(self.ini_dir, ini_name)
result = self._execute_single_ini(ini_path, timeout_min=self.timeout_per_test)
self.execution_log.append(result)
self._save_execution_log(self.execution_log)
self._kill_mt5()
time.sleep(3)
else:
logger.info("No pending INI files, waiting...")
time.sleep(check_interval)
except KeyboardInterrupt:
logger.info("Daemon stopped")
def generate_report(self) -> str:
logger.info("Generating summary report...")
from scripts.result_parser import ResultParser
parser = ResultParser(self.reports_dir)
results = parser.parse_all_reports(pattern="*.ht*")
if results:
from scripts.report_generator import ReportGenerator
generator = ReportGenerator(results)
os.makedirs("reports", exist_ok=True)
output = generator.generate_excel("reports/batch_summary.xlsx")
logger.info(f"Report saved: {output}")
return output
else:
logger.warning("No results to generate report")
return None
def main():
import argparse
parser = argparse.ArgumentParser(description="MT5 Auto Runner")
parser.add_argument("--config", "-c", default="config/ea_configs.yaml")
parser.add_argument("--mode", "-m", choices=["full", "daemon", "report", "init", "execute"],
default="full")
parser.add_argument("--interval", "-i", type=int, default=60)
parser.add_argument("--no-skip", action="store_true")
args = parser.parse_args()
runner = MT5AutoRunner(args.config)
if args.mode == "daemon":
runner.run_daemon(check_interval=args.interval)
elif args.mode == "report":
runner.generate_report()
elif args.mode == "init":
from scripts.ini_generator import INIGenerator
generator = INIGenerator(args.config)
ini_files = generator.generate_ini_files()
print(f"Generated {len(ini_files)} INI files")
elif args.mode == "execute":
from scripts.ini_generator import INIGenerator
generator = INIGenerator(args.config)
ini_files = generator.generate_ini_files()
results = runner.run_full_auto(ini_files, skip_if_exists=not args.no_skip)
completed = sum(1 for r in results if r["status"] == "completed")
print(f"Completed: {completed}/{len(results)}")
else:
from scripts.ini_generator import INIGenerator
generator = INIGenerator(args.config)
ini_files = generator.generate_ini_files()
logger.info(f"Generated {len(ini_files)} INI files")
results = runner.run_full_auto(ini_files, skip_if_exists=not args.no_skip)
runner.generate_report()
if __name__ == "__main__":
main()
+65
View File
@@ -0,0 +1,65 @@
"""
MT5 路径占位符解析(不做任何自动探测)。
支持占位符:
{PROJECT_ROOT} 项目根目录绝对路径
{APP_DATA_DIR} %APPDATA% (C:\\Users\\<user>\\AppData\\Roaming)
${APPDATA} 同上 (POSIX 风格)
${USERPROFILE} C:\\Users\\<user>
留空的路径会原样返回,由调用方/用户决定。
"""
import os
from typing import Optional
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def _expand_placeholders(value: str, project_root: str) -> str:
if not value:
return value
expanded = (
value
.replace("{PROJECT_ROOT}", project_root)
.replace("{APP_DATA_DIR}", os.environ.get("APPDATA", ""))
.replace("${APPDATA}", os.environ.get("APPDATA", ""))
.replace("${USERPROFILE}", os.environ.get("USERPROFILE", ""))
)
if "{" in expanded and "}" in expanded:
return expanded
return os.path.normpath(expanded)
def resolve_mt5_settings(mt5_settings: dict, project_root: Optional[str] = None) -> dict:
"""只做 {PROJECT_ROOT} 等占位符替换;空值原样保留,不做任何自动探测。"""
if project_root is None:
project_root = PROJECT_ROOT
resolved = dict(mt5_settings)
for key, default_sub in (("ini_dir", os.path.join("config", "generated")),
("reports_dir", "reports")):
v = resolved.get(key, "")
if not v:
resolved[key] = os.path.join(project_root, default_sub)
else:
expanded = _expand_placeholders(v, project_root)
if expanded and (os.path.isabs(expanded) or "{" not in expanded):
resolved[key] = expanded
else:
resolved[key] = os.path.join(project_root, expanded)
return resolved
if __name__ == "__main__":
import pprint
sample = {
"terminal_path": "",
"data_dir": "",
"ini_dir": "{PROJECT_ROOT}/config/generated",
"reports_dir": "{PROJECT_ROOT}/reports",
}
pprint.pp(resolve_mt5_settings(sample))
View File
+320
View File
@@ -0,0 +1,320 @@
import os
import re
import json
import glob
import shutil
from typing import List, Dict, Any, Optional
from bs4 import BeautifulSoup
import xml.etree.ElementTree as ET
class ResultAnalyzer:
def __init__(self, reports_dir: str = 'reports'):
self.reports_dir = reports_dir
def parse_mt5_html_report(self, report_path: str) -> Optional[Dict]:
if not os.path.exists(report_path):
return None
try:
with open(report_path, 'rb') as f:
raw_bytes = f.read()
if raw_bytes[:2] == b'\xff\xfe':
content = raw_bytes[2:].decode('utf-16-le')
else:
content = raw_bytes.decode('utf-8', errors='ignore')
content = content.encode('utf-8').decode('utf-8')
soup = BeautifulSoup(content, 'html.parser')
result = {}
pattern = r'<td[^>]*>([^<]+):</td>\s*<td[^>]*><b>([^<]*)</b></td>'
matches = re.findall(pattern, content)
plain_pattern = r'<td[^>]*nowrap[^>]*>([^<]+):</td>\s*<td[^>]*colspan=.10.[^>]*><b>([^<]*)</b></td>'
plain_matches = re.findall(plain_pattern, content)
all_matches = dict(plain_matches)
for k, v in matches:
if k not in all_matches:
all_matches[k] = v
for key, value in all_matches.items():
key = key.strip()
value = value.strip()
if 'Net Profit' in key or '\u603b\u51c0\u76c8\u5229' in key:
result['net_profit'] = self._extract_number(value)
elif 'Gross Profit' in key or '\u6bdb\u5229' in key:
result['gross_profit'] = self._extract_number(value)
elif 'Gross Loss' in key or '\u6bdb\u635f' in key:
result['gross_loss'] = self._extract_number(value)
elif 'Profit Factor' in key or '\u76c8\u5229\u56e0\u5b50' in key:
result['profit_factor'] = self._extract_number(value)
elif 'Total Trades' in key or '\u4ea4\u6613\u603b\u8ba1' in key:
result['total_trades'] = self._extract_number(value)
elif 'Sharpe Ratio' in key or '\u590f\u666e\u6bd4\u7387' in key:
result['sharpe_ratio'] = self._extract_number(value)
elif 'Maximal Drawdown' in key or '\u6700\u5927\u7ed3\u4f59\u4e8f\u635f' in key:
result['max_drawdown'] = self._extract_number(value)
elif 'Win Rate' in key or '\u76c8\u5229\u4ea4\u6613' in key:
result['win_rate'] = self._extract_percentage(value)
return result if result else None
except Exception as e:
print('Error parsing ' + report_path + ': ' + str(e))
return None
def _extract_number(self, text: str) -> float:
numbers = re.findall(r'[-+]?\d*\.?\d+', text.replace(',', '').replace(' ', ''))
if numbers:
try:
return float(numbers[0])
except:
return 0.0
return 0.0
def _extract_percentage(self, text: str) -> float:
numbers = re.findall(r'\d+\.?\d*%', text)
if numbers:
try:
return float(numbers[0].replace('%', ''))
except:
return 0.0
return 0.0
def parse_mt5_xml_optimization(self, xml_path: str) -> List[Dict]:
if not os.path.exists(xml_path):
return []
results = []
try:
tree = ET.parse(xml_path)
root = tree.getroot()
ns = {'ss': 'urn:schemas-microsoft-com:office:spreadsheet'}
worksheet = root.find('.//ss:Worksheet', ns)
if worksheet is None:
return []
table = worksheet.find('ss:Table', ns)
if table is None:
return []
rows = table.findall('ss:Row', ns)
if len(rows) < 2:
return []
header_row = rows[0]
headers = []
for cell in header_row.findall('ss:Cell', ns):
data = cell.find('ss:Data', ns)
if data is not None and data.text:
headers.append(data.text.strip().lower())
for row_idx, row in enumerate(rows[1:], start=1):
cells = row.findall('ss:Cell', ns)
if not cells:
continue
result = {'pass': row_idx}
for col_idx, cell in enumerate(cells):
if col_idx >= len(headers):
break
header = headers[col_idx]
data = cell.find('ss:Data', ns)
value = data.text.strip() if data is not None and data.text else ''
if 'pass' in header:
result['pass'] = int(value) if value.isdigit() else row_idx
elif 'result' in header:
result['result'] = self._extract_number(value)
elif 'profit' in header and 'expected' not in header and 'factor' not in header and 'drawdown' not in header:
result['net_profit'] = self._extract_number(value)
elif 'expected payoff' in header:
result['expected_payoff'] = self._extract_number(value)
elif 'profit factor' in header:
if value:
result['profit_factor'] = self._extract_number(value)
elif 'recovery factor' in header:
result['recovery_factor'] = self._extract_number(value)
elif 'sharpe ratio' in header:
result['sharpe_ratio'] = self._extract_number(value)
elif 'custom' in header:
result['custom'] = self._extract_number(value)
elif 'equity dd' in header or 'drawdown' in header:
result['max_drawdown'] = self._extract_number(value)
elif 'trade' in header:
result['total_trades'] = int(self._extract_number(value))
if result:
results.append(result)
except Exception as e:
import traceback
print('Error parsing MT5 XML ' + xml_path + ': ' + str(e))
traceback.print_exc()
return results
def copy_xml_results_to_reports(self, mt5_terminal_dir: str, ea_name: str, reports_dir: str) -> str:
xml_patterns = [
f'{ea_name}_optimization.xml',
f'{ea_name}_optimization[1].xml',
f'{ea_name}_optimization[2].xml',
]
os.makedirs(reports_dir, exist_ok=True)
latest_xml = None
latest_time = 0
for pattern in xml_patterns:
xml_path = os.path.join(mt5_terminal_dir, pattern)
if os.path.exists(xml_path):
mtime = os.path.getmtime(xml_path)
if mtime > latest_time:
latest_time = mtime
latest_xml = xml_path
if latest_xml:
dest_path = os.path.join(reports_dir, os.path.basename(latest_xml))
shutil.copy2(latest_xml, dest_path)
return dest_path
return None
def parse_all_reports(self, pattern: str = '*.htm*') -> List[Dict]:
results = []
search_path = os.path.join(self.reports_dir, pattern)
report_files = glob.glob(search_path)
for report_file in report_files:
parsed = self.parse_mt5_html_report(report_file)
if parsed:
parsed['report_file'] = os.path.basename(report_file)
results.append(parsed)
return results
def merge_parameters_and_results(self, param_mapping: List[Dict], results: List[Dict]) -> List[Dict]:
merged = []
result_map = {r.get('report_file', ''): r for r in results}
for param_set in param_mapping:
report_name = param_set.get('report_name', '')
if report_name in result_map:
combined = {**param_set, **result_map[report_name]}
merged.append(combined)
return merged
def find_optimal_params(self, results: List[Dict],
criterion: str = 'profit_factor',
min_trades: int = 10,
max_drawdown_pct: float = 50.0) -> List[Dict]:
filtered = []
for r in results:
trades = r.get('total_trades', 0)
dd = r.get('max_drawdown', 0)
pf = r.get('profit_factor', 0)
if trades >= min_trades and dd <= max_drawdown_pct and pf > 0:
filtered.append(r)
filtered.sort(key=lambda x: x.get(criterion, 0), reverse=True)
return filtered
def generate_report(self, results: List[Dict], output_path: str = 'results/optimization_report.txt'):
if not results:
return
os.makedirs(os.path.dirname(output_path), exist_ok=True)
lines = [
'=' * 60,
'MT5 EA Optimization Report',
'=' * 60,
'',
'Total Results: ' + str(len(results)),
'',
]
if results:
best = results[0]
lines.extend([
'Best Configuration:',
' Profit Factor: ' + str(best.get('profit_factor', 0)),
' Net Profit: ' + str(best.get('net_profit', 0)),
' Total Trades: ' + str(best.get('total_trades', 0)),
' Max Drawdown: ' + str(best.get('max_drawdown', 0)),
' Win Rate: ' + str(best.get('win_rate', 0)) + '%',
'',
])
lines.append('Top 10 Configurations:')
lines.append('-' * 60)
for i, r in enumerate(results[:10], 1):
lines.append(
str(i) + '. PF=' + str(r.get('profit_factor', 0)) +
' Net=' + str(r.get('net_profit', 0)) +
' Trades=' + str(r.get('total_trades', 0))
)
with open(output_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(lines))
return output_path
def analyze_walk_forward(self, mt5_terminal_dir: str, ea_name: str, reports_dir: str) -> Dict:
result = {
'ea_name': ea_name,
'in_sample': {},
'out_of_sample': {},
'overfitting_score': 0,
'verdict': 'Unknown'
}
xml_path = self.copy_xml_results_to_reports(mt5_terminal_dir, ea_name, reports_dir)
if xml_path:
is_results = self.parse_mt5_xml_optimization(xml_path)
if is_results:
best_is = sorted(is_results, key=lambda x: x.get('profit_factor', 0), reverse=True)[0]
result['in_sample'] = best_is
forward_xml = os.path.join(mt5_terminal_dir, f'{ea_name}_optimization.forward.xml')
if not os.path.exists(forward_xml):
forward_xml = os.path.join(mt5_terminal_dir, f'{ea_name}_optimization[1].forward.xml')
if os.path.exists(forward_xml):
oos_results = self.parse_mt5_xml_optimization(forward_xml)
if oos_results:
best_oos = sorted(oos_results, key=lambda x: x.get('profit_factor', 0), reverse=True)[0]
result['out_of_sample'] = best_oos
is_pf = result.get('in_sample', {}).get('profit_factor', 0)
oos_pf = result.get('out_of_sample', {}).get('profit_factor', 0)
if is_pf > 0 and oos_pf > 0:
decay = (is_pf - oos_pf) / is_pf
result['pf_decay'] = round(decay * 100, 1)
if oos_pf >= is_pf * 0.7 and oos_pf > 1.0:
result['verdict'] = 'Robust (\u7a33\u5065)'
elif oos_pf > 1.0:
result['verdict'] = 'Mild Overfit (\u8f7b\u5ea6\u8fc7\u62df\u5408)'
else:
result['verdict'] = 'Overfit (\u4e25\u91cd\u8fc7\u62df\u5408)'
elif oos_pf > 0:
result['pf_decay'] = 0
result['verdict'] = 'Valid (\u6709\u6548\u4f46IS\u65e0\u7ed3\u679c)'
is_dd = result.get('in_sample', {}).get('max_drawdown', 0)
oos_dd = result.get('out_of_sample', {}).get('max_drawdown', 0)
if is_dd > 0 and oos_dd > 0:
result['dd_increase'] = round((oos_dd / is_dd - 1) * 100, 1) if is_dd > 0 else 0
return result
+243
View File
@@ -0,0 +1,243 @@
import os
import sys
import json
import time
import subprocess
import logging
from datetime import datetime
from typing import List, Dict, Any
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
OPTIMIZER_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, OPTIMIZER_DIR)
from ea_config_parser import load_all_ea_configs, EAConfig
from smart_search import SmartSearch
from parameter_constraint import create_constraint_engine
from analyze_results import ResultAnalyzer
class BatchAutoOptimizer:
def __init__(self, base_dir: str = "optimizer", mt5_path: str = None):
self.base_dir = base_dir
self.mt5_path = mt5_path
self.configs_dir = os.path.join(base_dir, "configs")
self.set_files_dir = os.path.join(base_dir, "set_files")
self.reports_dir = os.path.join(base_dir, "reports")
self.results_dir = os.path.join(base_dir, "results")
self.scripts_dir = os.path.join(base_dir, "scripts")
for d in [self.configs_dir, self.set_files_dir, self.reports_dir,
self.results_dir, self.scripts_dir]:
os.makedirs(d, exist_ok=True)
def generate_set_files(self, ea_config: EAConfig, max_samples: int = 1000) -> List[Dict]:
logger.info(f"Generating SET files for {ea_config.ea_name}")
logger.info(f" Combinations: {ea_config.get_total_combinations()}")
searcher = SmartSearch(ea_config)
combinations = searcher.generate_combinations(max_samples)
logger.info(f" Generated {len(combinations)} samples")
engine = create_constraint_engine(ea_config)
valid_combos = engine.filter_valid_combinations(combinations)
logger.info(f" Valid combinations: {len(valid_combos)}")
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
os.makedirs(ea_set_dir, exist_ok=True)
param_mapping = []
for i, combo in enumerate(valid_combos, 1):
set_id = str(i).zfill(8)
set_filename = f"params_{set_id}.set"
set_path = os.path.join(ea_set_dir, set_filename)
self._write_set_file(set_path, combo)
param_mapping.append({
'set_id': set_id,
'set_file': set_filename,
'set_path': set_path,
'report_name': f"{ea_config.ea_name}_{set_id}.htm",
'params': combo
})
mapping_path = os.path.join(self.results_dir, f"{ea_config.ea_name}_param_mapping.json")
with open(mapping_path, 'w', encoding='utf-8') as f:
json.dump(param_mapping, f, ensure_ascii=False, indent=2)
logger.info(f" SET files saved to {ea_set_dir}")
return param_mapping
def _write_set_file(self, set_path: str, params: Dict):
lines = ["; MT5 EA Parameters", f"; Generated: {datetime.now()}", ""]
for name, value in params.items():
if isinstance(value, bool):
lines.append(f"{name} <true> <{'true' if value else 'false'}>")
else:
lines.append(f"{name} <{value}> <{value}>")
with open(set_path, 'w', encoding='utf-8') as f:
f.write("\n".join(lines))
def generate_ini(self, ea_config: EAConfig) -> str:
test_config = ea_config.test_config
ini_path = os.path.join(self.configs_dir, f"{ea_config.ea_name}.ini")
date_from = test_config.get("from_date", "2025.01.01")
date_to = test_config.get("to_date", "2025.12.31")
lines = [
"; MT5 Strategy Tester Configuration",
f"; Generated: {datetime.now()}",
"",
"[Tester]",
f"Expert={ea_config.ea_path}",
"ExpertParameters=",
f"Symbol={test_config.get('symbol', 'EURUSD')}",
f"Period={test_config.get('period', 'H1')}",
f"Model={test_config.get('model', 1)}",
f"FromDate={date_from}",
f"ToDate={date_to}",
f"Deposit={test_config.get('deposit', 10000)}",
f"Leverage={test_config.get('leverage', '1:100')}",
"ReplaceReport=true",
"ShutdownTerminal=true",
]
with open(ini_path, 'w', encoding='utf-8') as f:
f.write("\n".join(lines))
logger.info(f"INI saved: {ini_path}")
return ini_path
def generate_powershell_script(self, ea_config: EAConfig, param_mapping: List[Dict]) -> str:
ps_path = os.path.join(self.scripts_dir, f"run_{ea_config.ea_name}.ps1")
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
ea_report_dir = os.path.join(self.reports_dir, ea_config.ea_name)
os.makedirs(ea_report_dir, exist_ok=True)
ini_path = os.path.join(self.configs_dir, f"{ea_config.ea_name}.ini")
lines = [
f"# MT5 Batch Optimization for {ea_config.ea_name}",
f"# Generated: {datetime.now()}",
"",
f'$MT5Path = "{self.mt5_path or "terminal64.exe"}"',
f'$IniFile = "{ini_path}"',
f'$SetDir = "{ea_set_dir}"',
"",
f"Write-Host 'Starting batch optimization for {ea_config.ea_name}'",
"",
"$SetFiles = Get-ChildItem -Path $SetDir -Filter 'params_*.set'",
"$Total = $SetFiles.Count",
"$Current = 0",
"",
"foreach ($SetFile in $SetFiles) {",
" $Current++",
' Write-Host "[$Current/$Total] $($SetFile.Name)"',
"",
" $IniContent = Get-Content $IniFile",
' $IniContent = $IniContent -replace "ExpertParameters=.*", "ExpertParameters=$($SetFile.FullName)"',
' $TempIni = Join-Path $env:TEMP "temp_$([guid]::NewGuid().ToString().Substring(0,8)).ini"',
" $IniContent | Set-Content $TempIni -Encoding UTF8",
"",
" Start-Process -FilePath $MT5Path -ArgumentList '/portable',\"/config:$TempIni\" -Wait",
" Start-Sleep -Seconds 3",
"}",
"",
'Write-Host "Batch optimization completed!"',
]
with open(ps_path, 'w', encoding='utf-8') as f:
f.write("\n".join(lines))
logger.info(f"PowerShell script: {ps_path}")
return ps_path
def run_full_auto(self, ea_names: List[str] = None, max_samples: int = 1000):
logger.info("=" * 60)
logger.info("MT5 Batch Auto Optimizer")
logger.info("=" * 60)
configs = load_all_ea_configs(self.configs_dir)
if not configs:
logger.error(f"No configs found in {self.configs_dir}")
return
if ea_names:
configs = {k: v for k, v in configs.items() if k in ea_names}
for ea_name, ea_config in configs.items():
logger.info(f"\nProcessing: {ea_name}")
param_mapping = self.generate_set_files(ea_config, max_samples)
self.generate_ini(ea_config)
self.generate_powershell_script(ea_config, param_mapping)
logger.info(f" SET files: {len(param_mapping)}")
logger.info(f" Next: Run the PS1 script in MT5 terminal")
def analyze_all(self, ea_names: List[str] = None) -> List[Dict]:
logger.info("Analyzing results...")
all_results = []
search_dirs = [self.reports_dir] if not ea_names else [os.path.join(self.reports_dir, name) for name in ea_names]
for search_dir in search_dirs:
if not os.path.exists(search_dir):
continue
for ea_name in os.listdir(search_dir):
ea_report_dir = os.path.join(search_dir, ea_name)
if not os.path.isdir(ea_report_dir):
continue
analyzer = ResultAnalyzer(ea_report_dir)
results = analyzer.parse_all_reports()
for r in results:
r['ea_name'] = ea_name
all_results.extend(results)
logger.info(f" {ea_name}: {len(results)} reports")
if not all_results:
logger.warning("No results found")
return []
all_results.sort(key=lambda x: x.get('profit_factor', 0), reverse=True)
output_path = os.path.join(self.results_dir, "optimization_results.json")
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(all_results, f, indent=2, ensure_ascii=False)
logger.info(f"Results: {len(all_results)} total")
logger.info(f"Best PF: {all_results[0].get('profit_factor', 0) if all_results else 'N/A'}")
return all_results
def main():
import argparse
parser = argparse.ArgumentParser(description="MT5 Batch Auto Optimizer")
parser.add_argument("--base-dir", "-d", default="optimizer", help="Base directory")
parser.add_argument("--ea", "-e", nargs="+", help="EA names")
parser.add_argument("--max-samples", "-m", type=int, default=1000, help="Max samples per EA")
parser.add_argument("--mt5-path", "-p", help="MT5 terminal path")
parser.add_argument("--analyze", "-a", action="store_true", help="Analyze only")
args = parser.parse_args()
optimizer = BatchAutoOptimizer(args.base_dir, args.mt5_path)
if args.analyze:
optimizer.analyze_all(args.ea)
else:
optimizer.run_full_auto(args.ea, args.max_samples)
if __name__ == "__main__":
main()
+91
View File
@@ -0,0 +1,91 @@
{
"ea_name": "Grid Pro",
"ea_path": "Experts\\\\Grid\\\\GridPro.ex5",
"description": "Multi-period grid EA with genetic optimization",
"search_strategy": "genetic",
"optimization_criterion": "profit_factor",
"parameters": {
"GridLevels": {
"type": "int",
"default": 5,
"min": 3,
"max": 20,
"step": 1,
"description": "Grid levels"
},
"GridSpacing": {
"type": "int",
"default": 50,
"min": 10,
"max": 200,
"step": 10,
"description": "Grid spacing in points"
},
"GridSpacingMult": {
"type": "double",
"default": 1.5,
"min": 1.0,
"max": 3.0,
"step": 0.1,
"precision": 1,
"description": "Spacing multiplier"
},
"BaseLotsize": {
"type": "double",
"default": 0.1,
"min": 0.01,
"max": 1.0,
"step": 0.01,
"precision": 2,
"description": "Base lot size"
},
"MaxOpenLots": {
"type": "double",
"default": 5.0,
"min": 0.1,
"max": 20.0,
"step": 0.1,
"precision": 1,
"description": "Max open lots"
},
"MaxDrawdown": {
"type": "int",
"default": 30,
"min": 10,
"max": 100,
"step": 5,
"description": "Max drawdown percentage"
},
"TakeProfitGrid": {
"type": "int",
"default": 20,
"min": 5,
"max": 100,
"step": 5,
"description": "Take profit grid"
},
"TradingMode": {
"type": "enum",
"default": "BOTH",
"options": [
"BUY",
"SELL",
"BOTH"
],
"description": "Trading mode"
}
},
"test_config": {
"symbol": "EURUSD",
"period": "H1",
"from_date": "2024.01.01",
"to_date": "2025.12.31",
"model": 1,
"deposit": 5000,
"leverage": "1:100"
},
"walk_forward": {
"enabled": true,
"train_test_ratio": 0.75
}
}
+68
View File
@@ -0,0 +1,68 @@
{
"ea_name": "MA Cross Scalper",
"ea_path": "Experts\\\\MA_Cross.ex5",
"description": "Dual MA crossover scalping strategy",
"search_strategy": "grid",
"optimization_criterion": "profit_factor",
"parameters": {
"FastMA_Period": {
"type": "int",
"default": 14,
"min": 5,
"max": 30,
"step": 1,
"description": "Fast MA period"
},
"SlowMA_Period": {
"type": "int",
"default": 50,
"min": 20,
"max": 100,
"step": 5,
"description": "Slow MA period",
"condition": ">FastMA_Period"
},
"StopLoss": {
"type": "int",
"default": 50,
"min": 10,
"max": 200,
"step": 10,
"description": "Stop loss in points"
},
"TakeProfit": {
"type": "int",
"default": 100,
"min": 30,
"max": 300,
"step": 10,
"description": "Take profit in points"
},
"RiskPercent": {
"type": "double",
"default": 1.0,
"min": 0.5,
"max": 3.0,
"step": 0.5,
"precision": 1,
"description": "Risk percentage"
},
"EnableMM": {
"type": "bool",
"default": true,
"description": "Enable money management"
}
},
"test_config": {
"symbol": "EURUSD",
"period": "H1",
"from_date": "2025.01.01",
"to_date": "2025.12.31",
"model": 1,
"deposit": 10000,
"leverage": "1:100"
},
"walk_forward": {
"enabled": false
}
}
+87
View File
@@ -0,0 +1,87 @@
import os
import glob
import re
from typing import Dict, List, Any, Optional
class EAAutoScanner:
def __init__(self, mt5_data_dir: str = None):
self.mt5_data_dir = mt5_data_dir
self.experts_dir = os.path.join(mt5_data_dir, "MQL5", "Experts") if mt5_data_dir else None
def scan_experts(self) -> List[Dict]:
experts = []
if not self.experts_dir or not os.path.exists(self.experts_dir):
return experts
for root, dirs, files in os.walk(self.experts_dir):
for f in files:
if f.endswith('.ex5'):
full_path = os.path.join(root, f)
rel_path = os.path.relpath(full_path, self.experts_dir)
ea_name = os.path.splitext(f)[0]
experts.append({'name': ea_name, 'filename': f, 'path': rel_path, 'full_path': full_path})
return experts
def generate_json_config(self, ea_info: Dict, params: Dict = None) -> Dict:
ea_name = ea_info["name"]
config = {
"ea_name": ea_name,
"ea_path": ea_info["path"],
"description": "Auto-scanned EA",
"search_strategy": "auto",
"optimization_criterion": "profit_factor",
"parameters": {},
"test_config": {
"symbol": "EURUSD", "period": "H1",
"from_date": "2025.01.01", "to_date": "2025.12.31",
"model": 1, "deposit": 10000, "leverage": "1:100"
},
"walk_forward": {"enabled": False}
}
if params:
for pname, pinfo in params.items():
config["parameters"][pname] = {
"type": pinfo.get("type", "int"),
"default": pinfo.get("default", 0),
"min": pinfo.get("min", 0),
"max": pinfo.get("max", 100),
"step": pinfo.get("step", 1),
"description": "Auto-scanned"
}
return config
def auto_scan_all(self) -> List[Dict]:
configs = []
experts = self.scan_experts()
for expert in experts:
config = self.generate_json_config(expert)
configs.append(config)
return configs
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--data-dir", "-d")
parser.add_argument("--output", "-o", default="configs")
parser.add_argument("--list", "-l", action="store_true")
args = parser.parse_args()
scanner = EAAutoScanner(args.data_dir)
experts = scanner.scan_experts()
if args.list:
print(f"Found {len(experts)} EAs:")
for e in experts:
print(f" - {e['name']}")
configs = scanner.auto_scan_all()
print(f"Generated {len(configs)} configs")
if args.output and configs:
import json
import os
os.makedirs(args.output, exist_ok=True)
for cfg in configs:
fname = cfg["ea_name"].replace(" ", "_") + ".json"
with open(os.path.join(args.output, fname), "w", encoding="utf-8") as f:
json.dump(cfg, f, indent=4, ensure_ascii=False)
print(f"Saved: {fname}")
+328
View File
@@ -0,0 +1,328 @@
import os
import sys
import json
import time
import shutil
import subprocess
from datetime import datetime
from typing import List, Dict, Any, Optional
sys.path.insert(0, os.path.dirname(__file__))
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from ea_config_parser import load_ea_config, load_all_ea_configs, EAConfig
from parameter_constraint import create_constraint_engine
from smart_search import SmartSearch
from analyze_results import ResultAnalyzer
from mt5_paths import auto_detect_data_dir
class EABatchOptimizer:
def __init__(self, base_dir: str = 'optimizer'):
self.base_dir = base_dir
self.configs_dir = os.path.join(base_dir, 'configs')
self.set_files_dir = os.path.join(base_dir, 'set_files')
self.reports_dir = os.path.join(base_dir, 'reports')
self.results_dir = os.path.join(base_dir, 'results')
self.scripts_dir = os.path.join(base_dir, 'scripts')
self.logs_dir = os.path.join(base_dir, 'logs')
for d in [self.configs_dir, self.set_files_dir, self.reports_dir,
self.results_dir, self.scripts_dir, self.logs_dir]:
os.makedirs(d, exist_ok=True)
def generate_set_files(self, ea_config: EAConfig, max_samples: int = 2000) -> List[Dict]:
print('Generating SET files for ' + ea_config.ea_name)
print(' Total combinations: ' + str(ea_config.get_total_combinations()))
print(' Strategy: ' + SmartSearch(ea_config).select_strategy())
searcher = SmartSearch(ea_config)
combinations = searcher.generate_combinations(max_samples)
engine = create_constraint_engine(ea_config)
valid_combinations = engine.filter_valid_combinations(combinations)
print(' Valid combinations: ' + str(len(valid_combinations)))
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
os.makedirs(ea_set_dir, exist_ok=True)
param_mapping = []
for i, combo in enumerate(valid_combinations, 1):
set_id = str(i).zfill(8)
set_filename = 'params_' + set_id + '.set'
set_path = os.path.join(ea_set_dir, set_filename)
self._write_set_file(set_path, combo)
mapping = {
'set_id': set_id,
'set_file': set_filename,
'report_name': ea_config.ea_name + '_' + set_id + '.htm',
'params': combo
}
param_mapping.append(mapping)
mapping_path = os.path.join(self.results_dir, ea_config.ea_name + '_param_mapping.json')
with open(mapping_path, 'w', encoding='utf-8') as f:
json.dump(param_mapping, f, ensure_ascii=False, indent=2)
print(' SET files generated: ' + str(len(valid_combinations)))
return param_mapping
def _write_set_file(self, set_path: str, params: Dict):
lines = ['; MT5 EA Parameters SET File', '; Generated: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '']
for name, value in params.items():
if isinstance(value, bool):
lines.append(name + ' <true> <' + ('true' if value else 'false') + '>')
elif isinstance(value, float):
lines.append(name + ' <' + str(value) + '> <' + str(value) + '>')
else:
lines.append(name + ' <' + str(value) + '> <' + str(value) + '>')
with open(set_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(lines))
def _save_param_mapping(self, mapping: List[Dict], output_path: str):
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(output_path, 'w', encoding='utf-8') as f:
f.write('set_id,set_file,report_name\n')
for m in mapping:
f.write(m['set_id'] + ',' + m['set_file'] + ',' + m['report_name'] + '\n')
def generate_ini_for_ea(self, ea_config: EAConfig, param_mapping: List[Dict] = None):
test_config = ea_config.test_config
ini_filename = ea_config.ea_name + '.ini'
ini_path = os.path.join(self.configs_dir, ini_filename)
set_filename = ea_config.ea_name + '_optimization.set'
data_dir = auto_detect_data_dir() or r'C:\Users\Administrator\AppData\Roaming\MetaQuotes\Terminal\010E047102812FC0C18890992854220E'
mt5_tester_dir = os.path.join(data_dir, 'MQL5', 'Profiles', 'Tester')
set_path = os.path.join(mt5_tester_dir, set_filename)
date_from = test_config.get('from_date', '2025.01.01')
date_to = test_config.get('to_date', '2025.12.31')
criterion_map = {
'profit_factor': 1,
'net_profit': 0,
'sharpe_ratio': 5,
'expected_payoff': 2,
'drawdown': 3,
'recovery_factor': 4,
}
opt_criterion = test_config.get('optimization_criterion', 1)
set_lines = [
'; saved automatically',
'; this file contains last used input parameters for testing/optimizing ' + ea_config.ea_name + ' expert advisor',
'',
]
if ea_config.parameters:
for pname, param in ea_config.parameters.items():
if param.param_type == 'bool':
if not param.optimize:
set_lines.append(pname + '=false||false||false||false||N')
else:
set_lines.append(pname + '=false||false||false||false||Y')
elif param.param_type == 'int':
start = int(param.min_value) if param.min_value is not None else 0
step = int(param.step) if param.step is not None else 1
stop = int(param.max_value) if param.max_value is not None else 100
cur = int(param.default) if param.default is not None else start
if not param.optimize or start == stop or step == 0:
set_lines.append(pname + '=' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||N')
else:
set_lines.append(pname + '=' + str(cur) + '||' + str(start) + '||' + str(step) + '||' + str(stop) + '||Y')
elif param.param_type == 'double':
start = float(param.min_value) if param.min_value is not None else 0.0
step = float(param.step) if param.step is not None else 0.01
stop = float(param.max_value) if param.max_value is not None else 1.0
cur = float(param.default) if param.default is not None else start
if not param.optimize or start == stop or step == 0:
set_lines.append(pname + '=' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||N')
else:
set_lines.append(pname + '=' + str(cur) + '||' + str(start) + '||' + str(step) + '||' + str(stop) + '||Y')
with open(set_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(set_lines))
ini_lines = [
'; EA backtest',
'[Tester]',
'Expert=' + ea_config.ea_path,
'ExpertParameters=' + set_filename,
'Symbol=' + test_config.get('symbol', 'EURUSD'),
'Period=' + test_config.get('period', 'H1'),
'Model=' + str(test_config.get('model', 1)),
'FromDate=' + date_from,
'ToDate=' + date_to,
]
wf = ea_config.walk_forward
if wf.get('enabled', False):
fm = wf.get('forward_mode', 2)
ini_lines.append('ForwardMode=' + str(fm))
else:
ini_lines.append('ForwardMode=0')
ini_lines.extend([
'Deposit=' + str(test_config.get('deposit', 10000)),
'Currency=' + str(test_config.get('currency', 'USD')),
'Leverage=' + str(test_config.get('leverage', '1:100')),
'ExecutionMode=' + str(test_config.get('execution_delay', 0)),
'Optimization=' + str(test_config.get('optimization_mode', 2)),
'OptimizationCriterion=' + str(opt_criterion),
'Report=' + ea_config.ea_name + '_optimization',
'ReplaceReport=true',
'ShutdownTerminal=true',
'Visual=0',
'',
])
with open(ini_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(ini_lines))
print('INI saved: ' + ini_path)
print('SET saved: ' + set_path)
return ini_path
def build_powershell_script(self, ea_config: EAConfig, param_mapping: List[Dict], mt5_path: str):
ps_filename = 'run_' + ea_config.ea_name + '.ps1'
ps_path = os.path.join(self.scripts_dir, ps_filename)
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
ea_report_dir = os.path.join(self.reports_dir, ea_config.ea_name)
os.makedirs(ea_report_dir, exist_ok=True)
ini_path = os.path.join(self.configs_dir, ea_config.ea_name + '.ini')
lines = [
'# MT5 Batch Backtest Script',
'# Generated: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
'',
f'$MT5Path = "{mt5_path}"',
f'$IniFile = "{ini_path}"',
f'$SetDir = "{ea_set_dir}"',
'',
f"Write-Host 'Starting batch optimization for {ea_config.ea_name}'",
'',
'$SetFiles = Get-ChildItem -Path $SetDir -Filter "params_*.set"',
'$Total = $SetFiles.Count',
'$Current = 0',
'',
'foreach ($SetFile in $SetFiles) {',
' $Current++',
' Write-Host "[$Current/$Total] Processing: $($SetFile.Name)"',
'',
' $IniContent = Get-Content $IniFile',
' $IniContent = $IniContent -replace "ExpertParameters=.*", "ExpertParameters=$($SetFile.FullName)"',
' $TempIni = Join-Path $env:TEMP "temp_$([guid]::NewGuid().ToString().Substring(0,8)).ini"',
' $IniContent | Set-Content $TempIni -Encoding UTF8',
'',
' Start-Process -FilePath $MT5Path -ArgumentList "/config:$TempIni" -Wait',
'',
' Start-Sleep -Seconds 3',
'}',
'',
f"Write-Host 'Batch optimization completed for {ea_config.ea_name}!'",
]
with open(ps_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(lines))
print('PowerShell script: ' + ps_path)
return ps_path
def run_optimization_for_ea(self, ea_config: EAConfig, mt5_path: str = None,
max_samples: int = 2000):
print('\n' + '=' * 60)
print('Optimization for: ' + ea_config.ea_name)
print('=' * 60)
param_mapping = self.generate_set_files(ea_config, max_samples)
self.generate_ini_for_ea(ea_config, param_mapping)
if mt5_path:
ps_path = self.build_powershell_script(ea_config, param_mapping, mt5_path)
print('PowerShell script: ' + ps_path)
return param_mapping
def run_full_optimization(self, ea_names: List[str] = None,
mt5_path: str = None,
max_samples_per_ea: int = 2000):
configs = load_all_ea_configs(self.configs_dir)
if ea_names:
configs = {k: v for k, v in configs.items() if k in ea_names}
all_mapping = {}
for ea_name, ea_config in configs.items():
mapping = self.run_optimization_for_ea(ea_config, mt5_path, max_samples_per_ea)
all_mapping[ea_name] = mapping
return all_mapping
def analyze_results(self, ea_names: List[str] = None) -> List[Dict]:
all_results = []
if ea_names:
for ea_name in ea_names:
report_dir = os.path.join(self.reports_dir, ea_name)
if os.path.exists(report_dir):
analyzer = ResultAnalyzer(report_dir)
results = analyzer.parse_all_reports()
for r in results:
r['ea_name'] = ea_name
all_results.extend(results)
else:
for ea_name in os.listdir(self.reports_dir):
report_dir = os.path.join(self.reports_dir, ea_name)
if os.path.isdir(report_dir):
analyzer = ResultAnalyzer(report_dir)
results = analyzer.parse_all_reports()
for r in results:
r['ea_name'] = ea_name
all_results.extend(results)
return all_results
def find_optimal_params(self, results: List[Dict], criterion: str = 'profit_factor',
min_trades: int = 10) -> List[Dict]:
filtered = [r for r in results if r.get('total_trades', 0) >= min_trades
and r.get('profit_factor', 0) > 0]
filtered.sort(key=lambda x: x.get(criterion, 0), reverse=True)
return filtered
def main():
import argparse
parser = argparse.ArgumentParser(description='MT5 EA Batch Optimizer')
parser.add_argument('--base-dir', '-d', default='optimizer', help='Base directory')
parser.add_argument('--ea', '-e', nargs='+', help='EA names to optimize')
parser.add_argument('--max-samples', '-m', type=int, default=2000, help='Max samples')
parser.add_argument('--mt5-path', '-p', help='MT5 terminal path')
parser.add_argument('--analyze-only', '-a', action='store_true', help='Only analyze results')
args = parser.parse_args()
optimizer = EABatchOptimizer(args.base_dir)
if args.analyze_only:
results = optimizer.analyze_results(args.ea)
if results:
analyzer = ResultAnalyzer()
best = optimizer.find_optimal_params(results)
analyzer.generate_report(best)
print('Results: ' + str(len(results)))
print('Best: PF=' + str(best[0].get('profit_factor', 0)) if best else 'No results')
else:
optimizer.run_full_optimization(args.ea, args.mt5_path, args.max_samples)
print('\nOptimization files generated. Run MT5 tests, then use --analyze-only')
if __name__ == '__main__':
main()
+135
View File
@@ -0,0 +1,135 @@
import os
import json
from dataclasses import dataclass, field
from typing import Dict, List, Any
@dataclass
class ParameterDef:
name: str
param_type: str
default: Any = None
min_value: Any = None
max_value: Any = None
step: Any = None
precision: int = 0
optimize: bool = True
enum_options: List[Any] = field(default_factory=list)
description: str = ''
condition: str = ''
def expand_values(self) -> List[Any]:
if self.param_type == 'bool':
return [True, False]
elif self.param_type == 'enum':
return self.enum_options
elif self.param_type in ('int', 'double'):
if self.min_value is None or self.max_value is None:
return [self.default] if self.default is not None else []
values = []
current = self.min_value
while current <= self.max_value:
values.append(round(current, self.precision) if self.precision > 0 else int(current))
current += self.step
return values
return [self.default]
def is_valid_value(self, value: Any) -> bool:
if self.param_type == 'bool':
return isinstance(value, bool)
elif self.param_type == 'enum':
return value in self.enum_options
elif self.param_type in ('int', 'double'):
if self.min_value is not None and value < self.min_value:
return False
if self.max_value is not None and value > self.max_value:
return False
return True
return True
@dataclass
class EAConfig:
ea_name: str
ea_path: str
parameters: Dict[str, ParameterDef] = field(default_factory=dict)
search_strategy: str = 'auto'
optimization_criterion: str = 'profit_factor'
test_config: Dict[str, Any] = field(default_factory=dict)
walk_forward: Dict[str, Any] = field(default_factory=dict)
description: str = ''
def get_total_combinations(self) -> int:
total = 1
for param in self.parameters.values():
values = param.expand_values()
total *= len(values) if values else 1
return total
def estimate_search_time(self, tests_per_minute: float = 10) -> str:
total = self.get_total_combinations()
minutes = total / tests_per_minute
if minutes < 60:
return str(round(minutes, 1)) + ' minutes'
elif minutes < 1440:
return str(round(minutes/60, 1)) + ' hours'
else:
return str(round(minutes/1440, 1)) + ' days'
def is_complex(self) -> bool:
return len(self.parameters) > 6 or self.get_total_combinations() > 1_000_000
def load_ea_config(config_path: str) -> EAConfig:
with open(config_path, 'r', encoding='utf-8') as f:
config_data = json.load(f)
ea_name = config_data.get('ea_name', 'Unknown')
ea_path = config_data.get('ea_path', '')
description = config_data.get('description', '')
search_strategy = config_data.get('search_strategy', 'auto')
criterion = config_data.get('optimization_criterion', 'profit_factor')
test_config = config_data.get('test_config', {})
walk_forward = config_data.get('walk_forward', {})
parameters = {}
for param_name, param_data in config_data.get('parameters', {}).items():
param_type = param_data.get('type', 'int')
if param_type == 'enum':
param = ParameterDef(
name=param_name,
param_type='enum',
default=param_data.get('default'),
enum_options=param_data.get('options', []),
description=param_data.get('description', ''),
condition=param_data.get('condition', '')
)
else:
param = ParameterDef(
name=param_name,
param_type=param_type,
default=param_data.get('default'),
min_value=param_data.get('min'),
max_value=param_data.get('max'),
step=param_data.get('step', 1),
precision=param_data.get('precision', 0),
optimize=param_data.get('optimize', True),
description=param_data.get('description', ''),
condition=param_data.get('condition', '')
)
parameters[param_name] = param
return EAConfig(
ea_name=ea_name, ea_path=ea_path, parameters=parameters,
search_strategy=search_strategy, optimization_criterion=criterion,
test_config=test_config, walk_forward=walk_forward, description=description
)
def load_all_ea_configs(configs_dir: str) -> Dict[str, EAConfig]:
configs = {}
if not os.path.exists(configs_dir):
return configs
for filename in os.listdir(configs_dir):
if filename.endswith('.json'):
config_path = os.path.join(configs_dir, filename)
try:
config = load_ea_config(config_path)
configs[config.ea_name] = config
except Exception as e:
print('Error loading ' + config_path + ': ' + str(e))
return configs
+86
View File
@@ -0,0 +1,86 @@
import re
from typing import List, Dict, Any, Callable
class ParameterConstraintEngine:
def __init__(self):
self.constraints = []
self.condition_functions = []
def add_constraint(self, name: str, condition: str, error_message: str = ''):
self.constraints.append({
'name': name,
'condition': condition,
'error_message': error_message or 'Constraint violated: ' + name
})
def add_condition_from_config(self, param_name: str, condition_str: str):
if not condition_str:
return
condition_str = condition_str.strip()
self.constraints.append({
'name': param_name,
'condition': condition_str,
'error_message': 'Parameter constraint violated: ' + param_name
})
def _parse_condition(self, condition: str) -> Callable[[Dict], bool]:
condition = condition.strip()
patterns = [
(r'^(\w+)\s*>\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) > p.get(m.group(2), 0)),
(r'^(\w+)\s*<\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) < p.get(m.group(2), 0)),
(r'^(\w+)\s*>=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) >= p.get(m.group(2), 0)),
(r'^(\w+)\s*<=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) <= p.get(m.group(2), 0)),
(r'^(\w+)\s*==\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) == p.get(m.group(2), 0)),
(r'^(\w+)\s*!=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) != p.get(m.group(2), 0)),
(r'^(\w+)\s*>\s*(\d+)$', lambda m, p: p.get(m.group(1), 0) > int(m.group(2))),
(r'^(\w+)\s*<\s*(\d+)$', lambda m, p: p.get(m.group(1), 0) < int(m.group(2))),
]
for pattern, func in patterns:
match = re.match(pattern, condition)
if match:
return lambda params, m=match, f=func: f(m, params)
return lambda params: True
def _is_valid_combination(self, params: Dict) -> bool:
for constraint in self.constraints:
condition = constraint['condition']
if not condition:
continue
parse_func = self._parse_condition(condition)
if not parse_func(params):
return False
return True
def filter_valid_combinations(self, combinations: List[Dict]) -> List[Dict]:
valid = []
invalid_count = 0
for combo in combinations:
if self._is_valid_combination(combo):
valid.append(combo)
else:
invalid_count += 1
if invalid_count > 0:
print('Filtered ' + str(invalid_count) + ' invalid combinations')
return valid
def get_constraint_count(self) -> int:
return len(self.constraints)
def create_constraint_engine(ea_config) -> ParameterConstraintEngine:
engine = ParameterConstraintEngine()
for param_name, param in ea_config.parameters.items():
if param.condition:
engine.add_condition_from_config(param_name, param.condition)
return engine
+168
View File
@@ -0,0 +1,168 @@
import random
import math
from typing import List, Dict, Any, Tuple
from itertools import product
class SmartSearch:
def __init__(self, ea_config):
self.ea_config = ea_config
self.total_combinations = ea_config.get_total_combinations()
def select_strategy(self) -> str:
if self.ea_config.search_strategy != 'auto':
return self.ea_config.search_strategy
if self.total_combinations <= 10000:
return 'grid'
elif self.total_combinations <= 500000:
return 'latin'
else:
return 'genetic'
def generate_combinations(self, max_samples: int = 2000) -> List[Dict]:
strategy = self.select_strategy()
print('Selected strategy: ' + strategy + ' for ' + str(self.total_combinations) + ' combinations')
if strategy == 'grid':
return self._grid_search()
elif strategy == 'random':
return self._random_search(max_samples)
elif strategy == 'latin':
return self._latin_hypercube(max_samples)
elif strategy == 'genetic':
return self._genetic_search(max_samples)
return []
def _grid_search(self) -> List[Dict]:
param_names = list(self.ea_config.parameters.keys())
param_values = [p.expand_values() for p in self.ea_config.parameters.values()]
combinations = list(product(*param_values))
return [dict(zip(param_names, combo)) for combo in combinations]
def _random_search(self, max_samples: int) -> List[Dict]:
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
samples = []
for _ in range(min(max_samples, self.total_combinations)):
sample = {name: random.choice(values) for name, values in all_values.items()}
if sample not in samples:
samples.append(sample)
return samples
def _latin_hypercube(self, max_samples: int) -> List[Dict]:
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
n_params = len(all_values)
samples = []
for i in range(min(max_samples, self.total_combinations)):
sample = {}
for j, (name, values) in enumerate(all_values.items()):
idx = int((i / max_samples) * len(values)) % len(values)
sample[name] = values[idx]
if sample not in samples:
samples.append(sample)
return samples
def _genetic_search(self, max_samples: int) -> List[Dict]:
pop_size = min(50, max_samples)
n_generations = max_samples // pop_size
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
population = []
for _ in range(pop_size):
individual = {name: random.choice(values) for name, values in all_values.items()}
population.append(individual)
for gen in range(n_generations):
population = self._evolve(population, all_values)
return population[:max_samples]
def _evolve(self, population: List[Dict], all_values: Dict) -> List[Dict]:
crossover_rate = 0.8
mutation_rate = 0.15
offspring = []
for _ in range(len(population)):
parent1, parent2 = random.sample(population, 2)
if random.random() < crossover_rate:
child = self._crossover(parent1, parent2)
else:
child = parent1.copy()
if random.random() < mutation_rate:
child = self._mutate(child, all_values)
offspring.append(child)
return population[:5] + offspring[:len(population)-5]
def _crossover(self, parent1: Dict, parent2: Dict) -> Dict:
child = {}
for key in parent1.keys():
if random.random() < 0.5:
child[key] = parent1[key]
else:
child[key] = parent2[key]
return child
def _mutate(self, individual: Dict, all_values: Dict) -> Dict:
key = random.choice(list(all_values.keys()))
individual[key] = random.choice(all_values[key])
return individual
class GeneticOptimizer:
def __init__(self, ea_config, population_size: int = 50, generations: int = 30):
self.ea_config = ea_config
self.population_size = population_size
self.generations = generations
self.all_values = {name: p.expand_values() for name, p in ea_config.parameters.items()}
def create_individual(self) -> Dict:
return {name: random.choice(values) for name, values in self.all_values.items()}
def evaluate(self, individual: Dict) -> float:
return random.random() * 10
def tournament_select(self, population: List[Dict], k: int = 3) -> Dict:
tournament = random.sample(population, k)
return max(tournament, key=self.evaluate)
def crossover(self, parent1: Dict, parent2: Dict) -> Tuple[Dict, Dict]:
child1, child2 = {}, {}
for key in parent1.keys():
if random.random() < 0.5:
child1[key] = parent1[key]
child2[key] = parent2[key]
else:
child1[key] = parent2[key]
child2[key] = parent1[key]
return child1, child2
def mutate(self, individual: Dict, rate: float = 0.15) -> Dict:
for key in individual.keys():
if random.random() < rate:
individual[key] = random.choice(self.all_values[key])
return individual
def run(self) -> List[Dict]:
population = [self.create_individual() for _ in range(self.population_size)]
best_individuals = []
for gen in range(self.generations):
fitness_scores = [(ind, self.evaluate(ind)) for ind in population]
fitness_scores.sort(key=lambda x: x[1], reverse=True)
elite = [ind for ind, _ in fitness_scores[:5]]
best_individuals.extend(elite)
new_population = elite.copy()
while len(new_population) < self.population_size:
parent1 = self.tournament_select(population)
parent2 = self.tournament_select(population)
child1, child2 = self.crossover(parent1, parent2)
child1 = self.mutate(child1)
child2 = self.mutate(child2)
new_population.extend([child1, child2])
population = new_population[:self.population_size]
return best_individuals
+321
View File
@@ -0,0 +1,321 @@
import os
import json
from datetime import datetime
from typing import List, Dict, Optional
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class ReportGenerator:
def __init__(self, results: List[Dict] = None):
self.results = results or []
def generate_excel(self, output_path: str = "reports/batch_report.xlsx") -> str:
try:
import openpyxl
from openpyxl.styles import Font, Alignment, PatternFill, Border, Side
wb = openpyxl.Workbook()
ws = wb.active
ws.title = "Backtest Results"
headers = [
"EA Name", "Symbol", "Period", "Model",
"Total Trades", "Winning Trades", "Losing Trades",
"Win Rate %", "Gross Profit", "Gross Loss",
"Profit Factor", "Expected Payoff",
"Initial Deposit", "Final Balance"
]
header_fill = PatternFill(start_color="366092", end_color="366092", fill_type="solid")
header_font = Font(bold=True, color="FFFFFF")
thin_border = Border(
left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin')
)
for col, header in enumerate(headers, 1):
cell = ws.cell(row=1, column=col, value=header)
cell.fill = header_fill
cell.font = header_font
cell.alignment = Alignment(horizontal='center', vertical='center')
cell.border = thin_border
for row_idx, result in enumerate(self.results, 2):
metrics = result.get("metrics", {})
test_info = result.get("test_info", {})
row_data = [
test_info.get("expert", ""),
test_info.get("symbol", ""),
test_info.get("period", ""),
test_info.get("model", ""),
metrics.get("total_trades", 0),
metrics.get("winning_trades", 0),
metrics.get("losing_trades", 0),
metrics.get("win_rate", 0),
metrics.get("gross_profit", 0),
metrics.get("gross_loss", 0),
metrics.get("profit_factor", 0),
metrics.get("expected_payoff", 0),
metrics.get("initial_deposit", 0),
metrics.get("final_balance", 0)
]
for col, value in enumerate(row_data, 1):
cell = ws.cell(row=row_idx, column=col, value=value)
cell.border = thin_border
if col >= 5:
cell.number_format = '0.00'
for col in range(1, len(headers) + 1):
ws.column_dimensions[openpyxl.utils.get_column_letter(col)].width = 15
ws.auto_filter.ref = f"A1:N{len(self.results) + 1}"
os.makedirs(os.path.dirname(output_path), exist_ok=True)
wb.save(output_path)
logger.info(f"Excel report saved: {output_path}")
return output_path
except Exception as e:
logger.error(f"Error generating Excel report: {e}")
raise
def generate_csv(self, output_path: str = "reports/batch_report.csv") -> str:
try:
import csv
os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
headers = [
"EA Name", "Symbol", "Period", "Model",
"Total Trades", "Winning Trades", "Losing Trades",
"Win Rate %", "Gross Profit", "Gross Loss",
"Profit Factor", "Expected Payoff",
"Initial Deposit", "Final Balance"
]
with open(output_path, 'w', newline='', encoding='utf-8-sig') as f:
writer = csv.writer(f)
writer.writerow(headers)
for result in self.results:
metrics = result.get("metrics", {})
test_info = result.get("test_info", {})
row_data = [
test_info.get("expert", ""),
test_info.get("symbol", ""),
test_info.get("period", ""),
test_info.get("model", ""),
metrics.get("total_trades", 0),
metrics.get("winning_trades", 0),
metrics.get("losing_trades", 0),
metrics.get("win_rate", 0),
metrics.get("gross_profit", 0),
metrics.get("gross_loss", 0),
metrics.get("profit_factor", 0),
metrics.get("expected_payoff", 0),
metrics.get("initial_deposit", 0),
metrics.get("final_balance", 0)
]
writer.writerow(row_data)
logger.info(f"CSV report saved: {output_path}")
return output_path
except Exception as e:
logger.error(f"Error generating CSV report: {e}")
raise
def generate_html(self, output_path: str = "reports/batch_report.html") -> str:
try:
os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
html_content = f"""
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>MT5 Batch Backtest Report</title>
<style>
body {{ font-family: Arial, sans-serif; margin: 20px; }}
h1 {{ color: #366092; }}
.summary {{ background: #f0f4f8; padding: 15px; border-radius: 5px; margin: 20px 0; }}
table {{ border-collapse: collapse; width: 100%; margin: 20px 0; }}
th {{ background: #366092; color: white; padding: 12px; text-align: left; }}
td {{ border: 1px solid #ddd; padding: 10px; }}
tr:nth-child(even) {{ background: #f9f9f9; }}
.best {{ color: green; font-weight: bold; }}
.worst {{ color: red; }}
</style>
</head>
<body>
<h1>MT5 Batch Backtest Report</h1>
<p>Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</p>
<div class="summary">
<h2>Summary</h2>
<p>Total Tests: {len(self.results)}</p>
</div>
<table>
<tr>
<th>EA Name</th>
<th>Symbol</th>
<th>Period</th>
<th>Total Trades</th>
<th>Win Rate %</th>
<th>Profit Factor</th>
<th>Gross Profit</th>
<th>Gross Loss</th>
<th>Final Balance</th>
</tr>
"""
for result in self.results:
metrics = result.get("metrics", {})
test_info = result.get("test_info", {})
pf = metrics.get("profit_factor", 0)
row_class = "best" if pf >= 2.0 else ("worst" if pf < 1.0 else "")
html_content += f"""
<tr class="{row_class}">
<td>{test_info.get("expert", "")}</td>
<td>{test_info.get("symbol", "")}</td>
<td>{test_info.get("period", "")}</td>
<td>{metrics.get("total_trades", 0)}</td>
<td>{metrics.get("win_rate", 0):.2f}%</td>
<td>{pf:.2f}</td>
<td>{metrics.get("gross_profit", 0):.2f}</td>
<td>{metrics.get("gross_loss", 0):.2f}</td>
<td>{metrics.get("final_balance", 0):.2f}</td>
</tr>
"""
html_content += """
</table>
</body>
</html>
"""
with open(output_path, 'w', encoding='utf-8') as f:
f.write(html_content)
logger.info(f"HTML report saved: {output_path}")
return output_path
except Exception as e:
logger.error(f"Error generating HTML report: {e}")
raise
def generate_markdown(self, output_path: str = "reports/batch_report.md") -> str:
try:
os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
md_lines = [
"# MT5 Batch Backtest Report",
"",
f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
"",
f"**Total Tests:** {len(self.results)}",
""
]
if self.results:
best = max(self.results, key=lambda x: x.get("metrics", {}).get("profit_factor", 0))
worst = min(self.results, key=lambda x: x.get("metrics", {}).get("profit_factor", 0))
md_lines.append(f"**Best Test:** {best.get('test_info', {}).get('expert', 'N/A')} "
f"(PF: {best.get('metrics', {}).get('profit_factor', 0):.2f})")
md_lines.append(f"**Worst Test:** {worst.get('test_info', {}).get('expert', 'N/A')} "
f"(PF: {worst.get('metrics', {}).get('profit_factor', 0):.2f})")
md_lines.append("")
md_lines.append("## Results")
md_lines.append("")
md_lines.append("| EA Name | Symbol | Period | Total Trades | Win Rate | Profit Factor | "
"Gross Profit | Gross Loss | Final Balance |")
md_lines.append("|---------|--------|--------|--------------|----------|---------------|"
"--------------|------------|---------------|")
for result in self.results:
metrics = result.get("metrics", {})
test_info = result.get("test_info", {})
md_lines.append(
f"| {test_info.get('expert', '')} | "
f"{test_info.get('symbol', '')} | "
f"{test_info.get('period', '')} | "
f"{metrics.get('total_trades', 0)} | "
f"{metrics.get('win_rate', 0):.2f}% | "
f"{metrics.get('profit_factor', 0):.2f} | "
f"{metrics.get('gross_profit', 0):.2f} | "
f"{metrics.get('gross_loss', 0):.2f} | "
f"{metrics.get('final_balance', 0):.2f} |"
)
with open(output_path, 'w', encoding='utf-8') as f:
f.write("\n".join(md_lines))
logger.info(f"Markdown report saved: {output_path}")
return output_path
except Exception as e:
logger.error(f"Error generating Markdown report: {e}")
raise
def main():
import argparse
parser = argparse.ArgumentParser(description="MT5 Report Generator")
parser.add_argument("--input", "-i", default="reports/parsed_results.json",
help="Input JSON file from result_parser")
parser.add_argument("--output-dir", "-o", default="reports",
help="Output directory for reports")
parser.add_argument("--format", "-f", choices=["excel", "csv", "html", "markdown", "all"],
default="all", help="Output format")
args = parser.parse_args()
results = []
if os.path.exists(args.input):
with open(args.input, 'r', encoding='utf-8') as f:
data = json.load(f)
results = data.get("results", [])
else:
logger.warning(f"Input file not found: {args.input}")
logger.info("Use result_parser.py first to generate parsed results")
if not results:
print("No results to generate report")
return
generator = ReportGenerator(results)
base_name = os.path.join(args.output_dir, "batch_report")
if args.format in ["excel", "all"]:
generator.generate_excel(f"{base_name}.xlsx")
if args.format in ["csv", "all"]:
generator.generate_csv(f"{base_name}.csv")
if args.format in ["html", "all"]:
generator.generate_html(f"{base_name}.html")
if args.format in ["markdown", "all"]:
generator.generate_markdown(f"{base_name}.md")
print(f"\nReports generated in: {args.output_dir}")
if __name__ == "__main__":
main()
+483
View File
@@ -0,0 +1,483 @@
import os
import csv
import xml.etree.ElementTree as ET
from datetime import datetime
from typing import Dict, List, Optional
from bs4 import BeautifulSoup
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class ResultParser:
def __init__(self, results_dir: str = "results"):
self.results_dir = results_dir
def parse_xml_report(self, report_path: str) -> Optional[Dict]:
if not os.path.exists(report_path):
logger.warning(f"Report not found: {report_path}")
return None
try:
tree = ET.parse(report_path)
root = tree.getroot()
result = {
"report_file": os.path.basename(report_path),
"parse_time": datetime.now().isoformat(),
"test_info": {},
"metrics": {},
"trades": []
}
tester = root.find("Tester")
if tester is not None:
result["test_info"] = {
"expert": tester.findtext("Expert", ""),
"symbol": tester.findtext("Symbol", ""),
"period": tester.findtext("Period", ""),
"model": tester.findtext("Model", ""),
"from_date": tester.findtext("FromDate", ""),
"to_date": tester.findtext("ToDate", ""),
}
equity = root.find("Equity")
if equity is not None:
result["metrics"]["initial_deposit"] = float(equity.findtext("Initial", "0"))
result["metrics"]["final_balance"] = float(equity.findtext("Final", "0"))
result["metrics"]["gross_profit"] = float(equity.findtext("GrossProfit", "0"))
result["metrics"]["gross_loss"] = float(equity.findtext("GrossLoss", "0"))
result["metrics"]["profit_factor"] = float(equity.findtext("ProfitFactor", "0"))
result["metrics"]["expected_payoff"] = float(equity.findtext("ExpectedPayoff", "0"))
trades_elem = root.find("Trades")
if trades_elem is not None:
result["metrics"]["total_trades"] = int(trades_elem.findtext("Total", "0"))
result["metrics"]["short_positions"] = int(trades_elem.findtext("Short", "0"))
result["metrics"]["long_positions"] = int(trades_elem.findtext("Long", "0"))
result["metrics"]["winning_trades"] = int(trades_elem.findtext("ProfitTrades", "0"))
result["metrics"]["losing_trades"] = int(trades_elem.findtext("LossTrades", "0"))
if result["metrics"]["total_trades"] > 0:
result["metrics"]["win_rate"] = (
result["metrics"]["winning_trades"] / result["metrics"]["total_trades"] * 100
)
else:
result["metrics"]["win_rate"] = 0.0
return result
except Exception as e:
logger.error(f"Error parsing XML report {report_path}: {e}")
return None
def _detect_encoding(self, report_path: str) -> str:
with open(report_path, 'rb') as f:
bom = f.read(4)
if bom[:2] == b'\xff\xfe':
return 'utf-16-le'
elif bom[:2] == b'\xfe\xff':
return 'utf-16-be'
return 'utf-8'
def parse_html_report(self, report_path: str) -> Optional[Dict]:
if not os.path.exists(report_path):
logger.warning(f"Report not found: {report_path}")
return None
try:
import re
with open(report_path, 'rb') as f:
raw_bytes = f.read()
if raw_bytes[:2] == b'\xff\xfe':
content = raw_bytes[2:].decode('utf-16-le')
else:
content = raw_bytes.decode('utf-8')
result = {
"report_file": os.path.basename(report_path),
"parse_time": datetime.now().isoformat(),
"test_info": {},
"metrics": {},
"trades": []
}
plain_pattern = r'<td[^>]*nowrap[^>]*>([^<]+):</td>\s*<td[^>]*colspan=.10.[^>]*><b>([^<]*)</b></td>'
plain_matches = re.findall(plain_pattern, content)
for key, value in plain_matches:
key = key.strip()
value = value.strip()
if key in result["test_info"] and result["test_info"].get(key):
continue
self._parse_html_cell(key, value, result)
pattern = r'<td[^>]*>([^<]+):</td>\s*<td[^>]*><b>([^<]*)</b></td>'
matches = re.findall(pattern, content)
for key, value in matches:
key = key.strip()
value = value.strip()
if key in result["test_info"] and result["test_info"].get(key):
continue
if key in result["metrics"] and result["metrics"].get(key):
continue
self._parse_html_cell(key, value, result)
return result
except Exception as e:
logger.error(f"Error parsing HTML report {report_path}: {e}")
return None
try:
with open(report_path, 'r', encoding='utf-16') as f:
content = f.read()
content_utf8 = content.encode('utf-8').decode('utf-8')
soup = BeautifulSoup(content_utf8, 'html.parser')
result = {
"report_file": os.path.basename(report_path),
"parse_time": datetime.now().isoformat(),
"test_info": {},
"metrics": {},
"trades": []
}
tables = soup.find_all('table')
for table in tables:
rows = table.find_all('tr')
for row in rows:
cells = row.find_all(['td', 'th'])
if len(cells) >= 2:
non_empty = [c for c in cells if c.get_text(strip=True)]
if len(non_empty) >= 2:
key = non_empty[0].get_text(strip=True)
value = non_empty[1].get_text(strip=True)
self._parse_html_cell(key, value, result)
return result
except Exception as e:
logger.error(f"Error parsing HTML report {report_path}: {e}")
return None
def _extract_number(self, text: str) -> float:
import re
numbers = re.findall(r'[-+]?\d*\.?\d+', text.replace(',', '').replace(' ', ''))
if numbers:
try:
return float(numbers[0])
except:
return 0.0
return 0.0
def _extract_percentage(self, text: str) -> float:
import re
numbers = re.findall(r'\d+\.?\d*%', text)
if numbers:
try:
return float(numbers[0].replace('%', ''))
except:
return 0.0
return 0.0
def _parse_html_cell(self, key: str, value: str, result: Dict):
import re
key_lower = key.lower()
if '专家' in key or 'Expert' in key:
if not result["test_info"].get("expert"):
result["test_info"]["expert"] = value
elif '交易品种' in key or '交易品' in key or 'Symbol' in key or 'symbol' in key_lower:
current = result["test_info"].get("symbol")
if (not current or current in ('1', '0', '')) and value and value not in ('1', ''):
result["test_info"]["symbol"] = value
if not result["test_info"].get("symbol") or result["test_info"].get("symbol") == "0":
fn = result["report_file"]
m = re.search(r'_([A-Z]{5,6})_', fn)
if m:
result["test_info"]["symbol"] = m.group(1)
elif '期间' in key or 'Period' in key:
if not result["test_info"].get("period"):
result["test_info"]["period"] = value
elif '模型' in key or 'Model' in key:
result["test_info"]["model"] = value
elif '公司' in key or 'Company' in key:
result["test_info"]["company"] = value
elif '货币' in key or 'Currency' in key:
result["test_info"]["currency"] = value
elif '杠杆' in key or 'Leverage' in key:
result["test_info"]["leverage"] = value
elif '初始入金' in key or ('Initial' in key and 'Deposit' in key):
result["metrics"]["initial_deposit"] = self._extract_number(value)
elif '总净盈利' in key or 'Net Profit' in key:
result["metrics"]["net_profit"] = self._extract_number(value)
elif '毛利' in key or 'Gross Profit' in key:
result["metrics"]["gross_profit"] = self._extract_number(value)
elif '毛损' in key or 'Gross Loss' in key:
result["metrics"]["gross_loss"] = self._extract_number(value)
elif '盈利因子' in key or 'Profit Factor' in key:
result["metrics"]["profit_factor"] = self._extract_number(value)
elif '预期收益' in key or 'Expected Payoff' in key:
result["metrics"]["expected_payoff"] = self._extract_number(value)
elif ('' in key and '交易' in key.lower()) or ('Total' in key and 'trades' in key_lower):
result["metrics"]["total_trades"] = self._extract_number(value)
elif '采收率' in key or 'Recovery Factor' in key:
result["metrics"]["recovery_factor"] = self._extract_number(value)
elif '夏普比率' in key or 'Sharpe Ratio' in key:
result["metrics"]["sharpe_ratio"] = self._extract_number(value)
elif 'AHPR' in key:
result["metrics"]["ahpr"] = value
elif 'GHPR' in key:
result["metrics"]["ghpr"] = value
elif 'LR 相关性' in key or 'LR Correlation' in key:
result["metrics"]["lr_correlation"] = self._extract_number(value)
elif 'LR 标准误差' in key or 'LR Standard Error' in key:
result["metrics"]["lr_standard_error"] = self._extract_number(value)
elif '最大结余亏损' in key or 'Maximal Drawdown' in key:
result["metrics"]["max_drawdown"] = self._extract_number(value)
elif '最大净值亏损' in key or 'Max Equity Drawdown' in key:
result["metrics"]["max_equity_drawdown"] = self._extract_number(value)
elif '绝对结余亏损' in key or 'Absolute Drawdown' in key:
result["metrics"]["absolute_drawdown"] = self._extract_number(value)
elif '预付款维持率' in key or 'Margin Level' in key:
result["metrics"]["margin_level"] = self._extract_number(value)
elif '卖出交易' in key or 'Short Positions' in key:
result["metrics"]["short_trades"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
elif '买入交易' in key or 'Long Positions' in key:
result["metrics"]["long_trades"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
elif '总成交' in key or 'Total Deals' in key:
result["metrics"]["total_deals"] = self._extract_number(value)
elif '盈利交易' in key or 'Profit Trades' in key:
result["metrics"]["winning_trades"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
result["metrics"]["win_rate"] = self._extract_percentage(value)
elif '亏损交易' in key or 'Loss Trades' in key:
result["metrics"]["losing_trades"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
elif '最大 获利交易' in key or 'Max Profit Trade' in key:
result["metrics"]["max_profit_trade"] = self._extract_number(value)
elif '最大 亏损交易' in key or 'Max Loss Trade' in key:
result["metrics"]["max_loss_trade"] = self._extract_number(value)
elif '平均 获利交易' in key or 'Avg Profit Trade' in key:
result["metrics"]["avg_profit_trade"] = self._extract_number(value)
elif '平均 亏损交易' in key or 'Avg Loss Trade' in key:
result["metrics"]["avg_loss_trade"] = self._extract_number(value)
elif '最大值 连胜' in key or 'Longest Winning Streak' in key:
result["metrics"]["longest_win_streak"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
elif '最大值 连败' in key or 'Longest Losing Streak' in key:
result["metrics"]["longest_lose_streak"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
elif '平均 连胜' in key or 'Average Winning Streak' in key:
result["metrics"]["avg_win_streak"] = self._extract_number(value)
elif '平均 连败' in key or 'Average Losing Streak' in key:
result["metrics"]["avg_lose_streak"] = self._extract_number(value)
elif '最小持仓时间' in key or 'Min Hold Time' in key:
result["metrics"]["min_hold_time"] = value
elif '最大持仓时间' in key or 'Max Hold Time' in key:
result["metrics"]["max_hold_time"] = value
elif '平均持仓时间' in key or 'Avg Hold Time' in key:
result["metrics"]["avg_hold_time"] = value
elif '质量历史' in key or 'Quality' in key:
result["metrics"]["quality"] = self._extract_number(value)
elif '' in key or 'Bars' in key:
result["metrics"]["bars"] = self._extract_number(value)
elif '报价' in key or 'Quotes' in key:
result["metrics"]["quotes"] = self._extract_number(value)
elif '分值' in key or 'Score' in key:
result["metrics"]["score"] = self._extract_number(value.split('(')[0]) if '(' in value else self._extract_number(value)
elif 'OnTester结果' in key or 'OnTester' in key:
result["metrics"]["on_tester"] = self._extract_number(value)
def parse_all_reports(self, pattern: str = "*.xml") -> List[Dict]:
import glob
pattern_base = pattern.replace('*', '')
if pattern_base == '.xml':
report_files = glob.glob(os.path.join(self.results_dir, '*.xml'))
report_files.extend(glob.glob(os.path.join(self.results_dir, '*.htm')))
report_files.extend(glob.glob(os.path.join(self.results_dir, '*.html')))
else:
report_files = glob.glob(os.path.join(self.results_dir, pattern))
results = []
for report_file in report_files:
if report_file.endswith('.xml'):
parsed = self.parse_xml_report(report_file)
elif report_file.endswith(('.html', '.htm')):
parsed = self.parse_html_report(report_file)
else:
continue
if parsed:
parsed["file_path"] = os.path.abspath(report_file)
parsed.setdefault("report_file", os.path.basename(report_file))
results.append(parsed)
logger.info(f"Parsed: {os.path.basename(report_file)}")
else:
logger.warning(f"Failed to parse: {report_file}")
logger.info(f"Total reports parsed: {len(results)}")
return results
def get_summary(self, results: List[Dict]) -> Dict:
if not results:
return {}
summary = {
"total_tests": len(results),
"total_trades": 0,
"avg_win_rate": 0,
"avg_profit_factor": 0,
"best_test": None,
"worst_test": None
}
total_trades = sum(r["metrics"].get("total_trades", 0) for r in results)
win_rates = [r["metrics"].get("win_rate", 0) for r in results if "win_rate" in r["metrics"]]
profit_factors = [r["metrics"].get("profit_factor", 0) for r in results if "profit_factor" in r["metrics"]]
summary["total_trades"] = total_trades
if win_rates:
summary["avg_win_rate"] = sum(win_rates) / len(win_rates)
if profit_factors:
summary["avg_profit_factor"] = sum(profit_factors) / len(profit_factors)
completed = [r for r in results if r.get("test_info", {}).get("expert")]
if completed:
summary["best_test"] = max(completed, key=lambda x: x["metrics"].get("profit_factor", 0))
summary["worst_test"] = min(completed, key=lambda x: x["metrics"].get("profit_factor", 0))
return summary
def export_to_csv(self, results: List[Dict], output_path: str, sort_by: str = "profit_factor", reverse: bool = True):
if not results:
return
fieldnames = [
"expert", "symbol", "period", "company", "currency", "leverage",
"initial_deposit", "net_profit", "gross_profit", "gross_loss",
"profit_factor", "expected_payoff", "recovery_factor", "sharpe_ratio",
"ahpr", "ghpr", "lr_correlation", "lr_standard_error",
"max_drawdown", "max_equity_drawdown", "absolute_drawdown", "margin_level",
"total_trades", "total_deals", "short_trades", "long_trades",
"winning_trades", "losing_trades", "win_rate",
"max_profit_trade", "max_loss_trade", "avg_profit_trade", "avg_loss_trade",
"longest_win_streak", "longest_lose_streak", "avg_win_streak", "avg_lose_streak",
"min_hold_time", "max_hold_time", "avg_hold_time",
"quality", "bars", "quotes", "score", "on_tester"
]
rows = []
for r in results:
info = r.get("test_info", {})
metrics = r.get("metrics", {})
row = {
"expert": info.get("expert", ""),
"symbol": info.get("symbol", ""),
"period": info.get("period", ""),
"company": info.get("company", ""),
"currency": info.get("currency", ""),
"leverage": info.get("leverage", ""),
"initial_deposit": metrics.get("initial_deposit", ""),
"net_profit": metrics.get("net_profit", ""),
"gross_profit": metrics.get("gross_profit", ""),
"gross_loss": metrics.get("gross_loss", ""),
"profit_factor": metrics.get("profit_factor", ""),
"expected_payoff": metrics.get("expected_payoff", ""),
"recovery_factor": metrics.get("recovery_factor", ""),
"sharpe_ratio": metrics.get("sharpe_ratio", ""),
"ahpr": metrics.get("ahpr", ""),
"ghpr": metrics.get("ghpr", ""),
"lr_correlation": metrics.get("lr_correlation", ""),
"lr_standard_error": metrics.get("lr_standard_error", ""),
"max_drawdown": metrics.get("max_drawdown", ""),
"max_equity_drawdown": metrics.get("max_equity_drawdown", ""),
"absolute_drawdown": metrics.get("absolute_drawdown", ""),
"margin_level": metrics.get("margin_level", ""),
"total_trades": metrics.get("total_trades", ""),
"total_deals": metrics.get("total_deals", ""),
"short_trades": metrics.get("short_trades", ""),
"long_trades": metrics.get("long_trades", ""),
"winning_trades": metrics.get("winning_trades", ""),
"losing_trades": metrics.get("losing_trades", ""),
"win_rate": metrics.get("win_rate", ""),
"max_profit_trade": metrics.get("max_profit_trade", ""),
"max_loss_trade": metrics.get("max_loss_trade", ""),
"avg_profit_trade": metrics.get("avg_profit_trade", ""),
"avg_loss_trade": metrics.get("avg_loss_trade", ""),
"longest_win_streak": metrics.get("longest_win_streak", ""),
"longest_lose_streak": metrics.get("longest_lose_streak", ""),
"avg_win_streak": metrics.get("avg_win_streak", ""),
"avg_lose_streak": metrics.get("avg_lose_streak", ""),
"min_hold_time": metrics.get("min_hold_time", ""),
"max_hold_time": metrics.get("max_hold_time", ""),
"avg_hold_time": metrics.get("avg_hold_time", ""),
"quality": metrics.get("quality", ""),
"bars": metrics.get("bars", ""),
"quotes": metrics.get("quotes", ""),
"score": metrics.get("score", ""),
"on_tester": metrics.get("on_tester", ""),
}
rows.append(row)
rows.sort(key=lambda x: x.get(sort_by, ""), reverse=reverse)
os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else '.', exist_ok=True)
with open(output_path, 'w', newline='', encoding='utf-8-sig') as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
logger.info(f"CSV exported to: {output_path}")
def main():
import argparse
parser = argparse.ArgumentParser(description="MT5 Report Parser")
parser.add_argument("--results-dir", "-r", default="reports",
help="Directory containing report files")
parser.add_argument("--output", "-o", default="reports/parsed_results.json",
help="Output file for parsed results")
parser.add_argument("--csv", "-c", default=None,
help="CSV output path (e.g. reports/results.csv)")
parser.add_argument("--sort", "-s", default="profit_factor",
choices=["expert", "symbol", "period", "net_profit", "gross_profit",
"profit_factor", "total_trades", "win_rate", "max_drawdown"],
help="Field to sort by")
parser.add_argument("--asc", action="store_true",
help="Sort in ascending order (default: descending)")
args = parser.parse_args()
parser = ResultParser(args.results_dir)
results = parser.parse_all_reports()
if results:
summary = parser.get_summary(results)
print("\n" + "="*60)
print("PARSED RESULTS SUMMARY")
print("="*60)
print(f"Total tests parsed: {summary['total_tests']}")
print(f"Total trades: {summary['total_trades']}")
print(f"Average win rate: {summary['avg_win_rate']:.2f}%")
print(f"Average profit factor: {summary['avg_profit_factor']:.2f}")
if summary['best_test']:
print(f"\nBest test: {summary['best_test']['test_info'].get('expert', 'N/A')}")
print(f" Profit factor: {summary['best_test']['metrics'].get('profit_factor', 0):.2f}")
print(f" Win rate: {summary['best_test']['metrics'].get('win_rate', 0):.2f}%")
os.makedirs(os.path.dirname(args.output) if os.path.dirname(args.output) else '.', exist_ok=True)
import json
with open(args.output, 'w', encoding='utf-8') as f:
json.dump({"results": results, "summary": summary}, f, indent=2, ensure_ascii=False)
print(f"\nResults saved to: {args.output}")
if args.csv:
parser.export_to_csv(results, args.csv, sort_by=args.sort, reverse=not args.asc)
else:
print("No results to parse")
if __name__ == "__main__":
main()
+430
View File
@@ -0,0 +1,430 @@
import os
import re
import argparse
import unicodedata
from dataclasses import dataclass, field
from typing import List, Dict, Optional
from pathlib import Path
def sanitize_path_name(name: str) -> str:
name = unicodedata.normalize('NFKD', name)
name = name.encode('ascii', 'ignore').decode('ascii')
name = re.sub(r'[^\w\-_.]', '_', name)
name = re.sub(r'_{2,}', '_', name)
return name.strip('_')
@dataclass
class SetParameter:
name: str
param_type: str
default: float
min_value: float
max_value: float
step: float
optimize: bool
@dataclass
class SetFileResult:
file_path: str
ea_name: str
parameters: Dict[str, SetParameter] = field(default_factory=dict)
optimize_count: int = 0
fixed_count: int = 0
MT5_NATIVE_PATTERN = re.compile(r'^(\w+)\s+<([^>]+)>\s+<([^>]+)>')
OUR_FORMAT_PATTERN = re.compile(r'^(\w+)=([^|]+)\|\|([^|]+)\|\|([^|]+)\|\|([^|]+)\|\|([YN])')
def detect_encoding(file_path: str) -> str:
with open(file_path, 'rb') as f:
raw = f.read(4)
if raw[:2] == b'\xff\xfe':
return 'utf-16-le'
elif raw[:2] == b'\xfe\xff':
return 'utf-16-be'
elif raw[:3] == b'\xef\xbb\xbf':
return 'utf-8-sig'
return 'utf-8'
def detect_set_format(line: str) -> str:
if '=' in line and '||' in line:
return 'our_format'
elif '<' in line and '>' in line:
return 'mt5_native'
elif line.strip().startswith(';'):
return 'comment'
elif line.strip() == '':
return 'empty'
return 'unknown'
def parse_set_line(line: str) -> Optional[SetParameter]:
line = line.strip()
if not line or line.startswith(';'):
return None
fmt = detect_set_format(line)
if fmt == 'our_format':
m = OUR_FORMAT_PATTERN.match(line)
if m:
name, cur, start, step, stop, opt = m.groups()
cur_lower = cur.lower()
if cur_lower in ('true', 'false'):
is_true = cur_lower == 'true'
return SetParameter(
name=name,
param_type='bool',
default=1.0 if is_true else 0.0,
min_value=0.0,
max_value=1.0,
step=1.0,
optimize=(opt.upper() == 'Y')
)
return SetParameter(
name=name,
param_type='double',
default=float(cur),
min_value=float(start),
max_value=float(stop),
step=float(step),
optimize=(opt.upper() == 'Y')
)
elif fmt == 'mt5_native':
m = MT5_NATIVE_PATTERN.match(line)
if m:
name, val1, val2 = m.groups()
val1l = val1.lower().strip()
val2l = val2.lower().strip()
if val1l in ('true', 'false') or val2l in ('true', 'false'):
is_true = val1l == 'true' or val2l == 'true'
return SetParameter(
name=name,
param_type='bool',
default=1.0 if is_true else 0.0,
min_value=0.0,
max_value=1.0,
step=1.0,
optimize=False
)
try:
v1 = float(val1)
v2 = float(val2)
return SetParameter(
name=name,
param_type='double',
default=v1,
min_value=min(v1, v2),
max_value=max(v1, v2),
step=abs(v2 - v1) if v1 != v2 else 0,
optimize=False
)
except ValueError:
pass
return None
def parse_set_file(file_path: str) -> SetFileResult:
result = SetFileResult(file_path=file_path, ea_name='')
path = Path(file_path)
result.ea_name = path.stem.replace('_optimization', '').replace('_params', '')
enc = detect_encoding(file_path)
with open(file_path, 'r', encoding=enc) as f:
content = f.read()
for line in content.split('\n'):
param = parse_set_line(line)
if param:
result.parameters[param.name] = param
result.optimize_count = sum(1 for p in result.parameters.values() if p.optimize)
result.fixed_count = sum(1 for p in result.parameters.values() if not p.optimize)
return result
def parse_set_content(content: str) -> SetFileResult:
result = SetFileResult(file_path='memory', ea_name='')
for line in content.split('\n'):
param = parse_set_line(line)
if param:
result.parameters[param.name] = param
result.optimize_count = sum(1 for p in result.parameters.values() if p.optimize)
result.fixed_count = sum(1 for p in result.parameters.values() if not p.optimize)
return result
def scan_folder(folder_path: str) -> List[SetFileResult]:
results = []
folder = Path(folder_path)
if folder.is_file():
return [parse_set_file(str(folder))]
for set_file in folder.rglob('*.set'):
try:
result = parse_set_file(str(set_file))
results.append(result)
except Exception as e:
print(f'Warning: Failed to parse {set_file}: {e}')
return sorted(results, key=lambda x: x.ea_name)
def batch_scan(file_paths: List[str]) -> List[SetFileResult]:
results = []
for path in file_paths:
if os.path.exists(path):
try:
result = parse_set_file(path)
results.append(result)
except Exception as e:
print(f'Warning: Failed to parse {path}: {e}')
return results
def generate_prompt(results: List[SetFileResult], output_dir: str = '.') -> List[str]:
output_files = []
for result in results:
lines = []
lines.append('=' * 80)
lines.append(f'EA 参数优化分析请求 - {result.ea_name}')
lines.append('=' * 80)
lines.append('')
lines.append('## 基本信息')
lines.append(f'- EA名称: {result.ea_name}')
lines.append(f'- SET文件: {result.file_path}')
lines.append(f'- 参数总数: {len(result.parameters)}')
lines.append('')
lines.append('## 参数详情表')
lines.append('')
lines.append('| 参数名 | 类型 | 当前值 | 最小值 | 最大值 | 步进 | 建议范围 |')
lines.append('|--------|------|--------|--------|--------|------|----------|')
for name, param in sorted(result.parameters.items()):
suggest = f'{param.min_value}~{param.max_value}'
if param.param_type == 'int':
suggest = f'{int(param.min_value)}~{int(param.max_value)} (步进:{int(param.step)})'
elif param.param_type == 'double':
suggest = f'{param.min_value}~{param.max_value} (步进:{param.step})'
lines.append(f'| {name} | {param.param_type} | {param.default} | {param.min_value} | {param.max_value} | {param.step} | {suggest} |')
lines.append('')
lines.append('## 参数分析任务')
lines.append('')
lines.append('请分析以上所有参数,根据策略逻辑和交易逻辑判断:')
lines.append('')
lines.append('1. **哪些参数应该参与优化(optimize: true**')
lines.append(' - 通常是影响策略核心逻辑、盈亏比、风险的关键参数')
lines.append(' - 例如:手数、止损止盈倍数、加仓间隔、风控阈值等')
lines.append('')
lines.append('2. **哪些参数应该保持固定(optimize: false**')
lines.append(' - 通常是风控类、开关类、显示类参数')
lines.append(' - 例如:开关标识、魔数、面板位置等')
lines.append('')
lines.append('3. **推荐优化范围调整**')
lines.append(' - 检查当前范围是否合理')
lines.append(' - 给出你认为更合适的 min/max/step')
lines.append('')
lines.append('4. **重要参数优先级**')
lines.append(' - 哪些2-3个参数对策略影响最大,需要重点优化')
lines.append('')
lines.append('## 输出要求')
lines.append('')
lines.append('请输出符合以下格式的完整JSON配置文件:')
lines.append('')
lines.append('```json')
lines.append('{')
lines.append(f' "ea_name": "{result.ea_name}",')
safe_ea_name = sanitize_path_name(result.ea_name)
lines.append(f' "ea_path": "MY-EA\\\\{safe_ea_name}.ex5", // {result.ea_name}')
lines.append(' "description": "Edited via GUI",')
lines.append(' "search_strategy": "auto",')
lines.append(' "parameters": {')
param_list = list(result.parameters.items())
for i, (name, param) in enumerate(param_list):
comma = ',' if i < len(param_list) - 1 else ''
lines.append(f' "{name}": {{')
lines.append(f' "type": "{param.param_type}",')
lines.append(f' "default": {param.default},')
lines.append(f' "min": {param.min_value},')
lines.append(f' "max": {param.max_value},')
lines.append(f' "step": {param.step},')
lines.append(f' "optimize": true // 请根据分析填写 true 或 false')
lines.append(f' }}{comma}')
lines.append(' },')
lines.append(' "test_config": {')
lines.append(' "symbol": "XAUUSD", // 请填写交易品种')
lines.append(' "period": "M1", // 请填写周期: M1/M5/M15/H1/H4/D1')
lines.append(' "from_date": "2026.01.01", // 请填写开始日期')
lines.append(' "to_date": "2026.01.04", // 请填写结束日期')
lines.append(' "model": 0, // 建模方式: 0=Every Tick, 1=1分钟OHLC, 2=仅开盘价, 3=数学计算, 4=真实Tick')
lines.append(' "execution_delay": 0, // 执行延迟: 0=无延迟, -1=随机延迟, 正数=固定延迟毫秒')
lines.append(' "optimization_mode": 2, // 优化模式: 0=禁用, 1=慢速完整算法, 2=快速遗传算法, 3=MarketWatch所有符号')
lines.append(' "optimization_criterion": 1, // 优化指标: 0=余额最大, 1=盈利因子最大, 2=期望收益, 3=回撤最小, 4=恢复因子, 5=夏普比率, 6=自定义, 7=复合指标')
lines.append(' "deposit": 10000, // 初始保证金')
lines.append(' "leverage": "1:100" // 杠杆')
lines.append(' },')
lines.append(' "walk_forward": {')
lines.append(' "enabled": true, // 是否启用前向测试')
lines.append(' "forward_mode": 2 // 前向模式: 0=禁用, 1=OOS 50%, 2=OOS 33%, 3=OOS 25%')
lines.append(' }')
lines.append('}')
lines.append('```')
lines.append('')
lines.append('注意:')
lines.append(f'- EA原始名称: {result.ea_name}')
lines.append(f'- ea_path 使用ASCII安全名称: {safe_ea_name}')
lines.append('- 实际EA文件应放在 MY-EA 目录下,文件名需与 ea_path 中的名称一致')
lines.append('- 每个参数的 "optimize" 字段需要你根据策略分析来填写')
lines.append('- optimize: true = 参与优化, optimize: false = 保持固定')
lines.append('- test_config 和 walk_forward 部分请根据回测需求填写或调整')
lines.append('- ea_path 路径使用双反斜杠 `\\\\`')
lines.append('')
lines.append('=' * 80)
output_path = os.path.join(output_dir, f'{result.ea_name}_optimization_prompt.txt')
with open(output_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(lines))
output_files.append(output_path)
print(f'Generated: {output_path}')
return output_files
def interactive_scan():
print('请选择扫描模式:')
print('1. 扫描单个 SET 文件')
print('2. 扫描文件夹下所有 SET 文件')
print('3. 批量扫描多个 SET 文件')
print('4. 从剪贴板读取 SET 内容')
print('0. 退出')
print('')
choice = input('请输入选项 (0-4): ').strip()
if choice == '1':
path = input('请输入 SET 文件路径: ').strip()
if os.path.exists(path):
results = [parse_set_file(path)]
output_dir = os.path.dirname(path) or '.'
generate_prompt(results, output_dir)
else:
print('文件不存在')
elif choice == '2':
path = input('请输入文件夹路径: ').strip()
if os.path.exists(path):
results = scan_folder(path)
generate_prompt(results, path)
else:
print('文件夹不存在')
elif choice == '3':
print('请输入 SET 文件路径(每行一个,输入空行结束):')
paths = []
while True:
line = input().strip()
if not line:
break
if os.path.exists(line):
paths.append(line)
else:
print(f'文件不存在: {line}')
if paths:
results = batch_scan(paths)
generate_prompt(results, os.path.dirname(paths[0]) if len(paths) == 1 else '.')
elif choice == '4':
print('请粘贴 SET 文件内容(输入空行结束):')
lines = []
while True:
line = input()
if not line.strip():
break
lines.append(line)
if lines:
content = '\n'.join(lines)
result = parse_set_content(content)
result.ea_name = input('请输入 EA 名称: ').strip() or 'EA'
generate_prompt([result], '.')
def main():
parser = argparse.ArgumentParser(
description='SET 文件扫描工具 - 生成 AI 优化提示词',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='''
示例:
python scan_set_prompt.py --file C:\\path\\to\\EA.set
python scan_set_prompt.py --folder C:\\path\\to\\set_files
python scan_set_prompt.py --batch a.set b.set c.set
python scan_set_prompt.py --interactive
'''
)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--file', '-f', metavar='PATH', help='扫描单个 SET 文件')
group.add_argument('--folder', '-d', metavar='PATH', help='扫描文件夹下所有 SET 文件')
group.add_argument('--batch', '-b', nargs='+', metavar='PATH', help='批量扫描多个 SET 文件')
group.add_argument('--interactive', '-i', action='store_true', help='交互式扫描')
parser.add_argument('--output', '-o', metavar='DIR', default='.', help='输出目录 (默认当前目录)')
args = parser.parse_args()
if args.interactive:
interactive_scan()
return
results = []
if args.file:
if not os.path.exists(args.file):
print(f'Error: 文件不存在 {args.file}')
return
results = [parse_set_file(args.file)]
output_dir = os.path.dirname(args.file) or args.output
elif args.folder:
if not os.path.exists(args.folder):
print(f'Error: 文件夹不存在 {args.folder}')
return
results = scan_folder(args.folder)
output_dir = args.folder
elif args.batch:
existing = [p for p in args.batch if os.path.exists(p)]
if not existing:
print('Error: 所有文件都不存在')
return
results = batch_scan(existing)
output_dir = args.output
if not results:
print('Warning: 没有找到任何 SET 文件')
return
output_files = generate_prompt(results, output_dir)
print(f'\n完成!共生成 {len(output_files)} 个提示词文件')
if __name__ == '__main__':
main()