328 lines
14 KiB
Python
328 lines
14 KiB
Python
import os
|
|
import sys
|
|
import json
|
|
import time
|
|
import shutil
|
|
import subprocess
|
|
from datetime import datetime
|
|
from typing import List, Dict, Any, Optional
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from ea_config_parser import load_ea_config, load_all_ea_configs, EAConfig
|
|
from parameter_constraint import create_constraint_engine
|
|
from smart_search import SmartSearch
|
|
from analyze_results import ResultAnalyzer
|
|
from mt5_paths import auto_detect_data_dir
|
|
|
|
|
|
class EABatchOptimizer:
|
|
def __init__(self, base_dir: str = 'optimizer'):
|
|
self.base_dir = base_dir
|
|
self.configs_dir = os.path.join(base_dir, 'configs')
|
|
self.set_files_dir = os.path.join(base_dir, 'set_files')
|
|
self.reports_dir = os.path.join(base_dir, 'reports')
|
|
self.results_dir = os.path.join(base_dir, 'results')
|
|
self.scripts_dir = os.path.join(base_dir, 'scripts')
|
|
self.logs_dir = os.path.join(base_dir, 'logs')
|
|
|
|
for d in [self.configs_dir, self.set_files_dir, self.reports_dir,
|
|
self.results_dir, self.scripts_dir, self.logs_dir]:
|
|
os.makedirs(d, exist_ok=True)
|
|
|
|
def generate_set_files(self, ea_config: EAConfig, max_samples: int = 2000) -> List[Dict]:
|
|
print('Generating SET files for ' + ea_config.ea_name)
|
|
print(' Total combinations: ' + str(ea_config.get_total_combinations()))
|
|
print(' Strategy: ' + SmartSearch(ea_config).select_strategy())
|
|
|
|
searcher = SmartSearch(ea_config)
|
|
combinations = searcher.generate_combinations(max_samples)
|
|
|
|
engine = create_constraint_engine(ea_config)
|
|
valid_combinations = engine.filter_valid_combinations(combinations)
|
|
|
|
print(' Valid combinations: ' + str(len(valid_combinations)))
|
|
|
|
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
|
os.makedirs(ea_set_dir, exist_ok=True)
|
|
|
|
param_mapping = []
|
|
for i, combo in enumerate(valid_combinations, 1):
|
|
set_id = str(i).zfill(8)
|
|
set_filename = 'params_' + set_id + '.set'
|
|
set_path = os.path.join(ea_set_dir, set_filename)
|
|
|
|
self._write_set_file(set_path, combo)
|
|
|
|
mapping = {
|
|
'set_id': set_id,
|
|
'set_file': set_filename,
|
|
'report_name': ea_config.ea_name + '_' + set_id + '.htm',
|
|
'params': combo
|
|
}
|
|
param_mapping.append(mapping)
|
|
|
|
mapping_path = os.path.join(self.results_dir, ea_config.ea_name + '_param_mapping.json')
|
|
with open(mapping_path, 'w', encoding='utf-8') as f:
|
|
json.dump(param_mapping, f, ensure_ascii=False, indent=2)
|
|
|
|
print(' SET files generated: ' + str(len(valid_combinations)))
|
|
return param_mapping
|
|
|
|
def _write_set_file(self, set_path: str, params: Dict):
|
|
lines = ['; MT5 EA Parameters SET File', '; Generated: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '']
|
|
|
|
for name, value in params.items():
|
|
if isinstance(value, bool):
|
|
lines.append(name + ' <true> <' + ('true' if value else 'false') + '>')
|
|
elif isinstance(value, float):
|
|
lines.append(name + ' <' + str(value) + '> <' + str(value) + '>')
|
|
else:
|
|
lines.append(name + ' <' + str(value) + '> <' + str(value) + '>')
|
|
|
|
with open(set_path, 'w', encoding='utf-8') as f:
|
|
f.write('\n'.join(lines))
|
|
|
|
def _save_param_mapping(self, mapping: List[Dict], output_path: str):
|
|
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
|
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
f.write('set_id,set_file,report_name\n')
|
|
for m in mapping:
|
|
f.write(m['set_id'] + ',' + m['set_file'] + ',' + m['report_name'] + '\n')
|
|
|
|
def generate_ini_for_ea(self, ea_config: EAConfig, param_mapping: List[Dict] = None):
|
|
test_config = ea_config.test_config
|
|
ini_filename = ea_config.ea_name + '.ini'
|
|
ini_path = os.path.join(self.configs_dir, ini_filename)
|
|
set_filename = ea_config.ea_name + '_optimization.set'
|
|
|
|
data_dir = auto_detect_data_dir() or r'C:\Users\Administrator\AppData\Roaming\MetaQuotes\Terminal\010E047102812FC0C18890992854220E'
|
|
mt5_tester_dir = os.path.join(data_dir, 'MQL5', 'Profiles', 'Tester')
|
|
set_path = os.path.join(mt5_tester_dir, set_filename)
|
|
|
|
date_from = test_config.get('from_date', '2025.01.01')
|
|
date_to = test_config.get('to_date', '2025.12.31')
|
|
|
|
criterion_map = {
|
|
'profit_factor': 1,
|
|
'net_profit': 0,
|
|
'sharpe_ratio': 5,
|
|
'expected_payoff': 2,
|
|
'drawdown': 3,
|
|
'recovery_factor': 4,
|
|
}
|
|
opt_criterion = test_config.get('optimization_criterion', 1)
|
|
|
|
set_lines = [
|
|
'; saved automatically',
|
|
'; this file contains last used input parameters for testing/optimizing ' + ea_config.ea_name + ' expert advisor',
|
|
'',
|
|
]
|
|
|
|
if ea_config.parameters:
|
|
for pname, param in ea_config.parameters.items():
|
|
if param.param_type == 'bool':
|
|
if not param.optimize:
|
|
set_lines.append(pname + '=false||false||false||false||N')
|
|
else:
|
|
set_lines.append(pname + '=false||false||false||false||Y')
|
|
elif param.param_type == 'int':
|
|
start = int(param.min_value) if param.min_value is not None else 0
|
|
step = int(param.step) if param.step is not None else 1
|
|
stop = int(param.max_value) if param.max_value is not None else 100
|
|
cur = int(param.default) if param.default is not None else start
|
|
if not param.optimize or start == stop or step == 0:
|
|
set_lines.append(pname + '=' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||N')
|
|
else:
|
|
set_lines.append(pname + '=' + str(cur) + '||' + str(start) + '||' + str(step) + '||' + str(stop) + '||Y')
|
|
elif param.param_type == 'double':
|
|
start = float(param.min_value) if param.min_value is not None else 0.0
|
|
step = float(param.step) if param.step is not None else 0.01
|
|
stop = float(param.max_value) if param.max_value is not None else 1.0
|
|
cur = float(param.default) if param.default is not None else start
|
|
if not param.optimize or start == stop or step == 0:
|
|
set_lines.append(pname + '=' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||N')
|
|
else:
|
|
set_lines.append(pname + '=' + str(cur) + '||' + str(start) + '||' + str(step) + '||' + str(stop) + '||Y')
|
|
|
|
with open(set_path, 'w', encoding='utf-8') as f:
|
|
f.write('\n'.join(set_lines))
|
|
|
|
ini_lines = [
|
|
'; EA backtest',
|
|
'[Tester]',
|
|
'Expert=' + ea_config.ea_path,
|
|
'ExpertParameters=' + set_filename,
|
|
'Symbol=' + test_config.get('symbol', 'EURUSD'),
|
|
'Period=' + test_config.get('period', 'H1'),
|
|
'Model=' + str(test_config.get('model', 1)),
|
|
'FromDate=' + date_from,
|
|
'ToDate=' + date_to,
|
|
]
|
|
|
|
wf = ea_config.walk_forward
|
|
if wf.get('enabled', False):
|
|
fm = wf.get('forward_mode', 2)
|
|
ini_lines.append('ForwardMode=' + str(fm))
|
|
else:
|
|
ini_lines.append('ForwardMode=0')
|
|
|
|
ini_lines.extend([
|
|
'Deposit=' + str(test_config.get('deposit', 10000)),
|
|
'Currency=' + str(test_config.get('currency', 'USD')),
|
|
'Leverage=' + str(test_config.get('leverage', '1:100')),
|
|
'ExecutionMode=' + str(test_config.get('execution_delay', 0)),
|
|
'Optimization=' + str(test_config.get('optimization_mode', 2)),
|
|
'OptimizationCriterion=' + str(opt_criterion),
|
|
'Report=' + ea_config.ea_name + '_optimization',
|
|
'ReplaceReport=true',
|
|
'ShutdownTerminal=true',
|
|
'Visual=0',
|
|
'',
|
|
])
|
|
|
|
with open(ini_path, 'w', encoding='utf-8') as f:
|
|
f.write('\n'.join(ini_lines))
|
|
|
|
print('INI saved: ' + ini_path)
|
|
print('SET saved: ' + set_path)
|
|
return ini_path
|
|
|
|
def build_powershell_script(self, ea_config: EAConfig, param_mapping: List[Dict], mt5_path: str):
|
|
ps_filename = 'run_' + ea_config.ea_name + '.ps1'
|
|
ps_path = os.path.join(self.scripts_dir, ps_filename)
|
|
|
|
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
|
ea_report_dir = os.path.join(self.reports_dir, ea_config.ea_name)
|
|
os.makedirs(ea_report_dir, exist_ok=True)
|
|
|
|
ini_path = os.path.join(self.configs_dir, ea_config.ea_name + '.ini')
|
|
|
|
lines = [
|
|
'# MT5 Batch Backtest Script',
|
|
'# Generated: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
|
'',
|
|
f'$MT5Path = "{mt5_path}"',
|
|
f'$IniFile = "{ini_path}"',
|
|
f'$SetDir = "{ea_set_dir}"',
|
|
'',
|
|
f"Write-Host 'Starting batch optimization for {ea_config.ea_name}'",
|
|
'',
|
|
'$SetFiles = Get-ChildItem -Path $SetDir -Filter "params_*.set"',
|
|
'$Total = $SetFiles.Count',
|
|
'$Current = 0',
|
|
'',
|
|
'foreach ($SetFile in $SetFiles) {',
|
|
' $Current++',
|
|
' Write-Host "[$Current/$Total] Processing: $($SetFile.Name)"',
|
|
'',
|
|
' $IniContent = Get-Content $IniFile',
|
|
' $IniContent = $IniContent -replace "ExpertParameters=.*", "ExpertParameters=$($SetFile.FullName)"',
|
|
' $TempIni = Join-Path $env:TEMP "temp_$([guid]::NewGuid().ToString().Substring(0,8)).ini"',
|
|
' $IniContent | Set-Content $TempIni -Encoding UTF8',
|
|
'',
|
|
' Start-Process -FilePath $MT5Path -ArgumentList "/config:$TempIni" -Wait',
|
|
'',
|
|
' Start-Sleep -Seconds 3',
|
|
'}',
|
|
'',
|
|
f"Write-Host 'Batch optimization completed for {ea_config.ea_name}!'",
|
|
]
|
|
|
|
with open(ps_path, 'w', encoding='utf-8') as f:
|
|
f.write('\n'.join(lines))
|
|
|
|
print('PowerShell script: ' + ps_path)
|
|
return ps_path
|
|
|
|
def run_optimization_for_ea(self, ea_config: EAConfig, mt5_path: str = None,
|
|
max_samples: int = 2000):
|
|
print('\n' + '=' * 60)
|
|
print('Optimization for: ' + ea_config.ea_name)
|
|
print('=' * 60)
|
|
|
|
param_mapping = self.generate_set_files(ea_config, max_samples)
|
|
self.generate_ini_for_ea(ea_config, param_mapping)
|
|
|
|
if mt5_path:
|
|
ps_path = self.build_powershell_script(ea_config, param_mapping, mt5_path)
|
|
print('PowerShell script: ' + ps_path)
|
|
|
|
return param_mapping
|
|
|
|
def run_full_optimization(self, ea_names: List[str] = None,
|
|
mt5_path: str = None,
|
|
max_samples_per_ea: int = 2000):
|
|
configs = load_all_ea_configs(self.configs_dir)
|
|
|
|
if ea_names:
|
|
configs = {k: v for k, v in configs.items() if k in ea_names}
|
|
|
|
all_mapping = {}
|
|
for ea_name, ea_config in configs.items():
|
|
mapping = self.run_optimization_for_ea(ea_config, mt5_path, max_samples_per_ea)
|
|
all_mapping[ea_name] = mapping
|
|
|
|
return all_mapping
|
|
|
|
def analyze_results(self, ea_names: List[str] = None) -> List[Dict]:
|
|
all_results = []
|
|
|
|
if ea_names:
|
|
for ea_name in ea_names:
|
|
report_dir = os.path.join(self.reports_dir, ea_name)
|
|
if os.path.exists(report_dir):
|
|
analyzer = ResultAnalyzer(report_dir)
|
|
results = analyzer.parse_all_reports()
|
|
for r in results:
|
|
r['ea_name'] = ea_name
|
|
all_results.extend(results)
|
|
else:
|
|
for ea_name in os.listdir(self.reports_dir):
|
|
report_dir = os.path.join(self.reports_dir, ea_name)
|
|
if os.path.isdir(report_dir):
|
|
analyzer = ResultAnalyzer(report_dir)
|
|
results = analyzer.parse_all_reports()
|
|
for r in results:
|
|
r['ea_name'] = ea_name
|
|
all_results.extend(results)
|
|
|
|
return all_results
|
|
|
|
def find_optimal_params(self, results: List[Dict], criterion: str = 'profit_factor',
|
|
min_trades: int = 10) -> List[Dict]:
|
|
filtered = [r for r in results if r.get('total_trades', 0) >= min_trades
|
|
and r.get('profit_factor', 0) > 0]
|
|
filtered.sort(key=lambda x: x.get(criterion, 0), reverse=True)
|
|
return filtered
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description='MT5 EA Batch Optimizer')
|
|
parser.add_argument('--base-dir', '-d', default='optimizer', help='Base directory')
|
|
parser.add_argument('--ea', '-e', nargs='+', help='EA names to optimize')
|
|
parser.add_argument('--max-samples', '-m', type=int, default=2000, help='Max samples')
|
|
parser.add_argument('--mt5-path', '-p', help='MT5 terminal path')
|
|
parser.add_argument('--analyze-only', '-a', action='store_true', help='Only analyze results')
|
|
|
|
args = parser.parse_args()
|
|
|
|
optimizer = EABatchOptimizer(args.base_dir)
|
|
|
|
if args.analyze_only:
|
|
results = optimizer.analyze_results(args.ea)
|
|
if results:
|
|
analyzer = ResultAnalyzer()
|
|
best = optimizer.find_optimal_params(results)
|
|
analyzer.generate_report(best)
|
|
print('Results: ' + str(len(results)))
|
|
print('Best: PF=' + str(best[0].get('profit_factor', 0)) if best else 'No results')
|
|
else:
|
|
optimizer.run_full_optimization(args.ea, args.mt5_path, args.max_samples)
|
|
print('\nOptimization files generated. Run MT5 tests, then use --analyze-only')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main() |