first commit
This commit is contained in:
@@ -0,0 +1,320 @@
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import glob
|
||||
import shutil
|
||||
from typing import List, Dict, Any, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
class ResultAnalyzer:
|
||||
def __init__(self, reports_dir: str = 'reports'):
|
||||
self.reports_dir = reports_dir
|
||||
|
||||
def parse_mt5_html_report(self, report_path: str) -> Optional[Dict]:
|
||||
if not os.path.exists(report_path):
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(report_path, 'rb') as f:
|
||||
raw_bytes = f.read()
|
||||
|
||||
if raw_bytes[:2] == b'\xff\xfe':
|
||||
content = raw_bytes[2:].decode('utf-16-le')
|
||||
else:
|
||||
content = raw_bytes.decode('utf-8', errors='ignore')
|
||||
|
||||
content = content.encode('utf-8').decode('utf-8')
|
||||
soup = BeautifulSoup(content, 'html.parser')
|
||||
|
||||
result = {}
|
||||
|
||||
pattern = r'<td[^>]*>([^<]+):</td>\s*<td[^>]*><b>([^<]*)</b></td>'
|
||||
matches = re.findall(pattern, content)
|
||||
|
||||
plain_pattern = r'<td[^>]*nowrap[^>]*>([^<]+):</td>\s*<td[^>]*colspan=.10.[^>]*><b>([^<]*)</b></td>'
|
||||
plain_matches = re.findall(plain_pattern, content)
|
||||
|
||||
all_matches = dict(plain_matches)
|
||||
for k, v in matches:
|
||||
if k not in all_matches:
|
||||
all_matches[k] = v
|
||||
|
||||
for key, value in all_matches.items():
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
|
||||
if 'Net Profit' in key or '\u603b\u51c0\u76c8\u5229' in key:
|
||||
result['net_profit'] = self._extract_number(value)
|
||||
elif 'Gross Profit' in key or '\u6bdb\u5229' in key:
|
||||
result['gross_profit'] = self._extract_number(value)
|
||||
elif 'Gross Loss' in key or '\u6bdb\u635f' in key:
|
||||
result['gross_loss'] = self._extract_number(value)
|
||||
elif 'Profit Factor' in key or '\u76c8\u5229\u56e0\u5b50' in key:
|
||||
result['profit_factor'] = self._extract_number(value)
|
||||
elif 'Total Trades' in key or '\u4ea4\u6613\u603b\u8ba1' in key:
|
||||
result['total_trades'] = self._extract_number(value)
|
||||
elif 'Sharpe Ratio' in key or '\u590f\u666e\u6bd4\u7387' in key:
|
||||
result['sharpe_ratio'] = self._extract_number(value)
|
||||
elif 'Maximal Drawdown' in key or '\u6700\u5927\u7ed3\u4f59\u4e8f\u635f' in key:
|
||||
result['max_drawdown'] = self._extract_number(value)
|
||||
elif 'Win Rate' in key or '\u76c8\u5229\u4ea4\u6613' in key:
|
||||
result['win_rate'] = self._extract_percentage(value)
|
||||
|
||||
return result if result else None
|
||||
|
||||
except Exception as e:
|
||||
print('Error parsing ' + report_path + ': ' + str(e))
|
||||
return None
|
||||
|
||||
def _extract_number(self, text: str) -> float:
|
||||
numbers = re.findall(r'[-+]?\d*\.?\d+', text.replace(',', '').replace(' ', ''))
|
||||
if numbers:
|
||||
try:
|
||||
return float(numbers[0])
|
||||
except:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
def _extract_percentage(self, text: str) -> float:
|
||||
numbers = re.findall(r'\d+\.?\d*%', text)
|
||||
if numbers:
|
||||
try:
|
||||
return float(numbers[0].replace('%', ''))
|
||||
except:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
def parse_mt5_xml_optimization(self, xml_path: str) -> List[Dict]:
|
||||
if not os.path.exists(xml_path):
|
||||
return []
|
||||
|
||||
results = []
|
||||
try:
|
||||
tree = ET.parse(xml_path)
|
||||
root = tree.getroot()
|
||||
|
||||
ns = {'ss': 'urn:schemas-microsoft-com:office:spreadsheet'}
|
||||
worksheet = root.find('.//ss:Worksheet', ns)
|
||||
|
||||
if worksheet is None:
|
||||
return []
|
||||
|
||||
table = worksheet.find('ss:Table', ns)
|
||||
if table is None:
|
||||
return []
|
||||
|
||||
rows = table.findall('ss:Row', ns)
|
||||
if len(rows) < 2:
|
||||
return []
|
||||
|
||||
header_row = rows[0]
|
||||
headers = []
|
||||
for cell in header_row.findall('ss:Cell', ns):
|
||||
data = cell.find('ss:Data', ns)
|
||||
if data is not None and data.text:
|
||||
headers.append(data.text.strip().lower())
|
||||
|
||||
for row_idx, row in enumerate(rows[1:], start=1):
|
||||
cells = row.findall('ss:Cell', ns)
|
||||
if not cells:
|
||||
continue
|
||||
|
||||
result = {'pass': row_idx}
|
||||
for col_idx, cell in enumerate(cells):
|
||||
if col_idx >= len(headers):
|
||||
break
|
||||
header = headers[col_idx]
|
||||
data = cell.find('ss:Data', ns)
|
||||
value = data.text.strip() if data is not None and data.text else ''
|
||||
|
||||
if 'pass' in header:
|
||||
result['pass'] = int(value) if value.isdigit() else row_idx
|
||||
elif 'result' in header:
|
||||
result['result'] = self._extract_number(value)
|
||||
elif 'profit' in header and 'expected' not in header and 'factor' not in header and 'drawdown' not in header:
|
||||
result['net_profit'] = self._extract_number(value)
|
||||
elif 'expected payoff' in header:
|
||||
result['expected_payoff'] = self._extract_number(value)
|
||||
elif 'profit factor' in header:
|
||||
if value:
|
||||
result['profit_factor'] = self._extract_number(value)
|
||||
elif 'recovery factor' in header:
|
||||
result['recovery_factor'] = self._extract_number(value)
|
||||
elif 'sharpe ratio' in header:
|
||||
result['sharpe_ratio'] = self._extract_number(value)
|
||||
elif 'custom' in header:
|
||||
result['custom'] = self._extract_number(value)
|
||||
elif 'equity dd' in header or 'drawdown' in header:
|
||||
result['max_drawdown'] = self._extract_number(value)
|
||||
elif 'trade' in header:
|
||||
result['total_trades'] = int(self._extract_number(value))
|
||||
|
||||
if result:
|
||||
results.append(result)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
print('Error parsing MT5 XML ' + xml_path + ': ' + str(e))
|
||||
traceback.print_exc()
|
||||
|
||||
return results
|
||||
|
||||
def copy_xml_results_to_reports(self, mt5_terminal_dir: str, ea_name: str, reports_dir: str) -> str:
|
||||
xml_patterns = [
|
||||
f'{ea_name}_optimization.xml',
|
||||
f'{ea_name}_optimization[1].xml',
|
||||
f'{ea_name}_optimization[2].xml',
|
||||
]
|
||||
|
||||
os.makedirs(reports_dir, exist_ok=True)
|
||||
latest_xml = None
|
||||
latest_time = 0
|
||||
|
||||
for pattern in xml_patterns:
|
||||
xml_path = os.path.join(mt5_terminal_dir, pattern)
|
||||
if os.path.exists(xml_path):
|
||||
mtime = os.path.getmtime(xml_path)
|
||||
if mtime > latest_time:
|
||||
latest_time = mtime
|
||||
latest_xml = xml_path
|
||||
|
||||
if latest_xml:
|
||||
dest_path = os.path.join(reports_dir, os.path.basename(latest_xml))
|
||||
shutil.copy2(latest_xml, dest_path)
|
||||
return dest_path
|
||||
|
||||
return None
|
||||
|
||||
def parse_all_reports(self, pattern: str = '*.htm*') -> List[Dict]:
|
||||
results = []
|
||||
search_path = os.path.join(self.reports_dir, pattern)
|
||||
report_files = glob.glob(search_path)
|
||||
|
||||
for report_file in report_files:
|
||||
parsed = self.parse_mt5_html_report(report_file)
|
||||
if parsed:
|
||||
parsed['report_file'] = os.path.basename(report_file)
|
||||
results.append(parsed)
|
||||
|
||||
return results
|
||||
|
||||
def merge_parameters_and_results(self, param_mapping: List[Dict], results: List[Dict]) -> List[Dict]:
|
||||
merged = []
|
||||
result_map = {r.get('report_file', ''): r for r in results}
|
||||
|
||||
for param_set in param_mapping:
|
||||
report_name = param_set.get('report_name', '')
|
||||
if report_name in result_map:
|
||||
combined = {**param_set, **result_map[report_name]}
|
||||
merged.append(combined)
|
||||
|
||||
return merged
|
||||
|
||||
def find_optimal_params(self, results: List[Dict],
|
||||
criterion: str = 'profit_factor',
|
||||
min_trades: int = 10,
|
||||
max_drawdown_pct: float = 50.0) -> List[Dict]:
|
||||
filtered = []
|
||||
for r in results:
|
||||
trades = r.get('total_trades', 0)
|
||||
dd = r.get('max_drawdown', 0)
|
||||
pf = r.get('profit_factor', 0)
|
||||
|
||||
if trades >= min_trades and dd <= max_drawdown_pct and pf > 0:
|
||||
filtered.append(r)
|
||||
|
||||
filtered.sort(key=lambda x: x.get(criterion, 0), reverse=True)
|
||||
return filtered
|
||||
|
||||
def generate_report(self, results: List[Dict], output_path: str = 'results/optimization_report.txt'):
|
||||
if not results:
|
||||
return
|
||||
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
||||
lines = [
|
||||
'=' * 60,
|
||||
'MT5 EA Optimization Report',
|
||||
'=' * 60,
|
||||
'',
|
||||
'Total Results: ' + str(len(results)),
|
||||
'',
|
||||
]
|
||||
|
||||
if results:
|
||||
best = results[0]
|
||||
lines.extend([
|
||||
'Best Configuration:',
|
||||
' Profit Factor: ' + str(best.get('profit_factor', 0)),
|
||||
' Net Profit: ' + str(best.get('net_profit', 0)),
|
||||
' Total Trades: ' + str(best.get('total_trades', 0)),
|
||||
' Max Drawdown: ' + str(best.get('max_drawdown', 0)),
|
||||
' Win Rate: ' + str(best.get('win_rate', 0)) + '%',
|
||||
'',
|
||||
])
|
||||
|
||||
lines.append('Top 10 Configurations:')
|
||||
lines.append('-' * 60)
|
||||
for i, r in enumerate(results[:10], 1):
|
||||
lines.append(
|
||||
str(i) + '. PF=' + str(r.get('profit_factor', 0)) +
|
||||
' Net=' + str(r.get('net_profit', 0)) +
|
||||
' Trades=' + str(r.get('total_trades', 0))
|
||||
)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
return output_path
|
||||
|
||||
def analyze_walk_forward(self, mt5_terminal_dir: str, ea_name: str, reports_dir: str) -> Dict:
|
||||
result = {
|
||||
'ea_name': ea_name,
|
||||
'in_sample': {},
|
||||
'out_of_sample': {},
|
||||
'overfitting_score': 0,
|
||||
'verdict': 'Unknown'
|
||||
}
|
||||
|
||||
xml_path = self.copy_xml_results_to_reports(mt5_terminal_dir, ea_name, reports_dir)
|
||||
|
||||
if xml_path:
|
||||
is_results = self.parse_mt5_xml_optimization(xml_path)
|
||||
if is_results:
|
||||
best_is = sorted(is_results, key=lambda x: x.get('profit_factor', 0), reverse=True)[0]
|
||||
result['in_sample'] = best_is
|
||||
|
||||
forward_xml = os.path.join(mt5_terminal_dir, f'{ea_name}_optimization.forward.xml')
|
||||
if not os.path.exists(forward_xml):
|
||||
forward_xml = os.path.join(mt5_terminal_dir, f'{ea_name}_optimization[1].forward.xml')
|
||||
|
||||
if os.path.exists(forward_xml):
|
||||
oos_results = self.parse_mt5_xml_optimization(forward_xml)
|
||||
if oos_results:
|
||||
best_oos = sorted(oos_results, key=lambda x: x.get('profit_factor', 0), reverse=True)[0]
|
||||
result['out_of_sample'] = best_oos
|
||||
|
||||
is_pf = result.get('in_sample', {}).get('profit_factor', 0)
|
||||
oos_pf = result.get('out_of_sample', {}).get('profit_factor', 0)
|
||||
|
||||
if is_pf > 0 and oos_pf > 0:
|
||||
decay = (is_pf - oos_pf) / is_pf
|
||||
result['pf_decay'] = round(decay * 100, 1)
|
||||
if oos_pf >= is_pf * 0.7 and oos_pf > 1.0:
|
||||
result['verdict'] = 'Robust (\u7a33\u5065)'
|
||||
elif oos_pf > 1.0:
|
||||
result['verdict'] = 'Mild Overfit (\u8f7b\u5ea6\u8fc7\u62df\u5408)'
|
||||
else:
|
||||
result['verdict'] = 'Overfit (\u4e25\u91cd\u8fc7\u62df\u5408)'
|
||||
elif oos_pf > 0:
|
||||
result['pf_decay'] = 0
|
||||
result['verdict'] = 'Valid (\u6709\u6548\u4f46IS\u65e0\u7ed3\u679c)'
|
||||
|
||||
is_dd = result.get('in_sample', {}).get('max_drawdown', 0)
|
||||
oos_dd = result.get('out_of_sample', {}).get('max_drawdown', 0)
|
||||
if is_dd > 0 and oos_dd > 0:
|
||||
result['dd_increase'] = round((oos_dd / is_dd - 1) * 100, 1) if is_dd > 0 else 0
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,243 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPTIMIZER_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, OPTIMIZER_DIR)
|
||||
|
||||
from ea_config_parser import load_all_ea_configs, EAConfig
|
||||
from smart_search import SmartSearch
|
||||
from parameter_constraint import create_constraint_engine
|
||||
from analyze_results import ResultAnalyzer
|
||||
|
||||
|
||||
class BatchAutoOptimizer:
|
||||
def __init__(self, base_dir: str = "optimizer", mt5_path: str = None):
|
||||
self.base_dir = base_dir
|
||||
self.mt5_path = mt5_path
|
||||
self.configs_dir = os.path.join(base_dir, "configs")
|
||||
self.set_files_dir = os.path.join(base_dir, "set_files")
|
||||
self.reports_dir = os.path.join(base_dir, "reports")
|
||||
self.results_dir = os.path.join(base_dir, "results")
|
||||
self.scripts_dir = os.path.join(base_dir, "scripts")
|
||||
|
||||
for d in [self.configs_dir, self.set_files_dir, self.reports_dir,
|
||||
self.results_dir, self.scripts_dir]:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
def generate_set_files(self, ea_config: EAConfig, max_samples: int = 1000) -> List[Dict]:
|
||||
logger.info(f"Generating SET files for {ea_config.ea_name}")
|
||||
logger.info(f" Combinations: {ea_config.get_total_combinations()}")
|
||||
|
||||
searcher = SmartSearch(ea_config)
|
||||
combinations = searcher.generate_combinations(max_samples)
|
||||
logger.info(f" Generated {len(combinations)} samples")
|
||||
|
||||
engine = create_constraint_engine(ea_config)
|
||||
valid_combos = engine.filter_valid_combinations(combinations)
|
||||
logger.info(f" Valid combinations: {len(valid_combos)}")
|
||||
|
||||
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
||||
os.makedirs(ea_set_dir, exist_ok=True)
|
||||
|
||||
param_mapping = []
|
||||
for i, combo in enumerate(valid_combos, 1):
|
||||
set_id = str(i).zfill(8)
|
||||
set_filename = f"params_{set_id}.set"
|
||||
set_path = os.path.join(ea_set_dir, set_filename)
|
||||
|
||||
self._write_set_file(set_path, combo)
|
||||
|
||||
param_mapping.append({
|
||||
'set_id': set_id,
|
||||
'set_file': set_filename,
|
||||
'set_path': set_path,
|
||||
'report_name': f"{ea_config.ea_name}_{set_id}.htm",
|
||||
'params': combo
|
||||
})
|
||||
|
||||
mapping_path = os.path.join(self.results_dir, f"{ea_config.ea_name}_param_mapping.json")
|
||||
with open(mapping_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(param_mapping, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f" SET files saved to {ea_set_dir}")
|
||||
return param_mapping
|
||||
|
||||
def _write_set_file(self, set_path: str, params: Dict):
|
||||
lines = ["; MT5 EA Parameters", f"; Generated: {datetime.now()}", ""]
|
||||
for name, value in params.items():
|
||||
if isinstance(value, bool):
|
||||
lines.append(f"{name} <true> <{'true' if value else 'false'}>")
|
||||
else:
|
||||
lines.append(f"{name} <{value}> <{value}>")
|
||||
with open(set_path, 'w', encoding='utf-8') as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
def generate_ini(self, ea_config: EAConfig) -> str:
|
||||
test_config = ea_config.test_config
|
||||
ini_path = os.path.join(self.configs_dir, f"{ea_config.ea_name}.ini")
|
||||
|
||||
date_from = test_config.get("from_date", "2025.01.01")
|
||||
date_to = test_config.get("to_date", "2025.12.31")
|
||||
|
||||
lines = [
|
||||
"; MT5 Strategy Tester Configuration",
|
||||
f"; Generated: {datetime.now()}",
|
||||
"",
|
||||
"[Tester]",
|
||||
f"Expert={ea_config.ea_path}",
|
||||
"ExpertParameters=",
|
||||
f"Symbol={test_config.get('symbol', 'EURUSD')}",
|
||||
f"Period={test_config.get('period', 'H1')}",
|
||||
f"Model={test_config.get('model', 1)}",
|
||||
f"FromDate={date_from}",
|
||||
f"ToDate={date_to}",
|
||||
f"Deposit={test_config.get('deposit', 10000)}",
|
||||
f"Leverage={test_config.get('leverage', '1:100')}",
|
||||
"ReplaceReport=true",
|
||||
"ShutdownTerminal=true",
|
||||
]
|
||||
|
||||
with open(ini_path, 'w', encoding='utf-8') as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
logger.info(f"INI saved: {ini_path}")
|
||||
return ini_path
|
||||
|
||||
def generate_powershell_script(self, ea_config: EAConfig, param_mapping: List[Dict]) -> str:
|
||||
ps_path = os.path.join(self.scripts_dir, f"run_{ea_config.ea_name}.ps1")
|
||||
|
||||
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
||||
ea_report_dir = os.path.join(self.reports_dir, ea_config.ea_name)
|
||||
os.makedirs(ea_report_dir, exist_ok=True)
|
||||
|
||||
ini_path = os.path.join(self.configs_dir, f"{ea_config.ea_name}.ini")
|
||||
|
||||
lines = [
|
||||
f"# MT5 Batch Optimization for {ea_config.ea_name}",
|
||||
f"# Generated: {datetime.now()}",
|
||||
"",
|
||||
f'$MT5Path = "{self.mt5_path or "terminal64.exe"}"',
|
||||
f'$IniFile = "{ini_path}"',
|
||||
f'$SetDir = "{ea_set_dir}"',
|
||||
"",
|
||||
f"Write-Host 'Starting batch optimization for {ea_config.ea_name}'",
|
||||
"",
|
||||
"$SetFiles = Get-ChildItem -Path $SetDir -Filter 'params_*.set'",
|
||||
"$Total = $SetFiles.Count",
|
||||
"$Current = 0",
|
||||
"",
|
||||
"foreach ($SetFile in $SetFiles) {",
|
||||
" $Current++",
|
||||
' Write-Host "[$Current/$Total] $($SetFile.Name)"',
|
||||
"",
|
||||
" $IniContent = Get-Content $IniFile",
|
||||
' $IniContent = $IniContent -replace "ExpertParameters=.*", "ExpertParameters=$($SetFile.FullName)"',
|
||||
' $TempIni = Join-Path $env:TEMP "temp_$([guid]::NewGuid().ToString().Substring(0,8)).ini"',
|
||||
" $IniContent | Set-Content $TempIni -Encoding UTF8",
|
||||
"",
|
||||
" Start-Process -FilePath $MT5Path -ArgumentList '/portable',\"/config:$TempIni\" -Wait",
|
||||
" Start-Sleep -Seconds 3",
|
||||
"}",
|
||||
"",
|
||||
'Write-Host "Batch optimization completed!"',
|
||||
]
|
||||
|
||||
with open(ps_path, 'w', encoding='utf-8') as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
logger.info(f"PowerShell script: {ps_path}")
|
||||
return ps_path
|
||||
|
||||
def run_full_auto(self, ea_names: List[str] = None, max_samples: int = 1000):
|
||||
logger.info("=" * 60)
|
||||
logger.info("MT5 Batch Auto Optimizer")
|
||||
logger.info("=" * 60)
|
||||
|
||||
configs = load_all_ea_configs(self.configs_dir)
|
||||
if not configs:
|
||||
logger.error(f"No configs found in {self.configs_dir}")
|
||||
return
|
||||
|
||||
if ea_names:
|
||||
configs = {k: v for k, v in configs.items() if k in ea_names}
|
||||
|
||||
for ea_name, ea_config in configs.items():
|
||||
logger.info(f"\nProcessing: {ea_name}")
|
||||
|
||||
param_mapping = self.generate_set_files(ea_config, max_samples)
|
||||
self.generate_ini(ea_config)
|
||||
self.generate_powershell_script(ea_config, param_mapping)
|
||||
|
||||
logger.info(f" SET files: {len(param_mapping)}")
|
||||
logger.info(f" Next: Run the PS1 script in MT5 terminal")
|
||||
|
||||
def analyze_all(self, ea_names: List[str] = None) -> List[Dict]:
|
||||
logger.info("Analyzing results...")
|
||||
|
||||
all_results = []
|
||||
search_dirs = [self.reports_dir] if not ea_names else [os.path.join(self.reports_dir, name) for name in ea_names]
|
||||
|
||||
for search_dir in search_dirs:
|
||||
if not os.path.exists(search_dir):
|
||||
continue
|
||||
|
||||
for ea_name in os.listdir(search_dir):
|
||||
ea_report_dir = os.path.join(search_dir, ea_name)
|
||||
if not os.path.isdir(ea_report_dir):
|
||||
continue
|
||||
|
||||
analyzer = ResultAnalyzer(ea_report_dir)
|
||||
results = analyzer.parse_all_reports()
|
||||
|
||||
for r in results:
|
||||
r['ea_name'] = ea_name
|
||||
|
||||
all_results.extend(results)
|
||||
logger.info(f" {ea_name}: {len(results)} reports")
|
||||
|
||||
if not all_results:
|
||||
logger.warning("No results found")
|
||||
return []
|
||||
|
||||
all_results.sort(key=lambda x: x.get('profit_factor', 0), reverse=True)
|
||||
|
||||
output_path = os.path.join(self.results_dir, "optimization_results.json")
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.info(f"Results: {len(all_results)} total")
|
||||
logger.info(f"Best PF: {all_results[0].get('profit_factor', 0) if all_results else 'N/A'}")
|
||||
|
||||
return all_results
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="MT5 Batch Auto Optimizer")
|
||||
parser.add_argument("--base-dir", "-d", default="optimizer", help="Base directory")
|
||||
parser.add_argument("--ea", "-e", nargs="+", help="EA names")
|
||||
parser.add_argument("--max-samples", "-m", type=int, default=1000, help="Max samples per EA")
|
||||
parser.add_argument("--mt5-path", "-p", help="MT5 terminal path")
|
||||
parser.add_argument("--analyze", "-a", action="store_true", help="Analyze only")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
optimizer = BatchAutoOptimizer(args.base_dir, args.mt5_path)
|
||||
|
||||
if args.analyze:
|
||||
optimizer.analyze_all(args.ea)
|
||||
else:
|
||||
optimizer.run_full_auto(args.ea, args.max_samples)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"ea_name": "Grid Pro",
|
||||
"ea_path": "Experts\\\\Grid\\\\GridPro.ex5",
|
||||
"description": "Multi-period grid EA with genetic optimization",
|
||||
"search_strategy": "genetic",
|
||||
"optimization_criterion": "profit_factor",
|
||||
"parameters": {
|
||||
"GridLevels": {
|
||||
"type": "int",
|
||||
"default": 5,
|
||||
"min": 3,
|
||||
"max": 20,
|
||||
"step": 1,
|
||||
"description": "Grid levels"
|
||||
},
|
||||
"GridSpacing": {
|
||||
"type": "int",
|
||||
"default": 50,
|
||||
"min": 10,
|
||||
"max": 200,
|
||||
"step": 10,
|
||||
"description": "Grid spacing in points"
|
||||
},
|
||||
"GridSpacingMult": {
|
||||
"type": "double",
|
||||
"default": 1.5,
|
||||
"min": 1.0,
|
||||
"max": 3.0,
|
||||
"step": 0.1,
|
||||
"precision": 1,
|
||||
"description": "Spacing multiplier"
|
||||
},
|
||||
"BaseLotsize": {
|
||||
"type": "double",
|
||||
"default": 0.1,
|
||||
"min": 0.01,
|
||||
"max": 1.0,
|
||||
"step": 0.01,
|
||||
"precision": 2,
|
||||
"description": "Base lot size"
|
||||
},
|
||||
"MaxOpenLots": {
|
||||
"type": "double",
|
||||
"default": 5.0,
|
||||
"min": 0.1,
|
||||
"max": 20.0,
|
||||
"step": 0.1,
|
||||
"precision": 1,
|
||||
"description": "Max open lots"
|
||||
},
|
||||
"MaxDrawdown": {
|
||||
"type": "int",
|
||||
"default": 30,
|
||||
"min": 10,
|
||||
"max": 100,
|
||||
"step": 5,
|
||||
"description": "Max drawdown percentage"
|
||||
},
|
||||
"TakeProfitGrid": {
|
||||
"type": "int",
|
||||
"default": 20,
|
||||
"min": 5,
|
||||
"max": 100,
|
||||
"step": 5,
|
||||
"description": "Take profit grid"
|
||||
},
|
||||
"TradingMode": {
|
||||
"type": "enum",
|
||||
"default": "BOTH",
|
||||
"options": [
|
||||
"BUY",
|
||||
"SELL",
|
||||
"BOTH"
|
||||
],
|
||||
"description": "Trading mode"
|
||||
}
|
||||
},
|
||||
"test_config": {
|
||||
"symbol": "EURUSD",
|
||||
"period": "H1",
|
||||
"from_date": "2024.01.01",
|
||||
"to_date": "2025.12.31",
|
||||
"model": 1,
|
||||
"deposit": 5000,
|
||||
"leverage": "1:100"
|
||||
},
|
||||
"walk_forward": {
|
||||
"enabled": true,
|
||||
"train_test_ratio": 0.75
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"ea_name": "MA Cross Scalper",
|
||||
"ea_path": "Experts\\\\MA_Cross.ex5",
|
||||
"description": "Dual MA crossover scalping strategy",
|
||||
"search_strategy": "grid",
|
||||
"optimization_criterion": "profit_factor",
|
||||
"parameters": {
|
||||
"FastMA_Period": {
|
||||
"type": "int",
|
||||
"default": 14,
|
||||
"min": 5,
|
||||
"max": 30,
|
||||
"step": 1,
|
||||
"description": "Fast MA period"
|
||||
},
|
||||
"SlowMA_Period": {
|
||||
"type": "int",
|
||||
"default": 50,
|
||||
"min": 20,
|
||||
"max": 100,
|
||||
"step": 5,
|
||||
"description": "Slow MA period",
|
||||
"condition": ">FastMA_Period"
|
||||
},
|
||||
"StopLoss": {
|
||||
"type": "int",
|
||||
"default": 50,
|
||||
"min": 10,
|
||||
"max": 200,
|
||||
"step": 10,
|
||||
"description": "Stop loss in points"
|
||||
},
|
||||
"TakeProfit": {
|
||||
"type": "int",
|
||||
"default": 100,
|
||||
"min": 30,
|
||||
"max": 300,
|
||||
"step": 10,
|
||||
"description": "Take profit in points"
|
||||
},
|
||||
"RiskPercent": {
|
||||
"type": "double",
|
||||
"default": 1.0,
|
||||
"min": 0.5,
|
||||
"max": 3.0,
|
||||
"step": 0.5,
|
||||
"precision": 1,
|
||||
"description": "Risk percentage"
|
||||
},
|
||||
"EnableMM": {
|
||||
"type": "bool",
|
||||
"default": true,
|
||||
"description": "Enable money management"
|
||||
}
|
||||
},
|
||||
"test_config": {
|
||||
"symbol": "EURUSD",
|
||||
"period": "H1",
|
||||
"from_date": "2025.01.01",
|
||||
"to_date": "2025.12.31",
|
||||
"model": 1,
|
||||
"deposit": 10000,
|
||||
"leverage": "1:100"
|
||||
},
|
||||
"walk_forward": {
|
||||
"enabled": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import os
|
||||
import glob
|
||||
import re
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
class EAAutoScanner:
|
||||
def __init__(self, mt5_data_dir: str = None):
|
||||
self.mt5_data_dir = mt5_data_dir
|
||||
self.experts_dir = os.path.join(mt5_data_dir, "MQL5", "Experts") if mt5_data_dir else None
|
||||
|
||||
def scan_experts(self) -> List[Dict]:
|
||||
experts = []
|
||||
if not self.experts_dir or not os.path.exists(self.experts_dir):
|
||||
return experts
|
||||
for root, dirs, files in os.walk(self.experts_dir):
|
||||
for f in files:
|
||||
if f.endswith('.ex5'):
|
||||
full_path = os.path.join(root, f)
|
||||
rel_path = os.path.relpath(full_path, self.experts_dir)
|
||||
ea_name = os.path.splitext(f)[0]
|
||||
experts.append({'name': ea_name, 'filename': f, 'path': rel_path, 'full_path': full_path})
|
||||
return experts
|
||||
|
||||
def generate_json_config(self, ea_info: Dict, params: Dict = None) -> Dict:
|
||||
ea_name = ea_info["name"]
|
||||
config = {
|
||||
"ea_name": ea_name,
|
||||
"ea_path": ea_info["path"],
|
||||
"description": "Auto-scanned EA",
|
||||
"search_strategy": "auto",
|
||||
"optimization_criterion": "profit_factor",
|
||||
"parameters": {},
|
||||
"test_config": {
|
||||
"symbol": "EURUSD", "period": "H1",
|
||||
"from_date": "2025.01.01", "to_date": "2025.12.31",
|
||||
"model": 1, "deposit": 10000, "leverage": "1:100"
|
||||
},
|
||||
"walk_forward": {"enabled": False}
|
||||
}
|
||||
if params:
|
||||
for pname, pinfo in params.items():
|
||||
config["parameters"][pname] = {
|
||||
"type": pinfo.get("type", "int"),
|
||||
"default": pinfo.get("default", 0),
|
||||
"min": pinfo.get("min", 0),
|
||||
"max": pinfo.get("max", 100),
|
||||
"step": pinfo.get("step", 1),
|
||||
"description": "Auto-scanned"
|
||||
}
|
||||
return config
|
||||
|
||||
def auto_scan_all(self) -> List[Dict]:
|
||||
configs = []
|
||||
experts = self.scan_experts()
|
||||
for expert in experts:
|
||||
config = self.generate_json_config(expert)
|
||||
configs.append(config)
|
||||
return configs
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data-dir", "-d")
|
||||
parser.add_argument("--output", "-o", default="configs")
|
||||
parser.add_argument("--list", "-l", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
scanner = EAAutoScanner(args.data_dir)
|
||||
experts = scanner.scan_experts()
|
||||
|
||||
if args.list:
|
||||
print(f"Found {len(experts)} EAs:")
|
||||
for e in experts:
|
||||
print(f" - {e['name']}")
|
||||
|
||||
configs = scanner.auto_scan_all()
|
||||
print(f"Generated {len(configs)} configs")
|
||||
|
||||
if args.output and configs:
|
||||
import json
|
||||
import os
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
for cfg in configs:
|
||||
fname = cfg["ea_name"].replace(" ", "_") + ".json"
|
||||
with open(os.path.join(args.output, fname), "w", encoding="utf-8") as f:
|
||||
json.dump(cfg, f, indent=4, ensure_ascii=False)
|
||||
print(f"Saved: {fname}")
|
||||
@@ -0,0 +1,328 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from ea_config_parser import load_ea_config, load_all_ea_configs, EAConfig
|
||||
from parameter_constraint import create_constraint_engine
|
||||
from smart_search import SmartSearch
|
||||
from analyze_results import ResultAnalyzer
|
||||
from mt5_paths import auto_detect_data_dir
|
||||
|
||||
|
||||
class EABatchOptimizer:
|
||||
def __init__(self, base_dir: str = 'optimizer'):
|
||||
self.base_dir = base_dir
|
||||
self.configs_dir = os.path.join(base_dir, 'configs')
|
||||
self.set_files_dir = os.path.join(base_dir, 'set_files')
|
||||
self.reports_dir = os.path.join(base_dir, 'reports')
|
||||
self.results_dir = os.path.join(base_dir, 'results')
|
||||
self.scripts_dir = os.path.join(base_dir, 'scripts')
|
||||
self.logs_dir = os.path.join(base_dir, 'logs')
|
||||
|
||||
for d in [self.configs_dir, self.set_files_dir, self.reports_dir,
|
||||
self.results_dir, self.scripts_dir, self.logs_dir]:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
def generate_set_files(self, ea_config: EAConfig, max_samples: int = 2000) -> List[Dict]:
|
||||
print('Generating SET files for ' + ea_config.ea_name)
|
||||
print(' Total combinations: ' + str(ea_config.get_total_combinations()))
|
||||
print(' Strategy: ' + SmartSearch(ea_config).select_strategy())
|
||||
|
||||
searcher = SmartSearch(ea_config)
|
||||
combinations = searcher.generate_combinations(max_samples)
|
||||
|
||||
engine = create_constraint_engine(ea_config)
|
||||
valid_combinations = engine.filter_valid_combinations(combinations)
|
||||
|
||||
print(' Valid combinations: ' + str(len(valid_combinations)))
|
||||
|
||||
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
||||
os.makedirs(ea_set_dir, exist_ok=True)
|
||||
|
||||
param_mapping = []
|
||||
for i, combo in enumerate(valid_combinations, 1):
|
||||
set_id = str(i).zfill(8)
|
||||
set_filename = 'params_' + set_id + '.set'
|
||||
set_path = os.path.join(ea_set_dir, set_filename)
|
||||
|
||||
self._write_set_file(set_path, combo)
|
||||
|
||||
mapping = {
|
||||
'set_id': set_id,
|
||||
'set_file': set_filename,
|
||||
'report_name': ea_config.ea_name + '_' + set_id + '.htm',
|
||||
'params': combo
|
||||
}
|
||||
param_mapping.append(mapping)
|
||||
|
||||
mapping_path = os.path.join(self.results_dir, ea_config.ea_name + '_param_mapping.json')
|
||||
with open(mapping_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(param_mapping, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(' SET files generated: ' + str(len(valid_combinations)))
|
||||
return param_mapping
|
||||
|
||||
def _write_set_file(self, set_path: str, params: Dict):
|
||||
lines = ['; MT5 EA Parameters SET File', '; Generated: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '']
|
||||
|
||||
for name, value in params.items():
|
||||
if isinstance(value, bool):
|
||||
lines.append(name + ' <true> <' + ('true' if value else 'false') + '>')
|
||||
elif isinstance(value, float):
|
||||
lines.append(name + ' <' + str(value) + '> <' + str(value) + '>')
|
||||
else:
|
||||
lines.append(name + ' <' + str(value) + '> <' + str(value) + '>')
|
||||
|
||||
with open(set_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
def _save_param_mapping(self, mapping: List[Dict], output_path: str):
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write('set_id,set_file,report_name\n')
|
||||
for m in mapping:
|
||||
f.write(m['set_id'] + ',' + m['set_file'] + ',' + m['report_name'] + '\n')
|
||||
|
||||
def generate_ini_for_ea(self, ea_config: EAConfig, param_mapping: List[Dict] = None):
|
||||
test_config = ea_config.test_config
|
||||
ini_filename = ea_config.ea_name + '.ini'
|
||||
ini_path = os.path.join(self.configs_dir, ini_filename)
|
||||
set_filename = ea_config.ea_name + '_optimization.set'
|
||||
|
||||
data_dir = auto_detect_data_dir() or r'C:\Users\Administrator\AppData\Roaming\MetaQuotes\Terminal\010E047102812FC0C18890992854220E'
|
||||
mt5_tester_dir = os.path.join(data_dir, 'MQL5', 'Profiles', 'Tester')
|
||||
set_path = os.path.join(mt5_tester_dir, set_filename)
|
||||
|
||||
date_from = test_config.get('from_date', '2025.01.01')
|
||||
date_to = test_config.get('to_date', '2025.12.31')
|
||||
|
||||
criterion_map = {
|
||||
'profit_factor': 1,
|
||||
'net_profit': 0,
|
||||
'sharpe_ratio': 5,
|
||||
'expected_payoff': 2,
|
||||
'drawdown': 3,
|
||||
'recovery_factor': 4,
|
||||
}
|
||||
opt_criterion = test_config.get('optimization_criterion', 1)
|
||||
|
||||
set_lines = [
|
||||
'; saved automatically',
|
||||
'; this file contains last used input parameters for testing/optimizing ' + ea_config.ea_name + ' expert advisor',
|
||||
'',
|
||||
]
|
||||
|
||||
if ea_config.parameters:
|
||||
for pname, param in ea_config.parameters.items():
|
||||
if param.param_type == 'bool':
|
||||
if not param.optimize:
|
||||
set_lines.append(pname + '=false||false||false||false||N')
|
||||
else:
|
||||
set_lines.append(pname + '=false||false||false||false||Y')
|
||||
elif param.param_type == 'int':
|
||||
start = int(param.min_value) if param.min_value is not None else 0
|
||||
step = int(param.step) if param.step is not None else 1
|
||||
stop = int(param.max_value) if param.max_value is not None else 100
|
||||
cur = int(param.default) if param.default is not None else start
|
||||
if not param.optimize or start == stop or step == 0:
|
||||
set_lines.append(pname + '=' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||N')
|
||||
else:
|
||||
set_lines.append(pname + '=' + str(cur) + '||' + str(start) + '||' + str(step) + '||' + str(stop) + '||Y')
|
||||
elif param.param_type == 'double':
|
||||
start = float(param.min_value) if param.min_value is not None else 0.0
|
||||
step = float(param.step) if param.step is not None else 0.01
|
||||
stop = float(param.max_value) if param.max_value is not None else 1.0
|
||||
cur = float(param.default) if param.default is not None else start
|
||||
if not param.optimize or start == stop or step == 0:
|
||||
set_lines.append(pname + '=' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||' + str(cur) + '||N')
|
||||
else:
|
||||
set_lines.append(pname + '=' + str(cur) + '||' + str(start) + '||' + str(step) + '||' + str(stop) + '||Y')
|
||||
|
||||
with open(set_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(set_lines))
|
||||
|
||||
ini_lines = [
|
||||
'; EA backtest',
|
||||
'[Tester]',
|
||||
'Expert=' + ea_config.ea_path,
|
||||
'ExpertParameters=' + set_filename,
|
||||
'Symbol=' + test_config.get('symbol', 'EURUSD'),
|
||||
'Period=' + test_config.get('period', 'H1'),
|
||||
'Model=' + str(test_config.get('model', 1)),
|
||||
'FromDate=' + date_from,
|
||||
'ToDate=' + date_to,
|
||||
]
|
||||
|
||||
wf = ea_config.walk_forward
|
||||
if wf.get('enabled', False):
|
||||
fm = wf.get('forward_mode', 2)
|
||||
ini_lines.append('ForwardMode=' + str(fm))
|
||||
else:
|
||||
ini_lines.append('ForwardMode=0')
|
||||
|
||||
ini_lines.extend([
|
||||
'Deposit=' + str(test_config.get('deposit', 10000)),
|
||||
'Currency=' + str(test_config.get('currency', 'USD')),
|
||||
'Leverage=' + str(test_config.get('leverage', '1:100')),
|
||||
'ExecutionMode=' + str(test_config.get('execution_delay', 0)),
|
||||
'Optimization=' + str(test_config.get('optimization_mode', 2)),
|
||||
'OptimizationCriterion=' + str(opt_criterion),
|
||||
'Report=' + ea_config.ea_name + '_optimization',
|
||||
'ReplaceReport=true',
|
||||
'ShutdownTerminal=true',
|
||||
'Visual=0',
|
||||
'',
|
||||
])
|
||||
|
||||
with open(ini_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(ini_lines))
|
||||
|
||||
print('INI saved: ' + ini_path)
|
||||
print('SET saved: ' + set_path)
|
||||
return ini_path
|
||||
|
||||
def build_powershell_script(self, ea_config: EAConfig, param_mapping: List[Dict], mt5_path: str):
|
||||
ps_filename = 'run_' + ea_config.ea_name + '.ps1'
|
||||
ps_path = os.path.join(self.scripts_dir, ps_filename)
|
||||
|
||||
ea_set_dir = os.path.join(self.set_files_dir, ea_config.ea_name)
|
||||
ea_report_dir = os.path.join(self.reports_dir, ea_config.ea_name)
|
||||
os.makedirs(ea_report_dir, exist_ok=True)
|
||||
|
||||
ini_path = os.path.join(self.configs_dir, ea_config.ea_name + '.ini')
|
||||
|
||||
lines = [
|
||||
'# MT5 Batch Backtest Script',
|
||||
'# Generated: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
'',
|
||||
f'$MT5Path = "{mt5_path}"',
|
||||
f'$IniFile = "{ini_path}"',
|
||||
f'$SetDir = "{ea_set_dir}"',
|
||||
'',
|
||||
f"Write-Host 'Starting batch optimization for {ea_config.ea_name}'",
|
||||
'',
|
||||
'$SetFiles = Get-ChildItem -Path $SetDir -Filter "params_*.set"',
|
||||
'$Total = $SetFiles.Count',
|
||||
'$Current = 0',
|
||||
'',
|
||||
'foreach ($SetFile in $SetFiles) {',
|
||||
' $Current++',
|
||||
' Write-Host "[$Current/$Total] Processing: $($SetFile.Name)"',
|
||||
'',
|
||||
' $IniContent = Get-Content $IniFile',
|
||||
' $IniContent = $IniContent -replace "ExpertParameters=.*", "ExpertParameters=$($SetFile.FullName)"',
|
||||
' $TempIni = Join-Path $env:TEMP "temp_$([guid]::NewGuid().ToString().Substring(0,8)).ini"',
|
||||
' $IniContent | Set-Content $TempIni -Encoding UTF8',
|
||||
'',
|
||||
' Start-Process -FilePath $MT5Path -ArgumentList "/config:$TempIni" -Wait',
|
||||
'',
|
||||
' Start-Sleep -Seconds 3',
|
||||
'}',
|
||||
'',
|
||||
f"Write-Host 'Batch optimization completed for {ea_config.ea_name}!'",
|
||||
]
|
||||
|
||||
with open(ps_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
print('PowerShell script: ' + ps_path)
|
||||
return ps_path
|
||||
|
||||
def run_optimization_for_ea(self, ea_config: EAConfig, mt5_path: str = None,
|
||||
max_samples: int = 2000):
|
||||
print('\n' + '=' * 60)
|
||||
print('Optimization for: ' + ea_config.ea_name)
|
||||
print('=' * 60)
|
||||
|
||||
param_mapping = self.generate_set_files(ea_config, max_samples)
|
||||
self.generate_ini_for_ea(ea_config, param_mapping)
|
||||
|
||||
if mt5_path:
|
||||
ps_path = self.build_powershell_script(ea_config, param_mapping, mt5_path)
|
||||
print('PowerShell script: ' + ps_path)
|
||||
|
||||
return param_mapping
|
||||
|
||||
def run_full_optimization(self, ea_names: List[str] = None,
|
||||
mt5_path: str = None,
|
||||
max_samples_per_ea: int = 2000):
|
||||
configs = load_all_ea_configs(self.configs_dir)
|
||||
|
||||
if ea_names:
|
||||
configs = {k: v for k, v in configs.items() if k in ea_names}
|
||||
|
||||
all_mapping = {}
|
||||
for ea_name, ea_config in configs.items():
|
||||
mapping = self.run_optimization_for_ea(ea_config, mt5_path, max_samples_per_ea)
|
||||
all_mapping[ea_name] = mapping
|
||||
|
||||
return all_mapping
|
||||
|
||||
def analyze_results(self, ea_names: List[str] = None) -> List[Dict]:
|
||||
all_results = []
|
||||
|
||||
if ea_names:
|
||||
for ea_name in ea_names:
|
||||
report_dir = os.path.join(self.reports_dir, ea_name)
|
||||
if os.path.exists(report_dir):
|
||||
analyzer = ResultAnalyzer(report_dir)
|
||||
results = analyzer.parse_all_reports()
|
||||
for r in results:
|
||||
r['ea_name'] = ea_name
|
||||
all_results.extend(results)
|
||||
else:
|
||||
for ea_name in os.listdir(self.reports_dir):
|
||||
report_dir = os.path.join(self.reports_dir, ea_name)
|
||||
if os.path.isdir(report_dir):
|
||||
analyzer = ResultAnalyzer(report_dir)
|
||||
results = analyzer.parse_all_reports()
|
||||
for r in results:
|
||||
r['ea_name'] = ea_name
|
||||
all_results.extend(results)
|
||||
|
||||
return all_results
|
||||
|
||||
def find_optimal_params(self, results: List[Dict], criterion: str = 'profit_factor',
|
||||
min_trades: int = 10) -> List[Dict]:
|
||||
filtered = [r for r in results if r.get('total_trades', 0) >= min_trades
|
||||
and r.get('profit_factor', 0) > 0]
|
||||
filtered.sort(key=lambda x: x.get(criterion, 0), reverse=True)
|
||||
return filtered
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='MT5 EA Batch Optimizer')
|
||||
parser.add_argument('--base-dir', '-d', default='optimizer', help='Base directory')
|
||||
parser.add_argument('--ea', '-e', nargs='+', help='EA names to optimize')
|
||||
parser.add_argument('--max-samples', '-m', type=int, default=2000, help='Max samples')
|
||||
parser.add_argument('--mt5-path', '-p', help='MT5 terminal path')
|
||||
parser.add_argument('--analyze-only', '-a', action='store_true', help='Only analyze results')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
optimizer = EABatchOptimizer(args.base_dir)
|
||||
|
||||
if args.analyze_only:
|
||||
results = optimizer.analyze_results(args.ea)
|
||||
if results:
|
||||
analyzer = ResultAnalyzer()
|
||||
best = optimizer.find_optimal_params(results)
|
||||
analyzer.generate_report(best)
|
||||
print('Results: ' + str(len(results)))
|
||||
print('Best: PF=' + str(best[0].get('profit_factor', 0)) if best else 'No results')
|
||||
else:
|
||||
optimizer.run_full_optimization(args.ea, args.mt5_path, args.max_samples)
|
||||
print('\nOptimization files generated. Run MT5 tests, then use --analyze-only')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,135 @@
|
||||
import os
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Any
|
||||
|
||||
@dataclass
|
||||
class ParameterDef:
|
||||
name: str
|
||||
param_type: str
|
||||
default: Any = None
|
||||
min_value: Any = None
|
||||
max_value: Any = None
|
||||
step: Any = None
|
||||
precision: int = 0
|
||||
optimize: bool = True
|
||||
enum_options: List[Any] = field(default_factory=list)
|
||||
description: str = ''
|
||||
condition: str = ''
|
||||
|
||||
def expand_values(self) -> List[Any]:
|
||||
if self.param_type == 'bool':
|
||||
return [True, False]
|
||||
elif self.param_type == 'enum':
|
||||
return self.enum_options
|
||||
elif self.param_type in ('int', 'double'):
|
||||
if self.min_value is None or self.max_value is None:
|
||||
return [self.default] if self.default is not None else []
|
||||
values = []
|
||||
current = self.min_value
|
||||
while current <= self.max_value:
|
||||
values.append(round(current, self.precision) if self.precision > 0 else int(current))
|
||||
current += self.step
|
||||
return values
|
||||
return [self.default]
|
||||
|
||||
def is_valid_value(self, value: Any) -> bool:
|
||||
if self.param_type == 'bool':
|
||||
return isinstance(value, bool)
|
||||
elif self.param_type == 'enum':
|
||||
return value in self.enum_options
|
||||
elif self.param_type in ('int', 'double'):
|
||||
if self.min_value is not None and value < self.min_value:
|
||||
return False
|
||||
if self.max_value is not None and value > self.max_value:
|
||||
return False
|
||||
return True
|
||||
return True
|
||||
|
||||
@dataclass
|
||||
class EAConfig:
|
||||
ea_name: str
|
||||
ea_path: str
|
||||
parameters: Dict[str, ParameterDef] = field(default_factory=dict)
|
||||
search_strategy: str = 'auto'
|
||||
optimization_criterion: str = 'profit_factor'
|
||||
test_config: Dict[str, Any] = field(default_factory=dict)
|
||||
walk_forward: Dict[str, Any] = field(default_factory=dict)
|
||||
description: str = ''
|
||||
|
||||
def get_total_combinations(self) -> int:
|
||||
total = 1
|
||||
for param in self.parameters.values():
|
||||
values = param.expand_values()
|
||||
total *= len(values) if values else 1
|
||||
return total
|
||||
|
||||
def estimate_search_time(self, tests_per_minute: float = 10) -> str:
|
||||
total = self.get_total_combinations()
|
||||
minutes = total / tests_per_minute
|
||||
if minutes < 60:
|
||||
return str(round(minutes, 1)) + ' minutes'
|
||||
elif minutes < 1440:
|
||||
return str(round(minutes/60, 1)) + ' hours'
|
||||
else:
|
||||
return str(round(minutes/1440, 1)) + ' days'
|
||||
|
||||
def is_complex(self) -> bool:
|
||||
return len(self.parameters) > 6 or self.get_total_combinations() > 1_000_000
|
||||
|
||||
def load_ea_config(config_path: str) -> EAConfig:
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
config_data = json.load(f)
|
||||
ea_name = config_data.get('ea_name', 'Unknown')
|
||||
ea_path = config_data.get('ea_path', '')
|
||||
description = config_data.get('description', '')
|
||||
search_strategy = config_data.get('search_strategy', 'auto')
|
||||
criterion = config_data.get('optimization_criterion', 'profit_factor')
|
||||
test_config = config_data.get('test_config', {})
|
||||
walk_forward = config_data.get('walk_forward', {})
|
||||
parameters = {}
|
||||
for param_name, param_data in config_data.get('parameters', {}).items():
|
||||
param_type = param_data.get('type', 'int')
|
||||
if param_type == 'enum':
|
||||
param = ParameterDef(
|
||||
name=param_name,
|
||||
param_type='enum',
|
||||
default=param_data.get('default'),
|
||||
enum_options=param_data.get('options', []),
|
||||
description=param_data.get('description', ''),
|
||||
condition=param_data.get('condition', '')
|
||||
)
|
||||
else:
|
||||
param = ParameterDef(
|
||||
name=param_name,
|
||||
param_type=param_type,
|
||||
default=param_data.get('default'),
|
||||
min_value=param_data.get('min'),
|
||||
max_value=param_data.get('max'),
|
||||
step=param_data.get('step', 1),
|
||||
precision=param_data.get('precision', 0),
|
||||
optimize=param_data.get('optimize', True),
|
||||
description=param_data.get('description', ''),
|
||||
condition=param_data.get('condition', '')
|
||||
)
|
||||
parameters[param_name] = param
|
||||
return EAConfig(
|
||||
ea_name=ea_name, ea_path=ea_path, parameters=parameters,
|
||||
search_strategy=search_strategy, optimization_criterion=criterion,
|
||||
test_config=test_config, walk_forward=walk_forward, description=description
|
||||
)
|
||||
|
||||
def load_all_ea_configs(configs_dir: str) -> Dict[str, EAConfig]:
|
||||
configs = {}
|
||||
if not os.path.exists(configs_dir):
|
||||
return configs
|
||||
for filename in os.listdir(configs_dir):
|
||||
if filename.endswith('.json'):
|
||||
config_path = os.path.join(configs_dir, filename)
|
||||
try:
|
||||
config = load_ea_config(config_path)
|
||||
configs[config.ea_name] = config
|
||||
except Exception as e:
|
||||
print('Error loading ' + config_path + ': ' + str(e))
|
||||
return configs
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import re
|
||||
from typing import List, Dict, Any, Callable
|
||||
|
||||
class ParameterConstraintEngine:
|
||||
def __init__(self):
|
||||
self.constraints = []
|
||||
self.condition_functions = []
|
||||
|
||||
def add_constraint(self, name: str, condition: str, error_message: str = ''):
|
||||
self.constraints.append({
|
||||
'name': name,
|
||||
'condition': condition,
|
||||
'error_message': error_message or 'Constraint violated: ' + name
|
||||
})
|
||||
|
||||
def add_condition_from_config(self, param_name: str, condition_str: str):
|
||||
if not condition_str:
|
||||
return
|
||||
|
||||
condition_str = condition_str.strip()
|
||||
self.constraints.append({
|
||||
'name': param_name,
|
||||
'condition': condition_str,
|
||||
'error_message': 'Parameter constraint violated: ' + param_name
|
||||
})
|
||||
|
||||
def _parse_condition(self, condition: str) -> Callable[[Dict], bool]:
|
||||
condition = condition.strip()
|
||||
|
||||
patterns = [
|
||||
(r'^(\w+)\s*>\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) > p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*<\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) < p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*>=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) >= p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*<=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) <= p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*==\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) == p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*!=\s*(\w+)$', lambda m, p: p.get(m.group(1), 0) != p.get(m.group(2), 0)),
|
||||
(r'^(\w+)\s*>\s*(\d+)$', lambda m, p: p.get(m.group(1), 0) > int(m.group(2))),
|
||||
(r'^(\w+)\s*<\s*(\d+)$', lambda m, p: p.get(m.group(1), 0) < int(m.group(2))),
|
||||
]
|
||||
|
||||
for pattern, func in patterns:
|
||||
match = re.match(pattern, condition)
|
||||
if match:
|
||||
return lambda params, m=match, f=func: f(m, params)
|
||||
|
||||
return lambda params: True
|
||||
|
||||
def _is_valid_combination(self, params: Dict) -> bool:
|
||||
for constraint in self.constraints:
|
||||
condition = constraint['condition']
|
||||
if not condition:
|
||||
continue
|
||||
|
||||
parse_func = self._parse_condition(condition)
|
||||
if not parse_func(params):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def filter_valid_combinations(self, combinations: List[Dict]) -> List[Dict]:
|
||||
valid = []
|
||||
invalid_count = 0
|
||||
for combo in combinations:
|
||||
if self._is_valid_combination(combo):
|
||||
valid.append(combo)
|
||||
else:
|
||||
invalid_count += 1
|
||||
|
||||
if invalid_count > 0:
|
||||
print('Filtered ' + str(invalid_count) + ' invalid combinations')
|
||||
|
||||
return valid
|
||||
|
||||
def get_constraint_count(self) -> int:
|
||||
return len(self.constraints)
|
||||
|
||||
|
||||
def create_constraint_engine(ea_config) -> ParameterConstraintEngine:
|
||||
engine = ParameterConstraintEngine()
|
||||
|
||||
for param_name, param in ea_config.parameters.items():
|
||||
if param.condition:
|
||||
engine.add_condition_from_config(param_name, param.condition)
|
||||
|
||||
return engine
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import random
|
||||
import math
|
||||
from typing import List, Dict, Any, Tuple
|
||||
from itertools import product
|
||||
|
||||
class SmartSearch:
|
||||
def __init__(self, ea_config):
|
||||
self.ea_config = ea_config
|
||||
self.total_combinations = ea_config.get_total_combinations()
|
||||
|
||||
def select_strategy(self) -> str:
|
||||
if self.ea_config.search_strategy != 'auto':
|
||||
return self.ea_config.search_strategy
|
||||
if self.total_combinations <= 10000:
|
||||
return 'grid'
|
||||
elif self.total_combinations <= 500000:
|
||||
return 'latin'
|
||||
else:
|
||||
return 'genetic'
|
||||
|
||||
def generate_combinations(self, max_samples: int = 2000) -> List[Dict]:
|
||||
strategy = self.select_strategy()
|
||||
print('Selected strategy: ' + strategy + ' for ' + str(self.total_combinations) + ' combinations')
|
||||
|
||||
if strategy == 'grid':
|
||||
return self._grid_search()
|
||||
elif strategy == 'random':
|
||||
return self._random_search(max_samples)
|
||||
elif strategy == 'latin':
|
||||
return self._latin_hypercube(max_samples)
|
||||
elif strategy == 'genetic':
|
||||
return self._genetic_search(max_samples)
|
||||
return []
|
||||
|
||||
def _grid_search(self) -> List[Dict]:
|
||||
param_names = list(self.ea_config.parameters.keys())
|
||||
param_values = [p.expand_values() for p in self.ea_config.parameters.values()]
|
||||
combinations = list(product(*param_values))
|
||||
return [dict(zip(param_names, combo)) for combo in combinations]
|
||||
|
||||
def _random_search(self, max_samples: int) -> List[Dict]:
|
||||
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
|
||||
samples = []
|
||||
for _ in range(min(max_samples, self.total_combinations)):
|
||||
sample = {name: random.choice(values) for name, values in all_values.items()}
|
||||
if sample not in samples:
|
||||
samples.append(sample)
|
||||
return samples
|
||||
|
||||
def _latin_hypercube(self, max_samples: int) -> List[Dict]:
|
||||
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
|
||||
n_params = len(all_values)
|
||||
samples = []
|
||||
for i in range(min(max_samples, self.total_combinations)):
|
||||
sample = {}
|
||||
for j, (name, values) in enumerate(all_values.items()):
|
||||
idx = int((i / max_samples) * len(values)) % len(values)
|
||||
sample[name] = values[idx]
|
||||
if sample not in samples:
|
||||
samples.append(sample)
|
||||
return samples
|
||||
|
||||
def _genetic_search(self, max_samples: int) -> List[Dict]:
|
||||
pop_size = min(50, max_samples)
|
||||
n_generations = max_samples // pop_size
|
||||
all_values = {name: p.expand_values() for name, p in self.ea_config.parameters.items()}
|
||||
|
||||
population = []
|
||||
for _ in range(pop_size):
|
||||
individual = {name: random.choice(values) for name, values in all_values.items()}
|
||||
population.append(individual)
|
||||
|
||||
for gen in range(n_generations):
|
||||
population = self._evolve(population, all_values)
|
||||
|
||||
return population[:max_samples]
|
||||
|
||||
def _evolve(self, population: List[Dict], all_values: Dict) -> List[Dict]:
|
||||
crossover_rate = 0.8
|
||||
mutation_rate = 0.15
|
||||
|
||||
offspring = []
|
||||
for _ in range(len(population)):
|
||||
parent1, parent2 = random.sample(population, 2)
|
||||
if random.random() < crossover_rate:
|
||||
child = self._crossover(parent1, parent2)
|
||||
else:
|
||||
child = parent1.copy()
|
||||
|
||||
if random.random() < mutation_rate:
|
||||
child = self._mutate(child, all_values)
|
||||
|
||||
offspring.append(child)
|
||||
|
||||
return population[:5] + offspring[:len(population)-5]
|
||||
|
||||
def _crossover(self, parent1: Dict, parent2: Dict) -> Dict:
|
||||
child = {}
|
||||
for key in parent1.keys():
|
||||
if random.random() < 0.5:
|
||||
child[key] = parent1[key]
|
||||
else:
|
||||
child[key] = parent2[key]
|
||||
return child
|
||||
|
||||
def _mutate(self, individual: Dict, all_values: Dict) -> Dict:
|
||||
key = random.choice(list(all_values.keys()))
|
||||
individual[key] = random.choice(all_values[key])
|
||||
return individual
|
||||
|
||||
|
||||
class GeneticOptimizer:
|
||||
def __init__(self, ea_config, population_size: int = 50, generations: int = 30):
|
||||
self.ea_config = ea_config
|
||||
self.population_size = population_size
|
||||
self.generations = generations
|
||||
self.all_values = {name: p.expand_values() for name, p in ea_config.parameters.items()}
|
||||
|
||||
def create_individual(self) -> Dict:
|
||||
return {name: random.choice(values) for name, values in self.all_values.items()}
|
||||
|
||||
def evaluate(self, individual: Dict) -> float:
|
||||
return random.random() * 10
|
||||
|
||||
def tournament_select(self, population: List[Dict], k: int = 3) -> Dict:
|
||||
tournament = random.sample(population, k)
|
||||
return max(tournament, key=self.evaluate)
|
||||
|
||||
def crossover(self, parent1: Dict, parent2: Dict) -> Tuple[Dict, Dict]:
|
||||
child1, child2 = {}, {}
|
||||
for key in parent1.keys():
|
||||
if random.random() < 0.5:
|
||||
child1[key] = parent1[key]
|
||||
child2[key] = parent2[key]
|
||||
else:
|
||||
child1[key] = parent2[key]
|
||||
child2[key] = parent1[key]
|
||||
return child1, child2
|
||||
|
||||
def mutate(self, individual: Dict, rate: float = 0.15) -> Dict:
|
||||
for key in individual.keys():
|
||||
if random.random() < rate:
|
||||
individual[key] = random.choice(self.all_values[key])
|
||||
return individual
|
||||
|
||||
def run(self) -> List[Dict]:
|
||||
population = [self.create_individual() for _ in range(self.population_size)]
|
||||
best_individuals = []
|
||||
|
||||
for gen in range(self.generations):
|
||||
fitness_scores = [(ind, self.evaluate(ind)) for ind in population]
|
||||
fitness_scores.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
elite = [ind for ind, _ in fitness_scores[:5]]
|
||||
best_individuals.extend(elite)
|
||||
|
||||
new_population = elite.copy()
|
||||
while len(new_population) < self.population_size:
|
||||
parent1 = self.tournament_select(population)
|
||||
parent2 = self.tournament_select(population)
|
||||
child1, child2 = self.crossover(parent1, parent2)
|
||||
child1 = self.mutate(child1)
|
||||
child2 = self.mutate(child2)
|
||||
new_population.extend([child1, child2])
|
||||
|
||||
population = new_population[:self.population_size]
|
||||
|
||||
return best_individuals
|
||||
Reference in New Issue
Block a user