first commit
This commit is contained in:
@@ -0,0 +1,222 @@
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('logs/batch_executor.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BatchExecutor:
|
||||
def __init__(self, config_path: str):
|
||||
self.config = self._load_config(config_path)
|
||||
self.mt5_path = self.config["mt5_settings"]["terminal_path"]
|
||||
self.reports_dir = self.config["mt5_settings"]["reports_dir"]
|
||||
self.results_dir = self.config["mt5_settings"].get("results_dir", "results")
|
||||
os.makedirs(self.results_dir, exist_ok=True)
|
||||
os.makedirs("logs", exist_ok=True)
|
||||
|
||||
def _load_config(self, config_path: str) -> Dict:
|
||||
import yaml
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
def _wait_for_report(self, report_path: str, timeout: int = 600) -> bool:
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
if os.path.exists(report_path):
|
||||
time.sleep(2)
|
||||
try:
|
||||
with open(report_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
if len(content) > 100:
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
def _check_mt5_process(self) -> bool:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['tasklist', '/FI', 'IMAGENAME eq terminal64.exe'],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
return 'terminal64.exe' in result.stdout
|
||||
except:
|
||||
return False
|
||||
|
||||
def execute_single(self, ini_path: str, wait_time: int = 120) -> Dict:
|
||||
result = {
|
||||
"ini_file": os.path.basename(ini_path),
|
||||
"status": "pending",
|
||||
"start_time": None,
|
||||
"end_time": None,
|
||||
"duration": 0,
|
||||
"report_path": None,
|
||||
"error": None
|
||||
}
|
||||
|
||||
logger.info(f"Starting backtest: {ini_path}")
|
||||
|
||||
if not os.path.exists(self.mt5_path):
|
||||
result["status"] = "error"
|
||||
result["error"] = f"MT5 terminal not found: {self.mt5_path}"
|
||||
logger.error(result["error"])
|
||||
return result
|
||||
|
||||
result["start_time"] = datetime.now()
|
||||
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
[self.mt5_path, "/portable", f"/config:{ini_path}"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
|
||||
logger.info(f"MT5 process started with PID: {process.pid}")
|
||||
|
||||
ini_basename = os.path.splitext(os.path.basename(ini_path))[0]
|
||||
report_xml = os.path.join(self.results_dir, f"{ini_basename}_report.xml")
|
||||
|
||||
report_found = self._wait_for_report(report_xml, timeout=wait_time * 60)
|
||||
|
||||
if report_found:
|
||||
result["status"] = "completed"
|
||||
result["report_path"] = report_xml
|
||||
logger.info(f"Report generated: {report_xml}")
|
||||
else:
|
||||
if process.poll() is not None:
|
||||
result["status"] = "mt5_closed"
|
||||
logger.warning("MT5 closed before report was generated")
|
||||
else:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
result["status"] = "timeout"
|
||||
logger.warning(f"Timeout waiting for report: {ini_path}")
|
||||
|
||||
if process.poll() is None:
|
||||
try:
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
except:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
logger.error(f"Error executing backtest: {e}")
|
||||
|
||||
result["end_time"] = datetime.now()
|
||||
if result["start_time"] and result["end_time"]:
|
||||
result["duration"] = (result["end_time"] - result["start_time"]).total_seconds()
|
||||
|
||||
return result
|
||||
|
||||
def execute_batch(self, ini_files: List[str],
|
||||
max_parallel: int = 1,
|
||||
wait_time_per_test: int = 120) -> List[Dict]:
|
||||
results = []
|
||||
total = len(ini_files)
|
||||
|
||||
logger.info(f"Starting batch execution: {total} tests")
|
||||
logger.info(f"Parallel execution: {max_parallel}")
|
||||
|
||||
for idx, ini_path in enumerate(ini_files, 1):
|
||||
logger.info(f"[{idx}/{total}] Executing: {os.path.basename(ini_path)}")
|
||||
result = self.execute_single(ini_path, wait_time_per_test)
|
||||
results.append(result)
|
||||
|
||||
status = result["status"]
|
||||
duration = result["duration"]
|
||||
logger.info(f" Status: {status}, Duration: {duration:.1f}s")
|
||||
|
||||
if result["error"]:
|
||||
logger.error(f" Error: {result['error']}")
|
||||
|
||||
success_count = sum(1 for r in results if r["status"] == "completed")
|
||||
logger.info(f"Batch completed: {success_count}/{total} successful")
|
||||
|
||||
return results
|
||||
|
||||
def execute_validate_tasks(self, ini_files: List[str]) -> str:
|
||||
validate_dir = os.path.join(os.path.dirname(self.mt5_path),
|
||||
"MQL5", "Files", "ValidateTasks")
|
||||
os.makedirs(validate_dir, exist_ok=True)
|
||||
|
||||
for ini_file in ini_files:
|
||||
dest_path = os.path.join(validate_dir, os.path.basename(ini_file))
|
||||
with open(ini_file, 'r', encoding='utf-8') as src:
|
||||
content = src.read()
|
||||
with open(dest_path, 'w', encoding='utf-8') as dst:
|
||||
dst.write(content)
|
||||
logger.info(f"Copied to ValidateTasks: {os.path.basename(ini_file)}")
|
||||
|
||||
logger.info(f"All {len(ini_files)} tasks queued in ValidateTasks folder")
|
||||
return validate_dir
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="MT5 Batch Backtest Executor")
|
||||
parser.add_argument("--config", "-c", default="config/ea_configs.yaml",
|
||||
help="Path to config file")
|
||||
parser.add_argument("--ini-dir", "-i",
|
||||
help="Directory containing INI files to execute")
|
||||
parser.add_argument("--wait", "-w", type=int, default=120,
|
||||
help="Wait time per test in minutes")
|
||||
parser.add_argument("--validate", "-v", action="store_true",
|
||||
help="Use ValidateTasks method")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
executor = BatchExecutor(args.config)
|
||||
|
||||
if args.ini_dir:
|
||||
ini_files = [os.path.join(args.ini_dir, f)
|
||||
for f in os.listdir(args.ini_dir)
|
||||
if f.endswith('.ini')]
|
||||
else:
|
||||
generator = __import__('ini_generator', fromlist=['']).INIGenerator(args.config)
|
||||
ini_files = generator.generate_ini_files()
|
||||
|
||||
logger.info(f"Found {len(ini_files)} INI files to execute")
|
||||
|
||||
if args.validate:
|
||||
validate_dir = executor.execute_validate_tasks(ini_files)
|
||||
logger.info(f"ValidateTasks method: files copied to {validate_dir}")
|
||||
logger.info("Run the Validate EA in MT5 terminal to execute")
|
||||
else:
|
||||
results = executor.execute_batch(ini_files, wait_time_per_test=args.wait)
|
||||
|
||||
success = [r for r in results if r["status"] == "completed"]
|
||||
failed = [r for r in results if r["status"] != "completed"]
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("BATCH EXECUTION SUMMARY")
|
||||
print("="*60)
|
||||
print(f"Total tests: {len(results)}")
|
||||
print(f"Completed: {len(success)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print("="*60)
|
||||
|
||||
if failed:
|
||||
print("\nFailed tests:")
|
||||
for r in failed:
|
||||
print(f" - {r['ini_file']}: {r['status']} - {r['error']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user