修复缺失json导致的崩溃

This commit is contained in:
2026-07-10 06:52:49 +00:00
parent 29428b415b
commit def8cf8d84
+380 -362
View File
@@ -1,362 +1,380 @@
import os import os
import subprocess import subprocess
import time import time
import logging import logging
import json import json
import shutil import shutil
from datetime import datetime from datetime import datetime
from typing import List, Dict from typing import List, Dict
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s', format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[ handlers=[
logging.FileHandler('logs/batch_executor.log', encoding='utf-8'), logging.FileHandler('logs/batch_executor.log', encoding='utf-8'),
logging.StreamHandler() logging.StreamHandler()
] ]
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class MT5AutoRunner: class MT5AutoRunner:
def __init__(self, source): def __init__(self, config_path):
""" if isinstance(config_path, dict):
source: YAML 文件路径 str 或 已加载的 config dict(GUI 内存直接传入用)。 self.config = config_path
""" else:
if isinstance(source, dict): self.config = self._load_config(config_path)
self.config = source self.mt5_path = self.config["mt5_settings"]["terminal_path"]
else: self.data_dir = self.config["mt5_settings"].get("data_dir", "")
self.config = self._load_config(source) self.ini_dir = self.config["mt5_settings"]["ini_dir"]
self.reports_dir = os.path.abspath(self.config["mt5_settings"].get("reports_dir", "reports"))
# 占位符解析(不做自动探测) execution_cfg = self.config.get("execution", {}) or {}
from mt5_paths import resolve_mt5_settings self.timeout_per_test = int(execution_cfg.get("timeout_per_test", 30))
self.config["mt5_settings"] = resolve_mt5_settings(self.config.get("mt5_settings", {})) self.kill_between = bool(execution_cfg.get("kill_between", True))
self.skip_existing = bool(execution_cfg.get("skip_existing", True))
self.mt5_path = self.config["mt5_settings"]["terminal_path"] self.execution_log = []
self.data_dir = self.config["mt5_settings"].get("data_dir", "") os.makedirs("logs", exist_ok=True)
self.reports_dir = os.path.abspath(self.config["mt5_settings"].get("reports_dir", "reports")) os.makedirs(self.reports_dir, exist_ok=True)
self.ini_dir = os.path.abspath(self.config["mt5_settings"].get("ini_dir",
os.path.join(os.path.dirname(self.reports_dir), "config", "generated"))) def _load_config(self, config_path: str) -> Dict:
os.makedirs(self.reports_dir, exist_ok=True) import yaml
os.makedirs(self.ini_dir, exist_ok=True) with open(config_path, 'r', encoding='utf-8') as f:
return yaml.safe_load(f)
exec_cfg = self.config.get("execution", {}) or {}
self.kill_between = bool(exec_cfg.get("kill_between", True)) def _load_execution_log(self) -> List[Dict]:
self.skip_existing = bool(exec_cfg.get("skip_existing", True)) log_file = os.path.join(self.ini_dir, "..", "results", "execution_log.json")
self.timeout_per_test = int(exec_cfg.get("timeout_per_test", 30)) log_file = os.path.normpath(log_file)
if os.path.exists(log_file):
if not self.mt5_path or not os.path.isfile(self.mt5_path): try:
raise FileNotFoundError( with open(log_file, 'r', encoding='utf-8') as f:
f"MT5 终端路径未配置或不存在: {self.mt5_path!r}\n" content = f.read().strip()
"请在 GUI '回测配置' 标签页设置 terminal64.exe 的完整路径后点 '保存配置'" if not content:
) return []
return json.loads(content)
def _load_config(self, config_path: str) -> Dict: except (json.JSONDecodeError, OSError) as e:
import yaml logger.warning(f"Corrupt execution_log.json, ignoring: {e}")
with open(config_path, 'r', encoding='utf-8') as f: return []
return yaml.safe_load(f) return []
def _load_execution_log(self) -> List[Dict]: def _save_execution_log(self, log: List[Dict]):
log_file = os.path.join(self.ini_dir, "..", "results", "execution_log.json") log_file = os.path.join(self.ini_dir, "..", "results", "execution_log.json")
log_file = os.path.normpath(log_file) log_file = os.path.normpath(log_file)
if os.path.exists(log_file): os.makedirs(os.path.dirname(log_file), exist_ok=True)
with open(log_file, 'r', encoding='utf-8') as f: with open(log_file, 'w', encoding='utf-8') as f:
return json.load(f) json.dump(log, f, indent=2, ensure_ascii=False)
return []
def _get_report_path(self, ini_name: str) -> str:
def _save_execution_log(self, log: List[Dict]): base_name = os.path.splitext(ini_name)[0]
log_file = os.path.join(self.ini_dir, "..", "results", "execution_log.json")
log_file = os.path.normpath(log_file) project_report_path = os.path.join(self.reports_dir, f"{base_name}.htm")
os.makedirs(os.path.dirname(log_file), exist_ok=True) if os.path.exists(project_report_path):
with open(log_file, 'w', encoding='utf-8') as f: return project_report_path
json.dump(log, f, indent=2, ensure_ascii=False)
if self.data_dir and os.path.isdir(self.data_dir):
def _get_report_path(self, ini_name: str) -> str: target_names = {f"{base_name}.htm", f"{base_name}.xml", base_name}
base_name = os.path.splitext(ini_name)[0] for root, dirs, files in os.walk(self.data_dir):
return os.path.join(self.data_dir, f"{base_name}.htm") if self.data_dir else f"{base_name}.htm" for f in files:
if f in target_names:
def _copy_report_to_project(self, ini_name: str) -> str: found = os.path.join(root, f)
base_name = os.path.splitext(ini_name)[0] logger.info(f"Report found at: {found}")
src_path = self._get_report_path(ini_name) return found
if not os.path.exists(src_path): logger.warning(f"Report not found for {ini_name}, base={base_name}, data_dir={self.data_dir}")
logger.warning(f"Source report not found: {src_path}") return project_report_path
return None
def _get_report_siblings(self, src_path: str) -> list:
dest_path = os.path.join(self.reports_dir, f"{base_name}.htm") parent = os.path.dirname(src_path)
try: base = os.path.splitext(os.path.basename(src_path))[0]
shutil.copy2(src_path, dest_path) siblings = []
logger.info(f"Report copied to: {dest_path}") try:
return dest_path for f in os.listdir(parent):
except Exception as e: name, ext = os.path.splitext(f)
logger.error(f"Failed to copy report: {e}") if name == base and ext.lower() in (".htm", ".html", ".xml", ".png", ".gif", ".csv"):
return None siblings.append(os.path.join(parent, f))
except OSError:
def _kill_mt5(self): pass
try: return siblings
subprocess.run(['taskkill', '/F', '/IM', 'terminal64.exe'],
capture_output=True, text=True) def _copy_report_to_project(self, ini_name: str) -> str:
time.sleep(2) base_name = os.path.splitext(ini_name)[0]
except: src_path = self._get_report_path(ini_name)
pass
if not os.path.exists(src_path):
def _read_ini_expert(self, ini_path: str) -> str: logger.warning(f"Source report not found: {src_path}")
try: return None
with open(ini_path, 'r', encoding='utf-8') as f:
for line in f: dest_path = os.path.join(self.reports_dir, f"{base_name}.htm")
if line.startswith('Expert='): if os.path.abspath(src_path) == os.path.abspath(dest_path):
return line.split('=', 1)[1].strip() return dest_path
except:
pass try:
return None shutil.copy2(src_path, dest_path)
logger.info(f"Report copied to: {dest_path}")
def _execute_single_ini(self, ini_path: str, timeout_min: int = 30) -> Dict: return dest_path
result = { except Exception as e:
"ini_file": os.path.basename(ini_path), logger.error(f"Failed to copy report: {e}")
"status": "pending", return None
"start_time": datetime.now().isoformat(),
"end_time": None,
"duration_sec": 0, def _kill_mt5(self):
"report_found": False, try:
"report_path": None, subprocess.run(['taskkill', '/F', '/IM', 'terminal64.exe'],
"error": None capture_output=True, text=True)
} time.sleep(2)
except:
ini_name = os.path.basename(ini_path) pass
expected_report = self._get_report_path(ini_name)
def _read_ini_expert(self, ini_path: str) -> str:
if os.path.exists(expected_report): try:
result["status"] = "already_completed" with open(ini_path, 'r', encoding='utf-8') as f:
result["report_found"] = True for line in f:
result["report_path"] = expected_report if line.startswith('Expert='):
logger.info(f"Already completed: {ini_name}") return line.split('=', 1)[1].strip()
return result except:
pass
logger.info(f"Executing: {ini_name}") return None
try: def _execute_single_ini(self, ini_path: str, timeout_min: int = 30) -> Dict:
start_time = time.time() result = {
"ini_file": os.path.basename(ini_path),
proc = subprocess.Popen( "status": "pending",
[self.mt5_path, f"/config:{ini_path}"], "start_time": datetime.now().isoformat(),
stdout=subprocess.DEVNULL, "end_time": None,
stderr=subprocess.DEVNULL "duration_sec": 0,
) "report_found": False,
"report_path": None,
logger.info(f"MT5 started PID: {proc.pid}") "error": None
}
while True:
if proc.poll() is not None: ini_name = os.path.basename(ini_path)
logger.info("MT5 process ended") expected_report = self._get_report_path(ini_name)
break
if os.path.exists(expected_report):
if os.path.exists(expected_report): result["status"] = "already_completed"
try: result["report_found"] = True
with open(expected_report, 'r', encoding='utf-8') as f: result["report_path"] = expected_report
content = f.read() logger.info(f"Already completed: {ini_name}")
if len(content) > 1000: return result
break
except: logger.info(f"Executing: {ini_name}")
pass
try:
elapsed = time.time() - start_time start_time = time.time()
if elapsed > timeout_min * 60:
logger.warning(f"Timeout: {ini_name}") proc = subprocess.Popen(
break [self.mt5_path, f"/config:{ini_path}"],
stdout=subprocess.DEVNULL,
time.sleep(2) stderr=subprocess.DEVNULL
)
end_time = time.time()
result["duration_sec"] = end_time - start_time logger.info(f"MT5 started PID: {proc.pid}")
if os.path.exists(expected_report): while True:
result["status"] = "completed" if proc.poll() is not None:
result["report_found"] = True logger.info("MT5 process ended")
copied_path = self._copy_report_to_project(ini_name) break
result["report_path"] = copied_path or expected_report
logger.info(f"Success: {ini_name} ({result['duration_sec']:.0f}s)") if os.path.exists(expected_report):
else: try:
result["status"] = "failed" with open(expected_report, 'r', encoding='utf-8') as f:
logger.warning(f"Failed: {ini_name}") content = f.read()
if len(content) > 1000:
result["end_time"] = datetime.now().isoformat() break
except:
if proc.poll() is None: pass
proc.terminate()
try: elapsed = time.time() - start_time
proc.wait(timeout=5) if elapsed > timeout_min * 60:
except: logger.warning(f"Timeout: {ini_name}")
pass break
except Exception as e: time.sleep(2)
result["status"] = "error"
result["error"] = str(e) end_time = time.time()
logger.error(f"Error: {e}") result["duration_sec"] = end_time - start_time
return result actual_report = self._get_report_path(ini_name)
if os.path.exists(actual_report):
def run_full_auto(self, ini_files: List[str] = None, result["status"] = "completed"
skip_if_exists: bool = None, result["report_found"] = True
kill_between_tests: bool = None, copied_path = self._copy_report_to_project(ini_name)
timeout_min: int = None) -> List[Dict]: result["report_path"] = copied_path or actual_report
logger.info("="*60) logger.info(f"Success: {ini_name} ({result['duration_sec']:.0f}s)")
logger.info("MT5 Full Auto Batch Runner") else:
logger.info("="*60) result["status"] = "failed"
logger.warning(f"Failed: {ini_name}, looked at: {actual_report}")
if skip_if_exists is None:
skip_if_exists = self.skip_existing result["end_time"] = datetime.now().isoformat()
if kill_between_tests is None:
kill_between_tests = self.kill_between if proc.poll() is None:
if timeout_min is None: proc.terminate()
timeout_min = self.timeout_per_test try:
proc.wait(timeout=5)
logger.info(f"Timeout per test: {timeout_min} minutes, Kill between tests: {kill_between_tests}, Skip existing: {skip_if_exists}") except:
pass
self.execution_log = self._load_execution_log()
except Exception as e:
if ini_files is None: result["status"] = "error"
ini_files = [os.path.join(self.ini_dir, f) result["error"] = str(e)
for f in os.listdir(self.ini_dir) if f.endswith('.ini')] logger.error(f"Error: {e}")
total = len(ini_files) return result
completed = 0
failed = 0 def run_full_auto(self, ini_files: List[str] = None,
skip_if_exists: bool = None,
logger.info(f"Total INI files: {total}") kill_between_tests: bool = None,
timeout_min: int = None) -> List[Dict]:
# First run must also start from a clean tester state. Otherwise the first test logger.info("="*60)
# can inherit an already-open terminal's previous deposit/delay/symbol settings, logger.info("MT5 Full Auto Batch Runner")
# while later tests look correct only because kill_between_tests runs after them. logger.info("="*60)
if kill_between_tests:
self._kill_mt5() if skip_if_exists is None:
time.sleep(2) skip_if_exists = self.skip_existing
if kill_between_tests is None:
for idx, ini_path in enumerate(ini_files, 1): kill_between_tests = self.kill_between
logger.info(f"[{idx}/{total}] {os.path.basename(ini_path)}") if timeout_min is None:
timeout_min = self.timeout_per_test
result = self._execute_single_ini(ini_path, timeout_min=timeout_min)
logger.info(f"Timeout per test: {timeout_min} minutes, Kill between tests: {kill_between_tests}, Skip existing: {skip_if_exists}")
self.execution_log.append(result)
self._save_execution_log(self.execution_log) self.execution_log = self._load_execution_log()
if result["status"] == "completed": if ini_files is None:
completed += 1 ini_files = [os.path.join(self.ini_dir, f)
elif result["status"] == "already_completed": for f in os.listdir(self.ini_dir) if f.endswith('.ini')]
completed += 1
else: total = len(ini_files)
failed += 1 completed = 0
failed = 0
logger.info(f" Status: {result['status']}, Duration: {result['duration_sec']:.0f}s")
logger.info(f"Total INI files: {total}")
if kill_between_tests and result["status"] != "already_completed":
self._kill_mt5() for idx, ini_path in enumerate(ini_files, 1):
time.sleep(2) logger.info(f"[{idx}/{total}] {os.path.basename(ini_path)}")
logger.info("="*60) result = self._execute_single_ini(ini_path, timeout_min=timeout_min)
logger.info(f"DONE: {completed}/{total} completed, {failed} failed")
logger.info("="*60) self.execution_log.append(result)
self._save_execution_log(self.execution_log)
return self.execution_log
if result["status"] == "completed":
def run_daemon(self, check_interval: int = 60): completed += 1
logger.info("="*60) elif result["status"] == "already_completed":
logger.info("MT5 Auto Runner - DAEMON MODE") completed += 1
logger.info(f"Monitoring: {self.ini_dir}") else:
logger.info(f"Reports will be copied to: {self.reports_dir}") failed += 1
logger.info(f"Timeout per test: {self.timeout_per_test} minutes")
logger.info("Press Ctrl+C to stop") logger.info(f" Status: {result['status']}, Duration: {result['duration_sec']:.0f}s")
logger.info("="*60)
if kill_between_tests and result["status"] != "already_completed":
self.execution_log = self._load_execution_log() self._kill_mt5()
time.sleep(2)
try:
while True: logger.info("="*60)
pending_inis = [] logger.info(f"DONE: {completed}/{total} completed, {failed} failed")
for f in os.listdir(self.ini_dir): logger.info("="*60)
if f.endswith('.ini'):
is_completed = any( return self.execution_log
log.get("ini_file") == f and log.get("status") == "completed"
for log in self.execution_log def run_daemon(self, check_interval: int = 60):
) logger.info("="*60)
if not is_completed: logger.info("MT5 Auto Runner - DAEMON MODE")
pending_inis.append(f) logger.info(f"Monitoring: {self.ini_dir}")
logger.info(f"Reports will be copied to: {self.reports_dir}")
if pending_inis: logger.info(f"Timeout per test: {self.timeout_per_test} minutes")
logger.info(f"Found {len(pending_inis)} pending INI files") logger.info("Press Ctrl+C to stop")
for ini_name in pending_inis: logger.info("="*60)
ini_path = os.path.join(self.ini_dir, ini_name)
result = self._execute_single_ini(ini_path, timeout_min=self.timeout_per_test) self.execution_log = self._load_execution_log()
self.execution_log.append(result)
self._save_execution_log(self.execution_log) try:
self._kill_mt5() while True:
time.sleep(3) pending_inis = []
else: for f in os.listdir(self.ini_dir):
logger.info("No pending INI files, waiting...") if f.endswith('.ini'):
is_completed = any(
time.sleep(check_interval) log.get("ini_file") == f and log.get("status") == "completed"
for log in self.execution_log
except KeyboardInterrupt: )
logger.info("Daemon stopped") if not is_completed:
pending_inis.append(f)
def generate_report(self) -> str:
logger.info("Generating summary report...") if pending_inis:
logger.info(f"Found {len(pending_inis)} pending INI files")
from scripts.result_parser import ResultParser for ini_name in pending_inis:
parser = ResultParser(self.reports_dir) ini_path = os.path.join(self.ini_dir, ini_name)
results = parser.parse_all_reports(pattern="*.ht*") result = self._execute_single_ini(ini_path, timeout_min=self.timeout_per_test)
self.execution_log.append(result)
if results: self._save_execution_log(self.execution_log)
from scripts.report_generator import ReportGenerator self._kill_mt5()
generator = ReportGenerator(results) time.sleep(3)
os.makedirs("reports", exist_ok=True) else:
output = generator.generate_excel("reports/batch_summary.xlsx") logger.info("No pending INI files, waiting...")
logger.info(f"Report saved: {output}")
return output time.sleep(check_interval)
else:
logger.warning("No results to generate report") except KeyboardInterrupt:
return None logger.info("Daemon stopped")
def generate_report(self) -> str:
def main(): logger.info("Generating summary report...")
import argparse
parser = argparse.ArgumentParser(description="MT5 Auto Runner") from scripts.result_parser import ResultParser
parser.add_argument("--config", "-c", default="config/ea_configs.yaml") parser = ResultParser(self.reports_dir)
parser.add_argument("--mode", "-m", choices=["full", "daemon", "report", "init", "execute"], results = parser.parse_all_reports(pattern="*.ht*")
default="full")
parser.add_argument("--interval", "-i", type=int, default=60) if results:
parser.add_argument("--no-skip", action="store_true") from scripts.report_generator import ReportGenerator
generator = ReportGenerator(results)
args = parser.parse_args() os.makedirs("reports", exist_ok=True)
output = generator.generate_excel("reports/batch_summary.xlsx")
runner = MT5AutoRunner(args.config) logger.info(f"Report saved: {output}")
return output
if args.mode == "daemon": else:
runner.run_daemon(check_interval=args.interval) logger.warning("No results to generate report")
elif args.mode == "report": return None
runner.generate_report()
elif args.mode == "init":
from scripts.ini_generator import INIGenerator def main():
generator = INIGenerator(args.config) import argparse
ini_files = generator.generate_ini_files() parser = argparse.ArgumentParser(description="MT5 Auto Runner")
print(f"Generated {len(ini_files)} INI files") parser.add_argument("--config", "-c", default="config/ea_configs.yaml")
elif args.mode == "execute": parser.add_argument("--mode", "-m", choices=["full", "daemon", "report", "init", "execute"],
from scripts.ini_generator import INIGenerator default="full")
generator = INIGenerator(args.config) parser.add_argument("--interval", "-i", type=int, default=60)
ini_files = generator.generate_ini_files() parser.add_argument("--no-skip", action="store_true")
results = runner.run_full_auto(ini_files, skip_if_exists=not args.no_skip)
completed = sum(1 for r in results if r["status"] == "completed") args = parser.parse_args()
print(f"Completed: {completed}/{len(results)}")
else: runner = MT5AutoRunner(args.config)
from scripts.ini_generator import INIGenerator
generator = INIGenerator(args.config) if args.mode == "daemon":
ini_files = generator.generate_ini_files() runner.run_daemon(check_interval=args.interval)
logger.info(f"Generated {len(ini_files)} INI files") elif args.mode == "report":
runner.generate_report()
results = runner.run_full_auto(ini_files, skip_if_exists=not args.no_skip) elif args.mode == "init":
from scripts.ini_generator import INIGenerator
runner.generate_report() generator = INIGenerator(args.config)
ini_files = generator.generate_ini_files()
print(f"Generated {len(ini_files)} INI files")
if __name__ == "__main__": elif args.mode == "execute":
main() from scripts.ini_generator import INIGenerator
generator = INIGenerator(args.config)
ini_files = generator.generate_ini_files()
results = runner.run_full_auto(ini_files, skip_if_exists=not args.no_skip)
completed = sum(1 for r in results if r["status"] == "completed")
print(f"Completed: {completed}/{len(results)}")
else:
from scripts.ini_generator import INIGenerator
generator = INIGenerator(args.config)
ini_files = generator.generate_ini_files()
logger.info(f"Generated {len(ini_files)} INI files")
results = runner.run_full_auto(ini_files, skip_if_exists=not args.no_skip)
runner.generate_report()
if __name__ == "__main__":
main()