"""贝叶斯优化器（Optuna TPE） 高效替代 DEAP 遗传算法，50次试验 ≈ 2分钟收敛 关键改进： 1. TPE sampler 建模适应度曲面，采样效率高 3-5 倍 2. 权重基因通过 individual_weights → MarketStateAnalyzer.get_strategy_weights() 真正生效 3. evaluate_fitness 逐 bar 调用 get_market_state(bar_index) 获取每根bar的动态权重 """ import random import numpy as np import pandas as pd import sys import os import logging from datetime import datetime import optuna SEED = 42 random.seed(SEED) np.random.seed(SEED) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from core.data import MultiTimeframeDataStore, BacktestDataProvider from core.risk.market_state import MarketStateAnalyzer from core.risk.position import PositionManager from core.signal.registry import StrategyRegistry from core.signal.combiner import SignalCombiner from config import ( SYMBOL, TIMEFRAME, OPTIMIZER_COUNT, OPTIMIZER_START_DATE, OPTIMIZER_END_DATE, USE_DATE_RANGE, INITIAL_CAPITAL, SIGNAL_THRESHOLDS, DEFAULT_WEIGHTS, RISK_CONFIG, MARKET_STATE_CONFIG, TREND_INDICATOR_WEIGHTS, TREND_THRESHOLDS, CONFIDENCE_THRESHOLDS, DATA_PROVIDER_MODE, REMOTE_SERVER_HOST, REMOTE_SERVER_PORT ) from utils.constants import PERIOD_H1 from core.utils import get_rates, initialize, shutdown from core.data.remote import RemoteDataProvider from logger import logger # 导入策略模块确保 StrategyRegistry 已注册 import strategies # noqa: F401 # ── 全局缓存 ── _multi_tf: MultiTimeframeDataStore | None = None # MT5 结构化数组 dtype（用于远程 API JSON → numpy 转换） _MT5_RATES_DTYPE = np.dtype([ ('time', 'i8'), ('open', 'f8'), ('high', 'f8'), ('low', 'f8'), ('close', 'f8'), ('tick_volume', 'i8'), ('spread', 'i4'), ('real_volume', 'i8'), ]) # ── 参数定义 ── PARAMETER_DEFINITIONS = [ # MACrossStrategy {'name': 'ma_cross_short_window', 'type': 'int', 'min': 3, 'max': 15, 'strategy': 'MACrossStrategy'}, {'name': 'ma_cross_long_window', 'type': 'int', 'min': 10, 'max': 60, 'strategy': 'MACrossStrategy'}, # RSIStrategy {'name': 'rsi_period', 'type': 'int', 'min': 7, 'max': 21, 'strategy': 'RSIStrategy'}, {'name': 'rsi_overbought', 'type': 'int', 'min': 65, 'max': 80, 'strategy': 'RSIStrategy'}, {'name': 'rsi_oversold', 'type': 'int', 'min': 20, 'max': 35, 'strategy': 'RSIStrategy'}, # BollingerStrategy {'name': 'bollinger_period', 'type': 'int', 'min': 10, 'max': 50, 'strategy': 'BollingerStrategy'}, {'name': 'bollinger_std_dev', 'type': 'float', 'min': 1.5, 'max': 3.0, 'strategy': 'BollingerStrategy'}, # MACDStrategy {'name': 'macd_fast_ema', 'type': 'int', 'min': 8, 'max': 20, 'strategy': 'MACDStrategy'}, {'name': 'macd_slow_ema', 'type': 'int', 'min': 20, 'max': 35, 'strategy': 'MACDStrategy'}, {'name': 'macd_signal_period', 'type': 'int', 'min': 5, 'max': 15, 'strategy': 'MACDStrategy'}, # MeanReversionStrategy {'name': 'mean_reversion_period', 'type': 'int', 'min': 10, 'max': 50, 'strategy': 'MeanReversionStrategy'}, {'name': 'mean_reversion_std_dev', 'type': 'float', 'min': 1.5, 'max': 3.0, 'strategy': 'MeanReversionStrategy'}, # MomentumBreakoutStrategy {'name': 'momentum_breakout_period', 'type': 'int', 'min': 10, 'max': 50, 'strategy': 'MomentumBreakoutStrategy'}, {'name': 'momentum_breakout_momentum_period', 'type': 'int', 'min': 5, 'max': 30, 'strategy': 'MomentumBreakoutStrategy'}, # KDJStrategy {'name': 'kdj_period', 'type': 'int', 'min': 5, 'max': 21, 'strategy': 'KDJStrategy'}, # SwingPointRetestStrategy — 前高前低回踩 {'name': 'swing_point_left_bars', 'type': 'int', 'min': 2, 'max': 5, 'strategy': 'SwingPointRetestStrategy'}, {'name': 'swing_point_right_bars', 'type': 'int', 'min': 2, 'max': 5, 'strategy': 'SwingPointRetestStrategy'}, {'name': 'swing_point_tolerance_pct', 'type': 'float', 'min': 0.0003, 'max': 0.002, 'strategy': 'SwingPointRetestStrategy'}, {'name': 'swing_point_num_swings', 'type': 'int', 'min': 1, 'max': 3, 'strategy': 'SwingPointRetestStrategy'}, # WaveTheoryStrategy {'name': 'wave_ema_short', 'type': 'int', 'min': 3, 'max': 10, 'strategy': 'WaveTheoryStrategy'}, {'name': 'wave_ema_medium', 'type': 'int', 'min': 8, 'max': 20, 'strategy': 'WaveTheoryStrategy'}, {'name': 'wave_ema_long', 'type': 'int', 'min': 20, 'max': 50, 'strategy': 'WaveTheoryStrategy'}, {'name': 'wave_period', 'type': 'int', 'min': 10, 'max': 40, 'strategy': 'WaveTheoryStrategy'}, {'name': 'wave_range_period', 'type': 'int', 'min': 10, 'max': 40, 'strategy': 'WaveTheoryStrategy'}, {'name': 'wave_adx_period', 'type': 'int', 'min': 10, 'max': 25, 'strategy': 'WaveTheoryStrategy'}, {'name': 'wave_momentum_period', 'type': 'int', 'min': 7, 'max': 21, 'strategy': 'WaveTheoryStrategy'}, {'name': 'wave_range_threshold', 'type': 'float', 'min': 0.002, 'max': 0.01, 'strategy': 'WaveTheoryStrategy'}, {'name': 'wave_adx_threshold', 'type': 'int', 'min': 15, 'max': 35, 'strategy': 'WaveTheoryStrategy'}, # DailyBreakoutStrategy {'name': 'daily_breakout_bars_count', 'type': 'int', 'min': 720, 'max': 2880, 'strategy': 'DailyBreakoutStrategy'}, # Signal Thresholds {'name': 'buy_threshold', 'type': 'float', 'min': 0.5, 'max': 3.0, 'strategy': 'signal'}, {'name': 'sell_threshold', 'type': 'float', 'min': -3.0, 'max': -0.5, 'strategy': 'signal'}, # ★ Strategy Weights {'name': 'weight_MACrossStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_RSIStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_BollingerStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_MeanReversionStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_MomentumBreakoutStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_MACDStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_KDJStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_SwingPointRetestStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_DailyBreakoutStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, {'name': 'weight_WaveTheoryStrategy', 'type': 'float', 'min': 0.0, 'max': 2.0, 'strategy': 'weight'}, # Market State Analysis parameters {'name': 'market_trend_period', 'type': 'int', 'min': 20, 'max': 100, 'strategy': 'market_state'}, {'name': 'market_retracement_tolerance', 'type': 'float', 'min': 0.1, 'max': 0.5, 'strategy': 'market_state'}, {'name': 'market_volume_period', 'type': 'int', 'min': 10, 'max': 50, 'strategy': 'market_state'}, {'name': 'market_volume_ma_period', 'type': 'int', 'min': 5, 'max': 30, 'strategy': 'market_state'}, # Trend Indicator Weights {'name': 'trend_price_breakout_weight', 'type': 'float', 'min': 0.1, 'max': 0.5, 'strategy': 'trend_weights'}, {'name': 'trend_volume_confirmation_weight', 'type': 'float', 'min': 0.1, 'max': 0.5, 'strategy': 'trend_weights'}, {'name': 'trend_momentum_oscillator_weight', 'type': 'float', 'min': 0.1, 'max': 0.5, 'strategy': 'trend_weights'}, {'name': 'trend_moving_average_weight', 'type': 'float', 'min': 0.1, 'max': 0.5, 'strategy': 'trend_weights'}, # Trend Thresholds {'name': 'trend_strong_threshold', 'type': 'float', 'min': 0.4, 'max': 0.8, 'strategy': 'trend_thresholds'}, {'name': 'trend_weak_threshold', 'type': 'float', 'min': 0.2, 'max': 0.5, 'strategy': 'trend_thresholds'}, {'name': 'trend_volume_spike', 'type': 'float', 'min': 1.0, 'max': 3.0, 'strategy': 'trend_thresholds'}, {'name': 'trend_oversold', 'type': 'int', 'min': 20, 'max': 40, 'strategy': 'trend_thresholds'}, {'name': 'trend_overbought', 'type': 'int', 'min': 60, 'max': 80, 'strategy': 'trend_thresholds'}, # Confidence Thresholds {'name': 'confidence_high', 'type': 'float', 'min': 0.5, 'max': 0.9, 'strategy': 'confidence'}, {'name': 'confidence_medium', 'type': 'float', 'min': 0.3, 'max': 0.7, 'strategy': 'confidence'}, ] _class_name_to_config_key = { "MACrossStrategy": "ma_cross", "RSIStrategy": "rsi", "BollingerStrategy": "bollinger", "MeanReversionStrategy": "mean_reversion", "MomentumBreakoutStrategy": "momentum_breakout", "MACDStrategy": "macd", "KDJStrategy": "kdj", "SwingPointRetestStrategy": "swing_point", "DailyBreakoutStrategy": "daily_breakout", "WaveTheoryStrategy": "wave_theory", } def _trial_to_params(trial: optuna.Trial) -> dict: """从 Optuna trial 提取参数 → 兼容旧 _parse_individual 格式""" parsed = {} for param_def in PARAMETER_DEFINITIONS: name = param_def['name'] if param_def['type'] == 'int': val = trial.suggest_int(name, param_def['min'], param_def['max']) else: val = trial.suggest_float(name, param_def['min'], param_def['max']) parsed[name] = val return parsed def _extract_strategy_params(parsed: dict) -> dict: """从解析后的参数提取策略参数字典""" strategy_params = {} for param_def in PARAMETER_DEFINITIONS: strategy_name = param_def.get('strategy') if strategy_name in ('weight', 'signal', 'risk', 'market_state', 'trend_weights', 'trend_thresholds', 'confidence'): continue if strategy_name not in strategy_params: strategy_params[strategy_name] = {} param_name = param_def['name'] if param_name == 'ma_cross_short_window': param_name = 'short_window' elif param_name == 'ma_cross_long_window': param_name = 'long_window' elif param_name == 'rsi_period': param_name = 'period' elif param_name == 'rsi_overbought': param_name = 'overbought' elif param_name == 'rsi_oversold': param_name = 'oversold' elif param_name == 'bollinger_period': param_name = 'period' elif param_name == 'bollinger_std_dev': param_name = 'std_dev' elif param_name == 'macd_fast_ema': param_name = 'fast_ema' elif param_name == 'macd_slow_ema': param_name = 'slow_ema' elif param_name == 'macd_signal_period': param_name = 'signal_period' elif param_name == 'mean_reversion_period': param_name = 'period' elif param_name == 'mean_reversion_std_dev': param_name = 'std_dev' elif param_name == 'momentum_breakout_period': param_name = 'period' elif param_name == 'momentum_breakout_momentum_period': param_name = 'momentum_period' elif param_name == 'kdj_period': param_name = 'period' elif param_name == 'swing_point_left_bars': param_name = 'left_bars' elif param_name == 'swing_point_right_bars': param_name = 'right_bars' elif param_name == 'swing_point_tolerance_pct': param_name = 'tolerance_pct' elif param_name == 'swing_point_num_swings': param_name = 'num_swings' elif param_name.startswith("wave_"): param_name = param_name[5:] if param_name == "period": param_name = "wave_period" elif param_name == 'daily_breakout_bars_count': param_name = 'bars_count' strategy_params[strategy_name][param_name] = parsed[param_def['name']] return strategy_params def _extract_weight_genes(parsed: dict) -> dict: """提取权重基因 → {config_key: weight}""" weight_genes = {} for param_def in PARAMETER_DEFINITIONS: if param_def.get('strategy') != 'weight': continue class_name = param_def['name'].replace('weight_', '') config_key = _class_name_to_config_key.get(class_name, class_name.lower()) weight_genes[config_key] = parsed[param_def['name']] return weight_genes def _precompute_h1_states(multi_tf: MultiTimeframeDataStore, analyzer: MarketStateAnalyzer) -> list: """预计算H1市场状态序列 → 映射到M1时间轴""" h1_df = multi_tf.ensure_timeframe(PERIOD_H1) if h1_df is None or len(h1_df) == 0: return [("none", 0.0)] * multi_tf.length m1_index = multi_tf.main_df.index h1_index = h1_df.index h1_states = [] for i in range(len(h1_df)): lookback = analyzer.trend_period + 10 start = max(0, i - lookback) df_slice = h1_df.iloc[start:i + 1] state, conf = analyzer._calculate_state_from_df(df_slice) h1_states.append((state, conf)) m1_states = [] h1_pos = 0 first_h1_time = h1_index[0] for m1_time in m1_index: if m1_time < first_h1_time: m1_states.append(("none", 0.0)) continue while h1_pos < len(h1_index) - 1 and m1_time >= h1_index[h1_pos + 1]: h1_pos += 1 if h1_pos < len(h1_states): m1_states.append(h1_states[h1_pos]) else: m1_states.append(("none", 0.0)) return m1_states def evaluate_fitness(trial: optuna.Trial, multi_tf: MultiTimeframeDataStore) -> float: """★ Optuna 适应度函数 — 所有权重/策略/趋势参数真正生效""" parsed = _trial_to_params(trial) # 1. 策略参数 strategy_params = _extract_strategy_params(parsed) # 2. 权重基因 weight_genes = _extract_weight_genes(parsed) # 3. 风控参数（从 config 读，不进优化器） risk_params = { 'stop_loss_pct': RISK_CONFIG.get('stop_loss_pct', -0.50), 'profit_retracement_pct': RISK_CONFIG.get('profit_retracement_pct', 0.35), 'min_profit_for_trailing': RISK_CONFIG.get('min_profit_for_trailing', 0.70), 'take_profit_pct': RISK_CONFIG.get('take_profit_pct', 1.00), 'max_holding_minutes': RISK_CONFIG.get('max_holding_minutes', 0), 'min_profit_for_time_exit': RISK_CONFIG.get('min_profit_for_time_exit', 0.005), 'max_daily_loss': RISK_CONFIG.get('max_daily_loss', -0.30), } # 4. 市场状态和趋势参数 market_params = { 'trend_period': parsed.get('market_trend_period', 50), 'retracement_tolerance': parsed.get('market_retracement_tolerance', 0.30), 'volume_period': parsed.get('market_volume_period', 20), 'volume_ma_period': parsed.get('market_volume_ma_period', 10), 'hourly_data_count': 100, } trend_weights = { 'price_breakout': parsed.get('trend_price_breakout_weight', 0.35), 'volume_confirmation': parsed.get('trend_volume_confirmation_weight', 0.25), 'momentum oscillator': parsed.get('trend_momentum_oscillator_weight', 0.20), 'moving_average': parsed.get('trend_moving_average_weight', 0.20), } trend_thresholds = { 'strong_trend': parsed.get('trend_strong_threshold', 0.6), 'weak_trend': parsed.get('trend_weak_threshold', 0.3), 'volume_spike': parsed.get('trend_volume_spike', 1.5), 'oversold': parsed.get('trend_oversold', 30), 'overbought': parsed.get('trend_overbought', 70), } confidence_thresholds = { 'high_confidence': parsed.get('confidence_high', 0.7), 'medium_confidence': parsed.get('confidence_medium', 0.4), } # 5. MarketStateAnalyzer analyzer = MarketStateAnalyzer( timeframe=PERIOD_H1, market_state_params=market_params, trend_weights=trend_weights, trend_thresholds=trend_thresholds, confidence_thresholds=confidence_thresholds, ) analyzer._precomputed_states = _precompute_h1_states(multi_tf, analyzer) # 5.5 用个体策略参数重新计算信号 strategy_params_by_key = { _class_name_to_config_key.get(cn, cn.lower()): params for cn, params in strategy_params.items() } registry = StrategyRegistry() sig_instances = registry.instantiate_all(SYMBOL, TIMEFRAME, strategy_params_by_key) signals_df = pd.DataFrame(index=multi_tf.main_df.index) for config_key, strat in sig_instances.items(): try: sig = strat.run_backtest(multi_tf.main_df) signals_df[config_key] = sig if sig is not None else pd.Series(0, index=signals_df.index) except Exception: signals_df[config_key] = pd.Series(0, index=signals_df.index) # 6. 回测环境 data_provider = BacktestDataProvider(multi_tf, INITIAL_CAPITAL) pm = PositionManager(data_provider, trade_direction="both", risk_config=risk_params, persist_peaks=False) buy_th = parsed.get('buy_threshold', 1.5) sell_th = parsed.get('sell_threshold', -1.5) total_bars = multi_tf.length # 7. 逐 bar 回测 for bar_index in range(total_bars): current_price = data_provider.get_current_price(SYMBOL) if not current_price: data_provider.tick() continue state, conf = analyzer.get_market_state(bar_index) weights = analyzer.get_strategy_weights(state, conf, individual_weights=weight_genes) bar_signals = {col: signals_df[col].iloc[bar_index] for col in signals_df.columns} final_signal = SignalCombiner.combine_at_bar(bar_signals, weights, buy_th, sell_th) if final_signal == 1: pm.open_position("buy", current_price, 1.0, dry_run=True) elif final_signal == -1: pm.open_position("sell", current_price, 1.0, dry_run=True) pm.monitor_positions(current_price, dry_run=True) data_provider.tick() # 8. 适应度 = 总盈亏 return pm.total_equity - INITIAL_CAPITAL def _json_to_mt5_rates(rates_data): """远程API JSON → MT5 numpy 结构化数组""" if not rates_data: return None records = [(r['time'], r['open'], r['high'], r['low'], r['close'], r.get('tick_volume', 0), r.get('spread', 0), r.get('real_volume', 0)) for r in rates_data] return np.array(records, dtype=_MT5_RATES_DTYPE) def load_historical_data(): """一次性加载M1数据并返回 MultiTimeframeDataStore""" if DATA_PROVIDER_MODE == "remote": provider = RemoteDataProvider(host=REMOTE_SERVER_HOST, port=REMOTE_SERVER_PORT) if not provider.initialize(): raise RuntimeError("远程MT5 API初始化失败，请检查 Windows MT5 是否运行") rates_json = provider.get_historical_data(SYMBOL, TIMEFRAME, OPTIMIZER_COUNT) provider.shutdown() if not rates_json: raise RuntimeError("远程获取历史数据失败") rates = _json_to_mt5_rates(rates_json) else: initialize() rates = (get_rates(SYMBOL, TIMEFRAME, OPTIMIZER_COUNT, OPTIMIZER_START_DATE, OPTIMIZER_END_DATE) if USE_DATE_RANGE else get_rates(SYMBOL, TIMEFRAME, OPTIMIZER_COUNT)) shutdown() if rates is None or len(rates) == 0: raise RuntimeError("获取历史数据失败") if len(rates) > OPTIMIZER_COUNT: rates = rates[-OPTIMIZER_COUNT:] store = MultiTimeframeDataStore() store.load_m1_data(rates) store.ensure_timeframe(PERIOD_H1) return store def save_optimization_results(best_params: dict, best_fitness: float, study: optuna.Study): """保存优化结果（兼容旧格式）""" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") import json results = { 'best_fitness': best_fitness, 'best_params': best_params, 'n_trials': len(study.trials), 'optimizer': 'optuna_tpe', } json_path = f"optimization_results_{timestamp}.json" with open(json_path, 'w', encoding='utf-8') as f: json.dump(results, f, ensure_ascii=False, indent=2, default=str) logger.info(f"结果已保存: {json_path}") txt_path = f"optimization_report_{timestamp}.txt" with open(txt_path, 'w', encoding='utf-8') as f: f.write(f"优化完成时间: {datetime.now()}\n") f.write(f"最佳适应度: {best_fitness:.2f}\n") f.write(f"试验次数: {len(study.trials)}\n\n") f.write("最佳参数:\n") for key, value in best_params.items(): f.write(f" {key}: {value}\n") logger.info(f"报告已保存: {txt_path}") def run_optimizer(): """★ Optuna TPE 贝叶斯优化（替代 DEAP 遗传算法）""" try: # 1. 加载数据 logger.info("加载历史数据...") multi_tf = load_historical_data() logger.info(f"数据加载完成: {multi_tf.length} 条M1数据") # 2. Optuna study sampler = optuna.samplers.TPESampler(seed=SEED, n_startup_trials=10) study = optuna.create_study(direction="maximize", sampler=sampler) # 3. 优化（4进程并行，静默回测日志避免 I/O 争用） optuna.logging.set_verbosity(optuna.logging.WARNING) import logging as _logging _old_level = logger.level _old_handler_levels = [h.level for h in logger.handlers] logger.setLevel(_logging.WARNING) for h in logger.handlers: h.setLevel(_logging.WARNING) logger.info(f"开始贝叶斯优化: 50 次试验, TPE sampler (回测日志已静默)") try: study.optimize( lambda trial: evaluate_fitness(trial, multi_tf), n_trials=50, n_jobs=4, show_progress_bar=False, ) finally: logger.setLevel(_old_level) for h, lvl in zip(logger.handlers, _old_handler_levels): h.setLevel(lvl) # 4. 结果 best_fitness = study.best_value best_params = study.best_params logger.info(f"优化完成: 最佳适应度={best_fitness:.2f}, 试验次数={len(study.trials)}") save_optimization_results(best_params, best_fitness, study) return best_params, best_fitness except Exception as e: import traceback logger.error(f"优化器错误: {e}\n{traceback.format_exc()}") raise if __name__ == "__main__": run_optimizer()