From 144311f6ec67cfbb2f43f804b29def49f893923c Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Mon, 13 Apr 2026 15:52:03 +0200 Subject: [PATCH] fix(strategy): Re-evaluate Optuna-optimized strategies with full OHLCV backtest - Add _patch_strategy_code() to inject Optuna's best parameters into LLM-generated strategy code (handles window, entry_thresh, exit_thresh, signal_window, and .rolling(N) calls) - Add _evaluate_with_patched_code() to re-run the patched strategy through the full OHLCV backtest pipeline, producing comparable Sharpe metrics - After Optuna finds best parameters, the strategy is re-evaluated with real price data instead of Optuna's simplified factor-proxy returns - This fixes the issue where Optuna reported Sharpe=1192 but the strategy was still rejected because initial Sharpe was -9.82 (different calculation) - Now strategies can be rescued if Optuna finds parameters that produce positive Sharpe in the real backtest --- .../components/coder/strategy_orchestrator.py | 160 +++++++++++++++++- 1 file changed, 158 insertions(+), 2 deletions(-) diff --git a/rdagent/components/coder/strategy_orchestrator.py b/rdagent/components/coder/strategy_orchestrator.py index 8af8dd13..11601f70 100644 --- a/rdagent/components/coder/strategy_orchestrator.py +++ b/rdagent/components/coder/strategy_orchestrator.py @@ -1129,6 +1129,7 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) optimized = optimizer.optimize_strategy(result, factor_values) optimized_sharpe = optimized.get("sharpe_ratio", float('-inf')) optimized_status = optimized.get("status", "rejected") + best_params = optimized.get("best_params", {}) # Check if Optuna improved the strategy if optimized_sharpe > initial_sharpe: @@ -1137,7 +1138,24 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) f"Optuna {'RESCUED' if optimized_status == 'accepted' and initial_status == 'rejected' else 'improved'} " f"{strategy_name}: Sharpe {initial_sharpe:.4f} → {optimized_sharpe:.4f} (+{improvement:.4f})" ) - result.update(optimized) + + # Re-evaluate with best parameters to get comparable metrics + if best_params: + patched_code = self._patch_strategy_code(code, best_params) + re_eval = self._evaluate_with_patched_code(patched_code, strategy_name, factors) + if re_eval.get("sharpe_ratio", float('-inf')) > initial_sharpe: + result.update(re_eval) + result["code"] = patched_code + result["best_params"] = best_params + logger.info( + f"Re-evaluated {strategy_name} with best params: " + f"Sharpe {initial_sharpe:.4f} → {re_eval.get('sharpe_ratio', 0):.4f}" + ) + else: + result.update(optimized) + result["best_params"] = best_params + else: + result.update(optimized) else: logger.debug(f"Optuna did not improve {strategy_name}: {initial_sharpe:.4f} vs {optimized_sharpe:.4f}") else: @@ -1154,7 +1172,7 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) series = self.load_factor_values(fname) if series is not None: factor_values[fname] = series - + if factor_values: df = pd.DataFrame(factor_values) # Forward-fill to OHLCV index @@ -1164,6 +1182,144 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) return df.dropna() return None + def _patch_strategy_code(self, code: str, params: Dict[str, Any]) -> str: + """Patch strategy code with Optuna's best parameters.""" + import re + patched = code + + entry_thresh = params.get("entry_threshold", 0.8) + exit_thresh = params.get("exit_threshold", 0.3) + zscore_window = params.get("zscore_window", 50) + signal_window = params.get("signal_window", 3) + + param_patterns = [ + (r'entry_thresh\s*=\s*[\d.]+', f'entry_thresh = {entry_thresh}'), + (r'exit_thresh\s*=\s*[\d.]+', f'exit_thresh = {exit_thresh}'), + (r'window\s*=\s*\d+', f'window = {zscore_window}'), + (r'signal_window\s*=\s*\d+', f'signal_window = {signal_window}'), + ] + for pattern, replacement in param_patterns: + patched = re.sub(pattern, replacement, patched) + + # Patch .rolling(N) calls for common window sizes + rolling_pattern = r'\.rolling\((\d+)\)' + def replace_rolling(match): + val = int(match.group(1)) + if val in (20, 30, 50, 100, 200): + return f'.rolling({zscore_window})' + return match.group(0) + patched = re.sub(rolling_pattern, replace_rolling, patched) + + return patched + + def _evaluate_with_patched_code(self, patched_code: str, strategy_name: str, factors: List[Dict]) -> Dict[str, Any]: + """Re-evaluate strategy with patched parameters using full OHLCV backtest.""" + try: + factor_names = [f["factor_name"] for f in factors if f["factor_name"] != "timestamp"] + factor_values = {} + for fname in factor_names: + series = self.load_factor_values(fname) + if series is not None: + factor_values[fname] = series + + if not factor_values: + return {"sharpe_ratio": float('-inf'), "status": "rejected"} + + common_idx = None + for name, s in factor_values.items(): + if common_idx is None: + common_idx = s.index + else: + common_idx = common_idx.intersection(s.index) + + if common_idx is None or len(common_idx) < 100: + return {"sharpe_ratio": float('-inf'), "status": "rejected"} + + df_factors = pd.DataFrame({ + name: s.reindex(common_idx) for name, s in factor_values.items() + }).dropna() + + for col in df_factors.columns: + df_factors[col] = pd.to_numeric(df_factors[col], errors='coerce') + + close = self.load_ohlcv_close() + if close is not None: + df_factors = df_factors.reindex(close.index).ffill() + + df_factors = df_factors.dropna() + if len(df_factors) < 1000: + return {"sharpe_ratio": float('-inf'), "status": "rejected"} + + if close is not None: + close = close.reindex(df_factors.index) + + local_vars = {"factors": df_factors} + try: + exec(patched_code, {"np": np, "pd": pd, "numpy": np}, local_vars) + except Exception: + return {"sharpe_ratio": float('-inf'), "status": "rejected"} + + if "signal" not in local_vars: + return {"sharpe_ratio": float('-inf'), "status": "rejected"} + + signal = local_vars["signal"] + signal_index = signal.index + close_aligned = close.reindex(signal_index).ffill() + price_returns = close_aligned.pct_change().fillna(0) + signal_positions = signal.shift(1).fillna(0) + returns = price_returns * signal_positions + + SPREAD_COST = 0.00015 + signal_changes = signal_positions.diff().abs().fillna(0) + spread_costs = signal_changes * SPREAD_COST + returns = returns - spread_costs + + if returns.std() == 0 or len(returns) < 10: + return {"sharpe_ratio": float('-inf'), "status": "rejected"} + + total_return = float(returns.sum()) + n_periods = len(returns) + minutes_per_year = 252 * 1440 + ann_factor = np.sqrt(minutes_per_year) + years = max(n_periods / minutes_per_year, 0.1) + + if years >= 1 and (1 + total_return) > 0: + ann_return = (1 + total_return) ** (1 / years) - 1 + else: + ann_return = total_return / years + + volatility = float(returns.std() * ann_factor) + sharpe = ann_return / volatility if volatility > 0 else 0.0 + + returns = returns.fillna(0).replace([np.inf, -np.inf], 0) + cum_returns = (1 + returns).cumprod() + running_max = cum_returns.expanding().max() + drawdown = (cum_returns - running_max) / running_max.replace(0, np.nan) + drawdown = drawdown.fillna(0).replace([np.inf, -np.inf], 0) + max_dd = float(drawdown.min()) if len(drawdown) > 0 else 0.0 + + signal_changes_eval = signal.diff().fillna(0) + trades = signal_changes_eval[signal_changes_eval != 0] + win_rate = float((trades > 0).sum() / len(trades)) if len(trades) > 0 else 0.0 + + status = "accepted" if sharpe >= self.min_sharpe and max_dd >= self.max_drawdown and win_rate >= self.min_win_rate else "rejected" + + return { + "strategy_name": strategy_name, + "status": status, + "sharpe_ratio": round(sharpe, 4), + "annualized_return": round(ann_return, 6), + "max_drawdown": round(max_dd, 6), + "win_rate": round(win_rate, 4), + "volatility": round(volatility, 6), + "total_return": round(total_return, 6), + "num_periods": n_periods, + } + + except Exception as e: + logger.debug(f"Re-evaluation failed for {strategy_name}: {e}") + return {"sharpe_ratio": float('-inf'), "status": "rejected"} + def _save_strategy(self, result: Dict[str, Any]) -> None: """Save accepted strategy to JSON file.""" timestamp = int(time.time())