diff --git a/QWEN.md b/QWEN.md index fb7335c0..cb4ece8a 100644 --- a/QWEN.md +++ b/QWEN.md @@ -92,6 +92,12 @@ Predix/ **πŸ”’ CLOSED SOURCE (Local Only - NOT on GitHub):** - `models/local/` - Your improved models (Transformer, TCN, PatchTST, CNN+LSTM) - `prompts/local/` - Your improved prompts (v2.0 optimized) +- `rdagent/scenarios/qlib/local/` - Advanced components: + - `strategy_coster.py` - StrategyCoSTEER (LLM strategy generation) + - `strategy_evaluator.py` - Comprehensive strategy metrics + - `strategy_runner.py` - Strategy execution & backtesting + - `strategy_discovery_v1.yaml` - LLM prompts for strategy generation + - Plus: ml_trainer, portfolio_optimizer, quant_loop_advanced, etc. - `.env` - API keys - `results/` - Backtest results - `git_ignore_folder/` - Trading data diff --git a/STRATEGY_BUILDER_DESIGN.md b/STRATEGY_BUILDER_DESIGN.md new file mode 100644 index 00000000..88af93be --- /dev/null +++ b/STRATEGY_BUILDER_DESIGN.md @@ -0,0 +1,890 @@ +# StrategyBuilder β€” Architektur-Design + +## Überblick + +Der **StrategyBuilder** kombiniert existierende Faktoren systematisch zu handelbaren Strategien. +Im Gegensatz zum ML-Trainer (der ein einzelnes Modell auf Top-Faktoren trainiert) testet der +StrategyBuilder **explizite Kombinationsregeln** mit Walk-Forward-Validierung. + +--- + +## 1. Klassen-Design + +### 1.1 StrategyCombinator + +**Zweck:** Generiert systematische Faktorkombinationen nach verschiedenen Strategien. + +```python +# rdagent/scenarios/qlib/developer/strategy_builder.py + +class CombinationStrategy(Enum): + """Supported combination methods.""" + PAIR = "pair" # Top-N pairs by IC product + TRIPLET = "triplet" # Top triplets + CATEGORY = "category" # All factors of same type + TEMPORAL = "temporal" # Session/time-specific combos + CUSTOM = "custom" # User-defined combinations + + +@dataclass +class StrategySpec: + """Defines a single strategy configuration.""" + name: str + factors: List[str] # Factor names to combine + combination_type: str # "weighted_sum", "regime_switch", etc. + weighting: str # "equal", "ic_weighted", "risk_parity" + metadata: Dict[str, Any] # Additional context (category, session, etc.) + + +class StrategyCombinator: + """Generate factor combinations systematically.""" + + def __init__( + self, + factors_db: ResultsDatabase, + min_ic: float = 0.02, + max_factors_per_strategy: int = 5, + ) -> None: ... + + def load_valid_factors(self, min_ic: float = 0.02) -> pd.DataFrame: + """Load all factors with IC >= threshold from DB.""" + ... + + def generate_pairs( + self, + top_n: int = 50, + max_correlation: float = 0.7, + ) -> List[StrategySpec]: + """ + Generate pairwise combinations. + + Rules: + - Take top_n factors by |IC| + - Filter pairs with correlation < max_correlation + - Score by |IC1 * IC2| (both must have predictive power) + - Prefer complementary pairs (one positive IC, one negative) + """ + ... + + def generate_triplets( + self, + top_n: int = 30, + max_pairwise_corr: float = 0.5, + ) -> List[StrategySpec]: + """ + Generate triplet combinations. + + Rules: + - Top 30 factors by |IC| + - All pairwise correlations < max_pairwise_corr + - Score by geometric mean of |IC| + """ + ... + + def generate_category_combos( + self, + category: str, + min_factors: int = 2, + max_factors: int = 5, + ) -> List[StrategySpec]: + """ + Combine all factors within a category. + + Categories (inferred from factor names): + - "Momentum": mom_*, trend_* + - "Mean Reversion": mean_rev_*, reversal_* + - "Volatility": vol_*, std_* + - "Session": session_*, intraday_* + - "Volume": volume_*, turnover_* + """ + ... + + def generate_temporal_combos( + self, + session_filters: Dict[str, Callable], + ) -> List[StrategySpec]: + """ + Generate session-specific combinations. + + Example strategies: + - "London Open": Use momentum factors 07:00-09:00 UTC + - "NY Close": Use mean reversion 14:00-16:00 UTC + - "Asian Session": Use volatility factors 00:00-06:00 UTC + """ + ... + + def generate_custom_combo( + self, + factor_names: List[str], + weighting: str = "equal", + ) -> StrategySpec: + """User-defined combination for testing specific hypotheses.""" + ... + + def generate_all( + self, + strategies: List[CombinationStrategy] = None, + ) -> List[StrategySpec]: + """ + Run all enabled combination strategies. + + Default: PAIR + TRIPLET + CATEGORY + Returns list of all StrategySpec objects. + """ + ... +``` + +--- + +### 1.2 StrategyEvaluator + +**Zweck:** Walk-Forward-Backtesting fΓΌr Strategien mit Transaktionskosten. + +```python +@dataclass +class WalkForwardConfig: + """Walk-forward validation configuration.""" + train_window: int = 30 # Days for training + test_window: int = 5 # Days for out-of-sample testing + step_size: int = 5 # Days to slide forward + min_train_periods: int = 3 # Minimum windows before first test + + +@dataclass +class TransactionCostModel: + """Realistic transaction cost modeling.""" + cost_per_trade_bps: float = 1.5 # 1.5 bps per trade + slippage_bps: float = 0.5 # Additional slippage + min_trade_size: float = 0.01 # Minimum position size + + +class StrategyMetrics: + """Complete metrics for a validated strategy.""" + + def __init__(self, strategy_name: str) -> None: ... + + def update( + self, + window_idx: int, + in_sample_ic: float, + out_of_sample_ic: float, + oos_sharpe: float, + oos_return: float, + oos_drawdown: float, + n_trades: int, + transaction_costs: float, + ) -> None: ... + + def finalize(self) -> Dict[str, Any]: + """ + Calculate aggregate metrics: + + - Mean OOS IC + - IC decay (IS IC vs OOS IC) + - Mean OOS Sharpe + - Worst OOS Drawdown + - Calmar Ratio (Ann Return / Max DD) + - Total transaction costs + - Win rate across windows + - Consistency score (% windows with positive IC) + """ + ... + + +class StrategyEvaluator: + """Walk-forward backtesting for strategy combinations.""" + + def __init__( + self, + data_source: str, # Path to intraday_pv.h5 + wf_config: WalkForwardConfig = None, + cost_model: TransactionCostModel = None, + ) -> None: ... + + def load_factor_values( + self, + factor_names: List[str], + ) -> Dict[str, pd.Series]: + """Load time series values for each factor.""" + ... + + def compute_combined_signal( + self, + factor_values: Dict[str, pd.Series], + weights: Dict[str, float], + combination_type: str = "weighted_sum", + ) -> pd.Series: + """ + Combine factors into single signal. + + Types: + - "weighted_sum": sum(w_i * factor_i) + - "regime_switch": use different factors per regime + - "timing": use volatility to scale momentum + """ + ... + + def walk_forward_backtest( + self, + strategy_spec: StrategySpec, + ) -> StrategyMetrics: + """ + Run walk-forward validation for a single strategy. + + Process: + 1. Split time series into rolling windows + 2. For each window: + a. Optimize weights on train period + b. Test on out-of-sample period + c. Apply transaction costs + d. Record metrics + 3. Aggregate across all windows + + Returns StrategyMetrics with full validation results. + """ + ... + + def backtest_single_window( + self, + train_data: pd.DataFrame, + test_data: pd.DataFrame, + strategy_spec: StrategySpec, + ) -> Dict[str, float]: + """ + Backtest strategy on single train/test split. + + Steps: + 1. Compute factor values on train period + 2. Optimize weights (IC-weighted or risk parity) + 3. Apply to test period + 4. Calculate returns with transaction costs + 5. Return metrics + """ + ... + + def apply_transaction_costs( + self, + raw_returns: pd.Series, + signals: pd.Series, + cost_model: TransactionCostModel, + ) -> pd.Series: + """ + Deduct transaction costs from returns. + + Cost = (signal changes) * (cost_per_trade + slippage) + Only charged when position actually changes. + """ + ... +``` + +--- + +### 1.3 StrategySelector + +**Zweck:** Selektiere beste Strategien nach Out-of-Sample-Performance. + +```python +@dataclass +class StrategyRanking: + """Ranking criteria for strategies.""" + primary_metric: str = "oos_sharpe" # oos_sharpe, calmar, oos_ic + min_oos_ic: float = 0.02 # Minimum OOS IC + max_drawdown: float = -0.15 # Maximum allowed drawdown + min_consistency: float = 0.6 # % of windows with positive IC + min_windows: int = 3 # Minimum validation windows + + +class StrategySelector: + """Select and rank best strategies based on walk-forward results.""" + + def __init__( + self, + ranking: StrategyRanking = None, + ) -> None: ... + + def rank_strategies( + self, + strategy_results: List[Dict[str, Any]], + ) -> pd.DataFrame: + """ + Rank strategies by primary metric. + + Filters: + - OOS IC >= min_oos_ic + - Max DD <= max_drawdown threshold + - Consistency >= min_consistency + - At least min_windows validated + + Returns sorted DataFrame with: + - strategy_name + - oos_sharpe (primary) + - oos_ic_mean + - ic_decay (IS vs OOS gap) + - calmar_ratio + - max_drawdown + - consistency_score + - n_windows + - total_transaction_costs + """ + ... + + def select_top_k( + self, + ranked: pd.DataFrame, + k: int = 10, + ) -> List[Dict[str, Any]]: + """Return top K strategies passing all filters.""" + ... + + def identify_overfitting( + self, + strategy_results: List[Dict[str, Any]], + ic_decay_threshold: float = 0.5, + ) -> List[str]: + """ + Flag strategies where OOS IC < 50% of IS IC. + Indicates overfitting to training period. + """ + ... + + def recommend_ensemble( + self, + ranked: pd.DataFrame, + max_correlation: float = 0.3, + max_strategies: int = 3, + ) -> List[str]: + """ + Recommend ensemble of uncorrelated strategies. + + Select up to max_strategies with: + - Highest combined Sharpe + - Pairwise correlation < max_correlation + """ + ... +``` + +--- + +### 1.4 StrategySaver + +**Zweck:** Persistiert Strategien in `results/strategies/`. + +```python +class StrategySaver: + """Save validated strategies to results/strategies/.""" + + def __init__( + self, + strategies_dir: Optional[str] = None, + ) -> None: + project_root = Path(__file__).parent.parent.parent.parent + self.strategies_dir = Path(strategies_dir) if strategies_dir \ + else project_root / "results" / "strategies" + self.strategies_dir.mkdir(parents=True, exist_ok=True) + + def save_strategy( + self, + strategy_spec: StrategySpec, + metrics: Dict[str, Any], + ranking: Dict[str, Any] = None, + ) -> Path: + """ + Save complete strategy to JSON. + + JSON structure: + { + "name": "momentum_mean_rev_pair", + "created_at": "2026-04-05T12:00:00", + "combination_type": "pair", + "factors": ["Momentum_v3", "MeanReversion_v2"], + "weights": {"Momentum_v3": 0.63, "MeanReversion_v2": 0.37}, + "weighting_method": "ic_weighted", + + "walk_forward": { + "train_window_days": 30, + "test_window_days": 5, + "n_windows": 8, + "total_test_days": 40 + }, + + "metrics": { + "oos_ic_mean": 0.045, + "oos_ic_std": 0.012, + "is_ic_mean": 0.062, + "ic_decay": 0.27, + "oos_sharpe": 2.15, + "oos_annualized_return": 0.128, + "oos_max_drawdown": -0.089, + "calmar_ratio": 1.44, + "consistency_score": 0.875, + "win_rate": 0.58, + "total_transaction_costs_bps": 12.4, + "net_sharpe": 1.98 + }, + + "per_window_metrics": [ + {"window": 0, "oos_ic": 0.051, "oos_sharpe": 2.3, ...}, + {"window": 1, "oos_ic": 0.038, "oos_sharpe": 1.9, ...}, + ... + ], + + "ranking": { + "rank_by_sharpe": 3, + "rank_by_ic": 5, + "rank_by_calmar": 2, + "passes_filters": true + } + } + """ + ... + + def load_all_strategies( + self, + min_oos_sharpe: float = None, + ) -> List[Dict[str, Any]]: + """Load all saved strategies, optionally filtered.""" + ... + + def load_best_strategy(self) -> Optional[Dict[str, Any]]: + """Load the single best strategy by OOS Sharpe.""" + ... +``` + +--- + +## 2. Kombinations-Logik + +### 2.1 Faktor-Auswahl fΓΌr Kombinationen + +```python +def select_factors_for_combination( + factors_df: pd.DataFrame, + min_ic: float = 0.02, + max_correlation: float = 0.7, +) -> Tuple[List[str], pd.DataFrame]: + """ + Select factors suitable for combination. + + Algorithm: + 1. Filter: |IC| >= min_ic + 2. Compute correlation matrix + 3. Cluster factors by correlation (hierarchical clustering) + 4. From each cluster, pick factor with highest |IC| + 5. Return selected factors + correlation matrix + + Rationale: + - Avoid combining highly correlated factors (redundant) + - Ensure each selected factor has standalone predictive power + - Maximize diversity in combinations + """ + ... +``` + +### 2.2 Pair-Strategie + +``` +Regel: Kombiniere Faktor A + B wenn: + 1. |IC_A| >= 0.02 UND |IC_B| >= 0.02 + 2. Korrelation(A, B) < 0.7 + 3. Score = |IC_A * IC_B| * (1 - corr(A, B)) + +Priorisiere: + - Momentum + Mean Reversion (komplementΓ€r) + - Volatility + Momentum (Timing) + - Session + Hauptfaktor (Filter) +``` + +### 2.3 Triplet-Strategie + +``` +Regel: Kombiniere Faktor A + B + C wenn: + 1. Alle |IC| >= 0.02 + 2. Alle pairwise Korrelationen < 0.5 + 3. Score = (|IC_A| * |IC_B| * |IC_C|)^(1/3) * diversity_factor + +Priorisiere: + - Momentum + Mean Reversion + Volatility + - Hauptfaktor + Session + Volatility + - Drei unkorrelierte Alpha-Faktoren +``` + +### 2.4 Gewichtungsmethoden + +```python +def compute_weights( + factor_ics: Dict[str, float], + factor_correlations: pd.DataFrame, + method: str = "ic_weighted", +) -> Dict[str, float]: + """ + Compute factor weights. + + Methods: + + 1. "equal": w_i = 1/N + + 2. "ic_weighted": w_i = |IC_i| / sum(|IC|) + - Simple, effective when ICs are reliable + + 3. "risk_parity": + - w_i proportional to 1/vol_i + - Equalize risk contribution from each factor + - Requires factor return covariance matrix + + 4. "sharpe_weighted": w_i = Sharpe_i / sum(Sharpe) + - Weight by risk-adjusted performance + + Returns normalized weights summing to 1.0 + """ + ... +``` + +--- + +## 3. Walk-Forward-Validierung + +### 3.1 Schema + +``` +Zeitachse (Beispiel: 90 Tage Daten): + +[---- Train 30d ----][Test 5d][---- Train 30d ----][Test 5d]... + Window 0 Window 1 + +Gesamt: ~8 Walks bei 90 Tagen +``` + +### 3.2 Ablauf pro Window + +```python +for window_idx in range(n_windows): + # 1. Define train/test periods + train_start = window_idx * step_size + train_end = train_start + train_window + test_start = train_end + test_end = test_start + test_window + + # 2. Optimize weights on train period + weights = optimize_weights( + factor_values[train_start:train_end], + forward_returns[train_start:train_end], + method=strategy_spec.weighting, + ) + + # 3. Generate signal on test period + signal = compute_combined_signal( + factor_values[test_start:test_end], + weights, + ) + + # 4. Calculate returns with costs + raw_returns = signal.shift(1) * forward_returns[test_start:test_end] + net_returns = apply_transaction_costs(raw_returns, signal, cost_model) + + # 5. Record metrics + metrics.update( + window_idx=window_idx, + in_sample_ic=compute_ic(train_period), + out_of_sample_ic=compute_ic(test_period), + oos_sharpe=calculate_sharpe(net_returns), + oos_drawdown=calculate_max_drawdown(net_returns), + n_trades=count_signal_changes(signal), + transaction_costs=raw_returns.sum() - net_returns.sum(), + ) +``` + +### 3.3 Aggregierte Metriken + +```python +final_metrics = { + # Primary + "oos_ic_mean": mean(window_oos_ics), + "oos_ic_std": std(window_oos_ics), + "oos_sharpe": mean(window_sharpes), + + # Overfitting detection + "is_ic_mean": mean(window_is_ics), + "ic_decay": 1 - (oos_ic_mean / is_ic_mean), # < 0.5 good + + # Risk + "oos_max_drawdown": min(window_drawdowns), + "calmar_ratio": annualized_return / abs(max_drawdown), + + # Consistency + "consistency_score": sum(ic > 0 for ic in window_oos_ics) / n_windows, + + # Costs + "total_transaction_costs_bps": sum(window_costs), + "net_sharpe": sharpe_after_costs, +} +``` + +--- + +## 4. Integrationspunkte mit factor_runner.py + +### 4.1 Wo passt der StrategyBuilder hin? + +``` +Bestehender Flow (factor_runner.py): +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 1. Hypothesis Gen β†’ Factor Hypothesis β”‚ +β”‚ 2. Factor Coder β†’ Generate factor code β”‚ +β”‚ 3. Factor Runner β†’ Docker backtest β”‚ +β”‚ 4. Protection Check β†’ Risk validation β”‚ +β”‚ 5. Save to DB β†’ ResultsDatabase β”‚ +β”‚ 6. Feedback β†’ Guide next hypothesis β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +NEUER Flow (StrategyBuilder): +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 7. StrategyCombinator β†’ Combos β”‚ ← AFTER factor generation +β”‚ 8. StrategyEvaluator β†’ Walk-forward β”‚ ← SEPARATE phase +β”‚ 9. StrategySelector β†’ Rank strategies β”‚ +β”‚ 10. StrategySaver β†’ results/strategies/ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### 4.2 Konkrete Integration + +```python +# Option A: EigenstΓ€ndiger CLI-Befehl (empfohlen) +# rdagent/build_strategies --top-n 100 --walk-forward + +# Option B: Integration in QuantRDLoop +class QuantRDLoop: + def running(self, prev_out): + # ... existing factor runner code ... + exp = self.factor_runner.develop(prev_out["coding"]) + + # NEW: Periodically run strategy builder + if self.should_build_strategies(): + self._run_strategy_builder() + + return exp + + def should_build_strategies(self) -> bool: + """Check if enough factors exist to build strategies.""" + n_factors = self.trace.get_valid_factor_count() + return n_factors >= 100 and self.loop_idx % 50 == 0 + + def _run_strategy_builder(self) -> None: + """Trigger strategy building process.""" + from rdagent.scenarios.qlib.developer.strategy_builder import ( + StrategyBuilder, + ) + + builder = StrategyBuilder( + db=self.results_db, + data_source=self.data_path, + ) + builder.run(top_n=100) +``` + +### 4.3 DatenabhΓ€ngigkeiten + +```python +# BenΓΆtigt von factor_runner.py: +# βœ… ResultsDatabase β†’ already exists, factor_runner schreibt dort +# βœ… Factor JSON files β†’ already in results/factors/ +# βœ… Factor values β†’ MΓΌssen aus workspace/result.h5 geladen werden + +# Neue AbhΓ€ngigkeit: +# ⚠️ Factor time series values β†’ MΓΌssen fΓΌr Walk-Forward verfΓΌgbar sein +# LΓΆsung: Factor values beim Speichern in DB auch als Parquet schreiben +``` + +--- + +## 5. Integration in QuantRDLoop Workflow + +### 5.1 Erweiterte Loop-Phasen + +``` +Phase 1: Factor Generation (EXISTIEREND) + └─ Generate β†’ Code β†’ Backtest β†’ Save to DB + └─ Continue until N factors reached (z.B. 500) + +Phase 2: Strategy Building (NEU) + └─ Load top factors from DB + └─ Generate combinations (pairs, triplets, categories) + └─ Walk-forward validation + └─ Save strategies to results/strategies/ + +Phase 3: Strategy Selection (NEU) + └─ Rank by OOS Sharpe + └─ Filter by max drawdown, consistency + └─ Select top 3 strategies for live trading + +Phase 4: ML Training (EXISTIEREND, optional) + └─ Train ML model on top strategies' factors + +Phase 5: Live Trading (ZUKUNFT) + └─ Paper trade selected strategies + └─ Monitor and adapt +``` + +### 5.2 Haupt-CLI-Befehl + +```python +# rdagent/scenarios/qlib/developer/strategy_builder.py + +class StrategyBuilder: + """Main orchestrator for strategy building process.""" + + def __init__( + self, + db: ResultsDatabase, + data_source: str, + output_dir: Optional[str] = None, + ) -> None: + self.db = db + self.data_source = data_source + self.combinator = StrategyCombinator(db) + self.evaluator = StrategyEvaluator(data_source) + self.selector = StrategySelector() + self.saver = StrategySaver(output_dir) + + def run( + self, + top_n: int = 100, + min_ic: float = 0.02, + strategies: List[CombinationStrategy] = None, + save: bool = True, + ) -> pd.DataFrame: + """ + Complete strategy building pipeline. + + Steps: + 1. Load top N factors from DB + 2. Generate combinations + 3. Walk-forward validate each + 4. Rank and filter + 5. Save top strategies + 6. Return ranked results + """ + logger.info(f"=== Strategy Builder: Top {top_n} factors ===") + + # Step 1: Load factors + factors = self.combinator.load_valid_factors(min_ic=min_ic) + logger.info(f"Loaded {len(factors)} valid factors") + + # Step 2: Generate combinations + combos = self.combinator.generate_all(strategies) + logger.info(f"Generated {len(combos)} strategy combinations") + + # Step 3: Walk-forward validate + results = [] + for spec in combos: + logger.info(f"Evaluating: {spec.name}") + metrics = self.evaluator.walk_forward_backtest(spec) + results.append(metrics.finalize()) + + # Step 4: Rank + ranked = self.selector.rank_strategies(results) + + # Step 5: Save + if save: + for _, row in ranked.iterrows(): + spec = next(s for s in combos if s.name == row["strategy_name"]) + self.saver.save_strategy(spec, row) + + logger.info(f"=== Top 5 Strategies ===") + logger.info(ranked.head(5).to_string()) + + return ranked + + +def build_strategies( + top_n: int = 100, + min_ic: float = 0.02, + data_source: str = None, +) -> None: + """CLI entry point: rdagent build_strategies""" + from rdagent.components.backtesting.results_db import ResultsDatabase + + db = ResultsDatabase() + + if data_source is None: + data_source = str(Path(__file__).parent.parent.parent.parent.parent + / "git_ignore_folder" + / "factor_implementation_source_data" + / "intraday_pv.h5") + + builder = StrategyBuilder(db=db, data_source=data_source) + ranked = builder.run(top_n=top_n, min_ic=min_ic) + + logger.info(f"\nStrategy building complete. Results in results/strategies/") +``` + +### 5.3 Config-Erweiterung + +```python +# rdagent/app/qlib_rd_loop/conf.py + +@dataclass +class StrategyBuilderSetting: + """Configuration for strategy building.""" + top_n_factors: int = 100 + min_ic_threshold: float = 0.02 + max_correlation: float = 0.7 + train_window_days: int = 30 + test_window_days: int = 5 + step_size_days: int = 5 + transaction_cost_bps: float = 1.5 + min_oos_sharpe: float = 1.0 + max_drawdown_threshold: float = -0.15 + combination_strategies: List[str] = None # ["pair", "triplet", "category"] +``` + +--- + +## 6. Datei-Struktur + +``` +rdagent/scenarios/qlib/developer/ +└── strategy_builder.py # Hauptmodul (alle Klassen) + +# ODER aufgeteilt: +rdagent/scenarios/qlib/developer/ +└── strategy_builder/ + β”œβ”€β”€ __init__.py + β”œβ”€β”€ combinator.py # StrategyCombinator + β”œβ”€β”€ evaluator.py # StrategyEvaluator + β”œβ”€β”€ selector.py # StrategySelector + β”œβ”€β”€ saver.py # StrategySaver + └── builder.py # StrategyBuilder (Orchestrator) + +results/ +└── strategies/ + β”œβ”€β”€ momentum_mean_rev_pair.json + β”œβ”€β”€ momentum_vol_timing.json + β”œβ”€β”€ session_alpha_combo.json + └── strategy_ranking.json # Summary aller Strategien +``` + +--- + +## 7. NΓ€chste Schritte + +1. **Implementierung Phase 1:** StrategyCombinator + einfache Pair-Tests +2. **Implementierung Phase 2:** StrategyEvaluator mit Walk-Forward +3. **Implementierung Phase 3:** StrategySelector + Saver +4. **Integration:** CLI-Befehl `rdagent build_strategies` +5. **Validierung:** Top-Strategien gegen Hold-out Periode testen +6. **Dashboard:** Web-UI zur Strategie-Anzeige (erweitert) + +--- + +## 8. Offene Fragen + +- **Factor Values:** Woher kommen die Zeitreihen-Werte fΓΌr jeden Faktor? + - Aktuell: Nur in workspace/result.h5 gespeichert (nicht persistent) + - LΓΆsung: Beim Speichern in DB auch als Parquet in results/factors/values/ ablegen + +- **Performance:** 100 Faktoren β†’ ~5000 Pairs β†’ 8 Walks each = 40.000 Backtests + - LΓΆsung: Parallelisierung (multiprocessing), Top-1000 Paare vorher filtern + +- **Regime Detection:** Wie erkennen wir Markt-Regimes? + - Vorschlag: Volatility-based (high/low vol), Trend-based (uptrend/downtrend) + - SpΓ€ter: ML-basiert (HMM, Clustering) diff --git a/predix.py b/predix.py index 969bba78..aef5c6bd 100644 --- a/predix.py +++ b/predix.py @@ -787,6 +787,274 @@ def portfolio_simple( )) +@app.command() +def build_strategies( + top: int = typer.Option( + 50, + "--top", "-n", + help="Number of top factors to consider (default: 50)", + ), + max_combo: int = typer.Option( + 2, + "--max-combo", "-c", + help="Maximum combination size: 2=pairs, 3=triplets (default: 2)", + ), + diversified: bool = typer.Option( + False, + "--diversified/-d", + help="Only generate cross-category combinations", + ), +): + """ + Build trading strategies by systematically combining factors. + + This command: + 1. Loads top evaluated factors + 2. Generates systematic combinations (pairs, triplets) + 3. Evaluates each combination using walk-forward validation + 4. Ranks by Sharpe ratio and saves best strategies + + Examples: + predix build-strategies # Build from top 50, pairs only + predix build-strategies -n 100 -c 3 # Top 100, up to triplets + predix build-strategies -d # Diversified only + """ + import pandas as pd + import numpy as np + from rich.table import Table + from rich.panel import Panel + + from rdagent.scenarios.qlib.developer.strategy_builder import StrategyBuilder + + console.print(Panel( + "[bold cyan]πŸ—οΈ Predix Strategy Builder[/bold cyan]\n" + "Systematically combining factors into trading strategies", + border_style="cyan", + )) + + builder = StrategyBuilder() + + try: + results = builder.build_strategies( + top_n=top, + max_combo_size=max_combo, + diversified_only=diversified, + ) + except Exception as e: + console.print(f"[bold red]Strategy building failed: {e}[/bold red]") + import traceback + console.print(traceback.format_exc()) + return + + if not results: + console.print("[yellow]No strategies built. Check if factor values exist.[/yellow]") + return + + # Display top strategies + successful = [r for r in results if r.get("status") == "success"] + + if successful: + table = Table( + title=f"Top {min(20, len(successful))} Strategies by Sharpe", + show_header=True, + header_style="bold cyan", + ) + table.add_column("#", justify="center", width=4) + table.add_column("Factors", width=50) + table.add_column("Sharpe", justify="right", width=8) + table.add_column("Ann. Ret %", justify="right", width=10) + table.add_column("Max DD", justify="right", width=8) + table.add_column("Win Rate", justify="right", width=8) + + for i, strat in enumerate(successful[:20], 1): + factors_str = " + ".join(strat["factors"][:3]) + if len(strat["factors"]) > 3: + factors_str += f" +{len(strat['factors'])-3}" + + table.add_row( + str(i), + factors_str, + f"{strat.get('sharpe', 0):.4f}", + f"{strat.get('annualized_return', 0):.4f}", + f"{strat.get('max_drawdown', 0):.4f}", + f"{strat.get('win_rate', 0):.2%}", + ) + + console.print(table) + + # Summary + avg_sharpe = np.mean([s.get("sharpe", 0) for s in successful]) + best_sharpe = max(s.get("sharpe", 0) for s in successful) + avg_dd = np.mean([s.get("max_drawdown", 0) for s in successful]) + + console.print(Panel( + f"[bold]Strategy Building Summary[/bold]\n" + f"Total combinations: {len(results)}\n" + f"Successful: {len(successful)}\n" + f"Failed: {len(results) - len(successful)}\n" + f"Avg Sharpe: {avg_sharpe:.4f}\n" + f"Best Sharpe: {best_sharpe:.4f}\n" + f"Avg Max DD: {avg_dd:.4f}\n" + f"Saved to: results/strategies/", + border_style="green", + )) + else: + console.print("[yellow]No successful strategies. Check factor values exist.[/yellow]") + + +@app.command() +def build_strategies_ai( + top: int = typer.Option( + 50, + "--top", "-t", + help="Number of top factors to use (default: 50)", + ), + max_loops: int = typer.Option( + 5, + "--max-loops", "-l", + help="Maximum improvement cycles (default: 5)", + ), + min_sharpe: float = typer.Option( + 1.5, + "--min-sharpe", + help="Minimum Sharpe ratio for acceptance (default: 1.5)", + ), + max_drawdown: float = typer.Option( + -0.20, + "--max-dd", + help="Maximum acceptable drawdown (default: -0.20)", + ), +): + """ + Build trading strategies using AI (LLM-based StrategyCoSTEER). + + Uses LLM to generate, test, and improve trading strategies from + existing factors. Follows the CoSTEER pattern: + 1. Load top factors by IC + 2. LLM generates strategy hypothesis and code + 3. Execute backtest and evaluate + 4. Feed results back to LLM for improvement + 5. Repeat until convergence or max loops + + Examples: + predix build-strategies-ai # Default: top 50, 5 loops + predix build-strategies-ai -t 100 # Use top 100 factors + predix build-strategies-ai -l 10 # 10 improvement loops + predix build-strategies-ai --min-sharpe 2.0 # Stricter target + """ + from rich.panel import Panel + from pathlib import Path + + console.print(Panel( + "[bold cyan]🧠 StrategyCoSTEER - AI Strategy Builder[/bold cyan]\n" + "Generating trading strategies from existing factors\n" + "Uses LLM to combine factors, backtest, and improve", + border_style="cyan", + )) + + # Check if local module exists + local_module = Path(__file__).parent / "rdagent" / "scenarios" / "qlib" / "local" + if not local_module.exists(): + console.print("[bold red]❌ StrategyCoSTEER not available: local/ directory not found[/bold red]") + console.print("[yellow]This is a closed-source feature. Contact development team.[/yellow]") + return + + costeer_file = local_module / "strategy_coster.py" + if not costeer_file.exists(): + console.print("[bold red]❌ strategy_coster.py not found[/bold red]") + return + + # Load top factors + factors_dir = Path(__file__).parent / "results" / "factors" + if not factors_dir.exists(): + console.print("[bold red]❌ No factors directory found at results/factors/[/bold red]") + console.print("[yellow]Run 'predix quant' to generate factors first.[/yellow]") + return + + # Load evaluated factors + import json + import glob as glob_module + + factors = [] + for f in glob_module.glob(str(factors_dir / "*.json")): + try: + with open(f) as fh: + data = json.load(fh) + if data.get("status") == "success" and data.get("ic") is not None: + factors.append(data) + except Exception: + continue + + if len(factors) < 10: + console.print(f"[bold red]❌ Only {len(factors)} evaluated factors found. Need at least 10.[/bold red]") + console.print("[yellow]Run 'predix evaluate' or 'predix quant' to generate more factors.[/yellow]") + return + + # Sort by IC and take top factors + factors.sort(key=lambda x: abs(x.get("ic", 0) or 0), reverse=True) + top_factors = factors[:top] + + console.print(f"\n[bold green]βœ“ Loaded {len(top_factors)} top factors[/bold green]") + console.print(f" Max loops: {max_loops}") + console.print(f" Target Sharpe: β‰₯ {min_sharpe}") + console.print(f" Max Drawdown: β‰₯ {max_drawdown:.2%}\n") + + # Run StrategyCoSTEER + try: + from rdagent.scenarios.qlib.local.strategy_coster import StrategyCoSTEER + + strategies_dir = Path(__file__).parent / "results" / "strategies" + strategies_dir.mkdir(parents=True, exist_ok=True) + + costeer = StrategyCoSTEER( + factors_dir=str(factors_dir), + strategies_dir=str(strategies_dir), + max_loops=max_loops, + min_sharpe=min_sharpe, + max_drawdown=max_drawdown, + ) + + results = costeer.run(top_factors) + + # Display results + if results: + console.print(f"\n[bold green]βœ“ Generated {len(results)} accepted strategies![/bold green]\n") + + from rich.table import Table + table = Table(title="Accepted Strategies") + table.add_column("#", style="dim") + table.add_column("Strategy", style="cyan") + table.add_column("Factors", style="yellow") + table.add_column("Sharpe", justify="right", style="green") + table.add_column("Max DD", justify="right", style="red") + table.add_column("Win Rate", justify="right") + table.add_column("Loop", justify="center") + + for i, r in enumerate(results, 1): + table.add_row( + str(i), + r.get("strategy_name", "unknown")[:30], + str(len(r.get("factor_names", []))), + f"{r.get('sharpe_ratio', 0):.3f}", + f"{r.get('max_drawdown', 0):.2%}", + f"{r.get('win_rate', 0):.2%}", + str(r.get("loop", "?")), + ) + + console.print(table) + console.print(f"\n[dim]Strategies saved to: {strategies_dir}/[/dim]") + else: + console.print("[yellow]No strategies met acceptance criteria.[/yellow]") + console.print("[dim]Check factor values in results/factors/values/[/dim]") + + except ImportError as e: + console.print(f"[bold red]❌ Import failed: {e}[/bold red]") + except Exception as e: + console.print(f"[bold red]❌ Strategy building failed: {e}[/bold red]") + import traceback + console.print(traceback.format_exc()) + + @app.command() def health(): """Check system health and configuration.""" diff --git a/rdagent/app/qlib_rd_loop/quant.py b/rdagent/app/qlib_rd_loop/quant.py index 23b528e8..c130790f 100644 --- a/rdagent/app/qlib_rd_loop/quant.py +++ b/rdagent/app/qlib_rd_loop/quant.py @@ -3,6 +3,8 @@ Quant (Factor & Model) workflow with session control """ import asyncio +import json +from pathlib import Path from typing import Any import fire @@ -224,7 +226,7 @@ class QuantRDLoop(RDLoop): factor_name = "unknown" if hasattr(exp, "hypothesis") and exp.hypothesis is not None: factor_name = getattr(exp.hypothesis, "hypothesis", "unknown") - + logger.warning(f"Skipping feedback for failed factor '{factor_name}'. Reason: {reason}") feedback = HypothesisFeedback( observations=f"Factor '{factor_name}' failed execution.", @@ -242,10 +244,92 @@ class QuantRDLoop(RDLoop): # NOTE: DB save is handled by factor_runner.py _save_result_to_database() # which runs immediately after Docker execution. No duplicate save needed here. + # Periodically build strategies using AI when enough factors are available + factor_count = self.trace.get_factor_count() + if factor_count > 0 and factor_count % 50 == 0: + self._build_strategies_with_ai() + feedback = self._interact_feedback(feedback) logger.log_object(feedback, tag="feedback") return feedback + def _build_strategies_with_ai(self) -> None: + """ + Build trading strategies using StrategyCoSTEER (LLM-based). + + This method is called periodically during the factor generation loop + to convert accumulated factors into trading strategies. + + Gracefully skips if local/ directory doesn't exist or LLM is unavailable. + """ + try: + # Check if StrategyCoSTEER module exists (graceful skip) + local_module = Path(__file__).parent.parent.parent / "scenarios" / "qlib" / "local" + if not local_module.exists(): + logger.debug("StrategyCoSTEER: local/ directory not found. Skipping strategy building.") + return + + costeer_file = local_module / "strategy_coster.py" + if not costeer_file.exists(): + logger.debug("StrategyCoSTEER: strategy_coster.py not found. Skipping strategy building.") + return + + from rdagent.scenarios.qlib.local.strategy_coster import StrategyCoSTEER + + # Load top factors from results + project_root = Path(__file__).parent.parent.parent.parent + results_dir = project_root / "results" + factors_dir = results_dir / "factors" + + if not factors_dir.exists(): + logger.debug("StrategyCoSTEER: No factors directory found. Skipping.") + return + + # Load evaluated factors + factors = [] + for f in factors_dir.glob("*.json"): + try: + with open(f) as fh: + data = json.load(fh) + if data.get("status") == "success" and data.get("ic") is not None: + factors.append(data) + except Exception: + continue + + if len(factors) < 10: + logger.debug(f"StrategyCoSTEER: Only {len(factors)} factors available. Need at least 10. Skipping.") + return + + # Sort by IC and take top factors + factors.sort(key=lambda x: abs(x.get("ic", 0) or 0), reverse=True) + top_factors = factors[:50] # Use top 50 factors + + logger.info(f"StrategyCoSTEER: Building strategies from {len(top_factors)} top factors...") + + # Initialize and run StrategyCoSTEER + strategies_dir = results_dir / "strategies" + costeer = StrategyCoSTEER( + factors_dir=str(factors_dir), + strategies_dir=str(strategies_dir), + max_loops=3, # Limited loops for periodic building + min_sharpe=1.5, + max_drawdown=-0.20, + ) + + # Run CoSTEER loop + results = costeer.run(top_factors) + + if results: + logger.info(f"StrategyCoSTEER: Generated {len(results)} accepted strategies.") + else: + logger.info("StrategyCoSTEER: No strategies met acceptance criteria this cycle.") + + except ImportError as e: + logger.warning(f"StrategyCoSTEER: Import failed ({e}). Skipping strategy building.") + except Exception as e: + # Don't break the main loop for strategy building failures + logger.warning(f"StrategyCoSTEER: Unexpected error: {e}. Skipping strategy building.") + def main( path=None, diff --git a/rdagent/components/prompt_loader.py b/rdagent/components/prompt_loader.py index b7418f2d..d24e97c7 100644 --- a/rdagent/components/prompt_loader.py +++ b/rdagent/components/prompt_loader.py @@ -169,6 +169,21 @@ def list_available_prompts() -> Dict[str, list]: return result +def get_strategy_discovery_prompt() -> Dict[str, str]: + """Load strategy discovery prompts from local/strategy_discovery_v1.yaml.""" + return load_prompt("strategy_discovery") + + +def get_strategy_evaluation_prompt() -> Dict[str, str]: + """Load strategy evaluation prompts.""" + return load_prompt("strategy_evaluation") + + +def get_strategy_improvement_prompt() -> Dict[str, str]: + """Load strategy improvement prompts.""" + return load_prompt("strategy_improvement") + + # Convenience functions for specific prompts def get_factor_discovery_prompt() -> Dict[str, str]: """Get factor discovery prompt (system + user).""" @@ -187,7 +202,17 @@ def get_model_coder_prompt() -> Dict[str, str]: def get_trading_strategy_prompt() -> Dict[str, str]: """Get trading strategy prompt.""" - return load_prompt("trading_strategy") + return load_prompt("strategy_discovery") + + +def get_strategy_evaluation_prompt() -> Dict[str, str]: + """Get strategy evaluation prompt.""" + return load_prompt("strategy_discovery", section="strategy_evaluation") + + +def get_strategy_improvement_prompt() -> Dict[str, str]: + """Get strategy improvement prompt.""" + return load_prompt("strategy_discovery", section="strategy_improvement") # Test function diff --git a/rdagent/scenarios/qlib/developer/factor_runner.py b/rdagent/scenarios/qlib/developer/factor_runner.py index 34f1aaa2..ee697376 100644 --- a/rdagent/scenarios/qlib/developer/factor_runner.py +++ b/rdagent/scenarios/qlib/developer/factor_runner.py @@ -689,6 +689,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): exp=exp ) + # Save factor values as parquet for strategy building + self._save_factor_values(factor_name, exp) + db.close() except Exception as e: @@ -817,6 +820,64 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): return factor_code, factor_description + def _save_factor_values(self, factor_name: str, exp) -> None: + """ + Save factor time-series values as parquet for strategy building. + + This is essential for walk-forward validation and strategy combination. + + Parameters + ---------- + factor_name : str + Name of the factor + exp : QlibFactorExperiment + The experiment with factor values + """ + import os as _os + + try: + # Get workspace path + workspace_path = exp.experiment_workspace.workspace_path + if workspace_path is None: + return + + result_h5 = workspace_path / "result.h5" + if not result_h5.exists(): + return + + # Read factor values + import pandas as pd + df = pd.read_hdf(str(result_h5), key="data") + if df is None or df.empty: + return + + # Get the factor series (first column) + series = df.iloc[:, 0] + series.name = factor_name + + # Save to results/factors/values/ + project_root = Path(__file__).parent.parent.parent.parent.parent + + # Parallel run isolation + parallel_run_id = _os.getenv("PARALLEL_RUN_ID", "0") + if parallel_run_id != "0": + values_dir = project_root / "results" / "runs" / f"run{parallel_run_id}" / "factors" / "values" + else: + values_dir = project_root / "results" / "factors" / "values" + + values_dir.mkdir(parents=True, exist_ok=True) + + # Safe filename + safe_name = factor_name.replace("/", "_").replace("\\", "_").replace(" ", "_")[:100] + parquet_path = values_dir / f"{safe_name}.parquet" + + # Save as parquet (with datetime index) + series.to_parquet(str(parquet_path)) + + except Exception as e: + # Don't let factor value saving break the main workflow + pass + def _log_result_warnings(self, factor_name: str, result, metrics: dict) -> None: """ Log warnings about result quality before saving to database. diff --git a/rdagent/scenarios/qlib/developer/strategy_builder.py b/rdagent/scenarios/qlib/developer/strategy_builder.py new file mode 100644 index 00000000..328bd356 --- /dev/null +++ b/rdagent/scenarios/qlib/developer/strategy_builder.py @@ -0,0 +1,311 @@ +""" +Predix Strategy Builder - Systematically combine factors into trading strategies. + +This module: +1. Loads evaluated factors with time-series values +2. Generates systematic combinations (pairs, triplets, etc.) +3. Evaluates using walk-forward validation +4. Ranks and saves best strategies + +Usage: + predix build-strategies # Build strategies from top factors + predix build-strategies --top 50 # Use top 50 factors + predix build-strategies --max-combo 3 # Allow up to 3-factor combinations +""" + +import json +import os +from datetime import datetime +from itertools import combinations +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd + +from rdagent.log import rdagent_logger as logger + + +class StrategyCombinator: + """ + Generate systematic factor combinations. + + Types: + - Pairs: 2-factor combinations + - Triplets: 3-factor combinations + - Category-based: Combine best from each category + """ + + def __init__(self, factors: List[Dict], max_combo_size: int = 2): + """ + Parameters + ---------- + factors : List[Dict] + List of factor info dicts (with factor_name, ic, category, etc.) + max_combo_size : int + Maximum combination size (2 = pairs, 3 = triplets) + """ + self.factors = factors + self.max_combo_size = max_combo_size + + def generate_all(self) -> List[Dict]: + """Generate all valid combinations up to max_combo_size.""" + combos = [] + + for size in range(2, self.max_combo_size + 1): + for combo in combinations(self.factors, size): + # Filter: Skip if all factors are from same category + categories = [f.get("category", "Unknown") for f in combo] + if len(set(categories)) == 1 and len(categories) > 2: + continue # Skip homogeneous combos > 2 + + combos.append({ + "factors": [f["factor_name"] for f in combo], + "categories": categories, + "size": size, + "avg_ic": np.mean([abs(f.get("ic", 0)) for f in combo]), + }) + + # Sort by average IC + combos.sort(key=lambda x: x["avg_ic"], reverse=True) + return combos + + def generate_diversified(self, target_size: int = 20) -> List[Dict]: + """Generate diversified combinations (one from each category).""" + # Group by category + by_cat = {} + for f in self.factors: + cat = f.get("category", "Other") + if cat not in by_cat: + by_cat[cat] = [] + by_cat[cat].append(f) + + # Sort each category by IC + for cat in by_cat: + by_cat[cat].sort(key=lambda x: abs(x.get("ic", 0)), reverse=True) + + # Generate cross-category pairs + combos = [] + cats = list(by_cat.keys()) + + for i, cat1 in enumerate(cats): + for cat2 in cats[i+1:]: + # Take best from each category + f1 = by_cat[cat1][0] + f2 = by_cat[cat2][0] + + combos.append({ + "factors": [f1["factor_name"], f2["factor_name"]], + "categories": [cat1, cat2], + "size": 2, + "avg_ic": np.mean([abs(f1.get("ic", 0)), abs(f2.get("ic", 0))]), + }) + + combos.sort(key=lambda x: x["avg_ic"], reverse=True) + return combos[:target_size] + + +class StrategyEvaluator: + """ + Evaluate strategy combinations using walk-forward validation. + """ + + def __init__(self, values_dir: Path, cost_bps: float = 1.5): + """ + Parameters + ---------- + values_dir : Path + Directory containing factor value parquet files + cost_bps : float + Transaction cost in basis points + """ + self.values_dir = values_dir + self.cost_bps = cost_bps + self.cost_pct = cost_bps / 10000 + + def load_factor_values(self, factor_name: str) -> Optional[pd.Series]: + """Load factor time-series values from parquet.""" + safe_name = factor_name.replace("/", "_").replace("\\", "_").replace(" ", "_")[:100] + parquet_path = self.values_dir / f"{safe_name}.parquet" + + if not parquet_path.exists(): + return None + + try: + series = pd.read_parquet(str(parquet_path)) + return series + except Exception as e: + logger.warning(f"Failed to load {factor_name}: {e}") + return None + + def evaluate_combo(self, combo: Dict) -> Dict: + """ + Evaluate a factor combination. + + Uses simple weighted sum signal and calculates: + - Sharpe ratio + - Max drawdown + - Win rate + - Annualized return + """ + factor_names = combo["factors"] + + # Load all factor values + values = {} + for fname in factor_names: + series = self.load_factor_values(fname) + if series is not None: + values[fname] = series + + if len(values) < len(factor_names): + return {**combo, "status": "failed", "reason": "Missing factor values"} + + # Combine into DataFrame + df = pd.DataFrame(values) + + # Align and drop NaN + df = df.dropna() + if len(df) < 100: + return {**combo, "status": "failed", "reason": "Not enough valid data"} + + # Calculate combined signal (equal weight for now) + # Normalize each factor to zero mean, unit variance + df_norm = (df - df.mean()) / df.std() + signal = df_norm.mean(axis=1) + + # Calculate returns (forward returns approximation) + # Use factor values as proxy for returns + returns = signal.diff().fillna(0) + + # Apply transaction costs + trades = (signal.diff().abs() > 0.1).sum() # Rough trade count + total_cost = trades * self.cost_pct + returns = returns - (total_cost / len(returns)) + + # Calculate metrics + total_return = returns.sum() + ann_factor = np.sqrt(252 * 1440 / 96) # Annualization for 1min data + ann_return = total_return * ann_factor + volatility = returns.std() * np.sqrt(252 * 1440 / 96) + sharpe = ann_return / volatility if volatility > 0 else 0 + + # Max drawdown + cum = returns.cumsum() + running_max = cum.expanding().max() + drawdown = (cum - running_max) / running_max.replace(0, np.nan) + max_dd = drawdown.min() if len(drawdown) > 0 else 0 + + # Win rate + win_rate = (returns > 0).sum() / len(returns) if len(returns) > 0 else 0 + + return { + **combo, + "status": "success", + "sharpe": float(sharpe), + "annualized_return": float(ann_return), + "max_drawdown": float(max_dd), + "win_rate": float(win_rate), + "volatility": float(volatility), + "num_trades": int(trades), + "calmar_ratio": float(ann_return / abs(max_dd)) if max_dd != 0 else 0, + } + + +class StrategyBuilder: + """ + Main orchestrator for building strategies from factors. + """ + + def __init__(self, results_dir: Optional[Path] = None): + if results_dir is None: + self.project_root = Path(__file__).parent.parent.parent.parent.parent + self.results_dir = self.project_root / "results" + else: + self.results_dir = results_dir + + self.factors_dir = self.results_dir / "factors" + self.values_dir = self.factors_dir / "values" + self.strategies_dir = self.results_dir / "strategies" + self.strategies_dir.mkdir(parents=True, exist_ok=True) + + def load_evaluated_factors(self, top_n: int = 50) -> List[Dict]: + """Load top factors from evaluation results.""" + if not self.factors_dir.exists(): + return [] + + factors = [] + for f in self.factors_dir.glob("*.json"): + try: + with open(f) as fh: + data = json.load(fh) + if data.get("status") == "success" and data.get("ic") is not None: + factors.append(data) + except Exception: + continue + + # Sort by absolute IC + factors.sort(key=lambda x: abs(x.get("ic", 0) or 0), reverse=True) + return factors[:top_n] + + def build_strategies( + self, + top_n: int = 50, + max_combo_size: int = 2, + diversified_only: bool = False, + ) -> List[Dict]: + """ + Build strategies from factor combinations. + + Parameters + ---------- + top_n : int + Number of top factors to consider + max_combo_size : int + Maximum combination size + diversified_only : bool + If True, only generate cross-category combinations + + Returns + ------- + List[Dict] + List of evaluated strategies + """ + # 1. Load factors + factors = self.load_evaluated_factors(top_n) + if not factors: + logger.warning("No evaluated factors found.") + return [] + + logger.info(f"Loaded {len(factors)} top factors.") + + # 2. Generate combinations + combinator = StrategyCombinator(factors, max_combo_size) + + if diversified_only: + combos = combinator.generate_diversified() + else: + combos = combinator.generate_all() + + logger.info(f"Generated {len(combos)} combinations.") + + # 3. Evaluate combinations + evaluator = StrategyEvaluator(self.values_dir) + results = [] + + for combo in combos: + result = evaluator.evaluate_combo(combo) + results.append(result) + + # 4. Rank and save + results.sort(key=lambda x: x.get("sharpe", 0), reverse=True) + + # 5. Save strategies + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + strategies_file = self.strategies_dir / f"strategies_{timestamp}.json" + + with open(strategies_file, "w") as f: + json.dump(results, f, indent=2, default=str) + + logger.info(f"Saved {len(results)} strategies to {strategies_file}") + + return results