From 5b1ded36ca366cc8dc1cd5d5818d20a01b4aa42a Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Sat, 4 Apr 2026 23:09:29 +0200 Subject: [PATCH] feat: Add complete ML pipeline with graceful degradation (closed source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NEW ARCHITECTURE: ┌─────────────────────────────────────────────────┐ │ Phase 1: Factor Generation (Open Source) │ │ - Generate factors with LLM v3 prompt │ │ - Backtest each factor in Qlib Docker │ │ - Save to results/factors/ with code + desc │ │ - Continue until 5000+ valid factors │ └─────────────────────────────────────────────────┘ ↓ ┌─────────────────────────────────────────────────┐ │ Phase 2: ML Training (Closed Source - Local) │ │ - Load top 50 factors │ │ - Train LightGBM model │ │ - Validate (IC, Sharpe) │ │ - Save to results/models/ │ └─────────────────────────────────────────────────┘ ↓ ┌─────────────────────────────────────────────────┐ │ Phase 3: Portfolio Optimization (Closed Source) │ │ - Select uncorrelated factors (max corr 0.3) │ │ - Optimize weights by IC │ │ - Backtest portfolio │ │ - Save to results/portfolios/ │ └─────────────────────────────────────────────────┘ ↓ ┌─────────────────────────────────────────────────┐ │ Phase 4: Strategy Generation (Closed Source) │ │ - Generate trading rules │ │ - Add risk management │ │ - Save to results/strategies/ │ └─────────────────────────────────────────────────┘ ↓ ┌─────────────────────────────────────────────────┐ │ Phase 5: Iterative Improvement (Closed Source) │ │ - Use ML results as feedback │ │ - Generate better factors │ │ - Loop back to Phase 1 │ └─────────────────────────────────────────────────┘ FILES CREATED (Closed Source - NOT in Git): - rdagent/scenarios/qlib/local/ml_trainer.py - rdagent/scenarios/qlib/local/portfolio_optimizer.py - rdagent/scenarios/qlib/local/quant_loop_advanced.py - rdagent/scenarios/qlib/local/__init__.py FILES MODIFIED (Open Source - in Git): - rdagent/scenarios/qlib/quant_loop_factory.py - .gitignore (added local/ exclusion) GRACEFUL DEGRADATION: - If local/ components don't exist → Standard loop - If < 5000 factors → Standard loop - If LightGBM not installed → Falls back - Open source users get FULLY FUNCTIONAL system USAGE: # Standard (always works): rdagent fin_quant # Advanced (automatic if local components exist + 5000+ factors): # Same command - factory auto-selects appropriate loop --- .gitignore | 3 + rdagent/scenarios/qlib/quant_loop_factory.py | 131 +++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 rdagent/scenarios/qlib/quant_loop_factory.py diff --git a/.gitignore b/.gitignore index 57ed4d70..cbcab4d6 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,6 @@ models/local/ .env.test *.test.env test_credentials.py + +# Closed source local components +rdagent/scenarios/qlib/local/ diff --git a/rdagent/scenarios/qlib/quant_loop_factory.py b/rdagent/scenarios/qlib/quant_loop_factory.py new file mode 100644 index 00000000..37e8bb70 --- /dev/null +++ b/rdagent/scenarios/qlib/quant_loop_factory.py @@ -0,0 +1,131 @@ +""" +Predix Quant Loop Factory - Selects appropriate workflow based on available components. + +This module is the entry point for the quantitative trading loop. +It automatically selects between: +1. Standard Loop (Open Source) - Factor generation + backtesting +2. Advanced Loop (Local/Closed Source) - Full ML pipeline with portfolio & strategy + +The selection is based on: +- Availability of local components (ml_trainer, portfolio_optimizer) +- Number of valid factors (threshold: 5000 for advanced loop) + +Usage: + from rdagent.scenarios.qlib.quant_loop_factory import create_quant_loop + + loop = create_quant_loop(scenario) + loop.run() +""" + +from pathlib import Path +from typing import Optional + +from rdagent.log import rdagent_logger as logger + + +# Threshold for advanced loop activation +ADVANCED_LOOP_FACTOR_THRESHOLD = 5000 + + +def has_local_components() -> bool: + """ + Check if local (closed source) components are available. + + Returns True if: + - rdagent/scenarios/qlib/local/ml_trainer.py exists + - rdagent/scenarios/qlib/local/portfolio_optimizer.py exists + """ + local_dir = Path(__file__).parent / "local" + if not local_dir.exists(): + return False + + required_files = [ + "ml_trainer.py", + "portfolio_optimizer.py", + ] + + for fname in required_files: + if not (local_dir / fname).exists(): + return False + + return True + + +def count_valid_factors() -> int: + """ + Count the number of valid (successful) factors in results/factors/. + + Returns + ------- + int + Number of valid factors + """ + import json + from glob import glob + + project_root = Path(__file__).parent.parent.parent.parent + factors_dir = project_root / "results" / "factors" + + if not factors_dir.exists(): + return 0 + + count = 0 + for json_file in glob(str(factors_dir / "*.json")): + try: + with open(json_file) as f: + data = json.load(f) + if data.get("status") == "success" and data.get("ic") is not None: + count += 1 + except Exception: + continue + + return count + + +def create_quant_loop(scenario) -> "BaseQuantLoop": + """ + Create the appropriate QuantLoop based on available components. + + Priority: + 1. Advanced Loop (if local components exist AND 5000+ factors) + 2. Standard Loop (always available) + + Parameters + ---------- + scenario : Scenario + The trading scenario + + Returns + ------- + BaseQuantLoop + The appropriate quant loop instance + """ + local_available = has_local_components() + factor_count = count_valid_factors() + + logger.info( + f"Quant Loop Factory: local_components={local_available}, " + f"valid_factors={factor_count}, threshold={ADVANCED_LOOP_FACTOR_THRESHOLD}" + ) + + if local_available and factor_count >= ADVANCED_LOOP_FACTOR_THRESHOLD: + logger.info("Creating AdvancedQuantLoop (ML + Portfolio + Strategy)") + from rdagent.scenarios.qlib.local.quant_loop_advanced import AdvancedQuantLoop + return AdvancedQuantLoop(scenario) + else: + if not local_available: + logger.info("Local components not found — using StandardQuantLoop") + else: + logger.info( + f"Only {factor_count}/{ADVANCED_LOOP_FACTOR_THRESHOLD} factors — " + f"using StandardQuantLoop (need {ADVANCED_LOOP_FACTOR_THRESHOLD - factor_count} more)" + ) + + from rdagent.app.qlib_rd_loop.quant import QuantRDLoop + return QuantRDLoop + + +# Base class for type hints +class BaseQuantLoop: + """Base class for quant loops.""" + pass