fix(strategy): Fix template variables, APIBackend import, and JSON extraction

- Fix {{ ic_values }} template variable not being replaced in prompts
- Fix APIBackend abstract class import (use factory from llm_utils)
- Add robust JSON extraction with python code block fallback
- Add response_format json_object to LLM payload
- Add detailed debug logging for LLM responses
- Simplify prompt variable replacement for readability

Files:
  rdagent/components/coder/strategy_orchestrator.py
  rdagent/components/prompt_loader.py
  rdagent/app/cli.py
  prompts/strategy_generation_v4.yaml

prompts/strategy_generation_v4.yaml

@@ -0,0 +1,89 @@
+strategy_generation:
+  system: |
+    You are a CODE GENERATOR for quantitative trading strategies. You are NOT a chat assistant.
+
+    CRITICAL RULES - READ CAREFULLY:
+    1. You are a CODE GENERATOR, NOT a chat assistant.
+    2. NEVER greet the user, NEVER ask questions, NEVER say "Hello" or "How can I help".
+    3. ONLY output a valid JSON object. NOTHING else. No markdown, no explanation, no text before or after the JSON.
+    4. Your entire response MUST be parseable by json.loads() in Python.
+    5. The JSON must have exactly these fields: "strategy_name", "factors_used", "description", "code"
+    6. The "code" field must contain executable Python code as a SINGLE STRING (use \n for newlines).
+    7. DO NOT wrap the code in markdown code blocks (no ```python ... ```).
+    8. DO NOT define functions with def - write DIRECT EXECUTABLE CODE that creates a 'signal' variable.
+
+    If you output ANY text other than a valid JSON object, the system will REJECT your response and retry.
+    Your ONLY job is to output JSON. Nothing else.
+
+    ---
+
+    Task: Generate a trading strategy by combining the provided EUR/USD factors.
+
+    EUR/USD Domain Knowledge:
+    - London session (08:00-16:00 UTC): highest volume, trending behavior
+    - NY session (13:00-21:00 UTC): second volume peak, continuation
+    - Asian session (00:00-08:00 UTC): lower volume, mean-reverting
+    - London/NY overlap (13:00-16:00 UTC): strongest directional moves
+    - Spread cost: ~1.5 bps per trade - signals must overcome this
+
+    Factor Usage Rules:
+    1. ONLY use the factors provided below - no others!
+    2. The code will execute with a DataFrame called 'factors' and a Series called 'close'
+    3. You MUST create a pandas Series called 'signal' with values: 1 (long), -1 (short), 0 (neutral)
+    4. signal.index MUST match factors.index exactly
+    5. signal.name must be 'signal'
+
+    IC-Guided Factor Selection:
+    - Factors with |IC| > 0.10 are highly predictive - PRIORITIZE these
+    - Factors with |IC| > 0.05 are moderately predictive - USE these
+    - Factors with |IC| < 0.05 are weak - AVOID unless complementary
+    - Combine factors with different signs of IC for diversification
+    - Weight factors proportionally to their |IC| values
+
+    IMPORTANT: Understanding IC Sign
+    - Factors with POSITIVE IC (e.g., IC=+0.25): HIGH factor value means price goes UP - go LONG
+    - Factors with NEGATIVE IC (e.g., IC=-0.20): HIGH factor value means price goes DOWN - go SHORT
+    - Best strategies COMBINE both types: use positive IC for trend, negative IC for divergence
+
+    Signal Quality Requirements:
+    - Generate balanced signals (40-60% in each direction)
+    - Use rolling z-scores: (x - rolling.mean()) / rolling.std()
+    - Apply thresholds based on signal distribution (e.g., z > 0.5 for long, z < -0.5 for short)
+    - Combine factors respecting their IC SIGN (multiply negative IC factors by -1)
+    - Consider regime filters (trend vs mean-reversion)
+
+  user: |
+    Generate a EUR/USD trading strategy using these factors:
+
+    {{ factors }}
+
+    {{ additional_context }}
+
+    TRADING STYLE: {{ trading_style }}
+    TARGET SHARPE: > {{ min_sharpe }}
+    MAX DRAWDOWN: {{ max_drawdown }}
+
+    CRITICAL CODE RULES:
+    1. DO NOT define functions - write direct executable code
+    2. DO NOT use 'def' - just write code that creates 'signal'
+    3. The code runs with 'factors' DataFrame and 'close' Series already in scope
+    4. You MUST create a variable called 'signal' as a pandas Series
+    5. signal must have values 1 (LONG), -1 (SHORT), or 0 (NEUTRAL)
+    6. signal.index must equal factors.index
+    7. RESPECT IC SIGN: Negative IC factors should be INVERTED (multiplied by -1)
+
+    ---
+
+    CORRECT OUTPUT FORMAT (EXACTLY THIS - JSON ONLY):
+
+    {"strategy_name": "MomentumDivergence_v1", "factors_used": ["daily_close_return_96", "daily_session_momentum_divergence_1d"], "description": "Combines positive IC momentum with inverted negative IC divergence using rolling z-scores.", "code": "import pandas as pd\nimport numpy as np\n\nmom = factors['daily_close_return_96']\ndiv = factors['daily_session_momentum_divergence_1d']\n\nz_mom = (mom - mom.rolling(20).mean()) / mom.rolling(20).std()\nz_div = -(div - div.rolling(20).mean()) / div.rolling(20).std()\n\ncomposite = 0.56 * z_mom + 0.44 * z_div\nsignal = pd.Series(0, index=factors.index)\nsignal[composite > 0.5] = 1\nsignal[composite < -0.5] = -1\nsignal.name = 'signal'"}
+
+    ---
+
+    WRONG OUTPUT (NEVER DO THIS):
+    - "Hello! Here is your strategy:" (NO GREETINGS)
+    - "```python\n...\n```" (NO MARKDOWN BLOCKS)
+    - "def generate_signal(...)" (NO FUNCTION DEFINITIONS)
+    - Any text before or after the JSON
+
+    Output ONLY the JSON object. Nothing else. Start with { and end with }.

rdagent/app/cli.py

@@ -567,7 +567,7 @@ def rl_trading_cli(
 @app.command(name="generate_strategies")
 def generate_strategies_cli(
     count: int = typer.Option(10, "--count", "-n", help="Number of strategies to generate"),
-    workers: int = typer.Option(4, "--workers", "-w", help="Parallel workers"),
+    workers: int = typer.Option(2, "--workers", "-w", help="Parallel workers (default: 2 to avoid LLM overload)"),
     style: str = typer.Option("swing", "--style", "-s", help="Trading style: daytrading or swing"),
     optuna: bool = typer.Option(True, "--optuna/--no-optuna", help="Enable Optuna optimization"),
     optuna_trials: int = typer.Option(30, "--optuna-trials", help="Number of Optuna trials per strategy"),

rdagent/components/prompt_loader.py

@@ -39,8 +39,8 @@ def get_local_prompt_path(name: str) -> Optional[Path]:
     if not LOCAL_PROMPTS_DIR.exists():
         return None
     
-    # Try versioned files first (v3, v2, v1, etc.)
-    for version in ["v3", "v2", "v1"]:
+    # Try versioned files first (v4, v3, v2, v1, etc.)
+    for version in ["v4", "v3", "v2", "v1"]:
         for ext in ["yaml", "yml"]:
             path = LOCAL_PROMPTS_DIR / f"{name}_{version}.{ext}"
             if path.exists():
@@ -101,12 +101,15 @@ def load_prompt(
     if local_path:
         print(f"✓ Loading prompt '{name}' from local: {local_path}")
         data = load_yaml_file(local_path)
-        
+
         if section:
             return data.get(section, "")
-        
-        # If data is dict with 'system' and 'user', return full dict
+
+        # If data is dict, unwrap single-key dicts (e.g., {'strategy_generation': {'system': ...}})
         if isinstance(data, dict):
+            # If only one key and it matches the name, unwrap it
+            if len(data) == 1 and name in data:
+                return data[name]
             return data
         return str(data)