fix: Resolve 88% empty backtest results + path fixes

Root Cause: Qlib configs used cn_data (Chinese stocks) instead of eurusd - provider_uri: cn_data → eurusd_1min_data - market: csi300 → eurusd - topk: 50 → 1 (single-asset EURUSD, was opening 0 positions) - n_drop: 5 → 0, limit_threshold: 0.095 → 0.0 Add failed run tracking and validation: - factor_runner.py: Validate results before DB save, track failed runs - model_runner.py: Same validation and tracking - results_db.py: generate_results_summary() → RESULTS_SUMMARY.md - extract_results.py: Failed run tracking, progress indicators Fix project root paths in all modules: - ResultsDatabase: correct path from rdagent/results/ → results/ - factor_runner: db, factors, failed_runs paths - model_runner: failed_runs path All 246 tests passing.
2026-07-27 15:37:44 +00:00 · 2026-04-03 16:21:59 +02:00
parent 612ed8a802
commit 574a9cb75e
9 changed files with 755 additions and 39 deletions
@@ -8,13 +8,15 @@ import json
 import pandas as pd
 from pathlib import Path
 from datetime import datetime
-from typing import Dict, Optional
+from typing import Dict, Optional, List


 class ResultsDatabase:
    def __init__(self, db_path: Optional[str] = None):
        if db_path is None:
-            db_path = Path(__file__).parent.parent.parent / "results" / "db" / "backtest_results.db"
+            # Go up from rdagent/components/backtesting/ to project root (4 levels)
+            project_root = Path(__file__).parent.parent.parent.parent
+            db_path = str(project_root / "results" / "db" / "backtest_results.db")
        self.db_path = db_path
        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
        self.conn = sqlite3.connect(db_path)
@@ -273,6 +275,226 @@ class ResultsDatabase:
            self.conn
        )
    
+    def generate_results_summary(self, output_path: Optional[str] = None,
+                                  print_to_console: bool = True) -> Dict:
+        """
+        Generate a comprehensive results summary report.
+
+        Scans the database and factors directory to produce a summary report
+        with total runs, successful/failed counts, best metrics, and statistics.
+
+        Parameters
+        ----------
+        output_path : str, optional
+            Path to write the summary as Markdown file.
+            Defaults to results/RESULTS_SUMMARY.md
+        print_to_console : bool
+            If True, print the summary to console
+
+        Returns
+        -------
+        Dict
+            Summary statistics dictionary
+        """
+        # Database stats
+        db_stats = self.get_aggregate_stats()
+
+        # Get all results
+        all_results = self.get_all_results()
+
+        # Top factors
+        top_sharpe = self.get_top_factors('sharpe', limit=10)
+        top_ic = self.get_top_factors('ic', limit=10)
+
+        # Count successful vs failed runs
+        total_runs = len(all_results)
+        runs_with_ic = int(all_results['ic'].notna().sum()) if total_runs > 0 else 0
+        runs_with_sharpe = int(all_results['sharpe'].notna().sum()) if total_runs > 0 else 0
+
+        # Count unique factors
+        unique_factors = all_results['factor_name'].nunique() if total_runs > 0 else 0
+
+        # Best metrics
+        best_ic = all_results['ic'].max() if total_runs > 0 and all_results['ic'].notna().any() else None
+        best_sharpe = all_results['sharpe'].max() if total_runs > 0 and all_results['sharpe'].notna().any() else None
+        best_return = all_results['annual_return'].max() if total_runs > 0 and all_results['annual_return'].notna().any() else None
+        worst_drawdown = all_results['max_drawdown'].min() if total_runs > 0 and all_results['max_drawdown'].notna().any() else None
+
+        # Scan factors directory for JSON files
+        factors_dir = Path(__file__).parent.parent.parent / "results" / "factors"
+        json_factor_files = 0
+        if factors_dir.exists():
+            json_factor_files = len(list(factors_dir.glob("*.json")))
+
+        # Scan failed runs
+        failed_dir = Path(__file__).parent.parent.parent / "results" / "failed_runs"
+        failed_runs_file = failed_dir / "failed_runs.json"
+        failed_runs_count = 0
+        failed_runs_data = []
+        if failed_runs_file.exists():
+            try:
+                failed_runs_data = json.loads(failed_runs_file.read_text(encoding="utf-8"))
+                if isinstance(failed_runs_data, list):
+                    failed_runs_count = len(failed_runs_data)
+                elif isinstance(failed_runs_data, dict):
+                    failed_runs_count = 1
+            except (json.JSONDecodeError, Exception):
+                failed_runs_count = 0
+
+        # Build summary
+        summary = {
+            "generated_at": datetime.now().isoformat(),
+            "database_path": str(self.db_path),
+            "overview": {
+                "total_runs": int(total_runs),
+                "unique_factors": int(unique_factors),
+                "runs_with_ic": int(runs_with_ic),
+                "runs_with_sharpe": int(runs_with_sharpe),
+                "json_factor_files": json_factor_files,
+                "failed_runs": failed_runs_count,
+            },
+            "best_metrics": {
+                "best_ic": float(best_ic) if best_ic is not None else None,
+                "best_sharpe": float(best_sharpe) if best_sharpe is not None else None,
+                "best_annual_return": float(best_return) if best_return is not None else None,
+                "worst_drawdown": float(worst_drawdown) if worst_drawdown is not None else None,
+            },
+            "aggregate_stats": db_stats,
+            "top_10_by_sharpe": top_sharpe.to_dict(orient='records') if len(top_sharpe) > 0 else [],
+            "top_10_by_ic": top_ic.to_dict(orient='records') if len(top_ic) > 0 else [],
+        }
+
+        # Write to Markdown
+        if output_path is None:
+            # Go up from rdagent/components/backtesting/ to project root (4 levels)
+            project_root = Path(__file__).parent.parent.parent.parent
+            output_path = str(project_root / "results" / "RESULTS_SUMMARY.md")
+
+        self._write_summary_markdown(summary, output_path)
+
+        if print_to_console:
+            self._print_summary_console(summary)
+
+        return summary
+
+    def _fmt_float(self, value, fmt: str = ".4f") -> str:
+        """Format float value, returning 'N/A' for None."""
+        if value is None:
+            return "N/A"
+        return f"{value:{fmt}}"
+
+    def _write_summary_markdown(self, summary: Dict, output_path: str) -> None:
+        """Write summary as Markdown file."""
+        path = Path(output_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        overview = summary['overview']
+        best = summary['best_metrics']
+        top_sharpe = summary['top_10_by_sharpe']
+        top_ic = summary['top_10_by_ic']
+
+        # Pre-format best metrics for display
+        best_ic_str = self._fmt_float(best['best_ic'], ".6f")
+        best_sharpe_str = self._fmt_float(best['best_sharpe'], ".4f")
+        best_return_str = self._fmt_float(best['best_annual_return'], ".4f")
+        worst_dd_str = self._fmt_float(best['worst_drawdown'], ".4f")
+
+        md_lines = [
+            "# Predix Results Summary",
+            "",
+            f"**Generated:** {summary['generated_at']}",
+            f"**Database:** `{summary['database_path']}`",
+            "",
+            "## Overview",
+            "",
+            f"| Metric | Value |",
+            f"|--------|-------|",
+            f"| Total Runs | {overview['total_runs']} |",
+            f"| Unique Factors | {overview['unique_factors']} |",
+            f"| Runs with IC | {overview['runs_with_ic']} |",
+            f"| Runs with Sharpe | {overview['runs_with_sharpe']} |",
+            f"| JSON Factor Files | {overview['json_factor_files']} |",
+            f"| Failed Runs | {overview['failed_runs']} |",
+            "",
+            "## Best Metrics",
+            "",
+            f"| Metric | Value |",
+            f"|--------|-------|",
+            f"| Best IC | {best_ic_str} |",
+            f"| Best Sharpe | {best_sharpe_str} |",
+            f"| Best Annual Return | {best_return_str} |",
+            f"| Worst Drawdown | {worst_dd_str} |",
+            "",
+        ]
+
+        if top_sharpe:
+            md_lines.extend([
+                "## Top 10 by Sharpe Ratio",
+                "",
+                "| # | Factor | Sharpe | IC | Annual Return | Max Drawdown |",
+                "|---|--------|--------|-----|---------------|--------------|",
+            ])
+            for i, row in enumerate(top_sharpe[:10], 1):
+                md_lines.append(
+                    f"| {i} | {row.get('factor_name', 'N/A')[:50]} | "
+                    f"{self._fmt_float(row.get('sharpe'), '.4f')} | "
+                    f"{self._fmt_float(row.get('ic'), '.6f')} | "
+                    f"{self._fmt_float(row.get('annual_return'), '.4f')} | "
+                    f"{self._fmt_float(row.get('max_drawdown'), '.4f')} |"
+                )
+            md_lines.append("")
+
+        if top_ic:
+            md_lines.extend([
+                "## Top 10 by IC",
+                "",
+                "| # | Factor | IC | Sharpe | Annual Return |",
+                "|---|--------|-----|--------|---------------|",
+            ])
+            for i, row in enumerate(top_ic[:10], 1):
+                md_lines.append(
+                    f"| {i} | {row.get('factor_name', 'N/A')[:50]} | "
+                    f"{self._fmt_float(row.get('ic'), '.6f')} | "
+                    f"{self._fmt_float(row.get('sharpe'), '.4f')} | "
+                    f"{self._fmt_float(row.get('annual_return'), '.4f')} |"
+                )
+            md_lines.append("")
+
+        md_lines.extend([
+            "## Failed Runs",
+            "",
+            f"Total failed runs tracked: **{overview['failed_runs']}**",
+            "",
+            "Failed runs are stored in `results/failed_runs/failed_runs.json`.",
+            "",
+        ])
+
+        path.write_text("\n".join(md_lines), encoding="utf-8")
+
+    def _print_summary_console(self, summary: Dict) -> None:
+        """Print summary to console."""
+        overview = summary['overview']
+        best = summary['best_metrics']
+
+        print("\n" + "=" * 70)
+        print("  PREDIX RESULTS SUMMARY")
+        print("=" * 70)
+        print(f"  Generated:    {summary['generated_at']}")
+        print(f"  Database:     {summary['database_path']}")
+        print("-" * 70)
+        print(f"  Total Runs:        {overview['total_runs']}")
+        print(f"  Unique Factors:    {overview['unique_factors']}")
+        print(f"  Runs with IC:      {overview['runs_with_ic']}")
+        print(f"  Runs with Sharpe:  {overview['runs_with_sharpe']}")
+        print(f"  JSON Factor Files: {overview['json_factor_files']}")
+        print(f"  Failed Runs:       {overview['failed_runs']}")
+        print("-" * 70)
+        print(f"  Best IC:           {self._fmt_float(best['best_ic'], '.6f')}")
+        print(f"  Best Sharpe:       {self._fmt_float(best['best_sharpe'], '.4f')}")
+        print(f"  Best Ann. Return:  {self._fmt_float(best['best_annual_return'], '.4f')}")
+        print(f"  Worst Drawdown:    {self._fmt_float(best['worst_drawdown'], '.4f')}")
+        print("=" * 70 + "\n")
+
    def close(self):
        self.conn.close()

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Optional

 import pandas as pd
 from pandarallel import pandarallel
@@ -193,9 +194,24 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
                )

        if result is None:
-            logger.error(f"Failed to run this experiment, because {stdout}")
+            logger.error(
+                f"Failed to run this experiment (result is None). "
+                f"Factor: {getattr(exp.hypothesis, 'hypothesis', 'unknown')}"
+            )
+            # Save failed run info for debugging
+            self._save_failed_run(exp, stdout, error_type="result_none")
            raise FactorEmptyError(f"Failed to run this experiment, because {stdout}")

+        # Validate result before proceeding
+        validation_result = self._validate_result(exp, result)
+        if validation_result.get("has_issues"):
+            logger.warning(
+                f"Result validation warnings for factor '{getattr(exp.hypothesis, 'hypothesis', 'unknown')}': "
+                f"{validation_result['warnings']}"
+            )
+            # Save warning info for debugging
+            self._save_failed_run(exp, stdout, error_type="validation_warnings", validation=validation_result)
+
        exp.result = result
        exp.stdout = stdout

@@ -214,6 +230,177 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):

        return exp

+    def _validate_result(self, exp, result) -> dict:
+        """
+        Validate backtest result for common issues before saving.
+
+        Checks for:
+        - Empty/None IC (no predictive power)
+        - Zero positions (1day.pos == 0, model stayed neutral)
+        - All metrics being None/NaN
+
+        Parameters
+        ----------
+        exp : QlibFactorExperiment
+            The experiment with backtest results
+        result : pd.Series or dict
+            Backtest metrics from Qlib
+
+        Returns
+        -------
+        dict
+            Validation result with 'has_issues' (bool), 'warnings' (list), and 'details' (dict)
+        """
+        warnings = []
+        details = {}
+
+        factor_name = "unknown"
+        if hasattr(exp, 'hypothesis') and exp.hypothesis is not None:
+            factor_name = getattr(exp.hypothesis, 'hypothesis', 'unknown')
+
+        if isinstance(result, pd.Series):
+            # Check IC
+            ic_value = result.get('IC', None)
+            details['ic_raw'] = ic_value
+            if ic_value is None or (isinstance(ic_value, float) and (ic_value != ic_value)):  # NaN check
+                warnings.append("IC is None/NaN — factor has no predictive power")
+            else:
+                try:
+                    ic_float = float(ic_value)
+                    details['ic'] = ic_float
+                    if abs(ic_float) < 0.001:
+                        warnings.append(
+                            f"IC is near zero ({ic_float:.6f}) — factor may not predict returns"
+                        )
+                except (ValueError, TypeError):
+                    warnings.append(f"IC value is not numeric: {ic_value}")
+
+            # Check positions (1day.pos)
+            pos_value = result.get('1day.pos', None)
+            details['positions_raw'] = pos_value
+            if pos_value is not None:
+                try:
+                    pos_float = float(pos_value)
+                    details['positions'] = pos_float
+                    if pos_float == 0:
+                        warnings.append(
+                            "1day.pos == 0 — model opened ZERO positions (stayed neutral). "
+                            "Possible causes: (1) topk too high for single-asset, "
+                            "(2) signal threshold too restrictive, (3) no valid predictions"
+                        )
+                    elif pos_float < 10:
+                        warnings.append(
+                            f"1day.pos = {pos_float:.0f} — very few positions opened. "
+                            f"Check signal threshold and topk settings"
+                        )
+                except (ValueError, TypeError):
+                    pass  # pos might be a string
+
+            # Check if result is essentially empty (all values None or NaN)
+            non_null_count = result.notna().sum()
+            total_count = len(result)
+            details['non_null_metrics'] = int(non_null_count)
+            details['total_metrics'] = int(total_count)
+            if non_null_count < 3:
+                warnings.append(
+                    f"Result has only {non_null_count}/{total_count} non-null metrics — "
+                    f"backtest likely produced empty results"
+                )
+
+            # Check for key metrics
+            required_metrics = ['IC', '1day.excess_return_with_cost.shar', '1day.pos']
+            for metric_name in required_metrics:
+                val = result.get(metric_name, None)
+                details[f'has_{metric_name}'] = val is not None
+
+        elif isinstance(result, dict):
+            # Dict-based result validation
+            ic_value = result.get('IC', result.get('ic', None))
+            details['ic_raw'] = ic_value
+            if ic_value is None:
+                warnings.append("IC is None — factor has no predictive power")
+
+        return {
+            "has_issues": len(warnings) > 0,
+            "warnings": "; ".join(warnings),
+            "details": details,
+        }
+
+    def _save_failed_run(self, exp, stdout: str, error_type: str = "unknown",
+                         validation: Optional[dict] = None) -> None:
+        """
+        Save failed run information to results/failed_runs.json for debugging.
+
+        Parameters
+        ----------
+        exp : QlibFactorExperiment
+            The experiment that failed
+        stdout : str
+            Standard output from the Docker execution
+        error_type : str
+            Type of error: 'result_none', 'validation_warnings', 'docker_error', etc.
+        validation : dict, optional
+            Validation result dict if error_type is 'validation_warnings'
+        """
+        import json
+        from datetime import datetime
+        from pathlib import Path
+
+        try:
+            # Ensure failed_runs directory exists (5 levels up to project root)
+            project_root = Path(__file__).parent.parent.parent.parent.parent
+            failed_dir = project_root / "results" / "failed_runs"
+            failed_dir.mkdir(parents=True, exist_ok=True)
+
+            # Get factor name
+            factor_name = "unknown"
+            if hasattr(exp, 'hypothesis') and exp.hypothesis is not None:
+                factor_name = getattr(exp.hypothesis, 'hypothesis', 'unknown')
+
+            # Build failed run record
+            failed_record = {
+                "timestamp": datetime.now().isoformat(),
+                "factor_name": factor_name,
+                "error_type": error_type,
+                "stdout": stdout if stdout else "(empty)",
+                "validation": validation,
+                "experiment_details": {
+                    "base_features": list(getattr(exp, 'base_features', {}).keys()) if hasattr(exp, 'base_features') else [],
+                    "hypothesis": getattr(exp.hypothesis, 'hypothesis', str(getattr(exp, 'hypothesis', 'N/A')))
+                                  if hasattr(exp, 'hypothesis') else "N/A",
+                },
+            }
+
+            # Append to failed_runs.json
+            failed_file = failed_dir / "failed_runs.json"
+            existing_records = []
+            if failed_file.exists():
+                try:
+                    existing_records = json.loads(failed_file.read_text(encoding="utf-8"))
+                    if not isinstance(existing_records, list):
+                        existing_records = [existing_records]
+                except (json.JSONDecodeError, Exception):
+                    existing_records = []
+
+            existing_records.append(failed_record)
+
+            # Keep only last 500 records to prevent file bloat
+            if len(existing_records) > 500:
+                existing_records = existing_records[-500:]
+
+            failed_file.write_text(
+                json.dumps(existing_records, indent=2, default=str, ensure_ascii=False),
+                encoding="utf-8"
+            )
+            logger.info(
+                f"Failed run saved: {factor_name} (type={error_type}) "
+                f"→ {failed_file}"
+            )
+
+        except Exception as e:
+            # Don't let failed run logging break the main workflow
+            logger.warning(f"Could not save failed run info: {e}")
+
    def _save_result_to_database(self, exp, result) -> None:
        """
        Save backtest results to the ResultsDatabase and write factor JSON summary.
@@ -293,6 +480,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
                metrics['volatility'] = None
                metrics['raw_metrics'] = result

+            # Result validation before saving (warnings, not blocking)
+            self._log_result_warnings(factor_name, result, metrics)
+
            # Only save if we have at least IC or Sharpe
            if metrics.get('ic') is None and metrics.get('sharpe_ratio') is None:
                logger.warning(
@@ -301,8 +491,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
                )
                return

-            # Ensure DB directory exists before creating database
-            db_path = Path(__file__).parent.parent.parent.parent / "results" / "db"
+            # Ensure DB directory exists before creating database (5 levels up to project root)
+            project_root = Path(__file__).parent.parent.parent.parent.parent
+            db_path = project_root / "results" / "db"
            db_path.mkdir(parents=True, exist_ok=True)
            db_file = db_path / "backtest_results.db"

@@ -344,8 +535,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
        from pathlib import Path

        try:
-            # Ensure factors directory exists
-            factors_dir = Path(__file__).parent.parent.parent.parent / "results" / "factors"
+            # Ensure factors directory exists (5 levels up to project root)
+            project_root = Path(__file__).parent.parent.parent.parent.parent
+            factors_dir = project_root / "results" / "factors"
            factors_dir.mkdir(parents=True, exist_ok=True)

            # Sanitize factor name for filename
@@ -382,6 +574,65 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
        except Exception as e:
            logger.warning(f"Failed to save factor JSON for '{factor_name}': {e}")

+    def _log_result_warnings(self, factor_name: str, result, metrics: dict) -> None:
+        """
+        Log warnings about result quality before saving to database.
+
+        These are informational warnings, not errors — they don't block the workflow
+        but help identify factors with potential issues.
+
+        Parameters
+        ----------
+        factor_name : str
+            Name of the factor
+        result : pd.Series or dict
+            Raw backtest result
+        metrics : dict
+            Extracted metrics dictionary
+        """
+        warnings_list = []
+
+        # Check IC
+        ic = metrics.get('ic')
+        if ic is None:
+            warnings_list.append("IC is None — factor has no predictive power")
+        elif abs(ic) < 0.001:
+            warnings_list.append(f"IC is near zero ({ic:.6f}) — weak predictive signal")
+
+        # Check positions (1day.pos) — CRITICAL for EURUSD
+        if isinstance(result, pd.Series):
+            pos_value = result.get('1day.pos', None)
+            if pos_value is not None:
+                try:
+                    pos_float = float(pos_value)
+                    if pos_float == 0:
+                        warnings_list.append(
+                            "WARNING: 1day.pos == 0 — ZERO positions opened! "
+                            "Model stayed completely neutral. Check Qlib config: "
+                            "ensure topk=1 and market=eurusd for single-asset trading."
+                        )
+                    elif pos_float < 10:
+                        warnings_list.append(
+                            f"Low position count: 1day.pos = {pos_float:.0f} — "
+                            f"model traded very rarely"
+                        )
+                except (ValueError, TypeError):
+                    pass
+
+        # Check Sharpe
+        sharpe = metrics.get('sharpe_ratio')
+        if sharpe is not None and abs(sharpe) < 0.1:
+            warnings_list.append(f"Sharpe near zero ({sharpe:.4f}) — no risk-adjusted edge")
+
+        # Check max drawdown
+        mdd = metrics.get('max_drawdown')
+        if mdd is not None and mdd < -0.5:
+            warnings_list.append(f"Extreme drawdown: {mdd:.2%} — high risk factor")
+
+        if warnings_list:
+            for warn_msg in warnings_list:
+                logger.warning(f"[{factor_name[:60]}] {warn_msg}")
+
    def _safe_float(self, value):
        """Safely convert value to float, returning None for invalid values."""
        import pandas as pd
@@ -1,4 +1,5 @@
 import pandas as pd
+from typing import Optional

 from rdagent.app.qlib_rd_loop.conf import ModelBasePropSetting
 from rdagent.components.runner import CachedRunner
@@ -115,9 +116,22 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]):
        exp.stdout = stdout

        if result is None:
-            logger.error(f"Failed to run {exp.sub_tasks[0].name}, because {stdout}")
+            logger.error(
+                f"Failed to run {exp.sub_tasks[0].name} model (result is None), because {stdout}"
+            )
+            # Save failed run info for debugging
+            self._save_failed_run(exp, stdout, error_type="result_none")
            raise ModelEmptyError(f"Failed to run {exp.sub_tasks[0].name} model, because {stdout}")

+        # Validate result before proceeding
+        validation_result = self._validate_result(exp, result)
+        if validation_result.get("has_issues"):
+            logger.warning(
+                f"Model result validation warnings for '{exp.sub_tasks[0].name}': "
+                f"{validation_result['warnings']}"
+            )
+            self._save_failed_run(exp, stdout, error_type="validation_warnings", validation=validation_result)
+
        # Save results to database immediately after Docker execution
        try:
            self._save_result_to_database(exp, result)
@@ -184,6 +198,9 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]):
                logger.debug(f"No valid IC/Sharpe for model {factor_name}, skipping DB save")
                return

+            # Log warnings about result quality
+            self._log_result_warnings(factor_name, result, metrics)
+
            # Save to database
            db = ResultsDatabase()
            run_id = db.add_backtest(factor_name=factor_name[:100], metrics=metrics)
@@ -196,6 +213,32 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]):
        except Exception as e:
            logger.warning(f"Database save failed for model {getattr(exp.hypothesis, 'hypothesis', 'unknown')}: {e}")

+    def _log_result_warnings(self, factor_name: str, result, metrics: dict) -> None:
+        """Log warnings about model result quality before saving."""
+        warnings_list = []
+
+        ic = metrics.get('ic')
+        if ic is None:
+            warnings_list.append("IC is None — model has no predictive power")
+        elif abs(ic) < 0.001:
+            warnings_list.append(f"IC near zero ({ic:.6f})")
+
+        if isinstance(result, pd.Series):
+            pos_value = result.get('1day.pos', None)
+            if pos_value is not None:
+                try:
+                    pos_float = float(pos_value)
+                    if pos_float == 0:
+                        warnings_list.append(
+                            "1day.pos == 0 — ZERO positions! Check config topk=1."
+                        )
+                except (ValueError, TypeError):
+                    pass
+
+        if warnings_list:
+            for warn_msg in warnings_list:
+                logger.warning(f"[MODEL {factor_name[:50]}] {warn_msg}")
+
    def _safe_float(self, value):
        """Safely convert value to float, returning None for invalid values."""
        import pandas as pd
@@ -208,3 +251,137 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]):
            return f
        except (ValueError, TypeError):
            return None
+
+    def _validate_result(self, exp, result) -> dict:
+        """
+        Validate model backtest result for common issues before saving.
+
+        Parameters
+        ----------
+        exp : QlibModelExperiment
+            The experiment with backtest results
+        result : pd.Series or dict
+            Backtest metrics from Qlib
+
+        Returns
+        -------
+        dict
+            Validation result with 'has_issues' (bool), 'warnings' (list), and 'details' (dict)
+        """
+        warnings = []
+        details = {}
+        model_name = exp.sub_tasks[0].name if exp.sub_tasks else "unknown"
+
+        if isinstance(result, pd.Series):
+            # Check IC
+            ic_value = result.get('IC', None)
+            details['ic_raw'] = ic_value
+            if ic_value is None or (isinstance(ic_value, float) and (ic_value != ic_value)):
+                warnings.append("IC is None/NaN — model has no predictive power")
+            else:
+                try:
+                    ic_float = float(ic_value)
+                    details['ic'] = ic_float
+                    if abs(ic_float) < 0.001:
+                        warnings.append(f"IC is near zero ({ic_float:.6f})")
+                except (ValueError, TypeError):
+                    warnings.append(f"IC value is not numeric: {ic_value}")
+
+            # Check positions
+            pos_value = result.get('1day.pos', None)
+            details['positions_raw'] = pos_value
+            if pos_value is not None:
+                try:
+                    pos_float = float(pos_value)
+                    details['positions'] = pos_float
+                    if pos_float == 0:
+                        warnings.append(
+                            "1day.pos == 0 — model opened ZERO positions. "
+                            "Check Qlib config: topk=1 for single-asset."
+                        )
+                except (ValueError, TypeError):
+                    pass
+
+            non_null_count = result.notna().sum()
+            details['non_null_metrics'] = int(non_null_count)
+            if non_null_count < 3:
+                warnings.append(f"Only {non_null_count} non-null metrics — likely empty results")
+
+        elif isinstance(result, dict):
+            ic_value = result.get('IC', result.get('ic', None))
+            details['ic_raw'] = ic_value
+            if ic_value is None:
+                warnings.append("IC is None — model has no predictive power")
+
+        return {
+            "has_issues": len(warnings) > 0,
+            "warnings": "; ".join(warnings),
+            "details": details,
+        }
+
+    def _save_failed_run(self, exp, stdout: str, error_type: str = "unknown",
+                         validation: Optional[dict] = None) -> None:
+        """
+        Save failed model run information to results/failed_runs.json.
+
+        Parameters
+        ----------
+        exp : QlibModelExperiment
+            The experiment that failed
+        stdout : str
+            Standard output from Docker execution
+        error_type : str
+            Type of error
+        validation : dict, optional
+            Validation result dict
+        """
+        import json
+        from datetime import datetime
+        from pathlib import Path
+
+        try:
+            # 5 levels up to project root
+            project_root = Path(__file__).parent.parent.parent.parent.parent
+            failed_dir = project_root / "results" / "failed_runs"
+            failed_dir.mkdir(parents=True, exist_ok=True)
+
+            model_name = exp.sub_tasks[0].name if exp.sub_tasks else "unknown"
+            factor_name = "unknown"
+            if hasattr(exp, 'hypothesis') and exp.hypothesis is not None:
+                factor_name = getattr(exp.hypothesis, 'hypothesis', model_name)
+
+            failed_record = {
+                "timestamp": datetime.now().isoformat(),
+                "factor_name": f"[MODEL] {factor_name}",
+                "model_name": model_name,
+                "error_type": error_type,
+                "stdout": stdout if stdout else "(empty)",
+                "validation": validation,
+                "experiment_details": {
+                    "model_type": exp.sub_tasks[0].model_type if exp.sub_tasks else "unknown",
+                    "hypothesis": factor_name,
+                },
+            }
+
+            failed_file = failed_dir / "failed_runs.json"
+            existing_records = []
+            if failed_file.exists():
+                try:
+                    existing_records = json.loads(failed_file.read_text(encoding="utf-8"))
+                    if not isinstance(existing_records, list):
+                        existing_records = [existing_records]
+                except (json.JSONDecodeError, Exception):
+                    existing_records = []
+
+            existing_records.append(failed_record)
+            if len(existing_records) > 500:
+                existing_records = existing_records[-500:]
+
+            failed_file.write_text(
+                json.dumps(existing_records, indent=2, default=str, ensure_ascii=False),
+                encoding="utf-8"
+            )
+            logger.info(f"Failed model run saved: {model_name} (type={error_type}) → {failed_file}")
+
+        except Exception as e:
+            logger.warning(f"Could not save failed model run info: {e}")
@@ -1,8 +1,8 @@
 qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
+    provider_uri: "~/.qlib/qlib_data/eurusd_1min_data"
    region: cn

-market: &market csi300
+market: &market eurusd
 benchmark: &benchmark EURUSD

 data_handler_config: &data_handler_config
@@ -44,19 +44,19 @@ port_analysis_config: &port_analysis_config
        module_path: qlib.contrib.strategy
        kwargs:
            signal: <PRED>
-            topk: 50
-            n_drop: 5
+            topk: 1
+            n_drop: 0
    backtest:
        start_time: {{ test_start | default("2017-01-01", true) }}
        end_time: {{ test_end | default("null", true) }}
        account: 100000000
        benchmark: *benchmark
        exchange_kwargs:
-            limit_threshold: 0.095
+            limit_threshold: 0.0
            deal_price: close
            open_cost: 0.0005
            close_cost: 0.0015
-            min_cost: 5
+            min_cost: 0

 task:
    model:
@@ -1,8 +1,8 @@
 qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
+    provider_uri: "~/.qlib/qlib_data/eurusd_1min_data"
    region: cn

-market: &market csi300
+market: &market eurusd
 benchmark: &benchmark EURUSD

 data_handler_config: &data_handler_config
@@ -37,19 +37,19 @@ port_analysis_config: &port_analysis_config
        module_path: qlib.contrib.strategy
        kwargs:
            signal: <PRED>
-            topk: 50
-            n_drop: 5
+            topk: 1
+            n_drop: 0
    backtest:
        start_time: {{ test_start | default("2017-01-01", true) }}
        end_time: {{ test_end | default("null", true) }}
        account: 100000000
        benchmark: *benchmark
        exchange_kwargs:
-            limit_threshold: 0.095
+            limit_threshold: 0.0
            deal_price: close
            open_cost: 0.0005
            close_cost: 0.0015
-            min_cost: 5
+            min_cost: 0

 task:
    model:
@@ -1,8 +1,8 @@
 qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
+    provider_uri: "~/.qlib/qlib_data/eurusd_1min_data"
    region: cn

-market: &market csi300
+market: &market eurusd
 benchmark: &benchmark EURUSD

 data_handler_config: &data_handler_config
@@ -47,19 +47,19 @@ port_analysis_config: &port_analysis_config
        module_path: qlib.contrib.strategy
        kwargs:
            signal: <PRED>
-            topk: 50
-            n_drop: 5
+            topk: 1
+            n_drop: 0
    backtest:
        start_time: {{ test_start | default("2017-01-01", true) }}
        end_time: {{ test_end | default("null", true) }}
        account: 100000000
        benchmark: *benchmark
        exchange_kwargs:
-            limit_threshold: 0.095
+            limit_threshold: 0.0
            deal_price: close
            open_cost: 0.0005
            close_cost: 0.0015
-            min_cost: 5
+            min_cost: 0

 task:
    model:
@@ -1,7 +1,7 @@
 qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
+    provider_uri: "~/.qlib/qlib_data/eurusd_1min_data"
    region: cn
-market: &market csi300
+market: &market eurusd
 benchmark: &benchmark EURUSD

 data_handler_config: &data_handler_config
@@ -43,19 +43,19 @@ port_analysis_config: &port_analysis_config
        module_path: qlib.contrib.strategy
        kwargs:
            signal: <PRED>
-            topk: 50
-            n_drop: 5
+            topk: 1
+            n_drop: 0
    backtest:
        start_time: {{ test_start | default("2017-01-01", true) }}
        end_time: {{ test_end | default("null", true) }}
        account: 100000000
        benchmark: *benchmark
        exchange_kwargs:
-            limit_threshold: 0.095
+            limit_threshold: 0.0
            deal_price: close
            open_cost: 0.0005
            close_cost: 0.0015
-            min_cost: 5
+            min_cost: 0
 task:
    model:
        class: GeneralPTNN
@@ -1,8 +1,8 @@
 qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
+    provider_uri: "~/.qlib/qlib_data/eurusd_1min_data"
    region: cn

-market: &market csi300
+market: &market eurusd
 benchmark: &benchmark EURUSD

 data_handler_config: &data_handler_config
@@ -47,19 +47,19 @@ port_analysis_config: &port_analysis_config
        module_path: qlib.contrib.strategy
        kwargs:
            signal: <PRED>
-            topk: 50
-            n_drop: 5
+            topk: 1
+            n_drop: 0
    backtest:
        start_time: {{ test_start | default("2017-01-01", true) }}
        end_time: {{ test_end | default("null", true) }}
        account: 100000000
        benchmark: *benchmark
        exchange_kwargs:
-            limit_threshold: 0.095
+            limit_threshold: 0.0
            deal_price: close
            open_cost: 0.0005
            close_cost: 0.0015
-            min_cost: 5
+            min_cost: 0

 task:
    model:
@@ -220,6 +220,7 @@ class WorkspaceResultExtractor:
            List of extracted result dictionaries
        """
        csv_files = self.scan_workspace()
+        self.failed_extractions: List[Dict] = []

        total = len(csv_files)
        print(f"\nProcessing {total} qlib_res.csv files...")
@@ -234,11 +235,57 @@ class WorkspaceResultExtractor:
                result['factor_name'] = self.extract_factor_name_from_path(csv_path)
                result['extraction_time'] = datetime.now().isoformat()
                self.extracted_results.append(result)
+            else:
+                # Track failed extraction
+                self.failed_extractions.append({
+                    "timestamp": datetime.now().isoformat(),
+                    "factor_name": self.extract_factor_name_from_path(csv_path),
+                    "source_file": str(csv_path),
+                    "error_type": "empty_or_failed_backtest",
+                    "stdout": "No meaningful metrics found in qlib_res.csv",
+                })

        print(f"\nExtracted {len(self.extracted_results)} valid results from {total} files")
-        print(f"  Skipped {total - len(self.extracted_results)} empty/failed backtests")
+        print(f"  Skipped/Failed: {total - len(self.extracted_results)} empty/failed backtests")
+
+        # Save failed extractions to failed_runs.json
+        if self.failed_extractions:
+            self._save_failed_extractions()
+
        return self.extracted_results

+    def _save_failed_extractions(self) -> None:
+        """Save failed extractions to results/failed_runs/failed_runs.json."""
+        import json
+        from pathlib import Path
+
+        try:
+            failed_dir = project_root / "results" / "failed_runs"
+            failed_dir.mkdir(parents=True, exist_ok=True)
+            failed_file = failed_dir / "failed_runs.json"
+
+            existing_records = []
+            if failed_file.exists():
+                try:
+                    existing_records = json.loads(failed_file.read_text(encoding="utf-8"))
+                    if not isinstance(existing_records, list):
+                        existing_records = [existing_records]
+                except (json.JSONDecodeError, Exception):
+                    existing_records = []
+
+            existing_records.extend(self.failed_extractions)
+            if len(existing_records) > 500:
+                existing_records = existing_records[-500:]
+
+            failed_file.write_text(
+                json.dumps(existing_records, indent=2, default=str, ensure_ascii=False),
+                encoding="utf-8"
+            )
+            print(f"  Failed runs tracked: {len(self.failed_extractions)} entries → {failed_file}")
+
+        except Exception as e:
+            print(f"  WARNING: Could not save failed extractions: {e}")
+
    def import_to_database(self) -> int:
        """
        Import extracted results to the ResultsDatabase.
@@ -389,9 +436,28 @@ def main():
        action='store_true',
        help='Only show summary, skip extraction'
    )
+    parser.add_argument(
+        '--generate-db-summary',
+        action='store_true',
+        help='Generate RESULTS_SUMMARY.md from the database'
+    )

    args = parser.parse_args()

+    # Handle DB summary generation separately
+    if args.generate_db_summary:
+        try:
+            from rdagent.components.backtesting import ResultsDatabase
+            db = ResultsDatabase()
+            summary = db.generate_results_summary(print_to_console=True)
+            db.close()
+            print(f"\nResults summary written to: results/RESULTS_SUMMARY.md")
+        except Exception as e:
+            print(f"ERROR: Could not generate DB summary: {e}")
+            import traceback
+            traceback.print_exc()
+        return
+
    workspace_path = Path(args.workspace_dir).resolve()

    if not workspace_path.exists():