diff --git a/rdagent/components/backtesting/results_db.py b/rdagent/components/backtesting/results_db.py index a7dd95e1..8b8ab932 100644 --- a/rdagent/components/backtesting/results_db.py +++ b/rdagent/components/backtesting/results_db.py @@ -8,13 +8,15 @@ import json import pandas as pd from pathlib import Path from datetime import datetime -from typing import Dict, Optional +from typing import Dict, Optional, List class ResultsDatabase: def __init__(self, db_path: Optional[str] = None): if db_path is None: - db_path = Path(__file__).parent.parent.parent / "results" / "db" / "backtest_results.db" + # Go up from rdagent/components/backtesting/ to project root (4 levels) + project_root = Path(__file__).parent.parent.parent.parent + db_path = str(project_root / "results" / "db" / "backtest_results.db") self.db_path = db_path Path(db_path).parent.mkdir(parents=True, exist_ok=True) self.conn = sqlite3.connect(db_path) @@ -273,6 +275,226 @@ class ResultsDatabase: self.conn ) + def generate_results_summary(self, output_path: Optional[str] = None, + print_to_console: bool = True) -> Dict: + """ + Generate a comprehensive results summary report. + + Scans the database and factors directory to produce a summary report + with total runs, successful/failed counts, best metrics, and statistics. + + Parameters + ---------- + output_path : str, optional + Path to write the summary as Markdown file. + Defaults to results/RESULTS_SUMMARY.md + print_to_console : bool + If True, print the summary to console + + Returns + ------- + Dict + Summary statistics dictionary + """ + # Database stats + db_stats = self.get_aggregate_stats() + + # Get all results + all_results = self.get_all_results() + + # Top factors + top_sharpe = self.get_top_factors('sharpe', limit=10) + top_ic = self.get_top_factors('ic', limit=10) + + # Count successful vs failed runs + total_runs = len(all_results) + runs_with_ic = int(all_results['ic'].notna().sum()) if total_runs > 0 else 0 + runs_with_sharpe = int(all_results['sharpe'].notna().sum()) if total_runs > 0 else 0 + + # Count unique factors + unique_factors = all_results['factor_name'].nunique() if total_runs > 0 else 0 + + # Best metrics + best_ic = all_results['ic'].max() if total_runs > 0 and all_results['ic'].notna().any() else None + best_sharpe = all_results['sharpe'].max() if total_runs > 0 and all_results['sharpe'].notna().any() else None + best_return = all_results['annual_return'].max() if total_runs > 0 and all_results['annual_return'].notna().any() else None + worst_drawdown = all_results['max_drawdown'].min() if total_runs > 0 and all_results['max_drawdown'].notna().any() else None + + # Scan factors directory for JSON files + factors_dir = Path(__file__).parent.parent.parent / "results" / "factors" + json_factor_files = 0 + if factors_dir.exists(): + json_factor_files = len(list(factors_dir.glob("*.json"))) + + # Scan failed runs + failed_dir = Path(__file__).parent.parent.parent / "results" / "failed_runs" + failed_runs_file = failed_dir / "failed_runs.json" + failed_runs_count = 0 + failed_runs_data = [] + if failed_runs_file.exists(): + try: + failed_runs_data = json.loads(failed_runs_file.read_text(encoding="utf-8")) + if isinstance(failed_runs_data, list): + failed_runs_count = len(failed_runs_data) + elif isinstance(failed_runs_data, dict): + failed_runs_count = 1 + except (json.JSONDecodeError, Exception): + failed_runs_count = 0 + + # Build summary + summary = { + "generated_at": datetime.now().isoformat(), + "database_path": str(self.db_path), + "overview": { + "total_runs": int(total_runs), + "unique_factors": int(unique_factors), + "runs_with_ic": int(runs_with_ic), + "runs_with_sharpe": int(runs_with_sharpe), + "json_factor_files": json_factor_files, + "failed_runs": failed_runs_count, + }, + "best_metrics": { + "best_ic": float(best_ic) if best_ic is not None else None, + "best_sharpe": float(best_sharpe) if best_sharpe is not None else None, + "best_annual_return": float(best_return) if best_return is not None else None, + "worst_drawdown": float(worst_drawdown) if worst_drawdown is not None else None, + }, + "aggregate_stats": db_stats, + "top_10_by_sharpe": top_sharpe.to_dict(orient='records') if len(top_sharpe) > 0 else [], + "top_10_by_ic": top_ic.to_dict(orient='records') if len(top_ic) > 0 else [], + } + + # Write to Markdown + if output_path is None: + # Go up from rdagent/components/backtesting/ to project root (4 levels) + project_root = Path(__file__).parent.parent.parent.parent + output_path = str(project_root / "results" / "RESULTS_SUMMARY.md") + + self._write_summary_markdown(summary, output_path) + + if print_to_console: + self._print_summary_console(summary) + + return summary + + def _fmt_float(self, value, fmt: str = ".4f") -> str: + """Format float value, returning 'N/A' for None.""" + if value is None: + return "N/A" + return f"{value:{fmt}}" + + def _write_summary_markdown(self, summary: Dict, output_path: str) -> None: + """Write summary as Markdown file.""" + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + + overview = summary['overview'] + best = summary['best_metrics'] + top_sharpe = summary['top_10_by_sharpe'] + top_ic = summary['top_10_by_ic'] + + # Pre-format best metrics for display + best_ic_str = self._fmt_float(best['best_ic'], ".6f") + best_sharpe_str = self._fmt_float(best['best_sharpe'], ".4f") + best_return_str = self._fmt_float(best['best_annual_return'], ".4f") + worst_dd_str = self._fmt_float(best['worst_drawdown'], ".4f") + + md_lines = [ + "# Predix Results Summary", + "", + f"**Generated:** {summary['generated_at']}", + f"**Database:** `{summary['database_path']}`", + "", + "## Overview", + "", + f"| Metric | Value |", + f"|--------|-------|", + f"| Total Runs | {overview['total_runs']} |", + f"| Unique Factors | {overview['unique_factors']} |", + f"| Runs with IC | {overview['runs_with_ic']} |", + f"| Runs with Sharpe | {overview['runs_with_sharpe']} |", + f"| JSON Factor Files | {overview['json_factor_files']} |", + f"| Failed Runs | {overview['failed_runs']} |", + "", + "## Best Metrics", + "", + f"| Metric | Value |", + f"|--------|-------|", + f"| Best IC | {best_ic_str} |", + f"| Best Sharpe | {best_sharpe_str} |", + f"| Best Annual Return | {best_return_str} |", + f"| Worst Drawdown | {worst_dd_str} |", + "", + ] + + if top_sharpe: + md_lines.extend([ + "## Top 10 by Sharpe Ratio", + "", + "| # | Factor | Sharpe | IC | Annual Return | Max Drawdown |", + "|---|--------|--------|-----|---------------|--------------|", + ]) + for i, row in enumerate(top_sharpe[:10], 1): + md_lines.append( + f"| {i} | {row.get('factor_name', 'N/A')[:50]} | " + f"{self._fmt_float(row.get('sharpe'), '.4f')} | " + f"{self._fmt_float(row.get('ic'), '.6f')} | " + f"{self._fmt_float(row.get('annual_return'), '.4f')} | " + f"{self._fmt_float(row.get('max_drawdown'), '.4f')} |" + ) + md_lines.append("") + + if top_ic: + md_lines.extend([ + "## Top 10 by IC", + "", + "| # | Factor | IC | Sharpe | Annual Return |", + "|---|--------|-----|--------|---------------|", + ]) + for i, row in enumerate(top_ic[:10], 1): + md_lines.append( + f"| {i} | {row.get('factor_name', 'N/A')[:50]} | " + f"{self._fmt_float(row.get('ic'), '.6f')} | " + f"{self._fmt_float(row.get('sharpe'), '.4f')} | " + f"{self._fmt_float(row.get('annual_return'), '.4f')} |" + ) + md_lines.append("") + + md_lines.extend([ + "## Failed Runs", + "", + f"Total failed runs tracked: **{overview['failed_runs']}**", + "", + "Failed runs are stored in `results/failed_runs/failed_runs.json`.", + "", + ]) + + path.write_text("\n".join(md_lines), encoding="utf-8") + + def _print_summary_console(self, summary: Dict) -> None: + """Print summary to console.""" + overview = summary['overview'] + best = summary['best_metrics'] + + print("\n" + "=" * 70) + print(" PREDIX RESULTS SUMMARY") + print("=" * 70) + print(f" Generated: {summary['generated_at']}") + print(f" Database: {summary['database_path']}") + print("-" * 70) + print(f" Total Runs: {overview['total_runs']}") + print(f" Unique Factors: {overview['unique_factors']}") + print(f" Runs with IC: {overview['runs_with_ic']}") + print(f" Runs with Sharpe: {overview['runs_with_sharpe']}") + print(f" JSON Factor Files: {overview['json_factor_files']}") + print(f" Failed Runs: {overview['failed_runs']}") + print("-" * 70) + print(f" Best IC: {self._fmt_float(best['best_ic'], '.6f')}") + print(f" Best Sharpe: {self._fmt_float(best['best_sharpe'], '.4f')}") + print(f" Best Ann. Return: {self._fmt_float(best['best_annual_return'], '.4f')}") + print(f" Worst Drawdown: {self._fmt_float(best['worst_drawdown'], '.4f')}") + print("=" * 70 + "\n") + def close(self): self.conn.close() diff --git a/rdagent/scenarios/qlib/developer/factor_runner.py b/rdagent/scenarios/qlib/developer/factor_runner.py index 4ee1f1cf..b2e1b1b1 100644 --- a/rdagent/scenarios/qlib/developer/factor_runner.py +++ b/rdagent/scenarios/qlib/developer/factor_runner.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Optional import pandas as pd from pandarallel import pandarallel @@ -193,9 +194,24 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): ) if result is None: - logger.error(f"Failed to run this experiment, because {stdout}") + logger.error( + f"Failed to run this experiment (result is None). " + f"Factor: {getattr(exp.hypothesis, 'hypothesis', 'unknown')}" + ) + # Save failed run info for debugging + self._save_failed_run(exp, stdout, error_type="result_none") raise FactorEmptyError(f"Failed to run this experiment, because {stdout}") + # Validate result before proceeding + validation_result = self._validate_result(exp, result) + if validation_result.get("has_issues"): + logger.warning( + f"Result validation warnings for factor '{getattr(exp.hypothesis, 'hypothesis', 'unknown')}': " + f"{validation_result['warnings']}" + ) + # Save warning info for debugging + self._save_failed_run(exp, stdout, error_type="validation_warnings", validation=validation_result) + exp.result = result exp.stdout = stdout @@ -214,6 +230,177 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): return exp + def _validate_result(self, exp, result) -> dict: + """ + Validate backtest result for common issues before saving. + + Checks for: + - Empty/None IC (no predictive power) + - Zero positions (1day.pos == 0, model stayed neutral) + - All metrics being None/NaN + + Parameters + ---------- + exp : QlibFactorExperiment + The experiment with backtest results + result : pd.Series or dict + Backtest metrics from Qlib + + Returns + ------- + dict + Validation result with 'has_issues' (bool), 'warnings' (list), and 'details' (dict) + """ + warnings = [] + details = {} + + factor_name = "unknown" + if hasattr(exp, 'hypothesis') and exp.hypothesis is not None: + factor_name = getattr(exp.hypothesis, 'hypothesis', 'unknown') + + if isinstance(result, pd.Series): + # Check IC + ic_value = result.get('IC', None) + details['ic_raw'] = ic_value + if ic_value is None or (isinstance(ic_value, float) and (ic_value != ic_value)): # NaN check + warnings.append("IC is None/NaN — factor has no predictive power") + else: + try: + ic_float = float(ic_value) + details['ic'] = ic_float + if abs(ic_float) < 0.001: + warnings.append( + f"IC is near zero ({ic_float:.6f}) — factor may not predict returns" + ) + except (ValueError, TypeError): + warnings.append(f"IC value is not numeric: {ic_value}") + + # Check positions (1day.pos) + pos_value = result.get('1day.pos', None) + details['positions_raw'] = pos_value + if pos_value is not None: + try: + pos_float = float(pos_value) + details['positions'] = pos_float + if pos_float == 0: + warnings.append( + "1day.pos == 0 — model opened ZERO positions (stayed neutral). " + "Possible causes: (1) topk too high for single-asset, " + "(2) signal threshold too restrictive, (3) no valid predictions" + ) + elif pos_float < 10: + warnings.append( + f"1day.pos = {pos_float:.0f} — very few positions opened. " + f"Check signal threshold and topk settings" + ) + except (ValueError, TypeError): + pass # pos might be a string + + # Check if result is essentially empty (all values None or NaN) + non_null_count = result.notna().sum() + total_count = len(result) + details['non_null_metrics'] = int(non_null_count) + details['total_metrics'] = int(total_count) + if non_null_count < 3: + warnings.append( + f"Result has only {non_null_count}/{total_count} non-null metrics — " + f"backtest likely produced empty results" + ) + + # Check for key metrics + required_metrics = ['IC', '1day.excess_return_with_cost.shar', '1day.pos'] + for metric_name in required_metrics: + val = result.get(metric_name, None) + details[f'has_{metric_name}'] = val is not None + + elif isinstance(result, dict): + # Dict-based result validation + ic_value = result.get('IC', result.get('ic', None)) + details['ic_raw'] = ic_value + if ic_value is None: + warnings.append("IC is None — factor has no predictive power") + + return { + "has_issues": len(warnings) > 0, + "warnings": "; ".join(warnings), + "details": details, + } + + def _save_failed_run(self, exp, stdout: str, error_type: str = "unknown", + validation: Optional[dict] = None) -> None: + """ + Save failed run information to results/failed_runs.json for debugging. + + Parameters + ---------- + exp : QlibFactorExperiment + The experiment that failed + stdout : str + Standard output from the Docker execution + error_type : str + Type of error: 'result_none', 'validation_warnings', 'docker_error', etc. + validation : dict, optional + Validation result dict if error_type is 'validation_warnings' + """ + import json + from datetime import datetime + from pathlib import Path + + try: + # Ensure failed_runs directory exists (5 levels up to project root) + project_root = Path(__file__).parent.parent.parent.parent.parent + failed_dir = project_root / "results" / "failed_runs" + failed_dir.mkdir(parents=True, exist_ok=True) + + # Get factor name + factor_name = "unknown" + if hasattr(exp, 'hypothesis') and exp.hypothesis is not None: + factor_name = getattr(exp.hypothesis, 'hypothesis', 'unknown') + + # Build failed run record + failed_record = { + "timestamp": datetime.now().isoformat(), + "factor_name": factor_name, + "error_type": error_type, + "stdout": stdout if stdout else "(empty)", + "validation": validation, + "experiment_details": { + "base_features": list(getattr(exp, 'base_features', {}).keys()) if hasattr(exp, 'base_features') else [], + "hypothesis": getattr(exp.hypothesis, 'hypothesis', str(getattr(exp, 'hypothesis', 'N/A'))) + if hasattr(exp, 'hypothesis') else "N/A", + }, + } + + # Append to failed_runs.json + failed_file = failed_dir / "failed_runs.json" + existing_records = [] + if failed_file.exists(): + try: + existing_records = json.loads(failed_file.read_text(encoding="utf-8")) + if not isinstance(existing_records, list): + existing_records = [existing_records] + except (json.JSONDecodeError, Exception): + existing_records = [] + + existing_records.append(failed_record) + + # Keep only last 500 records to prevent file bloat + if len(existing_records) > 500: + existing_records = existing_records[-500:] + + failed_file.write_text( + json.dumps(existing_records, indent=2, default=str, ensure_ascii=False), + encoding="utf-8" + ) + logger.info( + f"Failed run saved: {factor_name} (type={error_type}) " + f"→ {failed_file}" + ) + + except Exception as e: + # Don't let failed run logging break the main workflow + logger.warning(f"Could not save failed run info: {e}") + def _save_result_to_database(self, exp, result) -> None: """ Save backtest results to the ResultsDatabase and write factor JSON summary. @@ -293,6 +480,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): metrics['volatility'] = None metrics['raw_metrics'] = result + # Result validation before saving (warnings, not blocking) + self._log_result_warnings(factor_name, result, metrics) + # Only save if we have at least IC or Sharpe if metrics.get('ic') is None and metrics.get('sharpe_ratio') is None: logger.warning( @@ -301,8 +491,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): ) return - # Ensure DB directory exists before creating database - db_path = Path(__file__).parent.parent.parent.parent / "results" / "db" + # Ensure DB directory exists before creating database (5 levels up to project root) + project_root = Path(__file__).parent.parent.parent.parent.parent + db_path = project_root / "results" / "db" db_path.mkdir(parents=True, exist_ok=True) db_file = db_path / "backtest_results.db" @@ -344,8 +535,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): from pathlib import Path try: - # Ensure factors directory exists - factors_dir = Path(__file__).parent.parent.parent.parent / "results" / "factors" + # Ensure factors directory exists (5 levels up to project root) + project_root = Path(__file__).parent.parent.parent.parent.parent + factors_dir = project_root / "results" / "factors" factors_dir.mkdir(parents=True, exist_ok=True) # Sanitize factor name for filename @@ -382,6 +574,65 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): except Exception as e: logger.warning(f"Failed to save factor JSON for '{factor_name}': {e}") + def _log_result_warnings(self, factor_name: str, result, metrics: dict) -> None: + """ + Log warnings about result quality before saving to database. + + These are informational warnings, not errors — they don't block the workflow + but help identify factors with potential issues. + + Parameters + ---------- + factor_name : str + Name of the factor + result : pd.Series or dict + Raw backtest result + metrics : dict + Extracted metrics dictionary + """ + warnings_list = [] + + # Check IC + ic = metrics.get('ic') + if ic is None: + warnings_list.append("IC is None — factor has no predictive power") + elif abs(ic) < 0.001: + warnings_list.append(f"IC is near zero ({ic:.6f}) — weak predictive signal") + + # Check positions (1day.pos) — CRITICAL for EURUSD + if isinstance(result, pd.Series): + pos_value = result.get('1day.pos', None) + if pos_value is not None: + try: + pos_float = float(pos_value) + if pos_float == 0: + warnings_list.append( + "WARNING: 1day.pos == 0 — ZERO positions opened! " + "Model stayed completely neutral. Check Qlib config: " + "ensure topk=1 and market=eurusd for single-asset trading." + ) + elif pos_float < 10: + warnings_list.append( + f"Low position count: 1day.pos = {pos_float:.0f} — " + f"model traded very rarely" + ) + except (ValueError, TypeError): + pass + + # Check Sharpe + sharpe = metrics.get('sharpe_ratio') + if sharpe is not None and abs(sharpe) < 0.1: + warnings_list.append(f"Sharpe near zero ({sharpe:.4f}) — no risk-adjusted edge") + + # Check max drawdown + mdd = metrics.get('max_drawdown') + if mdd is not None and mdd < -0.5: + warnings_list.append(f"Extreme drawdown: {mdd:.2%} — high risk factor") + + if warnings_list: + for warn_msg in warnings_list: + logger.warning(f"[{factor_name[:60]}] {warn_msg}") + def _safe_float(self, value): """Safely convert value to float, returning None for invalid values.""" import pandas as pd diff --git a/rdagent/scenarios/qlib/developer/model_runner.py b/rdagent/scenarios/qlib/developer/model_runner.py index 8693d735..61ddafc2 100644 --- a/rdagent/scenarios/qlib/developer/model_runner.py +++ b/rdagent/scenarios/qlib/developer/model_runner.py @@ -1,4 +1,5 @@ import pandas as pd +from typing import Optional from rdagent.app.qlib_rd_loop.conf import ModelBasePropSetting from rdagent.components.runner import CachedRunner @@ -115,9 +116,22 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]): exp.stdout = stdout if result is None: - logger.error(f"Failed to run {exp.sub_tasks[0].name}, because {stdout}") + logger.error( + f"Failed to run {exp.sub_tasks[0].name} model (result is None), because {stdout}" + ) + # Save failed run info for debugging + self._save_failed_run(exp, stdout, error_type="result_none") raise ModelEmptyError(f"Failed to run {exp.sub_tasks[0].name} model, because {stdout}") + # Validate result before proceeding + validation_result = self._validate_result(exp, result) + if validation_result.get("has_issues"): + logger.warning( + f"Model result validation warnings for '{exp.sub_tasks[0].name}': " + f"{validation_result['warnings']}" + ) + self._save_failed_run(exp, stdout, error_type="validation_warnings", validation=validation_result) + # Save results to database immediately after Docker execution try: self._save_result_to_database(exp, result) @@ -184,6 +198,9 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]): logger.debug(f"No valid IC/Sharpe for model {factor_name}, skipping DB save") return + # Log warnings about result quality + self._log_result_warnings(factor_name, result, metrics) + # Save to database db = ResultsDatabase() run_id = db.add_backtest(factor_name=factor_name[:100], metrics=metrics) @@ -196,6 +213,32 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]): except Exception as e: logger.warning(f"Database save failed for model {getattr(exp.hypothesis, 'hypothesis', 'unknown')}: {e}") + def _log_result_warnings(self, factor_name: str, result, metrics: dict) -> None: + """Log warnings about model result quality before saving.""" + warnings_list = [] + + ic = metrics.get('ic') + if ic is None: + warnings_list.append("IC is None — model has no predictive power") + elif abs(ic) < 0.001: + warnings_list.append(f"IC near zero ({ic:.6f})") + + if isinstance(result, pd.Series): + pos_value = result.get('1day.pos', None) + if pos_value is not None: + try: + pos_float = float(pos_value) + if pos_float == 0: + warnings_list.append( + "1day.pos == 0 — ZERO positions! Check config topk=1." + ) + except (ValueError, TypeError): + pass + + if warnings_list: + for warn_msg in warnings_list: + logger.warning(f"[MODEL {factor_name[:50]}] {warn_msg}") + def _safe_float(self, value): """Safely convert value to float, returning None for invalid values.""" import pandas as pd @@ -208,3 +251,137 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]): return f except (ValueError, TypeError): return None + + def _validate_result(self, exp, result) -> dict: + """ + Validate model backtest result for common issues before saving. + + Parameters + ---------- + exp : QlibModelExperiment + The experiment with backtest results + result : pd.Series or dict + Backtest metrics from Qlib + + Returns + ------- + dict + Validation result with 'has_issues' (bool), 'warnings' (list), and 'details' (dict) + """ + warnings = [] + details = {} + model_name = exp.sub_tasks[0].name if exp.sub_tasks else "unknown" + + if isinstance(result, pd.Series): + # Check IC + ic_value = result.get('IC', None) + details['ic_raw'] = ic_value + if ic_value is None or (isinstance(ic_value, float) and (ic_value != ic_value)): + warnings.append("IC is None/NaN — model has no predictive power") + else: + try: + ic_float = float(ic_value) + details['ic'] = ic_float + if abs(ic_float) < 0.001: + warnings.append(f"IC is near zero ({ic_float:.6f})") + except (ValueError, TypeError): + warnings.append(f"IC value is not numeric: {ic_value}") + + # Check positions + pos_value = result.get('1day.pos', None) + details['positions_raw'] = pos_value + if pos_value is not None: + try: + pos_float = float(pos_value) + details['positions'] = pos_float + if pos_float == 0: + warnings.append( + "1day.pos == 0 — model opened ZERO positions. " + "Check Qlib config: topk=1 for single-asset." + ) + except (ValueError, TypeError): + pass + + non_null_count = result.notna().sum() + details['non_null_metrics'] = int(non_null_count) + if non_null_count < 3: + warnings.append(f"Only {non_null_count} non-null metrics — likely empty results") + + elif isinstance(result, dict): + ic_value = result.get('IC', result.get('ic', None)) + details['ic_raw'] = ic_value + if ic_value is None: + warnings.append("IC is None — model has no predictive power") + + return { + "has_issues": len(warnings) > 0, + "warnings": "; ".join(warnings), + "details": details, + } + + def _save_failed_run(self, exp, stdout: str, error_type: str = "unknown", + validation: Optional[dict] = None) -> None: + """ + Save failed model run information to results/failed_runs.json. + + Parameters + ---------- + exp : QlibModelExperiment + The experiment that failed + stdout : str + Standard output from Docker execution + error_type : str + Type of error + validation : dict, optional + Validation result dict + """ + import json + from datetime import datetime + from pathlib import Path + + try: + # 5 levels up to project root + project_root = Path(__file__).parent.parent.parent.parent.parent + failed_dir = project_root / "results" / "failed_runs" + failed_dir.mkdir(parents=True, exist_ok=True) + + model_name = exp.sub_tasks[0].name if exp.sub_tasks else "unknown" + factor_name = "unknown" + if hasattr(exp, 'hypothesis') and exp.hypothesis is not None: + factor_name = getattr(exp.hypothesis, 'hypothesis', model_name) + + failed_record = { + "timestamp": datetime.now().isoformat(), + "factor_name": f"[MODEL] {factor_name}", + "model_name": model_name, + "error_type": error_type, + "stdout": stdout if stdout else "(empty)", + "validation": validation, + "experiment_details": { + "model_type": exp.sub_tasks[0].model_type if exp.sub_tasks else "unknown", + "hypothesis": factor_name, + }, + } + + failed_file = failed_dir / "failed_runs.json" + existing_records = [] + if failed_file.exists(): + try: + existing_records = json.loads(failed_file.read_text(encoding="utf-8")) + if not isinstance(existing_records, list): + existing_records = [existing_records] + except (json.JSONDecodeError, Exception): + existing_records = [] + + existing_records.append(failed_record) + if len(existing_records) > 500: + existing_records = existing_records[-500:] + + failed_file.write_text( + json.dumps(existing_records, indent=2, default=str, ensure_ascii=False), + encoding="utf-8" + ) + logger.info(f"Failed model run saved: {model_name} (type={error_type}) → {failed_file}") + + except Exception as e: + logger.warning(f"Could not save failed model run info: {e}") diff --git a/rdagent/scenarios/qlib/experiment/factor_template/conf_baseline.yaml b/rdagent/scenarios/qlib/experiment/factor_template/conf_baseline.yaml index af553d5b..26b85623 100644 --- a/rdagent/scenarios/qlib/experiment/factor_template/conf_baseline.yaml +++ b/rdagent/scenarios/qlib/experiment/factor_template/conf_baseline.yaml @@ -1,8 +1,8 @@ qlib_init: - provider_uri: "~/.qlib/qlib_data/cn_data" + provider_uri: "~/.qlib/qlib_data/eurusd_1min_data" region: cn -market: &market csi300 +market: &market eurusd benchmark: &benchmark EURUSD data_handler_config: &data_handler_config @@ -44,19 +44,19 @@ port_analysis_config: &port_analysis_config module_path: qlib.contrib.strategy kwargs: signal: - topk: 50 - n_drop: 5 + topk: 1 + n_drop: 0 backtest: start_time: {{ test_start | default("2017-01-01", true) }} end_time: {{ test_end | default("null", true) }} account: 100000000 benchmark: *benchmark exchange_kwargs: - limit_threshold: 0.095 + limit_threshold: 0.0 deal_price: close open_cost: 0.0005 close_cost: 0.0015 - min_cost: 5 + min_cost: 0 task: model: diff --git a/rdagent/scenarios/qlib/experiment/factor_template/conf_combined_factors.yaml b/rdagent/scenarios/qlib/experiment/factor_template/conf_combined_factors.yaml index 8114ddbf..ba85f24e 100644 --- a/rdagent/scenarios/qlib/experiment/factor_template/conf_combined_factors.yaml +++ b/rdagent/scenarios/qlib/experiment/factor_template/conf_combined_factors.yaml @@ -1,8 +1,8 @@ qlib_init: - provider_uri: "~/.qlib/qlib_data/cn_data" + provider_uri: "~/.qlib/qlib_data/eurusd_1min_data" region: cn -market: &market csi300 +market: &market eurusd benchmark: &benchmark EURUSD data_handler_config: &data_handler_config @@ -37,19 +37,19 @@ port_analysis_config: &port_analysis_config module_path: qlib.contrib.strategy kwargs: signal: - topk: 50 - n_drop: 5 + topk: 1 + n_drop: 0 backtest: start_time: {{ test_start | default("2017-01-01", true) }} end_time: {{ test_end | default("null", true) }} account: 100000000 benchmark: *benchmark exchange_kwargs: - limit_threshold: 0.095 + limit_threshold: 0.0 deal_price: close open_cost: 0.0005 close_cost: 0.0015 - min_cost: 5 + min_cost: 0 task: model: diff --git a/rdagent/scenarios/qlib/experiment/factor_template/conf_combined_factors_sota_model.yaml b/rdagent/scenarios/qlib/experiment/factor_template/conf_combined_factors_sota_model.yaml index 94f76a7b..8b50abed 100644 --- a/rdagent/scenarios/qlib/experiment/factor_template/conf_combined_factors_sota_model.yaml +++ b/rdagent/scenarios/qlib/experiment/factor_template/conf_combined_factors_sota_model.yaml @@ -1,8 +1,8 @@ qlib_init: - provider_uri: "~/.qlib/qlib_data/cn_data" + provider_uri: "~/.qlib/qlib_data/eurusd_1min_data" region: cn -market: &market csi300 +market: &market eurusd benchmark: &benchmark EURUSD data_handler_config: &data_handler_config @@ -47,19 +47,19 @@ port_analysis_config: &port_analysis_config module_path: qlib.contrib.strategy kwargs: signal: - topk: 50 - n_drop: 5 + topk: 1 + n_drop: 0 backtest: start_time: {{ test_start | default("2017-01-01", true) }} end_time: {{ test_end | default("null", true) }} account: 100000000 benchmark: *benchmark exchange_kwargs: - limit_threshold: 0.095 + limit_threshold: 0.0 deal_price: close open_cost: 0.0005 close_cost: 0.0015 - min_cost: 5 + min_cost: 0 task: model: diff --git a/rdagent/scenarios/qlib/experiment/model_template/conf_baseline_factors_model.yaml b/rdagent/scenarios/qlib/experiment/model_template/conf_baseline_factors_model.yaml index b5a919bb..b9423d29 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/conf_baseline_factors_model.yaml +++ b/rdagent/scenarios/qlib/experiment/model_template/conf_baseline_factors_model.yaml @@ -1,7 +1,7 @@ qlib_init: - provider_uri: "~/.qlib/qlib_data/cn_data" + provider_uri: "~/.qlib/qlib_data/eurusd_1min_data" region: cn -market: &market csi300 +market: &market eurusd benchmark: &benchmark EURUSD data_handler_config: &data_handler_config @@ -43,19 +43,19 @@ port_analysis_config: &port_analysis_config module_path: qlib.contrib.strategy kwargs: signal: - topk: 50 - n_drop: 5 + topk: 1 + n_drop: 0 backtest: start_time: {{ test_start | default("2017-01-01", true) }} end_time: {{ test_end | default("null", true) }} account: 100000000 benchmark: *benchmark exchange_kwargs: - limit_threshold: 0.095 + limit_threshold: 0.0 deal_price: close open_cost: 0.0005 close_cost: 0.0015 - min_cost: 5 + min_cost: 0 task: model: class: GeneralPTNN diff --git a/rdagent/scenarios/qlib/experiment/model_template/conf_sota_factors_model.yaml b/rdagent/scenarios/qlib/experiment/model_template/conf_sota_factors_model.yaml index 94f76a7b..8b50abed 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/conf_sota_factors_model.yaml +++ b/rdagent/scenarios/qlib/experiment/model_template/conf_sota_factors_model.yaml @@ -1,8 +1,8 @@ qlib_init: - provider_uri: "~/.qlib/qlib_data/cn_data" + provider_uri: "~/.qlib/qlib_data/eurusd_1min_data" region: cn -market: &market csi300 +market: &market eurusd benchmark: &benchmark EURUSD data_handler_config: &data_handler_config @@ -47,19 +47,19 @@ port_analysis_config: &port_analysis_config module_path: qlib.contrib.strategy kwargs: signal: - topk: 50 - n_drop: 5 + topk: 1 + n_drop: 0 backtest: start_time: {{ test_start | default("2017-01-01", true) }} end_time: {{ test_end | default("null", true) }} account: 100000000 benchmark: *benchmark exchange_kwargs: - limit_threshold: 0.095 + limit_threshold: 0.0 deal_price: close open_cost: 0.0005 close_cost: 0.0015 - min_cost: 5 + min_cost: 0 task: model: diff --git a/scripts/extract_results.py b/scripts/extract_results.py index 199c2b48..26628323 100644 --- a/scripts/extract_results.py +++ b/scripts/extract_results.py @@ -220,6 +220,7 @@ class WorkspaceResultExtractor: List of extracted result dictionaries """ csv_files = self.scan_workspace() + self.failed_extractions: List[Dict] = [] total = len(csv_files) print(f"\nProcessing {total} qlib_res.csv files...") @@ -234,11 +235,57 @@ class WorkspaceResultExtractor: result['factor_name'] = self.extract_factor_name_from_path(csv_path) result['extraction_time'] = datetime.now().isoformat() self.extracted_results.append(result) + else: + # Track failed extraction + self.failed_extractions.append({ + "timestamp": datetime.now().isoformat(), + "factor_name": self.extract_factor_name_from_path(csv_path), + "source_file": str(csv_path), + "error_type": "empty_or_failed_backtest", + "stdout": "No meaningful metrics found in qlib_res.csv", + }) print(f"\nExtracted {len(self.extracted_results)} valid results from {total} files") - print(f" Skipped {total - len(self.extracted_results)} empty/failed backtests") + print(f" Skipped/Failed: {total - len(self.extracted_results)} empty/failed backtests") + + # Save failed extractions to failed_runs.json + if self.failed_extractions: + self._save_failed_extractions() + return self.extracted_results + def _save_failed_extractions(self) -> None: + """Save failed extractions to results/failed_runs/failed_runs.json.""" + import json + from pathlib import Path + + try: + failed_dir = project_root / "results" / "failed_runs" + failed_dir.mkdir(parents=True, exist_ok=True) + failed_file = failed_dir / "failed_runs.json" + + existing_records = [] + if failed_file.exists(): + try: + existing_records = json.loads(failed_file.read_text(encoding="utf-8")) + if not isinstance(existing_records, list): + existing_records = [existing_records] + except (json.JSONDecodeError, Exception): + existing_records = [] + + existing_records.extend(self.failed_extractions) + if len(existing_records) > 500: + existing_records = existing_records[-500:] + + failed_file.write_text( + json.dumps(existing_records, indent=2, default=str, ensure_ascii=False), + encoding="utf-8" + ) + print(f" Failed runs tracked: {len(self.failed_extractions)} entries → {failed_file}") + + except Exception as e: + print(f" WARNING: Could not save failed extractions: {e}") + def import_to_database(self) -> int: """ Import extracted results to the ResultsDatabase. @@ -389,9 +436,28 @@ def main(): action='store_true', help='Only show summary, skip extraction' ) + parser.add_argument( + '--generate-db-summary', + action='store_true', + help='Generate RESULTS_SUMMARY.md from the database' + ) args = parser.parse_args() + # Handle DB summary generation separately + if args.generate_db_summary: + try: + from rdagent.components.backtesting import ResultsDatabase + db = ResultsDatabase() + summary = db.generate_results_summary(print_to_console=True) + db.close() + print(f"\nResults summary written to: results/RESULTS_SUMMARY.md") + except Exception as e: + print(f"ERROR: Could not generate DB summary: {e}") + import traceback + traceback.print_exc() + return + workspace_path = Path(args.workspace_dir).resolve() if not workspace_path.exists():