From 1b193fa2e7df70819dfc507a12b81ee22f006010 Mon Sep 17 00:00:00 2001
From: XianBW <36835909+XianBW@users.noreply.github.com>
Date: Sat, 8 Feb 2025 19:03:29 +0800
Subject: [PATCH] fix: add scores.csv checking in ensemble_test (#567)

* add scores.csv checking in ensemble_test

* text modify
---
 .../ensemble/eval_tests/ensemble_test.txt     |  9 +++++++-
 .../coder/data_science/workflow/eval.py       | 21 ++++++++++---------
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt
index 396f2c46..b9b0042f 100644
--- a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt
+++ b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt
@@ -67,8 +67,15 @@ if isinstance(final_pred, (list, np.ndarray, pd.DataFrame, torch.Tensor, tf.Tens
         f"vs. len(test_X)={len(test_X)}"
     )
 
-# check if scores.csv is generated
+# check scores.csv
 assert Path("scores.csv").exists(), "scores.csv is not generated"
+score_df = pd.read_csv("scores.csv", index_col=0)
+model_set_in_scores = set(score_df.index)
+for model in {{model_names}}:
+    if model not in model_set_in_scores:
+        print(f"\nModel {model} is not evaluated in the scores.csv.")
+print("Please check scores dataframe's format, it's:")
+print(score_df)
 
 print("Ensemble test passed successfully.")
 print(f"Output shape: {final_pred.shape}")
diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py
index 59fde915..d25dfa73 100644
--- a/rdagent/components/coder/data_science/workflow/eval.py
+++ b/rdagent/components/coder/data_science/workflow/eval.py
@@ -80,16 +80,17 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         if not score_fp.exists():
             stdout += "\nMetrics file (scores.csv) is not generated."
         else:
-            score_df = pd.read_csv(score_fp, index_col=0)
-            model_set_in_scores = set(score_df.index)
-            model_set_in_folder = set(
-                f[:-3] for f in implementation.file_dict.keys() if re.match(r"^model_(?!test)\w+\.py$", f)
-            )
-            for model in model_set_in_folder:
-                if model not in model_set_in_scores:
-                    stdout += (
-                        f"\nModel {model} is not evaluated in the scores.csv. The scores.csv has {model_set_in_scores}."
-                    )
+            try:
+                score_df = pd.read_csv(score_fp, index_col=0)
+                model_set_in_scores = set(score_df.index)
+                model_set_in_folder = set(
+                    f[:-3] for f in implementation.file_dict.keys() if re.match(r"^model_(?!test)\w+\.py$", f)
+                )
+                for model in model_set_in_folder:
+                    if model not in model_set_in_scores:
+                        stdout += f"\nModel {model} is not evaluated in the scores.csv. The scores.csv has {model_set_in_scores}."
+            except Exception as e:
+                stdout += f"\nError in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
 
         # Check submission file
         submission_fp = implementation.workspace_path / "submission.csv"