fix: add metric name check for valid scores (#724)

* update metric_name

* fix some bugs

* add an evaluation in workflow

* add an evalution in runner

* fix ci

* test change

* fix CI

---------

Co-authored-by: TPLin22 <tplin2@163.com>
Co-authored-by: yuanteli <1957922024@qq.com>

rdagent/components/coder/data_science/ensemble/eval.py

@@ -31,6 +31,8 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
     ) -> EnsembleEvalFeedback:
 
         target_task_information = target_task.get_task_information()
+        metric_name = self.scen.metric_name
+
         if (
             queried_knowledge is not None
             and target_task_information in queried_knowledge.success_task_to_knowledge_dict
@@ -55,7 +57,8 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
             .render(
                 model_names=[
                     fn[:-3] for fn in implementation.file_dict.keys() if fn.startswith("model_") and "test" not in fn
-                ]
+                ],
+                metric_name=metric_name,
             )
         )
 
@@ -73,6 +76,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
         system_prompt = T(".prompts:ensemble_eval.system").r(
             task_desc=target_task_information,
             test_code=test_code,
+            metric_name=metric_name,
             code=implementation.file_dict["ensemble.py"],
             workflow_stdout=workflow_stdout,
             workflow_code=implementation.all_codes,

rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt

@@ -123,6 +123,7 @@ assert model_set_in_scores == set({{model_names}}).union({"ensemble"}), (
     f"The scores dataframe does not contain the correct model names as index.\ncorrect model names are: {{model_names}} + ['ensemble']\nscore_df is:\n{score_df}"
 )
 assert score_df.index.is_unique, "The scores dataframe has duplicate model names."
-assert len(score_df.columns) == 1, f"The scores dataframe should have exactly one column for the scores of the evaluation indicator, but has these columns: {score_df.columns.tolist()}"
+assert score_df.columns.tolist() == ["{{metric_name}}"], f"The column names of the scores dataframe should be ['{{metric_name}}'], but is '{score_df.columns.tolist()}'"
+
 
 print("Ensemble test end.")

rdagent/components/coder/data_science/ensemble/prompts.yaml

@@ -96,9 +96,13 @@ ensemble_eval:
     You should evaluate both the ensemble test results and the overall workflow results. **Approve the code only if both tests pass.**
     {% endif %}
 
+    The metric used for scoring the predictions:
+    **{{ metric_name }}**
+
     ## Evaluation Criteria
     - You will be given the standard output (`stdout`) from the ensemble test and, if applicable, the workflow test.
     - Code should have no try-except blocks because they can hide errors.
+    - Check whether the code implement the scoring process using the given metric.
     - The stdout includes the local variable values from the ensemble code execution. Check whether the validation score is calculated correctly.
     
     Please respond with your feedback in the following JSON format and order

rdagent/components/coder/data_science/ensemble/test.py

@@ -44,7 +44,7 @@ def develop_one_competition(competition: str):
         """,
     )
 
-    exp = EnsembleExperiment(sub_tasks=[task])
+    exp = EnsembleExperiment(pending_tasks_list=[task])
 
     # Injecting the corresponding specification
     exp.experiment_workspace.inject_files(**{"spec/ensemble.md": ensemble_spec})

rdagent/components/coder/data_science/workflow/eval.py

@@ -68,6 +68,8 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         implementation.execute(env=env, entry=f"rm submission.csv scores.csv")
 
         stdout = implementation.execute(env=env, entry=f"python main.py")
+
+        # remove EDA part
         stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
 
         # Check score file
@@ -85,9 +87,17 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 model_set_in_folder = set(
                     f[:-3] for f in implementation.file_dict.keys() if re.match(r"^model_(?!test)\w+\.py$", f)
                 )
+
+                # Check model names (index)
                 if model_set_in_scores != model_set_in_folder.union({"ensemble"}):
                     score_check_text += f"\n[Error] The scores dataframe does not contain the correct model names as index.\ncorrect model names are: {model_set_in_folder.union({'ensemble'})}\nscore_df is:\n{score_df}"
                     score_ret_code = 1
+
+                # Check metric name (columns)
+                if score_df.columns.tolist() != [self.scen.metric_name]:
+                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
+                    score_ret_code = 1
+
             except Exception as e:
                 score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
                 score_ret_code = 1
@@ -101,17 +111,6 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         )
         stdout += "\n" + submission_check_out
 
-        # MLEBench Check
-        # !!! Since we are running on a sampled dataset, mlebench check is not required.
-        # mle_check_code = (
-        #     (DIRNAME / "eval_tests" / "mle_submission_format_test.txt")
-        #     .read_text()
-        #     .replace("<competition_id>", self.scen.competition)
-        # )
-        # implementation.inject_files(**{"test/mle_submission_format_test.py": mle_check_code})
-        # stdout += "----Submission Check 2-----\n"
-        # stdout += implementation.execute(env=mde, entry=f"python test/mle_submission_format_test.py")
-
         system_prompt = T(".prompts:workflow_eval.system").r(
             scenario=self.scen.get_scenario_all_desc(),
             task_desc=target_task.get_task_information(),