fix: move mlebench check into runner (#556)

* abandon mlebench check in workflow

* move mlebench check to runner and put it into the exp

rdagent/components/coder/data_science/workflow/eval.py

@@ -61,13 +61,13 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         }
         de = DockerEnv(conf=ds_docker_conf)
 
-        # DockerEnv for MLEBench submission validation
-        mle_de_conf = MLEBDockerConf()
-        mle_de_conf.extra_volumes = {
-            f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
-        }
-        mde = DockerEnv(conf=mle_de_conf)
-        mde.prepare()
+        # # DockerEnv for MLEBench submission validation
+        # mle_de_conf = MLEBDockerConf()
+        # mle_de_conf.extra_volumes = {
+        #     f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
+        # }
+        # mde = DockerEnv(conf=mle_de_conf)
+        # mde.prepare()
 
         # Clean the scores.csv & submission.csv.
         stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv")
@@ -98,18 +98,19 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         else:
             base_check_code = (DIRNAME / "eval_tests" / "submission_check.txt").read_text()
             implementation.inject_files(**{"submission_check.py": base_check_code})
-            stdout += "----Submission Check 1-----\n"
+            # stdout += "----Submission Check 1-----\n"
             stdout += implementation.execute(env=de, entry="python submission_check.py")
 
             # MLEBench Check
-            mle_check_code = (
-                (DIRNAME / "eval_tests" / "mle_submission_check.txt")
-                .read_text()
-                .replace("<competition_id>", self.scen.competition)
-            )
-            implementation.inject_files(**{"mle_submission_check.py": mle_check_code})
-            stdout += "----Submission Check 2-----\n"
-            stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py")
+            # !!! Since we are running on a sampled dataset, mlebench check is not required.
+            # mle_check_code = (
+            #     (DIRNAME / "eval_tests" / "mle_submission_check.txt")
+            #     .read_text()
+            #     .replace("<competition_id>", self.scen.competition)
+            # )
+            # implementation.inject_files(**{"mle_submission_check.py": mle_check_code})
+            # stdout += "----Submission Check 2-----\n"
+            # stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py")
 
         system_prompt = T(".prompts:workflow_eval.system").r(
             scenario=self.scen.get_scenario_all_desc(),

rdagent/scenarios/data_science/dev/feedback.py

@@ -65,6 +65,7 @@ class DSExperiment2Feedback(Experiment2Feedback):
             sota_desc=sota_desc,
             cur_exp=exp,
             diff_edition=diff_edition,
+            format_check=exp.format_check_result,
             feedback_desc=feedback_desc,
         )
 

rdagent/scenarios/data_science/dev/prompts.yaml

@@ -48,6 +48,11 @@ exp_feedback:
     ### Complete Code of current solution
     {{cur_exp.experiment_workspace.all_codes}}
 
+    {% if format_check is not none %}
+    Submission format check to current solution:
+    {{ format_check }}
+    {% endif %}
+
     {{feedback_desc}}
 
     Please refer to these hypotheses and feedback to help you recommend new experiment and hypothesis

rdagent/scenarios/data_science/dev/runner.py

@@ -9,7 +9,7 @@ from rdagent.core.developer import Developer
 from rdagent.core.exception import RunnerError
 from rdagent.log import rdagent_logger as logger
 from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
-from rdagent.utils.env import DockerEnv, DSDockerConf
+from rdagent.utils.env import DockerEnv, DSDockerConf, MLEBDockerConf
 
 
 class DSRunner(Developer[DSExperiment]):
@@ -36,6 +36,22 @@ class DSRunner(Developer[DSExperiment]):
         if not submission_fp.exists():
             logger.error("Submission file (submission.csv) is not generated.")
             raise RunnerError(f"Submission file (submission.csv) is not generated, log is:\n{stdout}")
+        else:
+            # DockerEnv for MLEBench submission validation
+            mle_de_conf = MLEBDockerConf()
+            mle_de_conf.extra_volumes = {
+                f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
+            }
+            mde = DockerEnv(conf=mle_de_conf)
+            mde.prepare()
+            # MLEBench Check
+            mle_check_code = (
+                (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_check.txt")
+                .read_text()
+                .replace("<competition_id>", self.scen.competition)
+            )
+            exp.experiment_workspace.inject_files(**{"mle_submission_check.py": mle_check_code})
+            exp.format_check_result = exp.experiment_workspace.execute(env=mde, entry=f"python mle_submission_check.py")
 
         exp.result = pd.read_csv(score_fp, index_col=0)
 

rdagent/scenarios/data_science/experiment/experiment.py

@@ -13,6 +13,7 @@ class DSExperiment(Experiment[Task, FBWorkspace, FBWorkspace]):
         super().__init__(sub_tasks=[], *args, **kwargs)
         self.experiment_workspace = FBWorkspace()
         self.pending_tasks_list = pending_tasks_list
+        self.format_check_result = None
 
     def next_component_required(self) -> COMPONENT | None:
         files = list(self.experiment_workspace.file_dict.keys())