fix: move mlebench check into runner (#556)

* abandon mlebench check in workflow

* move mlebench check to runner and put it into the exp

rdagent/components/coder/data_science/workflow/eval.py

@@ -61,13 +61,13 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         }
         de = DockerEnv(conf=ds_docker_conf)
 
-        # DockerEnv for MLEBench submission validation
-        mle_de_conf = MLEBDockerConf()
-        mle_de_conf.extra_volumes = {
-            f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
-        }
-        mde = DockerEnv(conf=mle_de_conf)
-        mde.prepare()
+        # # DockerEnv for MLEBench submission validation
+        # mle_de_conf = MLEBDockerConf()
+        # mle_de_conf.extra_volumes = {
+        #     f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
+        # }
+        # mde = DockerEnv(conf=mle_de_conf)
+        # mde.prepare()
 
         # Clean the scores.csv & submission.csv.
         stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv")
@@ -98,18 +98,19 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         else:
             base_check_code = (DIRNAME / "eval_tests" / "submission_check.txt").read_text()
             implementation.inject_files(**{"submission_check.py": base_check_code})
-            stdout += "----Submission Check 1-----\n"
+            # stdout += "----Submission Check 1-----\n"
             stdout += implementation.execute(env=de, entry="python submission_check.py")
 
             # MLEBench Check
-            mle_check_code = (
-                (DIRNAME / "eval_tests" / "mle_submission_check.txt")
-                .read_text()
-                .replace("<competition_id>", self.scen.competition)
-            )
-            implementation.inject_files(**{"mle_submission_check.py": mle_check_code})
-            stdout += "----Submission Check 2-----\n"
-            stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py")
+            # !!! Since we are running on a sampled dataset, mlebench check is not required.
+            # mle_check_code = (
+            #     (DIRNAME / "eval_tests" / "mle_submission_check.txt")
+            #     .read_text()
+            #     .replace("<competition_id>", self.scen.competition)
+            # )
+            # implementation.inject_files(**{"mle_submission_check.py": mle_check_code})
+            # stdout += "----Submission Check 2-----\n"
+            # stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py")
 
         system_prompt = T(".prompts:workflow_eval.system").r(
             scenario=self.scen.get_scenario_all_desc(),

rdagent/components/coder/data_science/workflow/eval_tests/mle_submission_check.txt

@@ -1,12 +0,0 @@
-from pathlib import Path
-
-from mlebench.grade import validate_submission
-from mlebench.registry import registry
-
-COMPETITION_ID = "<competition_id>"
-new_registry = registry.set_data_dir(Path("/mle/data"))
-competition = new_registry.get_competition(COMPETITION_ID)
-
-is_valid, message = validate_submission(Path("submission.csv"), competition)
-
-print(message)