From b2d56e39e5ab0cfce246d802078d81e30eacd43f Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Thu, 6 Feb 2025 15:37:41 +0800 Subject: [PATCH] fix: move mlebench check into runner (#556) * abandon mlebench check in workflow * move mlebench check to runner and put it into the exp --- .../coder/data_science/workflow/eval.py | 33 ++++++++++--------- .../dev}/eval_tests/mle_submission_check.txt | 0 .../scenarios/data_science/dev/feedback.py | 1 + .../scenarios/data_science/dev/prompts.yaml | 5 +++ rdagent/scenarios/data_science/dev/runner.py | 18 +++++++++- .../data_science/experiment/experiment.py | 1 + 6 files changed, 41 insertions(+), 17 deletions(-) rename rdagent/{components/coder/data_science/workflow => scenarios/data_science/dev}/eval_tests/mle_submission_check.txt (100%) diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index 254d364f..59fde915 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -61,13 +61,13 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): } de = DockerEnv(conf=ds_docker_conf) - # DockerEnv for MLEBench submission validation - mle_de_conf = MLEBDockerConf() - mle_de_conf.extra_volumes = { - f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data", - } - mde = DockerEnv(conf=mle_de_conf) - mde.prepare() + # # DockerEnv for MLEBench submission validation + # mle_de_conf = MLEBDockerConf() + # mle_de_conf.extra_volumes = { + # f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data", + # } + # mde = DockerEnv(conf=mle_de_conf) + # mde.prepare() # Clean the scores.csv & submission.csv. stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv") @@ -98,18 +98,19 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): else: base_check_code = (DIRNAME / "eval_tests" / "submission_check.txt").read_text() implementation.inject_files(**{"submission_check.py": base_check_code}) - stdout += "----Submission Check 1-----\n" + # stdout += "----Submission Check 1-----\n" stdout += implementation.execute(env=de, entry="python submission_check.py") # MLEBench Check - mle_check_code = ( - (DIRNAME / "eval_tests" / "mle_submission_check.txt") - .read_text() - .replace("", self.scen.competition) - ) - implementation.inject_files(**{"mle_submission_check.py": mle_check_code}) - stdout += "----Submission Check 2-----\n" - stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py") + # !!! Since we are running on a sampled dataset, mlebench check is not required. + # mle_check_code = ( + # (DIRNAME / "eval_tests" / "mle_submission_check.txt") + # .read_text() + # .replace("", self.scen.competition) + # ) + # implementation.inject_files(**{"mle_submission_check.py": mle_check_code}) + # stdout += "----Submission Check 2-----\n" + # stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py") system_prompt = T(".prompts:workflow_eval.system").r( scenario=self.scen.get_scenario_all_desc(), diff --git a/rdagent/components/coder/data_science/workflow/eval_tests/mle_submission_check.txt b/rdagent/scenarios/data_science/dev/eval_tests/mle_submission_check.txt similarity index 100% rename from rdagent/components/coder/data_science/workflow/eval_tests/mle_submission_check.txt rename to rdagent/scenarios/data_science/dev/eval_tests/mle_submission_check.txt diff --git a/rdagent/scenarios/data_science/dev/feedback.py b/rdagent/scenarios/data_science/dev/feedback.py index 7e794eb5..7141d110 100644 --- a/rdagent/scenarios/data_science/dev/feedback.py +++ b/rdagent/scenarios/data_science/dev/feedback.py @@ -65,6 +65,7 @@ class DSExperiment2Feedback(Experiment2Feedback): sota_desc=sota_desc, cur_exp=exp, diff_edition=diff_edition, + format_check=exp.format_check_result, feedback_desc=feedback_desc, ) diff --git a/rdagent/scenarios/data_science/dev/prompts.yaml b/rdagent/scenarios/data_science/dev/prompts.yaml index bbc9ac2f..4fcc4da7 100644 --- a/rdagent/scenarios/data_science/dev/prompts.yaml +++ b/rdagent/scenarios/data_science/dev/prompts.yaml @@ -48,6 +48,11 @@ exp_feedback: ### Complete Code of current solution {{cur_exp.experiment_workspace.all_codes}} + {% if format_check is not none %} + Submission format check to current solution: + {{ format_check }} + {% endif %} + {{feedback_desc}} Please refer to these hypotheses and feedback to help you recommend new experiment and hypothesis diff --git a/rdagent/scenarios/data_science/dev/runner.py b/rdagent/scenarios/data_science/dev/runner.py index f221d8b6..cd317c3c 100644 --- a/rdagent/scenarios/data_science/dev/runner.py +++ b/rdagent/scenarios/data_science/dev/runner.py @@ -9,7 +9,7 @@ from rdagent.core.developer import Developer from rdagent.core.exception import RunnerError from rdagent.log import rdagent_logger as logger from rdagent.scenarios.data_science.experiment.experiment import DSExperiment -from rdagent.utils.env import DockerEnv, DSDockerConf +from rdagent.utils.env import DockerEnv, DSDockerConf, MLEBDockerConf class DSRunner(Developer[DSExperiment]): @@ -36,6 +36,22 @@ class DSRunner(Developer[DSExperiment]): if not submission_fp.exists(): logger.error("Submission file (submission.csv) is not generated.") raise RunnerError(f"Submission file (submission.csv) is not generated, log is:\n{stdout}") + else: + # DockerEnv for MLEBench submission validation + mle_de_conf = MLEBDockerConf() + mle_de_conf.extra_volumes = { + f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data", + } + mde = DockerEnv(conf=mle_de_conf) + mde.prepare() + # MLEBench Check + mle_check_code = ( + (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_check.txt") + .read_text() + .replace("", self.scen.competition) + ) + exp.experiment_workspace.inject_files(**{"mle_submission_check.py": mle_check_code}) + exp.format_check_result = exp.experiment_workspace.execute(env=mde, entry=f"python mle_submission_check.py") exp.result = pd.read_csv(score_fp, index_col=0) diff --git a/rdagent/scenarios/data_science/experiment/experiment.py b/rdagent/scenarios/data_science/experiment/experiment.py index b468970e..4f19f390 100644 --- a/rdagent/scenarios/data_science/experiment/experiment.py +++ b/rdagent/scenarios/data_science/experiment/experiment.py @@ -13,6 +13,7 @@ class DSExperiment(Experiment[Task, FBWorkspace, FBWorkspace]): super().__init__(sub_tasks=[], *args, **kwargs) self.experiment_workspace = FBWorkspace() self.pending_tasks_list = pending_tasks_list + self.format_check_result = None def next_component_required(self) -> COMPONENT | None: files = list(self.experiment_workspace.file_dict.keys())