From 8a11bfbf5356b6360b473db324aba549b8facf7e Mon Sep 17 00:00:00 2001 From: you-n-g Date: Wed, 9 Apr 2025 13:34:48 +0800 Subject: [PATCH] refactor: update MLEBench submission validation logic in runner and eval (#769) * refactor: Update MLEBench submission validation logic in runner and eval * lint * Delete docs/scens/data_science.rst * fix: update variable name in feedback prompt * define a variable: submission_check_out * fix: update submission check handling for MLE data * refactor: reformat if condition for clarity --------- Co-authored-by: Linlang --- .../data_science/dev/runner/__init__.py | 24 ++++----- .../scenarios/data_science/dev/runner/eval.py | 51 +++++++++++-------- .../data_science/experiment/experiment.py | 3 ++ .../proposal/exp_gen/prompts.yaml | 2 +- 4 files changed, 44 insertions(+), 36 deletions(-) diff --git a/rdagent/scenarios/data_science/dev/runner/__init__.py b/rdagent/scenarios/data_science/dev/runner/__init__.py index e3c0d20b..045dcfe6 100644 --- a/rdagent/scenarios/data_science/dev/runner/__init__.py +++ b/rdagent/scenarios/data_science/dev/runner/__init__.py @@ -111,28 +111,22 @@ class DSCoSTEERRunner(CoSTEER): description=f"The whole workflow of the solution has finished with some execution error, please check the error message and debug the whole code repo.\nCurrent code repo md5: {md5_hash(exp.experiment_workspace.all_codes)}", ) ] - exp = super().develop(exp) + exp = super().develop(exp) # run strategy(code implementation & evaluation loops) exp.sub_tasks = bak_sub_tasks + # NOTE: after running the loops, we expect some results are generated + # + # 1) scores of the models and ensemble score_fp = exp.experiment_workspace.workspace_path / "scores.csv" if not score_fp.exists(): logger.error("Metrics file (scores.csv) is not generated.") raise RunnerError(f"Metrics file (scores.csv) is not generated") exp.result = pd.read_csv(score_fp, index_col=0) + # 2) if mle-bench, then the submission format checking will be used. # DockerEnv for MLEBench submission validation - mde = get_ds_env(conf_type="mlebench") - mde.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"} - mde.prepare() - # MLEBench Check - mle_check_code = ( - (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt") - .read_text() - .replace("", self.scen.competition) - ) - exp.experiment_workspace.inject_files(**{"test/mle_submission_format_test.py": mle_check_code}) - exp.format_check_result = exp.experiment_workspace.execute( - env=mde, entry=f"python test/mle_submission_format_test.py" - ) - + if DS_RD_SETTING.if_using_mle_data: + score_fp = exp.experiment_workspace.workspace_path / "test" / "mle_submission_format_test.output" + with score_fp.open() as f: + exp.format_check_result = f.read() return exp diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 992a417f..6860de68 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -91,37 +91,48 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): score_ret_code = 1 # DockerEnv for MLEBench submission validation - mde = get_ds_env("mlebench") - mde.conf.extra_volumes = { - f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data", - } - mde.prepare() - # MLEBench Check - mle_check_code = ( - (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt") - .read_text() - .replace("", self.scen.competition) - ) - implementation.inject_files(**{"test/mle_submission_format_test.py": mle_check_code}) - submission_check_out, submission_ret_code = implementation.execute_ret_code( - env=mde, entry="python test/mle_submission_format_test.py" - ) + submission_check_out = "" + + if DS_RD_SETTING.if_using_mle_data: + mde = get_ds_env("mlebench") + mde.conf.extra_volumes = { + f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data", + } + mde.prepare() + # MLEBench Check + mle_check_code = ( + (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt") + .read_text() + .replace("", self.scen.competition) + ) + implementation.inject_files(**{"test/mle_submission_format_test.py": mle_check_code}) + submission_check_out, submission_ret_code = implementation.execute_ret_code( + env=mde, entry="python test/mle_submission_format_test.py" + ) + stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. " + implementation.inject_files(**{"test/mle_submission_format_test.output": submission_check_out}) + if DS_RD_SETTING.rule_base_eval: - if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0: + if DS_RD_SETTING.if_using_mle_data: + score_check_text = score_check_text + "\n" + submission_check_out + if ( + execute_ret_code == 0 + and score_ret_code == 0 + and (not DS_RD_SETTING.if_using_mle_data or submission_ret_code == 0) + ): return DSCoSTEEREvalFeedback( execution=stdout, - return_checking=score_check_text + "\n" + submission_check_out, + return_checking=score_check_text, code="Code evaluation is not available.", final_decision=True, ) else: return DSCoSTEEREvalFeedback( execution=stdout, - return_checking=score_check_text + "\n" + submission_check_out, + return_checking=score_check_text, code="Code evaluation is not available.", final_decision=False, ) - stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. " system_prompt = T(".prompts:DSCoSTEER_eval.system").r( scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)), @@ -181,7 +192,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): if score_ret_code != 0: feedback.final_decision = False feedback.return_checking += "\n" + score_check_text - if submission_ret_code != 0: + if DS_RD_SETTING.if_using_mle_data and submission_ret_code != 0: feedback.final_decision = False feedback.return_checking += "\nSubmission file check failed." return feedback diff --git a/rdagent/scenarios/data_science/experiment/experiment.py b/rdagent/scenarios/data_science/experiment/experiment.py index 7e5201e0..80649e84 100644 --- a/rdagent/scenarios/data_science/experiment/experiment.py +++ b/rdagent/scenarios/data_science/experiment/experiment.py @@ -18,7 +18,10 @@ class DSExperiment(Experiment[Task, FBWorkspace, FBWorkspace]): # the initial workspace or the successful new version after coding self.experiment_workspace = FBWorkspace() self.pending_tasks_list = pending_tasks_list + self.format_check_result = None + # this field is optional. It is not none only when we have a format checker. Currently, only following cases are supported. + # - mle-bench def is_ready_to_run(self) -> bool: """ diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/prompts.yaml b/rdagent/scenarios/data_science/proposal/exp_gen/prompts.yaml index 29edbf4f..86e99f3b 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/prompts.yaml +++ b/rdagent/scenarios/data_science/proposal/exp_gen/prompts.yaml @@ -273,7 +273,7 @@ component_gen: user: |- Here are the former experiments and their feedbacks: - {{ exp_and_feedback_desc }} + {{ exp_and_feedback_list_desc }} Please choose the most proper component to focus on based on the information above. Please balance the exploration and exploitation. Avoid selecting the same component more than 5 times in a row to ensure that the chosen component is not overly repetitive.