From a6ae88fb429e1449f055aa67bd98935b066b89d9 Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 10 Apr 2025 17:56:57 +0800 Subject: [PATCH] chore: fit more competition (#723) 1. Remove potential tags from the generated code. 2. Use return codes in data_loader, feature, and model. 3. Configure the debug timeout. --- rdagent/components/coder/data_science/conf.py | 8 +++++++- .../coder/data_science/ensemble/eval.py | 5 +++-- .../coder/data_science/feature/eval.py | 12 +++++++----- .../components/coder/data_science/model/eval.py | 10 +++++++--- .../coder/data_science/pipeline/eval.py | 5 +++-- .../data_science/raw_data_loader/__init__.py | 6 ++++-- .../coder/data_science/raw_data_loader/eval.py | 12 +++++++----- .../coder/data_science/workflow/eval.py | 5 +++-- rdagent/log/mle_summary.py | 3 +-- .../scenarios/data_science/dev/runner/eval.py | 17 ++++++++++------- rdagent/utils/agent/ret.py | 2 +- 11 files changed, 53 insertions(+), 32 deletions(-) diff --git a/rdagent/components/coder/data_science/conf.py b/rdagent/components/coder/data_science/conf.py index d60552cb..049e20fc 100644 --- a/rdagent/components/coder/data_science/conf.py +++ b/rdagent/components/coder/data_science/conf.py @@ -24,7 +24,11 @@ class DSCoderCoSTEERSettings(CoSTEERSettings): # TODO: extract a function for env and conf. -def get_ds_env(conf_type: Literal["kaggle", "mlebench"] = "kaggle") -> Env: +def get_ds_env( + conf_type: Literal["kaggle", "mlebench"] = "kaggle", + extra_volumes: dict = {}, + running_timeout_period: int = DS_RD_SETTING.debug_timeout, +) -> Env: """ Retrieve the appropriate environment configuration based on the env_type setting. @@ -48,6 +52,8 @@ def get_ds_env(conf_type: Literal["kaggle", "mlebench"] = "kaggle") -> Env: ) else: raise ValueError(f"Unknown env type: {conf.env_type}") + env.conf.extra_volumes = extra_volumes + env.conf.running_timeout_period = running_timeout_period return env diff --git a/rdagent/components/coder/data_science/ensemble/eval.py b/rdagent/components/coder/data_science/ensemble/eval.py index 761c02f6..37924a48 100644 --- a/rdagent/components/coder/data_science/ensemble/eval.py +++ b/rdagent/components/coder/data_science/ensemble/eval.py @@ -46,8 +46,9 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env() - env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + env = get_ds_env( + extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + ) fname = "test/ensemble_test.txt" test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text() diff --git a/rdagent/components/coder/data_science/feature/eval.py b/rdagent/components/coder/data_science/feature/eval.py index 895c0e96..d2b1aa85 100644 --- a/rdagent/components/coder/data_science/feature/eval.py +++ b/rdagent/components/coder/data_science/feature/eval.py @@ -20,7 +20,6 @@ FeatureEvalFeedback = CoSTEERSingleFeedback class FeatureCoSTEEREvaluator(CoSTEEREvaluator): - def evaluate( self, target_task: Task, @@ -29,7 +28,6 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> FeatureEvalFeedback: - target_task_information = target_task.get_task_information() if ( queried_knowledge is not None @@ -44,8 +42,9 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env() - env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + env = get_ds_env( + extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + ) # TODO: do we need to clean the generated temporary content? fname = "test/feature_test.py" @@ -72,9 +71,12 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): workflow_stdout=workflow_stdout, ) - return build_cls_from_json_with_retry( + fb = build_cls_from_json_with_retry( FeatureEvalFeedback, system_prompt=system_prompt, user_prompt=user_prompt, init_kwargs_update_func=FeatureEvalFeedback.val_and_update_init_dict, ) + fb.final_decision = fb.final_decision and ret_code == 0 + + return fb diff --git a/rdagent/components/coder/data_science/model/eval.py b/rdagent/components/coder/data_science/model/eval.py index 7f7939e8..b47d768f 100644 --- a/rdagent/components/coder/data_science/model/eval.py +++ b/rdagent/components/coder/data_science/model/eval.py @@ -56,8 +56,9 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env() - env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + env = get_ds_env( + extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + ) if_model_removed = False @@ -107,9 +108,12 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): workflow_stdout=workflow_stdout, ) - return build_cls_from_json_with_retry( + fb = build_cls_from_json_with_retry( ModelSingleFeedback, system_prompt=system_prompt, user_prompt=user_prompt, init_kwargs_update_func=ModelSingleFeedback.val_and_update_init_dict, ) + fb.final_decision = fb.final_decision and ret_code == 0 + + return fb diff --git a/rdagent/components/coder/data_science/pipeline/eval.py b/rdagent/components/coder/data_science/pipeline/eval.py index d59a1eb2..4333d654 100644 --- a/rdagent/components/coder/data_science/pipeline/eval.py +++ b/rdagent/components/coder/data_science/pipeline/eval.py @@ -51,8 +51,9 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env() - env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + env = get_ds_env( + extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + ) # Clean the scores.csv & submission.csv. implementation.execute(env=env, entry=get_clear_ws_cmd()) diff --git a/rdagent/components/coder/data_science/raw_data_loader/__init__.py b/rdagent/components/coder/data_science/raw_data_loader/__init__.py index 4f2a57e8..bed69226 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/__init__.py +++ b/rdagent/components/coder/data_science/raw_data_loader/__init__.py @@ -225,8 +225,10 @@ class DataLoaderCoSTEER(CoSTEER): def develop(self, exp): new_exp = super().develop(exp) - env = get_ds_env() - env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": "/kaggle/input"} + env = get_ds_env( + extra_volumes={f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": "/kaggle/input"}, + running_timeout_period=DS_RD_SETTING.full_timeout, + ) stdout = new_exp.experiment_workspace.execute(env=env, entry=f"python test/data_loader_test.py") match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL) diff --git a/rdagent/components/coder/data_science/raw_data_loader/eval.py b/rdagent/components/coder/data_science/raw_data_loader/eval.py index 693211f9..c66f6bd0 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/eval.py +++ b/rdagent/components/coder/data_science/raw_data_loader/eval.py @@ -23,7 +23,6 @@ DataLoaderEvalFeedback = CoSTEERSingleFeedback class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): - def evaluate( self, target_task: Task, @@ -32,7 +31,6 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): queried_knowledge: CoSTEERQueriedKnowledgeV2 = None, **kwargs, ) -> DataLoaderEvalFeedback: - target_task_information = target_task.get_task_information() if ( queried_knowledge is not None @@ -47,8 +45,9 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env() - env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + env = get_ds_env( + extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + ) # TODO: do we need to clean the generated temporary content? fname = "test/data_loader_test.py" @@ -80,9 +79,12 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): workflow_stdout=workflow_stdout, ) - return build_cls_from_json_with_retry( + fb = build_cls_from_json_with_retry( DataLoaderEvalFeedback, system_prompt=system_prompt, user_prompt=user_prompt, init_kwargs_update_func=DataLoaderEvalFeedback.val_and_update_init_dict, ) + fb.final_decision = fb.final_decision and ret_code == 0 + + return fb diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index 2207f66c..579e752b 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -54,8 +54,9 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env() - env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + env = get_ds_env( + extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"} + ) # # DockerEnv for MLEBench submission validation # mle_de_conf = MLEBDockerConf() diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index dd079053..a6be5b84 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -16,8 +16,7 @@ from rdagent.scenarios.data_science.experiment.experiment import DSExperiment from rdagent.scenarios.kaggle.kaggle_crawler import score_rank from rdagent.utils.env import DockerEnv, MLEBDockerConf -de = get_ds_env("mlebench") -de.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"} +de = get_ds_env(conf_type="mlebench", extra_volumes={f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"}) de.prepare() diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 1ab84a14..b02c538a 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -33,9 +33,10 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): **kwargs, ) -> DSCoSTEEREvalFeedback: - env = get_ds_env() - env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": "/kaggle/input"} - env.conf.running_timeout_period = DS_RD_SETTING.full_timeout + env = get_ds_env( + extra_volumes={f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": "/kaggle/input"}, + running_timeout_period=DS_RD_SETTING.full_timeout, + ) stdout = implementation.execute( env=env, entry=get_clear_ws_cmd() @@ -94,10 +95,12 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): submission_check_out = "" if DS_RD_SETTING.if_using_mle_data: - mde = get_ds_env("mlebench") - mde.conf.extra_volumes = { - f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data", - } + mde = get_ds_env( + conf_type="mlebench", + extra_volumes={ + f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data", + }, + ) mde.prepare() # MLEBench Check mle_check_code = ( diff --git a/rdagent/utils/agent/ret.py b/rdagent/utils/agent/ret.py index ff16183f..b82df18a 100644 --- a/rdagent/utils/agent/ret.py +++ b/rdagent/utils/agent/ret.py @@ -34,6 +34,7 @@ class PythonAgentOut(AgentOut): match = re.search(r".*```[Pp]ython\n(.*)\n```.*", resp, re.DOTALL) if match: code = match.group(1) + code = re.sub(r"", "", code, flags=re.IGNORECASE) return code return resp @@ -51,7 +52,6 @@ class BatchEditOut(AgentOut): class PythonBatchEditOut(AgentOut): - @classmethod def get_spec(cls, with_del=True): return T(".tpl:PythonBatchEditOut").r(with_del=with_del)