From bc8e9dc23a903de82f3f224fd867758f7c11d11e Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Wed, 2 Apr 2025 00:18:42 -0600 Subject: [PATCH] costeer max loop as hyperp & skip workflow in eval (#737) --- rdagent/app/data_science/conf.py | 3 +++ .../coder/data_science/ensemble/__init__.py | 11 ++++++++++- .../components/coder/data_science/ensemble/eval.py | 2 +- .../components/coder/data_science/feature/__init__.py | 11 ++++++++++- rdagent/components/coder/data_science/feature/eval.py | 4 ++-- .../components/coder/data_science/model/__init__.py | 11 ++++++++++- rdagent/components/coder/data_science/model/eval.py | 5 +++-- .../coder/data_science/raw_data_loader/__init__.py | 11 ++++++++++- .../coder/data_science/raw_data_loader/eval.py | 4 ++-- .../coder/data_science/workflow/__init__.py | 11 ++++++++++- rdagent/scenarios/data_science/dev/runner/__init__.py | 9 ++++++++- 11 files changed, 69 insertions(+), 13 deletions(-) diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index 4f390a6a..73c5a0aa 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -26,5 +26,8 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): proposal_version: str = "v1" + coder_max_loop: int = 10 + runner_max_loop: int = 3 + DS_RD_SETTING = DataScienceBasePropSetting() diff --git a/rdagent/components/coder/data_science/ensemble/__init__.py b/rdagent/components/coder/data_science/ensemble/__init__.py index fd0f8cfb..f25c5f56 100644 --- a/rdagent/components/coder/data_science/ensemble/__init__.py +++ b/rdagent/components/coder/data_science/ensemble/__init__.py @@ -154,4 +154,13 @@ class EnsembleCoSTEER(CoSTEER): eva = CoSTEERMultiEvaluator(EnsembleCoSTEEREvaluator(scen=scen), scen=scen) es = EnsembleMultiProcessEvolvingStrategy(scen=scen, settings=settings) - super().__init__(*args, settings=settings, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs) + super().__init__( + *args, + settings=settings, + eva=eva, + es=es, + evolving_version=2, + scen=scen, + max_loop=DS_RD_SETTING.coder_max_loop, + **kwargs, + ) diff --git a/rdagent/components/coder/data_science/ensemble/eval.py b/rdagent/components/coder/data_science/ensemble/eval.py index bddeb6ff..761c02f6 100644 --- a/rdagent/components/coder/data_science/ensemble/eval.py +++ b/rdagent/components/coder/data_science/ensemble/eval.py @@ -67,7 +67,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): stdout += f"\nNOTE: the above scripts run with return code {ret_code}" - if "main.py" in implementation.file_dict: + if "main.py" in implementation.file_dict and ret_code == 0: workflow_stdout = implementation.execute(env=env, entry="python main.py") workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) else: diff --git a/rdagent/components/coder/data_science/feature/__init__.py b/rdagent/components/coder/data_science/feature/__init__.py index 83e21f7b..38691aff 100644 --- a/rdagent/components/coder/data_science/feature/__init__.py +++ b/rdagent/components/coder/data_science/feature/__init__.py @@ -130,4 +130,13 @@ class FeatureCoSTEER(CoSTEER): ) # Please specify whether you agree running your eva in parallel or not es = FeatureMultiProcessEvolvingStrategy(scen=scen, settings=settings) - super().__init__(*args, settings=settings, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs) + super().__init__( + *args, + settings=settings, + eva=eva, + es=es, + evolving_version=2, + scen=scen, + max_loop=DS_RD_SETTING.coder_max_loop, + **kwargs, + ) diff --git a/rdagent/components/coder/data_science/feature/eval.py b/rdagent/components/coder/data_science/feature/eval.py index 148a511c..895c0e96 100644 --- a/rdagent/components/coder/data_science/feature/eval.py +++ b/rdagent/components/coder/data_science/feature/eval.py @@ -52,9 +52,9 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): test_code = (DIRNAME / "eval_tests" / "feature_test.txt").read_text() implementation.inject_files(**{fname: test_code}) - stdout = implementation.execute(env=env, entry=f"python {fname}") + stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}") - if "main.py" in implementation.file_dict: + if "main.py" in implementation.file_dict and ret_code == 0: workflow_stdout = implementation.execute(env=env, entry="python main.py") workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) else: diff --git a/rdagent/components/coder/data_science/model/__init__.py b/rdagent/components/coder/data_science/model/__init__.py index 09bb211e..e35729e2 100644 --- a/rdagent/components/coder/data_science/model/__init__.py +++ b/rdagent/components/coder/data_science/model/__init__.py @@ -162,4 +162,13 @@ class ModelCoSTEER(CoSTEER): # eva = ModelGeneralCaseSpecEvaluator(scen=scen) es = ModelMultiProcessEvolvingStrategy(scen=scen, settings=settings) - super().__init__(*args, settings=settings, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs) + super().__init__( + *args, + settings=settings, + eva=eva, + es=es, + evolving_version=2, + scen=scen, + max_loop=DS_RD_SETTING.coder_max_loop, + **kwargs, + ) diff --git a/rdagent/components/coder/data_science/model/eval.py b/rdagent/components/coder/data_science/model/eval.py index 98e1d7ed..7f7939e8 100644 --- a/rdagent/components/coder/data_science/model/eval.py +++ b/rdagent/components/coder/data_science/model/eval.py @@ -67,17 +67,18 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name) ) # only check the model changed this time implementation.inject_files(**{fname: test_code}) - stdout = implementation.execute(env=env, entry=f"python {fname}") + stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}") if stdout is None: raise CoderError( "The execution output contains too many progress bars and results in the LLM's token size exceeding the limit." ) else: + ret_code = 0 if_model_removed = True stdout = f"Model {target_task.name} removal succeeded." - if "main.py" in implementation.file_dict: + if "main.py" in implementation.file_dict and ret_code == 0: workflow_stdout = implementation.execute(env=env, entry="python main.py") workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) else: diff --git a/rdagent/components/coder/data_science/raw_data_loader/__init__.py b/rdagent/components/coder/data_science/raw_data_loader/__init__.py index 5fb2fcfe..5f4c19f4 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/__init__.py +++ b/rdagent/components/coder/data_science/raw_data_loader/__init__.py @@ -211,7 +211,16 @@ class DataLoaderCoSTEER(CoSTEER): ) # Please specify whether you agree running your eva in parallel or not es = DataLoaderMultiProcessEvolvingStrategy(scen=scen, settings=settings) - super().__init__(*args, settings=settings, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs) + super().__init__( + *args, + settings=settings, + eva=eva, + es=es, + evolving_version=2, + scen=scen, + max_loop=DS_RD_SETTING.coder_max_loop, + **kwargs, + ) def develop(self, exp): new_exp = super().develop(exp) diff --git a/rdagent/components/coder/data_science/raw_data_loader/eval.py b/rdagent/components/coder/data_science/raw_data_loader/eval.py index 8a6f6972..693211f9 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/eval.py +++ b/rdagent/components/coder/data_science/raw_data_loader/eval.py @@ -54,14 +54,14 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): fname = "test/data_loader_test.py" test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text() implementation.inject_files(**{fname: test_code}) - stdout = implementation.execute(env=env, entry=f"python {fname}") + stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}") match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===(.*)", stdout, re.DOTALL) stdout_part_1, eda_output, stdout_part_2 = match.groups() if match else (stdout, None, "") stdout = stdout_part_1 + stdout_part_2 if eda_output is not None and len(eda_output.split(" ")) > 10000: eda_output += "Length of EDA output is too long, truncated. Please reject this implementation and motivate it to reduce the length of EDA output." - if "main.py" in implementation.file_dict: + if "main.py" in implementation.file_dict and ret_code == 0: workflow_stdout = implementation.execute(env=env, entry="python main.py") workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) else: diff --git a/rdagent/components/coder/data_science/workflow/__init__.py b/rdagent/components/coder/data_science/workflow/__init__.py index 72d9a2f2..e63c543b 100644 --- a/rdagent/components/coder/data_science/workflow/__init__.py +++ b/rdagent/components/coder/data_science/workflow/__init__.py @@ -123,4 +123,13 @@ class WorkflowCoSTEER(CoSTEER): WorkflowGeneralCaseSpecEvaluator(scen=scen), scen=scen ) # Please specify whether you agree running your eva in parallel or not es = WorkflowMultiProcessEvolvingStrategy(scen=scen, settings=settings) - super().__init__(*args, settings=settings, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs) + super().__init__( + *args, + settings=settings, + eva=eva, + es=es, + evolving_version=2, + scen=scen, + max_loop=DS_RD_SETTING.coder_max_loop, + **kwargs, + ) diff --git a/rdagent/scenarios/data_science/dev/runner/__init__.py b/rdagent/scenarios/data_science/dev/runner/__init__.py index 5b808302..e3c0d20b 100644 --- a/rdagent/scenarios/data_science/dev/runner/__init__.py +++ b/rdagent/scenarios/data_science/dev/runner/__init__.py @@ -93,7 +93,14 @@ class DSCoSTEERRunner(CoSTEER): # In runner, we don't need very big loops, so we set max_loop to 3 super().__init__( - *args, settings=CoSTEER_SETTINGS, eva=eva, es=es, evolving_version=2, scen=scen, max_loop=3, **kwargs + *args, + settings=CoSTEER_SETTINGS, + eva=eva, + es=es, + evolving_version=2, + scen=scen, + max_loop=DS_RD_SETTING.runner_max_loop, + **kwargs, ) def develop(self, exp):