From d46d27375d29b20c0537157bb9e0b6d8c875a64a Mon Sep 17 00:00:00 2001 From: you-n-g Date: Thu, 17 Apr 2025 16:19:26 +0800 Subject: [PATCH] refactor: use remove_eda_part for EDA cleanup, fix diff eval (#800) --- rdagent/components/coder/data_science/ensemble/eval.py | 3 ++- rdagent/components/coder/data_science/feature/eval.py | 3 ++- rdagent/components/coder/data_science/model/eval.py | 3 ++- rdagent/components/coder/data_science/pipeline/eval.py | 3 ++- .../components/coder/data_science/raw_data_loader/eval.py | 3 ++- rdagent/components/coder/data_science/workflow/eval.py | 3 ++- rdagent/scenarios/data_science/dev/feedback.py | 7 +++---- rdagent/scenarios/data_science/dev/runner/eval.py | 7 +++++-- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/rdagent/components/coder/data_science/ensemble/eval.py b/rdagent/components/coder/data_science/ensemble/eval.py index c318a6e7..3dfb295e 100644 --- a/rdagent/components/coder/data_science/ensemble/eval.py +++ b/rdagent/components/coder/data_science/ensemble/eval.py @@ -10,6 +10,7 @@ from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERSingleFeedback, ) from rdagent.components.coder.data_science.conf import get_ds_env +from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.evolving_framework import QueriedKnowledge from rdagent.core.experiment import FBWorkspace, Task from rdagent.utils.agent.tpl import T @@ -74,7 +75,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): if "main.py" in implementation.file_dict and ret_code == 0: workflow_stdout = implementation.execute(env=env, entry="python main.py") - workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) + workflow_stdout = remove_eda_part(workflow_stdout) else: workflow_stdout = None diff --git a/rdagent/components/coder/data_science/feature/eval.py b/rdagent/components/coder/data_science/feature/eval.py index cf5b812a..bef2266e 100644 --- a/rdagent/components/coder/data_science/feature/eval.py +++ b/rdagent/components/coder/data_science/feature/eval.py @@ -8,6 +8,7 @@ from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERSingleFeedback, ) from rdagent.components.coder.data_science.conf import get_ds_env +from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.evolving_framework import QueriedKnowledge from rdagent.core.experiment import FBWorkspace, Task from rdagent.utils.agent.tpl import T @@ -59,7 +60,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): if "main.py" in implementation.file_dict and ret_code == 0: workflow_stdout = implementation.execute(env=env, entry="python main.py") - workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) + workflow_stdout = remove_eda_part(workflow_stdout) else: workflow_stdout = None diff --git a/rdagent/components/coder/data_science/model/eval.py b/rdagent/components/coder/data_science/model/eval.py index dcb02d77..3280043e 100644 --- a/rdagent/components/coder/data_science/model/eval.py +++ b/rdagent/components/coder/data_science/model/eval.py @@ -13,6 +13,7 @@ from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERSingleFeedback, ) from rdagent.components.coder.data_science.conf import get_ds_env +from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.evolving_framework import QueriedKnowledge from rdagent.core.exception import CoderError from rdagent.core.experiment import FBWorkspace, Task @@ -85,7 +86,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): if "main.py" in implementation.file_dict and ret_code == 0: workflow_stdout = implementation.execute(env=env, entry="python main.py") - workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) + workflow_stdout = remove_eda_part(workflow_stdout) else: workflow_stdout = None diff --git a/rdagent/components/coder/data_science/pipeline/eval.py b/rdagent/components/coder/data_science/pipeline/eval.py index 6c30156b..caed8b7d 100644 --- a/rdagent/components/coder/data_science/pipeline/eval.py +++ b/rdagent/components/coder/data_science/pipeline/eval.py @@ -16,6 +16,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledgeV2, ) from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env +from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.experiment import FBWorkspace, Task from rdagent.utils.agent.tpl import T from rdagent.utils.agent.workflow import build_cls_from_json_with_retry @@ -62,7 +63,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): # Clean the scores.csv & submission.csv. implementation.execute(env=env, entry=get_clear_ws_cmd()) stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python main.py") - stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout) + stdout = remove_eda_part(stdout) score_fp = implementation.workspace_path / "scores.csv" score_ret_code = 0 diff --git a/rdagent/components/coder/data_science/raw_data_loader/eval.py b/rdagent/components/coder/data_science/raw_data_loader/eval.py index be696ffc..a977bd5c 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/eval.py +++ b/rdagent/components/coder/data_science/raw_data_loader/eval.py @@ -13,6 +13,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledgeV2, ) from rdagent.components.coder.data_science.conf import get_ds_env +from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.experiment import FBWorkspace, Task from rdagent.utils.agent.tpl import T from rdagent.utils.agent.workflow import build_cls_from_json_with_retry @@ -66,7 +67,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): if "main.py" in implementation.file_dict and ret_code == 0: workflow_stdout = implementation.execute(env=env, entry="python main.py") - workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) + workflow_stdout = remove_eda_part(workflow_stdout) else: workflow_stdout = None diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index 8ed32d07..2b773a4f 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -11,6 +11,7 @@ from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERSingleFeedback, ) from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env +from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.evolving_framework import QueriedKnowledge from rdagent.core.experiment import FBWorkspace, Task from rdagent.log import rdagent_logger as logger @@ -76,7 +77,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): stdout = implementation.execute(env=env, entry=f"python -m coverage run main.py") # remove EDA part - stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout) + stdout = remove_eda_part(stdout) # Check score file score_fp = implementation.workspace_path / "scores.csv" diff --git a/rdagent/scenarios/data_science/dev/feedback.py b/rdagent/scenarios/data_science/dev/feedback.py index 5dacae8d..ce3be70e 100644 --- a/rdagent/scenarios/data_science/dev/feedback.py +++ b/rdagent/scenarios/data_science/dev/feedback.py @@ -39,12 +39,11 @@ class DSExperiment2Feedback(Experiment2Feedback): # - Should we choose between the diff from last experiment or last sota ? # Retrieve the last experiment from the history - last_exp = trace.hist[-1][0] if trace.hist else None - if last_exp and last_exp.experiment_workspace and exp.experiment_workspace: + if sota_exp and sota_exp.experiment_workspace and exp.experiment_workspace: # Generate a diff between the two workspaces - last_exp_files = last_exp.experiment_workspace.file_dict + sota_exp_files = sota_exp.experiment_workspace.file_dict current_exp_files = exp.experiment_workspace.file_dict - diff_edition = generate_diff_from_dict(last_exp_files, current_exp_files) + diff_edition = generate_diff_from_dict(sota_exp_files, current_exp_files) else: diff_edition = [] diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 5be21c76..b00675e2 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -10,6 +10,7 @@ from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERSingleFeedback, ) from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env +from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.evolving_framework import QueriedKnowledge from rdagent.core.experiment import FBWorkspace, Task from rdagent.log import rdagent_logger as logger @@ -50,8 +51,10 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry="python -m coverage run main.py") match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL) eda_output = match.groups()[1] if match else None - self.scen.eda_output = eda_output - stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout) + if eda_output is None: + eda_output = "No EDA output." + implementation.inject_files(**{"EDA.md": eda_output}) + stdout = remove_eda_part(stdout) # Check score file score_fp = implementation.workspace_path / "scores.csv"