refactor: use remove_eda_part for EDA cleanup, fix diff eval (#800)

rdagent/components/coder/data_science/ensemble/eval.py

@@ -10,6 +10,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERSingleFeedback,
 )
 from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.utils.agent.tpl import T
@@ -74,7 +75,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict and ret_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
-            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
+            workflow_stdout = remove_eda_part(workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/feature/eval.py

@@ -8,6 +8,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERSingleFeedback,
 )
 from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.utils.agent.tpl import T
@@ -59,7 +60,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict and ret_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
-            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
+            workflow_stdout = remove_eda_part(workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/model/eval.py

@@ -13,6 +13,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERSingleFeedback,
 )
 from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace, Task
@@ -85,7 +86,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict and ret_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
-            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
+            workflow_stdout = remove_eda_part(workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/pipeline/eval.py

@@ -16,6 +16,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERQueriedKnowledgeV2,
 )
 from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
@@ -62,7 +63,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
         # Clean the scores.csv & submission.csv.
         implementation.execute(env=env, entry=get_clear_ws_cmd())
         stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python main.py")
-        stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
+        stdout = remove_eda_part(stdout)
 
         score_fp = implementation.workspace_path / "scores.csv"
         score_ret_code = 0

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -13,6 +13,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERQueriedKnowledgeV2,
 )
 from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
@@ -66,7 +67,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict and ret_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
-            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
+            workflow_stdout = remove_eda_part(workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/workflow/eval.py

@@ -11,6 +11,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERSingleFeedback,
 )
 from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.log import rdagent_logger as logger
@@ -76,7 +77,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         stdout = implementation.execute(env=env, entry=f"python -m coverage run main.py")
 
         # remove EDA part
-        stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
+        stdout = remove_eda_part(stdout)
 
         # Check score file
         score_fp = implementation.workspace_path / "scores.csv"

rdagent/scenarios/data_science/dev/feedback.py

@@ -39,12 +39,11 @@ class DSExperiment2Feedback(Experiment2Feedback):
         # -  Should we choose between the diff from last experiment or last sota ?
 
         # Retrieve the last experiment from the history
-        last_exp = trace.hist[-1][0] if trace.hist else None
-        if last_exp and last_exp.experiment_workspace and exp.experiment_workspace:
+        if sota_exp and sota_exp.experiment_workspace and exp.experiment_workspace:
             # Generate a diff between the two workspaces
-            last_exp_files = last_exp.experiment_workspace.file_dict
+            sota_exp_files = sota_exp.experiment_workspace.file_dict
             current_exp_files = exp.experiment_workspace.file_dict
-            diff_edition = generate_diff_from_dict(last_exp_files, current_exp_files)
+            diff_edition = generate_diff_from_dict(sota_exp_files, current_exp_files)
         else:
             diff_edition = []
 

rdagent/scenarios/data_science/dev/runner/eval.py

@@ -10,6 +10,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERSingleFeedback,
 )
 from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.log import rdagent_logger as logger
@@ -50,8 +51,10 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
         stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry="python -m coverage run main.py")
         match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL)
         eda_output = match.groups()[1] if match else None
-        self.scen.eda_output = eda_output
-        stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
+        if eda_output is None:
+            eda_output = "No EDA output."
+        implementation.inject_files(**{"EDA.md": eda_output})
+        stdout = remove_eda_part(stdout)
 
         # Check score file
         score_fp = implementation.workspace_path / "scores.csv"