add feedback to workspace and ds runner base on costeer

rdagent/components/coder/CoSTEER/evaluators.py

@@ -6,8 +6,7 @@ from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evaluation import Evaluator, Feedback
 from rdagent.core.evolving_framework import QueriedKnowledge
-from rdagent.core.experiment import Workspace
-from rdagent.core.scenario import Task
+from rdagent.core.experiment import Task, Workspace
 from rdagent.core.utils import multiprocessing_wrapper
 from rdagent.log import rdagent_logger as logger
 
@@ -165,6 +164,9 @@ class CoSTEERMultiEvaluator(Evaluator):
             n=RD_AGENT_SETTINGS.multi_proc_n,
         )
 
+        for index in range(len(evo.sub_tasks)):
+            evo.sub_workspace_list[index].feedback = multi_implementation_feedback[index]
+
         final_decision = [
             None if single_feedback is None else single_feedback.final_decision
             for single_feedback in multi_implementation_feedback

rdagent/components/coder/CoSTEER/evolvable_subjects.py

@@ -1,6 +1,5 @@
 from rdagent.core.evolving_framework import EvolvableSubjects
-from rdagent.core.experiment import Experiment, FBWorkspace
-from rdagent.core.scenario import Task
+from rdagent.core.experiment import Experiment, FBWorkspace, Task
 from rdagent.log import rdagent_logger as logger
 
 

rdagent/components/coder/CoSTEER/evolving_agent.py

@@ -1,4 +1,3 @@
-from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedbackDeprecated
 from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
 from rdagent.core.evolving_agent import RAGEvoAgent
 from rdagent.core.evolving_framework import EvolvableSubjects
@@ -7,15 +6,14 @@ from rdagent.core.exception import CoderError
 
 class FilterFailedRAGEvoAgent(RAGEvoAgent):
 
-    def filter_evolvable_subjects_by_feedback(
-        self, evo: EvolvableSubjects, feedback: CoSTEERSingleFeedbackDeprecated
-    ) -> EvolvableSubjects:
+    def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: list) -> EvolvableSubjects:
         assert isinstance(evo, EvolvingItem)
         # FIXME: the list does not align with the annotation; It should be MultipleFeedback instead of a list of feedbacks
         assert isinstance(feedback, list)
         assert len(evo.sub_workspace_list) == len(feedback)
 
         for index in range(len(evo.sub_workspace_list)):
+            evo.sub_workspace_list[index].feedback = None
             if evo.sub_workspace_list[index] is not None and feedback[index] is not None and not feedback[index]:
                 evo.sub_workspace_list[index].clear()
 

rdagent/components/coder/CoSTEER/evolving_strategy.py

@@ -10,11 +10,10 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
 )
 from rdagent.components.coder.CoSTEER.scheduler import random_select
 from rdagent.core.conf import RD_AGENT_SETTINGS
-from rdagent.core.evaluation import Scenario
 from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
-from rdagent.core.experiment import FBWorkspace
+from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.core.prompts import Prompts
-from rdagent.core.scenario import Task
+from rdagent.core.scenario import Scenario
 from rdagent.core.utils import multiprocessing_wrapper
 
 implement_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
@@ -111,4 +110,8 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
         evo = self.assign_code_list_to_evo(code_list, evo)
         evo.corresponding_selection = to_be_finished_task_index
 
+        # After implementation, the feedback should be reset
+        for workspace in evo.sub_workspace_list:
+            workspace.feedback = None
+
         return evo

rdagent/components/coder/CoSTEER/knowledge_management.py

@@ -25,9 +25,8 @@ from rdagent.core.evolving_framework import (
     QueriedKnowledge,
     RAGStrategy,
 )
-from rdagent.core.experiment import FBWorkspace
+from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.core.prompts import Prompts
-from rdagent.core.scenario import Task
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import (
     APIBackend,

rdagent/components/coder/CoSTEER/scheduler.py

@@ -4,7 +4,7 @@ from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
 from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERQueriedKnowledge,
 )
-from rdagent.core.evaluation import Scenario
+from rdagent.core.scenario import Scenario
 from rdagent.log import rdagent_logger as logger
 
 

rdagent/components/coder/data_science/ensemble/__init__.py

@@ -52,21 +52,14 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             if queried_knowledge is not None
             else []
         )
-        latest_code_feedback = [
-            knowledge.feedback
-            for knowledge in queried_former_failed_knowledge[0]
-            if knowledge.implementation.file_dict.get("ensemble.py") is not None
-            and knowledge.implementation.file_dict.get("ensemble.py") == workspace.file_dict.get("ensemble.py")
-        ]
-        if len(latest_code_feedback) > 0:
-            queried_former_failed_knowledge = (
-                [
-                    knowledge
-                    for knowledge in queried_former_failed_knowledge[0]
-                    if knowledge.implementation.file_dict.get("ensemble.py") != workspace.file_dict.get("ensemble.py")
-                ],
-                queried_former_failed_knowledge[1],
-            )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("ensemble.py") != workspace.file_dict.get("ensemble.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
 
         # Generate code with knowledge integration
         competition_info = self.scen.get_scenario_all_desc()
@@ -81,7 +74,7 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         user_prompt = T(".prompts:ensemble_coder.user").r(
             ensemble_spec=workspace.file_dict["spec/ensemble.md"],
             latest_code=workspace.file_dict.get("ensemble.py"),
-            latest_code_feedback=latest_code_feedback[0] if len(latest_code_feedback) > 0 else None,
+            latest_code_feedback=workspace.feedback,
         )
 
         for _ in range(5):

rdagent/components/coder/data_science/feature/__init__.py

@@ -40,21 +40,14 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             if queried_knowledge is not None
             else []
         )
-        latest_code_feedback = [
-            knowledge.feedback
-            for knowledge in queried_former_failed_knowledge[0]
-            if knowledge.implementation.file_dict.get("feature.py") is not None
-            and knowledge.implementation.file_dict.get("feature.py") == workspace.file_dict.get("feature.py")
-        ]
-        if len(latest_code_feedback) > 0:
-            queried_former_failed_knowledge = (
-                [
-                    knowledge
-                    for knowledge in queried_former_failed_knowledge[0]
-                    if knowledge.implementation.file_dict.get("feature.py") != workspace.file_dict.get("feature.py")
-                ],
-                queried_former_failed_knowledge[1],
-            )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("feature.py") != workspace.file_dict.get("feature.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
 
         # 2. code
         system_prompt = T(".prompts:feature_coder.system").r(
@@ -66,7 +59,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         user_prompt = T(".prompts:feature_coder.user").r(
             feature_spec=workspace.file_dict["spec/feature.md"],
             latest_code=workspace.file_dict.get("feature.py"),
-            latest_code_feedback=latest_code_feedback[0] if len(latest_code_feedback) > 0 else None,
+            latest_code_feedback=workspace.feedback,
         )
 
         for _ in range(5):

rdagent/components/coder/data_science/model/__init__.py

@@ -44,23 +44,15 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             if queried_knowledge is not None
             else []
         )
-        latest_code_feedback = [
-            knowledge.feedback
-            for knowledge in queried_former_failed_knowledge[0]
-            if knowledge.implementation.file_dict.get(f"{target_task.name}.py") is not None
-            and knowledge.implementation.file_dict.get(f"{target_task.name}.py")
-            == workspace.file_dict.get(f"{target_task.name}.py")
-        ]
-        if len(latest_code_feedback) > 0:
-            queried_former_failed_knowledge = (
-                [
-                    knowledge
-                    for knowledge in queried_former_failed_knowledge[0]
-                    if knowledge.implementation.file_dict.get(f"{target_task.name}.py")
-                    != workspace.file_dict.get(f"{target_task.name}.py")
-                ],
-                queried_former_failed_knowledge[1],
-            )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get(f"{target_task.name}.py")
+                != workspace.file_dict.get(f"{target_task.name}.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
 
         # 2. code
         system_prompt = T(".prompts:model_coder.system").r(
@@ -82,7 +74,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             latest_model_code=workspace.get_codes(
                 r"^model_(?!test)\w+\.py$"
             ),  # TODO: If we have high failure rate here, we should clean this step with less information.
-            latest_code_feedback=latest_code_feedback[0] if len(latest_code_feedback) > 0 else None,
+            latest_code_feedback=workspace.feedback,
         )
 
         for _ in range(5):

rdagent/components/coder/data_science/raw_data_loader/__init__.py

@@ -69,21 +69,14 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             if queried_knowledge is not None
             else []
         )
-        latest_code_feedback = [
-            knowledge.feedback
-            for knowledge in queried_former_failed_knowledge[0]
-            if knowledge.implementation.file_dict.get("load_data.py") is not None
-            and knowledge.implementation.file_dict.get("load_data.py") == workspace.file_dict.get("load_data.py")
-        ]
-        if len(latest_code_feedback) > 0:
-            queried_former_failed_knowledge = (
-                [
-                    knowledge
-                    for knowledge in queried_former_failed_knowledge[0]
-                    if knowledge.implementation.file_dict.get("load_data.py") != workspace.file_dict.get("load_data.py")
-                ],
-                queried_former_failed_knowledge[1],
-            )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("load_data.py") != workspace.file_dict.get("load_data.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
 
         # 1. specifications
         # TODO: We may move spec into a separated COSTEER task
@@ -141,7 +134,7 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             data_loader_spec=data_loader_spec,
             folder_spec=data_folder_info,
             latest_code=workspace.file_dict.get("load_data.py"),
-            latest_code_feedback=latest_code_feedback[0] if len(latest_code_feedback) > 0 else None,
+            latest_code_feedback=workspace.feedback,
         )
 
         for _ in range(5):

rdagent/components/coder/data_science/workflow/__init__.py

@@ -41,21 +41,14 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             if queried_knowledge is not None
             else []
         )
-        latest_code_feedback = [
-            knowledge.feedback
-            for knowledge in queried_former_failed_knowledge[0]
-            if knowledge.implementation.file_dict.get("main.py") is not None
-            and knowledge.implementation.file_dict.get("main.py") == workspace.file_dict.get("main.py")
-        ]
-        if len(latest_code_feedback) > 0:
-            queried_former_failed_knowledge = (
-                [
-                    knowledge
-                    for knowledge in queried_former_failed_knowledge[0]
-                    if knowledge.implementation.file_dict.get("main.py") != workspace.file_dict.get("main.py")
-                ],
-                queried_former_failed_knowledge[1],
-            )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("main.py") != workspace.file_dict.get("main.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
 
         # 2. code
         system_prompt = T(".prompts:workflow_coder.system").r(
@@ -71,7 +64,7 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             ensemble_code=workspace.file_dict["ensemble.py"],
             latest_code=workspace.file_dict.get("main.py"),
             workflow_spec=workspace.file_dict["spec/workflow.md"],
-            latest_code_feedback=latest_code_feedback[0] if len(latest_code_feedback) > 0 else None,
+            latest_code_feedback=workspace.feedback,
         )
 
         for _ in range(5):

rdagent/components/coder/data_science/workflow/eval.py

@@ -97,21 +97,21 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         if not submission_fp.exists():
             stdout += "\nSubmission file (submission.csv) is not generated."
         else:
-            base_check_code = (DIRNAME / "eval_tests" / "submission_check.txt").read_text()
-            implementation.inject_files(**{"submission_check.py": base_check_code})
+            base_check_code = (DIRNAME / "eval_tests" / "submission_format_test.txt").read_text()
+            implementation.inject_files(**{"submission_format_test.py": base_check_code})
             # stdout += "----Submission Check 1-----\n"
-            stdout += implementation.execute(env=de, entry="python submission_check.py")
+            stdout += implementation.execute(env=de, entry="python submission_format_test.py")
 
             # MLEBench Check
             # !!! Since we are running on a sampled dataset, mlebench check is not required.
             # mle_check_code = (
-            #     (DIRNAME / "eval_tests" / "mle_submission_check.txt")
+            #     (DIRNAME / "eval_tests" / "mle_submission_format_test.txt")
             #     .read_text()
             #     .replace("<competition_id>", self.scen.competition)
             # )
-            # implementation.inject_files(**{"mle_submission_check.py": mle_check_code})
+            # implementation.inject_files(**{"mle_submission_format_test.py": mle_check_code})
             # stdout += "----Submission Check 2-----\n"
-            # stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py")
+            # stdout += implementation.execute(env=mde, entry=f"python mle_submission_format_test.py")
 
         system_prompt = T(".prompts:workflow_eval.system").r(
             scenario=self.scen.get_scenario_all_desc(),