Several update on the repo (see desc) (#76)

* ignore result csv file

* fix app scripts

* rename taskgenerator to developer and generate to develop

* fix a config bug in coder

* fix a small bug in factor coder evaluators

* remove a single logger in factor coder evaluators

* fix a small bug in model coder main.py

* rename Implementation to Workspace

* move the prepare the inject_code into FBWorkspace to align all the behavior

* fix a small bug in model feedback

* remove debug lines for multi processing and simplify evaluators multi proc

* add a copy function to workspace to freeze the workspace && add config prefix to speed up debugging

* make hypothesisgen a abc class

* use Qlib***Experiment

* fix a small bug

* rename Imp to Ws

* rename sub_implementations to sub_workspace_list

* fix a bug in feedback not presented as content in prompts

* move proposal pys to proposal folder

* reformat the folder

* align factor and model qlib workspace and use template to handle the workspace

* add a filter to evoagent to filter out false evo

* align multi_proc_n into RDAGENT seeting

* handle when runner gets empty experiment

* fix logger merge remaining problems

* fix black and isort automatically

rdagent/components/coder/model_coder/CoSTEER/__init__.py

@@ -8,6 +8,7 @@ from rdagent.components.coder.model_coder.CoSTEER.evaluators import (
 from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
     ModelEvolvingItem,
 )
+from rdagent.components.coder.model_coder.CoSTEER.evolving_agent import ModelRAGEvoAgent
 from rdagent.components.coder.model_coder.CoSTEER.evolving_strategy import (
     ModelCoderEvolvingStrategy,
 )
@@ -16,17 +17,18 @@ from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import (
     ModelRAGStrategy,
 )
 from rdagent.components.coder.model_coder.model import ModelExperiment
+from rdagent.core.developer import Developer
 from rdagent.core.evolving_agent import RAGEvoAgent
-from rdagent.core.task_generator import TaskGenerator
 
 
-class ModelCoSTEER(TaskGenerator[ModelExperiment]):
+class ModelCoSTEER(Developer[ModelExperiment]):
     def __init__(
         self,
         *args,
         with_knowledge: bool = True,
         with_feedback: bool = True,
         knowledge_self_gen: bool = True,
+        filter_final_evo: bool = True,
         **kwargs,
     ) -> None:
         super().__init__(*args, **kwargs)
@@ -44,6 +46,7 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
         self.with_knowledge = with_knowledge
         self.with_feedback = with_feedback
         self.knowledge_self_gen = knowledge_self_gen
+        self.filter_final_evo = filter_final_evo
         self.evolving_strategy = ModelCoderEvolvingStrategy(scen=self.scen)
         self.model_evaluator = ModelCoderMultiEvaluator(scen=self.scen)
 
@@ -57,7 +60,7 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
 
         return model_knowledge_base
 
-    def generate(self, exp: ModelExperiment) -> ModelExperiment:
+    def develop(self, exp: ModelExperiment) -> ModelExperiment:
         # init knowledge base
         model_knowledge_base = self.load_or_init_knowledge_base(
             former_knowledge_base_path=self.knowledge_base_path,
@@ -69,7 +72,9 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
         # init intermediate items
         model_experiment = ModelEvolvingItem(sub_tasks=exp.sub_tasks)
 
-        self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
+        self.evolve_agent = ModelRAGEvoAgent(
+            max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag
+        )
 
         model_experiment = self.evolve_agent.multistep_evolve(
             model_experiment,
@@ -77,11 +82,11 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
             with_knowledge=self.with_knowledge,
             with_feedback=self.with_feedback,
             knowledge_self_gen=self.knowledge_self_gen,
+            filter_final_evo=self.filter_final_evo,
         )
 
         # save new knowledge base
         if self.new_knowledge_base_path is not None:
             pickle.dump(model_knowledge_base, open(self.new_knowledge_base_path, "wb"))
-        self.knowledge_base = model_knowledge_base
-        model_experiment.based_experiments = exp.based_experiments
-        return model_experiment
+        exp.sub_workspace_list = model_experiment.sub_workspace_list
+        return exp

rdagent/components/coder/model_coder/CoSTEER/evaluators.py

@@ -11,14 +11,14 @@ from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
 from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
     ModelEvolvingItem,
 )
-from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evaluation import Evaluator
 from rdagent.core.evolving_framework import QueriedKnowledge
-from rdagent.core.experiment import Implementation, Task
-from rdagent.log import rdagent_logger as logger
+from rdagent.core.experiment import Task, Workspace
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -62,15 +62,15 @@ class ModelCodeEvaluator(Evaluator):
     def evaluate(
         self,
         target_task: Task,
-        implementation: Implementation,
-        gt_implementation: Implementation,
+        implementation: Workspace,
+        gt_implementation: Workspace,
         model_execution_feedback: str = "",
         model_value_feedback: str = "",
     ):
         assert isinstance(target_task, ModelTask)
-        assert isinstance(implementation, ModelImplementation)
+        assert isinstance(implementation, ModelFBWorkspace)
         if gt_implementation is not None:
-            assert isinstance(gt_implementation, ModelImplementation)
+            assert isinstance(gt_implementation, ModelFBWorkspace)
 
         model_task_information = target_task.get_task_information()
         code = implementation.code
@@ -120,16 +120,16 @@ class ModelFinalEvaluator(Evaluator):
     def evaluate(
         self,
         target_task: Task,
-        implementation: Implementation,
-        gt_implementation: Implementation,
+        implementation: Workspace,
+        gt_implementation: Workspace,
         model_execution_feedback: str,
         model_value_feedback: str,
         model_code_feedback: str,
     ):
         assert isinstance(target_task, ModelTask)
-        assert isinstance(implementation, ModelImplementation)
+        assert isinstance(implementation, ModelFBWorkspace)
         if gt_implementation is not None:
-            assert isinstance(gt_implementation, ModelImplementation)
+            assert isinstance(gt_implementation, ModelFBWorkspace)
 
         system_prompt = (
             Environment(undefined=StrictUndefined)
@@ -219,8 +219,8 @@ class ModelCoderEvaluator(Evaluator):
     def evaluate(
         self,
         target_task: Task,
-        implementation: Implementation,
-        gt_implementation: Implementation,
+        implementation: Workspace,
+        gt_implementation: Workspace,
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> ModelCoderFeedback:
@@ -248,7 +248,7 @@ class ModelCoderEvaluator(Evaluator):
         input_value = 0.4
         param_init_value = 0.6
 
-        assert isinstance(implementation, ModelImplementation)
+        assert isinstance(implementation, ModelFBWorkspace)
         model_execution_feedback, gen_tensor = implementation.execute(
             batch_size=batch_size,
             num_features=num_features,
@@ -257,7 +257,7 @@ class ModelCoderEvaluator(Evaluator):
             param_init_value=param_init_value,
         )
         if gt_implementation is not None:
-            assert isinstance(gt_implementation, ModelImplementation)
+            assert isinstance(gt_implementation, ModelFBWorkspace)
             _, gt_tensor = gt_implementation.execute(
                 batch_size=batch_size,
                 num_features=num_features,
@@ -303,26 +303,21 @@ class ModelCoderMultiEvaluator(Evaluator):
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> List[ModelCoderFeedback]:
-        multi_implementation_feedback = []
-
-        calls = []
-        for index in range(len(evo.sub_tasks)):
-            corresponding_implementation = evo.sub_implementations[index]
-            corresponding_gt_implementation = (
-                evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
-            )
-            calls.append(
+        multi_implementation_feedback = multiprocessing_wrapper(
+            [
                 (
                     ModelCoderEvaluator(scen=self.scen).evaluate,
                     (
                         evo.sub_tasks[index],
-                        corresponding_implementation,
-                        corresponding_gt_implementation,
+                        evo.sub_workspace_list[index],
+                        evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
                         queried_knowledge,
                     ),
-                ),
-            )
-        multi_implementation_feedback = multiprocessing_wrapper(calls, n=MODEL_IMPL_SETTINGS.evo_multi_proc_n)
+                )
+                for index in range(len(evo.sub_tasks))
+            ],
+            n=RD_AGENT_SETTINGS.multi_proc_n,
+        )
 
         final_decision = [
             None if single_feedback is None else single_feedback.final_decision

rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py

@@ -1,6 +1,6 @@
 from rdagent.components.coder.model_coder.model import (
     ModelExperiment,
-    ModelImplementation,
+    ModelFBWorkspace,
     ModelTask,
 )
 from rdagent.core.evolving_framework import EvolvableSubjects
@@ -15,7 +15,7 @@ class ModelEvolvingItem(ModelExperiment, EvolvableSubjects):
     def __init__(
         self,
         sub_tasks: list[ModelTask],
-        sub_gt_implementations: list[ModelImplementation] = None,
+        sub_gt_implementations: list[ModelFBWorkspace] = None,
     ):
         ModelExperiment.__init__(self, sub_tasks=sub_tasks)
         if sub_gt_implementations is not None and len(

rdagent/components/coder/model_coder/CoSTEER/evolving_agent.py

@@ -0,0 +1,19 @@
+from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
+from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
+    ModelEvolvingItem,
+)
+from rdagent.core.evaluation import Feedback
+from rdagent.core.evolving_agent import RAGEvoAgent
+from rdagent.core.evolving_framework import EvolvableSubjects
+
+
+class ModelRAGEvoAgent(RAGEvoAgent):
+    def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects:
+        assert isinstance(evo, ModelEvolvingItem)
+        assert isinstance(feedback, list)
+        assert len(evo.sub_workspace_list) == len(feedback)
+
+        for index in range(len(evo.sub_workspace_list)):
+            if not feedback[index].final_decision:
+                evo.sub_workspace_list[index].clear()
+        return evo

rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py

@@ -11,7 +11,7 @@ from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
 from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import (
     ModelQueriedKnowledge,
 )
-from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evolving_framework import EvolvingStrategy
 from rdagent.core.prompts import Prompts
@@ -26,7 +26,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
         self,
         target_task: ModelTask,
         queried_knowledge: ModelQueriedKnowledge = None,
-    ) -> ModelImplementation:
+    ) -> str:
         model_information_str = target_task.get_task_information()
 
         if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -86,19 +86,15 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
                     queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
 
             code = json.loads(
-                APIBackend(use_chat_cache=False).build_messages_and_create_chat_completion(
+                APIBackend(
+                    use_chat_cache=MODEL_IMPL_SETTINGS.coder_use_cache
+                ).build_messages_and_create_chat_completion(
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                     json_mode=True,
                 ),
             )["code"]
-            model_implementation = ModelImplementation(
-                target_task,
-            )
-            model_implementation.prepare()
-            model_implementation.inject_code(**{"model.py": code})
-
-            return model_implementation
+            return code
 
     def evolve(
         self,
@@ -107,14 +103,12 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
         queried_knowledge: ModelQueriedKnowledge | None = None,
         **kwargs,
     ) -> ModelEvolvingItem:
-        new_evo = deepcopy(evo)
-
         # 1.找出需要evolve的model
         to_be_finished_task_index = []
-        for index, target_model_task in enumerate(new_evo.sub_tasks):
+        for index, target_model_task in enumerate(evo.sub_tasks):
             target_model_task_desc = target_model_task.get_task_information()
             if target_model_task_desc in queried_knowledge.success_task_to_knowledge_dict:
-                new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
+                evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[
                     target_model_task_desc
                 ].implementation
             elif (
@@ -125,20 +119,17 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
 
         result = multiprocessing_wrapper(
             [
-                (self.implement_one_model, (new_evo.sub_tasks[target_index], queried_knowledge))
+                (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge))
                 for target_index in to_be_finished_task_index
             ],
-            n=MODEL_IMPL_SETTINGS.evo_multi_proc_n,
+            n=RD_AGENT_SETTINGS.multi_proc_n,
         )
 
         for index, target_index in enumerate(to_be_finished_task_index):
-            new_evo.sub_implementations[target_index] = result[index]
+            if evo.sub_workspace_list[target_index] is None:
+                evo.sub_workspace_list[target_index] = ModelFBWorkspace(target_task=evo.sub_tasks[target_index])
+            evo.sub_workspace_list[target_index].inject_code(**{"model.py": result[index]})
 
-        # for target_index in to_be_finished_task_index:
-        #     new_evo.sub_implementations[target_index] = self.implement_one_model(
-        #         new_evo.sub_tasks[target_index], queried_knowledge
-        #     )
+        evo.corresponding_selection = to_be_finished_task_index
 
-        new_evo.corresponding_selection = to_be_finished_task_index
-
-        return new_evo
+        return evo

rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py

@@ -9,7 +9,7 @@ from rdagent.core.evolving_framework import (
     QueriedKnowledge,
     RAGStrategy,
 )
-from rdagent.core.experiment import Implementation
+from rdagent.core.experiment import Workspace
 from rdagent.oai.llm_utils import calculate_embedding_distance_between_str_list
 
 
@@ -17,7 +17,7 @@ class ModelKnowledge(Knowledge):
     def __init__(
         self,
         target_task: ModelTask,
-        implementation: Implementation,
+        implementation: Workspace,
         feedback: ModelCoderFeedback,
     ) -> None:
         """
@@ -30,7 +30,7 @@ class ModelKnowledge(Knowledge):
             None
         """
         self.target_task = target_task
-        self.implementation = implementation
+        self.implementation = implementation.copy()
         self.feedback = feedback
 
     def get_implementation_and_feedback_str(self) -> str:
@@ -87,7 +87,7 @@ class ModelRAGStrategy(RAGStrategy):
                 for task_index in range(len(implementations.sub_tasks)):
                     target_task = implementations.sub_tasks[task_index]
                     target_task_information = target_task.get_task_information()
-                    implementation = implementations.sub_implementations[task_index]
+                    implementation = implementations.sub_workspace_list[task_index]
                     single_feedback = feedback[task_index]
                     if single_feedback is None:
                         continue
@@ -121,9 +121,9 @@ class ModelRAGStrategy(RAGStrategy):
         for target_model_task in evo.sub_tasks:
             target_model_task_information = target_model_task.get_task_information()
             if target_model_task_information in self.knowledgebase.success_task_info_set:
-                queried_knowledge.success_task_to_knowledge_dict[target_model_task_information] = (
-                    self.knowledgebase.implementation_trace[target_model_task_information][-1]
-                )
+                queried_knowledge.success_task_to_knowledge_dict[
+                    target_model_task_information
+                ] = self.knowledgebase.implementation_trace[target_model_task_information][-1]
             elif (
                 len(
                     self.knowledgebase.implementation_trace.setdefault(
@@ -135,12 +135,14 @@ class ModelRAGStrategy(RAGStrategy):
             ):
                 queried_knowledge.failed_task_info_set.add(target_model_task_information)
             else:
-                queried_knowledge.working_task_to_former_failed_knowledge_dict[target_model_task_information] = (
-                    self.knowledgebase.implementation_trace.setdefault(
-                        target_model_task_information,
-                        [],
-                    )[-query_former_trace_limit:]
-                )
+                queried_knowledge.working_task_to_former_failed_knowledge_dict[
+                    target_model_task_information
+                ] = self.knowledgebase.implementation_trace.setdefault(
+                    target_model_task_information,
+                    [],
+                )[
+                    -query_former_trace_limit:
+                ]
 
                 knowledge_base_success_task_list = list(
                     self.knowledgebase.success_task_info_set,
@@ -161,7 +163,7 @@ class ModelRAGStrategy(RAGStrategy):
                     )[-1]
                     for index in similar_indexes
                 ]
-                queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_model_task_information] = (
-                    similar_successful_knowledge
-                )
+                queried_knowledge.working_task_to_similar_successful_knowledge_dict[
+                    target_model_task_information
+                ] = similar_successful_knowledge
         return queried_knowledge