First version of factor idea proposal (#46)

* update all code

* save code

* update first version of factor proposal

* change a comment

* remove a useless comment

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/components/idea_proposal/factor_proposal.py

@@ -1,13 +1,18 @@
 from abc import abstractmethod
 from pathlib import Path
+from typing import Tuple
 
 from jinja2 import Environment, StrictUndefined
 
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorExperiment,
+)
 from rdagent.core.prompts import Prompts
 from rdagent.core.proposal import (
     Hypothesis,
-    Hypothesis2Task,
+    Hypothesis2Experiment,
     HypothesisGen,
+    HypothesisSet,
     Scenario,
     Trace,
 )
@@ -22,40 +27,71 @@ FactorHypothesis = Hypothesis
 class FactorHypothesisGen(HypothesisGen):
     def __init__(self, scen: Scenario):
         super().__init__(scen)
-        self.gen_context_flag = False
-        self.gen_context_dict = None
-        self.gen_json_flag = False
 
     # The following methods are scenario related so they should be implemented in the subclass
     @abstractmethod
-    def prepare_gen_context(self, trace: Trace) -> None: ...
+    def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ...
 
     @abstractmethod
-    def gen_response_to_hypothesis_list(self, response: str) -> FactorHypothesis: ...
+    def convert_response(self, response: str) -> FactorHypothesis: ...
 
     def gen(self, trace: Trace) -> FactorHypothesis:
-        assert self.gen_context_flag, "Please call prepare_gen_context before calling gen."
-        self.gen_context_flag = False  # reset the flag
+        context_dict, json_flag = self.prepare_context(trace)
 
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(prompt_dict["factor_hypothesis_gen"]["system_prompt"])
-            .render(scenario=self.scen.get_scenario_all_desc())
+            .render(
+                scenario=self.scen.get_scenario_all_desc(),
+                hypothesis_output_format=context_dict["hypothesis_output_format"],
+            )
         )
         user_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(prompt_dict["factor_hypothesis_gen"]["user_prompt"])
-            .render(self.gen_context_dict)
+            .render(
+                hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
+                RAG=context_dict["RAG"],
+            )
         )
 
-        resp = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt, system_prompt, json_mode=self.gen_json_flag
-        )
+        resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
 
-        hypothesis = self.gen_response_to_hypothesis_list(resp)
+        hypothesis = self.convert_response(resp)
 
         return hypothesis
 
 
-class FactorHypothesis2Task(Hypothesis2Task):
-    def convert(self, bs: FactorHypothesis) -> None: ...
+class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]):
+    def __init__(self) -> None:
+        super().__init__()
+
+    @abstractmethod
+    def prepare_context(self, hs: HypothesisSet) -> Tuple[dict, bool]: ...
+
+    @abstractmethod
+    def convert_response(self, response: str) -> FactorExperiment: ...
+
+    def convert(self, hs: HypothesisSet) -> FactorExperiment:
+        context, json_flag = self.prepare_context(hs)
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["factor_hypothesis2experiment"]["system_prompt"])
+            .render(
+                scenario=hs.trace.scen.get_scenario_all_desc(),
+                experiment_output_format=context["experiment_output_format"],
+            )
+        )
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["factor_hypothesis2experiment"]["user_prompt"])
+            .render(
+                hypothesis_and_feedback=context["hypothesis_and_feedback"],
+                factor_list=context["factor_list"],
+                RAG=context["RAG"],
+            )
+        )
+
+        resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
+
+        return self.convert_response(resp)

rdagent/components/idea_proposal/prompts.yaml

@@ -3,20 +3,36 @@ factor_hypothesis_gen:
     The user is trying to generate new hypothesis on the factors in data-driven research and development.
     The factors are used in a certain scenario, the scenario is as follows:
     {{ scenario }}
-    The user has made several hypothesis on this sencario and did several evaluation on them. The user will provide this information to you.
+    The user has made several hypothesis on this scenario and did several evaluation on them. The user will provide this information to you.
     To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new factors.
+    Please generate the output following the format below:
+    {{ hypothesis_output_format }}
   user_prompt: |-
-    The user has made several hypothesis on this sencario and did several evaluation on them.
+    The user has made several hypothesis on this scenario and did several evaluation on them.
     The former hypothesis and the corresponding feedbacks are as follows:
     {{ hypothesis_and_feedback }}
     To help you generate new factors, we have prepared the following information for you:
     {{ RAG }}
-    Please generate the new hypothesis based on the information above and generate the output following the format below:
-    {{ factor_output_format }}
+    Please generate the new hypothesis based on the information above.
 
-factor_hypothesis_to_tasks:
+factor_hypothesis2experiment:
   system_prompt: |-
     The user is trying to generate new factors based on the hypothesis generated in the previous step. 
     The factors are used in certain scenario, the scenario is as follows:
     {{ scenario }}
-
+    The user will use the factors generated to do some experiments. The user will provide this information to you:
+    1. The hypothesis generated in the previous steps and their corresponding feedbacks.
+    2. Former proposed factors on similar hypothesis.
+    3. Some additional information to help you generate new factors.
+    Please generate the output following the format below:
+    {{ experiment_output_format }}
+    
+  user_prompt: |-
+    The user has made several hypothesis on this scenario and did several evaluation on them.
+    The former hypothesis and the corresponding feedbacks are as follows:
+    {{ hypothesis_and_feedback }}
+    The former proposed factors on similar hypothesis are as follows:
+    {{ factor_list }}
+    To help you generate new factors, we have prepared the following information for you:
+    {{ RAG }}
+    Please generate the new factors based on the information above.

rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py

@@ -7,7 +7,7 @@ from rdagent.core.evolving_framework import EvolvableSubjects
 from rdagent.core.log import RDAgentLog
 
 
-class FactorEvolvingItem(FactorExperiment[FactorTask, FileBasedFactorImplementation], EvolvableSubjects):
+class FactorEvolvingItem(FactorExperiment, EvolvableSubjects):
     """
     Intermediate item of factor implementation.
     """

rdagent/components/task_implementation/factor_implementation/factor.py

@@ -220,4 +220,4 @@ class FileBasedFactorImplementation(FBImplementation):
         return FileBasedFactorImplementation(task, code=code, **kwargs)
 
 
-FactorExperiment = Experiment
+class FactorExperiment(Experiment[FactorTask, FileBasedFactorImplementation]): ...