refine class design and inheritance first version code (#41)

* refine class design and inheritance first version code

* fix all typos

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/app/CI/run.py

@@ -23,9 +23,9 @@ from rich.table import Table
 from rich.text import Text
 from tree_sitter import Language, Node, Parser
 
+from rdagent.core.evaluation import Evaluator
+from rdagent.core.evolving_agent import EvoAgent
 from rdagent.core.evolving_framework import (
-    Evaluator,
-    EvoAgent,
     EvolvableSubjects,
     EvolvingStrategy,
     EvoStep,

rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py

@@ -1,18 +1,18 @@
 # %%
 from dotenv import load_dotenv
 
+from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
+    FactorImplementationExperimentLoaderFromPDFfiles,
+)
 from rdagent.scenarios.qlib.factor_task_implementation import (
     COSTEERFG_QUANT_FACTOR_IMPLEMENTATION,
 )
-from rdagent.scenarios.qlib.factor_task_loader.pdf_loader import (
-    FactorImplementationTaskLoaderFromPDFfiles,
-)
 
 assert load_dotenv()
 
 
 def extract_factors_and_implement(report_file_path: str) -> None:
-    factor_tasks = FactorImplementationTaskLoaderFromPDFfiles().load(report_file_path)
+    factor_tasks = FactorImplementationExperimentLoaderFromPDFfiles().load(report_file_path)
     implementation_result = COSTEERFG_QUANT_FACTOR_IMPLEMENTATION().generate(factor_tasks)
     # Qlib to run the implementation
     return implementation_result

rdagent/app/model_implementation/eval.py

@@ -2,13 +2,19 @@ from pathlib import Path
 
 DIRNAME = Path(__file__).absolute().resolve().parent
 
-from rdagent.components.task_implementation.model_implementation.benchmark.eval import ModelImpValEval
-from rdagent.components.task_implementation.model_implementation.one_shot import ModelTaskGen
-from rdagent.components.task_implementation.model_implementation.task import  ModelImpLoader, ModelTaskLoderJson
-
+from rdagent.components.task_implementation.model_implementation.benchmark.eval import (
+    ModelImpValEval,
+)
+from rdagent.components.task_implementation.model_implementation.one_shot import (
+    ModelTaskGen,
+)
+from rdagent.components.task_implementation.model_implementation.task import (
+    ModelImpLoader,
+    ModelTaskLoaderJson,
+)
 
 bench_folder = DIRNAME.parent.parent / "components" / "task_implementation" / "model_implementation" / "benchmark"
-mtl = ModelTaskLoderJson(str(bench_folder / "model_dict.json"))
+mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
 
 task_l = mtl.load()
 

rdagent/app/model_proposal/run.py

@@ -5,32 +5,37 @@ TODO: move the following code to a new class: Model_RD_Agent
 
 # import_from
 from rdagent.app.model_proposal.conf import MODEL_PROP_SETTING
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.proposal import Belief2Task, BeliefSet, Imp2Feedback, Trace
+from rdagent.core.proposal import (
+    Experiment2Feedback,
+    Hypothesis2Experiment,
+    HypothesisSet,
+    Trace,
+)
+from rdagent.core.task_generator import TaskGenerator
 
 # load_from_cls_uri
 
 
 scen = load_from_cls_uri(MODEL_PROP_SETTING.scen)()
 
-belief_gen = load_from_cls_uri(MODEL_PROP_SETTING.belief_gen)(scen)
+hypothesis_gen = load_from_cls_uri(MODEL_PROP_SETTING.hypothesis_gen)(scen)
 
-belief2task: Belief2Task = load_from_cls_uri(MODEL_PROP_SETTING.belief2task)()
+hypothesis2task: Hypothesis2Experiment = load_from_cls_uri(MODEL_PROP_SETTING.hypothesis2task)()
 
 task_gen: TaskGenerator = load_from_cls_uri(MODEL_PROP_SETTING.task_gen)(scen)  # for implementation
 
-imp2feedback: Imp2Feedback = load_from_cls_uri(MODEL_PROP_SETTING.imp2feedback)(scen)  # for implementation
+imp2feedback: Experiment2Feedback = load_from_cls_uri(MODEL_PROP_SETTING.imp2feedback)(scen)  # for implementation
 
 
 iter_n = MODEL_PROP_SETTING.iter_n
 
 trace = Trace()
 
-belief_set = BeliefSet()
+hypothesis_set = HypothesisSet()
 for _ in range(iter_n):
-    belief = belief_gen.gen(trace)
-    task = belief2task.convert(belief)
+    hypothesis = hypothesis_gen.gen(trace)
+    task = hypothesis2task.convert(hypothesis)
     imp = task_gen.gen(task)
     imp.execute()
     feedback = imp2feedback.summarize(imp)
-    trace.hist.append((belief, feedback))
+    trace.hist.append((hypothesis, feedback))

rdagent/components/benchmark/eval_method.py

@@ -4,6 +4,9 @@ from typing import List, Tuple, Union
 
 from tqdm import tqdm
 
+from rdagent.components.task_implementation.factor_implementation.config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
 from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
     FactorImplementationCorrelationEvaluator,
     FactorImplementationEvaluator,
@@ -14,18 +17,25 @@ from rdagent.components.task_implementation.factor_implementation.evolving.evalu
     FactorImplementationSingleColumnEvaluator,
     FactorImplementationValuesEvaluator,
 )
-from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
+from rdagent.components.task_implementation.factor_implementation.factor import (
     FileBasedFactorImplementation,
 )
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
-)
 from rdagent.core.exception import ImplementRunException
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.task import TaskImplementation, TestCase
+from rdagent.core.experiment import Implementation, Task
+from rdagent.core.task_generator import TaskGenerator
 from rdagent.core.utils import multiprocessing_wrapper
 
 
+class TestCase:
+    def __init__(
+        self,
+        target_task: list[Task] = [],
+        ground_truth: list[Implementation] = [],
+    ):
+        self.ground_truth = ground_truth
+        self.target_task = target_task
+
+
 class BaseEval:
     """
     The benchmark benchmark evaluation.
@@ -56,7 +66,7 @@ class BaseEval:
         self,
         path: Union[Path, str],
         **kwargs,
-    ) -> List[TaskImplementation]:
+    ) -> List[Implementation]:
         path = Path(path)
         fi_l = []
         for tc in self.test_cases:
@@ -69,8 +79,8 @@ class BaseEval:
 
     def eval_case(
         self,
-        case_gt: TaskImplementation,
-        case_gen: TaskImplementation,
+        case_gt: Implementation,
+        case_gen: Implementation,
     ) -> List[Union[Tuple[FactorImplementationEvaluator, object], Exception]]:
         """Parameters
         ----------
@@ -137,11 +147,11 @@ class FactorImplementEval(BaseEval):
                 print("Manually interrupted the evaluation. Saving existing results")
                 break
 
-            if len(gen_factor_l.corresponding_implementations) != len(self.test_cases.ground_truth):
+            if len(gen_factor_l.sub_implementations) != len(self.test_cases.ground_truth):
                 raise ValueError(
                     "The number of cases to eval should be equal to the number of test cases.",
                 )
-            gen_factor_l_all_rounds.extend(gen_factor_l.corresponding_implementations)
+            gen_factor_l_all_rounds.extend(gen_factor_l.sub_implementations)
             test_cases_all_rounds.extend(self.test_cases.ground_truth)
 
         eval_res_list = multiprocessing_wrapper(

rdagent/components/idea_proposal/factor_proposal.py

@@ -0,0 +1,61 @@
+from abc import abstractmethod
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import (
+    Hypothesis,
+    Hypothesis2Task,
+    HypothesisGen,
+    Scenario,
+    Trace,
+)
+from rdagent.oai.llm_utils import APIBackend
+
+prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+
+
+FactorHypothesis = Hypothesis
+
+
+class FactorHypothesisGen(HypothesisGen):
+    def __init__(self, scen: Scenario):
+        super().__init__(scen)
+        self.gen_context_flag = False
+        self.gen_context_dict = None
+        self.gen_json_flag = False
+
+    # The following methods are scenario related so they should be implemented in the subclass
+    @abstractmethod
+    def prepare_gen_context(self, trace: Trace) -> None: ...
+
+    @abstractmethod
+    def gen_response_to_hypothesis_list(self, response: str) -> FactorHypothesis: ...
+
+    def gen(self, trace: Trace) -> FactorHypothesis:
+        assert self.gen_context_flag, "Please call prepare_gen_context before calling gen."
+        self.gen_context_flag = False  # reset the flag
+
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["factor_hypothesis_gen"]["system_prompt"])
+            .render(scenario=self.scen.get_scenario_all_desc())
+        )
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["factor_hypothesis_gen"]["user_prompt"])
+            .render(self.gen_context_dict)
+        )
+
+        resp = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt, system_prompt, json_mode=self.gen_json_flag
+        )
+
+        hypothesis = self.gen_response_to_hypothesis_list(resp)
+
+        return hypothesis
+
+
+class FactorHypothesis2Task(Hypothesis2Task):
+    def convert(self, bs: FactorHypothesis) -> None: ...

rdagent/components/idea_proposal/prompts.yaml

@@ -0,0 +1,22 @@
+factor_hypothesis_gen:
+  system_prompt: |-
+    The user is trying to generate new hypothesis on the factors in data-driven research and development.
+    The factors are used in a certain scenario, the scenario is as follows:
+    {{ scenario }}
+    The user has made several hypothesis on this sencario and did several evaluation on them. The user will provide this information to you.
+    To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new factors.
+  user_prompt: |-
+    The user has made several hypothesis on this sencario and did several evaluation on them.
+    The former hypothesis and the corresponding feedbacks are as follows:
+    {{ hypothesis_and_feedback }}
+    To help you generate new factors, we have prepared the following information for you:
+    {{ RAG }}
+    Please generate the new hypothesis based on the information above and generate the output following the format below:
+    {{ factor_output_format }}
+
+factor_hypothesis_to_tasks:
+  system_prompt: |-
+    The user is trying to generate new factors based on the hypothesis generated in the previous step. 
+    The factors are used in certain scenario, the scenario is as follows:
+    {{ scenario }}
+

rdagent/components/loader/experiment_loader.py

@@ -0,0 +1,12 @@
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorExperiment,
+)
+from rdagent.core.experiment import Loader
+
+
+class FactorExperimentLoader(Loader[FactorExperiment]):
+    pass
+
+
+class ModelExperimentLoader(Loader[FactorExperiment]):
+    pass

rdagent/components/loader/task_loader.py

@@ -0,0 +1,12 @@
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorTask,
+)
+from rdagent.core.experiment import Loader
+
+
+class FactorTaskLoader(Loader[FactorTask]):
+    pass
+
+
+class ModelTaskLoader(Loader[FactorTask]):
+    pass

rdagent/components/task_implementation/factor_implementation/CoSTEER.py

@@ -2,31 +2,33 @@ import pickle
 from pathlib import Path
 from typing import List
 
+from rdagent.components.task_implementation.factor_implementation.config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
 from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
     FactorImplementationEvaluatorV1,
     FactorImplementationsMultiEvaluator,
 )
+from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
+    FactorEvolvingItem,
+)
 from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
     FactorEvolvingStrategyWithGraph,
 )
-from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
-    FactorEvovlingItem,
-    FactorImplementTask,
-)
 from rdagent.components.task_implementation.factor_implementation.evolving.knowledge_management import (
     FactorImplementationGraphKnowledgeBase,
     FactorImplementationGraphRAGStrategy,
     FactorImplementationKnowledgeBaseV1,
 )
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorExperiment,
 )
 from rdagent.core.evolving_agent import RAGEvoAgent
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.task import TaskImplementation
+from rdagent.core.experiment import Experiment
+from rdagent.core.task_generator import TaskGenerator
 
 
-class CoSTEERFG(TaskGenerator):
+class CoSTEERFG(TaskGenerator[FactorExperiment]):
     def __init__(
         self,
         with_knowledge: bool = True,
@@ -74,7 +76,7 @@ class CoSTEERFG(TaskGenerator):
             )
         return factor_knowledge_base
 
-    def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]:
+    def generate(self, exp: FactorExperiment) -> FactorExperiment:
         # init knowledge base
         factor_knowledge_base = self.load_or_init_knowledge_base(
             former_knowledge_base_path=self.knowledge_base_path,
@@ -83,8 +85,8 @@ class CoSTEERFG(TaskGenerator):
         # init rag method
         self.rag = FactorImplementationGraphRAGStrategy(factor_knowledge_base)
 
-        # init indermediate items
-        factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks)
+        # init intermediate items
+        factor_implementations = FactorEvolvingItem(sub_tasks=exp.sub_tasks)
 
         self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
 
@@ -100,5 +102,5 @@ class CoSTEERFG(TaskGenerator):
         if self.new_knowledge_base_path is not None:
             pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
         self.knowledge_base = factor_knowledge_base
-        self.latest_factor_implementations = tasks
+        self.latest_factor_implementations = exp.sub_tasks
         return factor_implementations

rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py

@@ -5,21 +5,23 @@ from pathlib import Path
 from typing import List, Tuple
 
 import pandas as pd
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
 
-from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
-    FactorEvovlingItem,
-    FactorImplementTask,
-)
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
+from rdagent.components.task_implementation.factor_implementation.config import (
     FACTOR_IMPLEMENT_SETTINGS,
 )
+from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
+    FactorEvolvingItem,
+)
+from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
+    FactorTask,
+)
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evaluation import Evaluator
 from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
+from rdagent.core.experiment import Implementation
 from rdagent.core.log import RDAgentLog
 from rdagent.core.prompts import Prompts
-from rdagent.core.task import TaskImplementation
 from rdagent.core.utils import multiprocessing_wrapper
 from rdagent.oai.llm_utils import APIBackend
 
@@ -33,8 +35,8 @@ class FactorImplementationEvaluator(Evaluator):
     @abstractmethod
     def evaluate(
         self,
-        gt: TaskImplementation,
-        gen: TaskImplementation,
+        gt: Implementation,
+        gen: Implementation,
     ) -> Tuple[str, object]:
         """You can get the dataframe by
 
@@ -52,7 +54,7 @@ class FactorImplementationEvaluator(Evaluator):
         """
         raise NotImplementedError("Please implement the `evaluator` method")
 
-    def _get_df(self, gt: TaskImplementation, gen: TaskImplementation):
+    def _get_df(self, gt: Implementation, gen: Implementation):
         _, gt_df = gt.execute()
         _, gen_df = gen.execute()
         if isinstance(gen_df, pd.Series):
@@ -65,11 +67,11 @@ class FactorImplementationEvaluator(Evaluator):
 class FactorImplementationCodeEvaluator(Evaluator):
     def evaluate(
         self,
-        target_task: FactorImplementTask,
-        implementation: TaskImplementation,
+        target_task: FactorTask,
+        implementation: Implementation,
         execution_feedback: str,
         factor_value_feedback: str = "",
-        gt_implementation: TaskImplementation = None,
+        gt_implementation: Implementation = None,
         **kwargs,
     ):
         factor_information = target_task.get_factor_information()
@@ -78,14 +80,18 @@ class FactorImplementationCodeEvaluator(Evaluator):
         system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"]
 
         execution_feedback_to_render = execution_feedback
-        user_prompt = Template(
-            evaluate_prompts["evaluator_code_feedback_v1_user"],
-        ).render(
-            factor_information=factor_information,
-            code=code,
-            execution_feedback=execution_feedback_to_render,
-            factor_value_feedback=factor_value_feedback,
-            gt_code=gt_implementation.code if gt_implementation else None,
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(
+                evaluate_prompts["evaluator_code_feedback_v1_user"],
+            )
+            .render(
+                factor_information=factor_information,
+                code=code,
+                execution_feedback=execution_feedback_to_render,
+                factor_value_feedback=factor_value_feedback,
+                gt_code=gt_implementation.code if gt_implementation else None,
+            )
         )
         while (
             APIBackend().build_messages_and_calculate_token(
@@ -96,14 +102,18 @@ class FactorImplementationCodeEvaluator(Evaluator):
             > RD_AGENT_SETTINGS.chat_token_limit
         ):
             execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
-            user_prompt = Template(
-                evaluate_prompts["evaluator_code_feedback_v1_user"],
-            ).render(
-                factor_information=factor_information,
-                code=code,
-                execution_feedback=execution_feedback_to_render,
-                factor_value_feedback=factor_value_feedback,
-                gt_code=gt_implementation.code if gt_implementation else None,
+            user_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    evaluate_prompts["evaluator_code_feedback_v1_user"],
+                )
+                .render(
+                    factor_information=factor_information,
+                    code=code,
+                    execution_feedback=execution_feedback_to_render,
+                    factor_value_feedback=factor_value_feedback,
+                    gt_code=gt_implementation.code if gt_implementation else None,
+                )
             )
         critic_response = APIBackend().build_messages_and_create_chat_completion(
             user_prompt=user_prompt,
@@ -117,8 +127,8 @@ class FactorImplementationCodeEvaluator(Evaluator):
 class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
     def evaluate(
         self,
-        gt: TaskImplementation,
-        gen: TaskImplementation,
+        gt: Implementation,
+        gen: Implementation,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt, gen)
 
@@ -143,8 +153,8 @@ class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
 class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
     def evaluate(
         self,
-        gt: TaskImplementation,
-        gen: TaskImplementation,
+        gt: Implementation,
+        gen: Implementation,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt, gen)
         idx_name_right = gen_df.index.names == ("datetime", "instrument")
@@ -166,8 +176,8 @@ class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
 class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
     def evaluate(
         self,
-        gt: TaskImplementation,
-        gen: TaskImplementation,
+        gt: Implementation,
+        gen: Implementation,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt, gen)
 
@@ -186,8 +196,8 @@ class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
 class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
     def evaluate(
         self,
-        gt: TaskImplementation,
-        gen: TaskImplementation,
+        gt: Implementation,
+        gen: Implementation,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt, gen)
 
@@ -206,8 +216,8 @@ class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
 class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
     def evaluate(
         self,
-        gt: TaskImplementation,
-        gen: TaskImplementation,
+        gt: Implementation,
+        gen: Implementation,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt, gen)
 
@@ -226,8 +236,8 @@ class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
 class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
     def evaluate(
         self,
-        gt: TaskImplementation,
-        gen: TaskImplementation,
+        gt: Implementation,
+        gen: Implementation,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt, gen)
 
@@ -259,8 +269,8 @@ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
 
     def evaluate(
         self,
-        gt: TaskImplementation,
-        gen: TaskImplementation,
+        gt: Implementation,
+        gen: Implementation,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt, gen)
 
@@ -293,7 +303,7 @@ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
 
 
 class FactorImplementationValEvaluator(FactorImplementationEvaluator):
-    def evaluate(self, gt: TaskImplementation, gen: TaskImplementation):
+    def evaluate(self, gt: Implementation, gen: Implementation):
         _, gt_df = gt.execute()
         _, gen_df = gen.execute()
         # FIXME: refactor the two classes
@@ -473,7 +483,7 @@ def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: i
 class FactorImplementationFinalDecisionEvaluator(Evaluator):
     def evaluate(
         self,
-        target_task: FactorImplementTask,
+        target_task: FactorTask,
         execution_feedback: str,
         value_feedback: str,
         code_feedback: str,
@@ -483,17 +493,21 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
             "evaluator_final_decision_v1_system"
         ]
         execution_feedback_to_render = execution_feedback
-        user_prompt = Template(
-            evaluate_prompts["evaluator_final_decision_v1_user"],
-        ).render(
-            factor_information=target_task.get_factor_information(),
-            execution_feedback=execution_feedback_to_render,
-            code_feedback=code_feedback,
-            factor_value_feedback=(
-                value_feedback
-                if value_feedback is not None
-                else "No Ground Truth Value provided, so no evaluation on value is performed."
-            ),
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(
+                evaluate_prompts["evaluator_final_decision_v1_user"],
+            )
+            .render(
+                factor_information=target_task.get_factor_information(),
+                execution_feedback=execution_feedback_to_render,
+                code_feedback=code_feedback,
+                factor_value_feedback=(
+                    value_feedback
+                    if value_feedback is not None
+                    else "No Ground Truth Value provided, so no evaluation on value is performed."
+                ),
+            )
         )
         while (
             APIBackend().build_messages_and_calculate_token(
@@ -504,17 +518,21 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
             > RD_AGENT_SETTINGS.chat_token_limit
         ):
             execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
-            user_prompt = Template(
-                evaluate_prompts["evaluator_final_decision_v1_user"],
-            ).render(
-                factor_information=target_task.get_factor_information(),
-                execution_feedback=execution_feedback_to_render,
-                code_feedback=code_feedback,
-                factor_value_feedback=(
-                    value_feedback
-                    if value_feedback is not None
-                    else "No Ground Truth Value provided, so no evaluation on value is performed."
-                ),
+            user_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    evaluate_prompts["evaluator_final_decision_v1_user"],
+                )
+                .render(
+                    factor_information=target_task.get_factor_information(),
+                    execution_feedback=execution_feedback_to_render,
+                    code_feedback=code_feedback,
+                    factor_value_feedback=(
+                        value_feedback
+                        if value_feedback is not None
+                        else "No Ground Truth Value provided, so no evaluation on value is performed."
+                    ),
+                )
             )
 
         final_evaluation_dict = json.loads(
@@ -584,9 +602,9 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
 
     def evaluate(
         self,
-        target_task: FactorImplementTask,
-        implementation: TaskImplementation,
-        gt_implementation: TaskImplementation = None,
+        target_task: FactorTask,
+        implementation: Implementation,
+        gt_implementation: Implementation = None,
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> FactorImplementationSingleFeedback:
@@ -681,23 +699,23 @@ class FactorImplementationsMultiEvaluator(Evaluator):
 
     def evaluate(
         self,
-        evo: FactorEvovlingItem,
+        evo: FactorEvolvingItem,
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> FactorImplementationsMultiFeedback:
         multi_implementation_feedback = FactorImplementationsMultiFeedback()
 
-        # for index in range(len(evo.target_factor_tasks)):
-        #     corresponding_implementation = evo.corresponding_implementations[index]
+        # for index in range(len(evo.sub_tasks)):
+        #     corresponding_implementation = evo.sub_implementations[index]
         #     corresponding_gt_implementation = (
-        #         evo.corresponding_gt_implementations[index]
-        #         if evo.corresponding_gt_implementations is not None
+        #         evo.sub_gt_implementations[index]
+        #         if evo.sub_gt_implementations is not None
         #         else None
         #     )
 
         #     multi_implementation_feedback.append(
         #         self.single_factor_implementation_evaluator.evaluate(
-        #             target_task=evo.target_factor_tasks[index],
+        #             target_task=evo.sub_tasks[index],
         #             implementation=corresponding_implementation,
         #             gt_implementation=corresponding_gt_implementation,
         #             queried_knowledge=queried_knowledge,
@@ -705,18 +723,16 @@ class FactorImplementationsMultiEvaluator(Evaluator):
         #     )
 
         calls = []
-        for index in range(len(evo.target_factor_tasks)):
-            corresponding_implementation = evo.corresponding_implementations[index]
+        for index in range(len(evo.sub_tasks)):
+            corresponding_implementation = evo.sub_implementations[index]
             corresponding_gt_implementation = (
-                evo.corresponding_gt_implementations[index]
-                if evo.corresponding_gt_implementations is not None
-                else None
+                evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
             )
             calls.append(
                 (
                     self.single_factor_implementation_evaluator.evaluate,
                     (
-                        evo.target_factor_tasks[index],
+                        evo.sub_tasks[index],
                         corresponding_implementation,
                         corresponding_gt_implementation,
                         queried_knowledge,

rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py

@@ -0,0 +1,30 @@
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorExperiment,
+    FactorTask,
+    FileBasedFactorImplementation,
+)
+from rdagent.core.evolving_framework import EvolvableSubjects
+from rdagent.core.log import RDAgentLog
+
+
+class FactorEvolvingItem(FactorExperiment[FactorTask, FileBasedFactorImplementation], EvolvableSubjects):
+    """
+    Intermediate item of factor implementation.
+    """
+
+    def __init__(
+        self,
+        sub_tasks: list[FactorTask],
+        sub_gt_implementations: list[FileBasedFactorImplementation] = None,
+    ):
+        FactorExperiment.__init__(self, sub_tasks=sub_tasks)
+        self.corresponding_selection: list = None
+        if sub_gt_implementations is not None and len(
+            sub_gt_implementations,
+        ) != len(self.sub_tasks):
+            self.sub_gt_implementations = None
+            RDAgentLog().warning(
+                "The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
+            )
+        else:
+            self.sub_gt_implementations = sub_gt_implementations

rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py

@@ -6,27 +6,29 @@ from copy import deepcopy
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
 
-from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
-    FactorEvovlingItem,
-    FactorImplementTask,
-    FileBasedFactorImplementation,
+from rdagent.components.task_implementation.factor_implementation.config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
+from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
+    FactorEvolvingItem,
 )
 from rdagent.components.task_implementation.factor_implementation.evolving.scheduler import (
     LLMSelect,
     RandomSelect,
 )
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorTask,
+    FileBasedFactorImplementation,
 )
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import (
+from rdagent.components.task_implementation.factor_implementation.utils import (
     get_data_folder_intro,
 )
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
+from rdagent.core.experiment import Implementation
 from rdagent.core.prompts import Prompts
-from rdagent.core.task import TaskImplementation
 from rdagent.core.utils import multiprocessing_wrapper
 from rdagent.oai.llm_utils import APIBackend
 
@@ -43,27 +45,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
     @abstractmethod
     def implement_one_factor(
         self,
-        target_task: FactorImplementTask,
+        target_task: FactorTask,
         queried_knowledge: QueriedKnowledge = None,
-    ) -> TaskImplementation:
+    ) -> Implementation:
         raise NotImplementedError
 
     def evolve(
         self,
         *,
-        evo: FactorEvovlingItem,
+        evo: FactorEvolvingItem,
         queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
         **kwargs,
-    ) -> FactorEvovlingItem:
+    ) -> FactorEvolvingItem:
         self.num_loop += 1
         new_evo = deepcopy(evo)
 
         # 1.找出需要evolve的factor
         to_be_finished_task_index = []
-        for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
+        for index, target_factor_task in enumerate(new_evo.sub_tasks):
             target_factor_task_desc = target_factor_task.get_factor_information()
             if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
-                new_evo.corresponding_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
+                new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
                     target_factor_task_desc
                 ].implementation
             elif (
@@ -95,18 +97,18 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 
         result = multiprocessing_wrapper(
             [
-                (self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge))
+                (self.implement_one_factor, (new_evo.sub_tasks[target_index], queried_knowledge))
                 for target_index in to_be_finished_task_index
             ],
             n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
         )
 
         for index, target_index in enumerate(to_be_finished_task_index):
-            new_evo.corresponding_implementations[target_index] = result[index]
+            new_evo.sub_implementations[target_index] = result[index]
 
         # for target_index in to_be_finished_task_index:
-        #     new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
-        #         new_evo.target_factor_tasks[target_index], queried_knowledge
+        #     new_evo.sub_implementations[target_index] = self.implement_one_factor(
+        #         new_evo.sub_tasks[target_index], queried_knowledge
         #     )
 
         new_evo.corresponding_selection = to_be_finished_task_index
@@ -117,9 +119,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
     def implement_one_factor(
         self,
-        target_task: FactorImplementTask,
+        target_task: FactorTask,
         queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
-    ) -> TaskImplementation:
+    ) -> Implementation:
         factor_information_str = target_task.get_factor_information()
 
         if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -140,11 +142,15 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
 
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
-            system_prompt = Template(
-                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
-            ).render(
-                data_info=get_data_folder_intro(),
-                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+            system_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    implement_prompts["evolving_strategy_factor_implementation_v1_system"],
+                )
+                .render(
+                    data_info=get_data_folder_intro(),
+                    queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+                )
             )
             session = APIBackend(use_chat_cache=False).build_chat_session(
                 session_system_prompt=system_prompt,
@@ -153,7 +159,8 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
             queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
             while True:
                 user_prompt = (
-                    Template(
+                    Environment(undefined=StrictUndefined)
+                    .from_string(
                         implement_prompts["evolving_strategy_factor_implementation_v1_user"],
                     )
                     .render(
@@ -196,9 +203,9 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
 
     def implement_one_factor(
         self,
-        target_task: FactorImplementTask,
+        target_task: FactorTask,
         queried_knowledge,
-    ) -> TaskImplementation:
+    ) -> Implementation:
         error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
         # 1. 提取因子的背景信息
         target_factor_task_information = target_task.get_factor_information()
@@ -231,11 +238,15 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
 
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
-            system_prompt = Template(
-                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
-            ).render(
-                data_info=get_data_folder_intro(),
-                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+            system_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    implement_prompts["evolving_strategy_factor_implementation_v1_system"],
+                )
+                .render(
+                    data_info=get_data_folder_intro(),
+                    queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+                )
             )
 
             session = APIBackend(use_chat_cache=False).build_chat_session(
@@ -254,7 +265,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     and len(queried_former_failed_knowledge_to_render) != 0
                 ):
                     error_summary_system_prompt = (
-                        Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
+                        Environment(undefined=StrictUndefined)
+                        .from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
                         .render(
                             factor_information_str=target_factor_task_information,
                             code_and_feedback=queried_former_failed_knowledge_to_render[
@@ -268,7 +280,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     )
                     while True:
                         error_summary_user_prompt = (
-                            Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
+                            Environment(undefined=StrictUndefined)
+                            .from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
                             .render(
                                 queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
                             )
@@ -287,7 +300,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     )
                 # 构建user_prompt。开始写代码
                 user_prompt = (
-                    Template(
+                    Environment(undefined=StrictUndefined)
+                    .from_string(
                         implement_prompts["evolving_strategy_factor_implementation_v2_user"],
                     )
                     .render(

rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py

@@ -8,20 +8,20 @@ from itertools import combinations
 from pathlib import Path
 from typing import Union
 
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
 
 from rdagent.components.knowledge_management.graph import (
     UndirectedGraph,
     UndirectedNode,
 )
+from rdagent.components.task_implementation.factor_implementation.config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
 from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
     FactorImplementationSingleFeedback,
 )
 from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
-    FactorImplementTask,
-)
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
+    FactorTask,
 )
 from rdagent.core.evolving_framework import (
     EvolvableSubjects,
@@ -31,9 +31,9 @@ from rdagent.core.evolving_framework import (
     QueriedKnowledge,
     RAGStrategy,
 )
+from rdagent.core.experiment import Implementation
 from rdagent.core.log import RDAgentLog
 from rdagent.core.prompts import Prompts
-from rdagent.core.task import TaskImplementation
 from rdagent.oai.llm_utils import (
     APIBackend,
     calculate_embedding_distance_between_str_list,
@@ -43,8 +43,8 @@ from rdagent.oai.llm_utils import (
 class FactorImplementationKnowledge(Knowledge):
     def __init__(
         self,
-        target_task: FactorImplementTask,
-        implementation: TaskImplementation,
+        target_task: FactorTask,
+        implementation: Implementation,
         feedback: FactorImplementationSingleFeedback,
     ) -> None:
         """
@@ -116,10 +116,10 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
                 evo_step = evolving_trace[trace_index]
                 implementations = evo_step.evolvable_subjects
                 feedback = evo_step.feedback
-                for task_index in range(len(implementations.target_factor_tasks)):
-                    target_task = implementations.target_factor_tasks[task_index]
+                for task_index in range(len(implementations.sub_tasks)):
+                    target_task = implementations.sub_tasks[task_index]
                     target_task_information = target_task.get_factor_information()
-                    implementation = implementations.corresponding_implementations[task_index]
+                    implementation = implementations.sub_implementations[task_index]
                     single_feedback = feedback[task_index]
                     if single_feedback is None:
                         continue
@@ -150,12 +150,12 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
         fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
 
         queried_knowledge = FactorImplementationQueriedKnowledgeV1()
-        for target_factor_task in evo.target_factor_tasks:
+        for target_factor_task in evo.sub_tasks:
             target_factor_task_information = target_factor_task.get_factor_information()
             if target_factor_task_information in self.knowledgebase.success_task_info_set:
-                queried_knowledge.success_task_to_knowledge_dict[
-                    target_factor_task_information
-                ] = self.knowledgebase.implementation_trace[target_factor_task_information][-1]
+                queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
+                    self.knowledgebase.implementation_trace[target_factor_task_information][-1]
+                )
             elif (
                 len(
                     self.knowledgebase.implementation_trace.setdefault(
@@ -167,14 +167,12 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
             ):
                 queried_knowledge.failed_task_info_set.add(target_factor_task_information)
             else:
-                queried_knowledge.working_task_to_former_failed_knowledge_dict[
-                    target_factor_task_information
-                ] = self.knowledgebase.implementation_trace.setdefault(
-                    target_factor_task_information,
-                    [],
-                )[
-                    -v1_query_former_trace_limit:
-                ]
+                queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
+                    self.knowledgebase.implementation_trace.setdefault(
+                        target_factor_task_information,
+                        [],
+                    )[-v1_query_former_trace_limit:]
+                )
 
                 knowledge_base_success_task_list = list(
                     self.knowledgebase.success_task_info_set,
@@ -195,9 +193,9 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
                     )[-1]
                     for index in similar_indexes
                 ]
-                queried_knowledge.working_task_to_similar_successful_knowledge_dict[
-                    target_factor_task_information
-                ] = similar_successful_knowledge
+                queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = (
+                    similar_successful_knowledge
+                )
         return queried_knowledge
 
 
@@ -236,11 +234,11 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                 evo_step = evolving_trace[trace_index]
                 implementations = evo_step.evolvable_subjects
                 feedback = evo_step.feedback
-                for task_index in range(len(implementations.target_factor_tasks)):
+                for task_index in range(len(implementations.sub_tasks)):
                     single_feedback = feedback[task_index]
-                    target_task = implementations.target_factor_tasks[task_index]
+                    target_task = implementations.sub_tasks[task_index]
                     target_task_information = target_task.get_factor_information()
-                    implementation = implementations.corresponding_implementations[task_index]
+                    implementation = implementations.sub_implementations[task_index]
                     single_feedback = feedback[task_index]
                     if single_feedback is None:
                         continue
@@ -323,8 +321,12 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
         all_component_content = ""
         for _, component_node in enumerate(all_component_nodes):
             all_component_content += f"{component_node.content}, \n"
-        analyze_component_system_prompt = Template(self.prompt["analyze_component_prompt_v1_system"]).render(
-            all_component_content=all_component_content,
+        analyze_component_system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(self.prompt["analyze_component_prompt_v1_system"])
+            .render(
+                all_component_content=all_component_content,
+            )
         )
 
         analyze_component_user_prompt = target_factor_task_information
@@ -396,7 +398,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
         """
         fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
 
-        for target_factor_task in evo.target_factor_tasks:
+        for target_factor_task in evo.sub_tasks:
             target_factor_task_information = target_factor_task.get_factor_information()
             if (
                 target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
@@ -427,9 +429,9 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                     else:
                         current_index += 1
 
-                factor_implementation_queried_graph_knowledge.former_traces[
-                    target_factor_task_information
-                ] = former_trace_knowledge[-v2_query_former_trace_limit:]
+                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
+                    former_trace_knowledge[-v2_query_former_trace_limit:]
+                )
             else:
                 factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
 
@@ -443,7 +445,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
         knowledge_sampler: float = 1.0,
     ) -> QueriedKnowledge | None:
         # queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge()
-        for target_factor_task in evo.target_factor_tasks:
+        for target_factor_task in evo.sub_tasks:
             target_factor_task_information = target_factor_task.get_factor_information()
             if (
                 target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
@@ -583,7 +585,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
         knowledge_sampler: float = 1.0,
     ) -> QueriedKnowledge | None:
         # queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge()
-        for task_index, target_factor_task in enumerate(evo.target_factor_tasks):
+        for task_index, target_factor_task in enumerate(evo.sub_tasks):
             target_factor_task_information = target_factor_task.get_factor_information()
             factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
             if (

rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py

@@ -1,12 +1,12 @@
 import json
 from pathlib import Path
 
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
 
-from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
-    FactorEvovlingItem,
+from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
+    FactorEvolvingItem,
 )
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import (
+from rdagent.components.task_implementation.factor_implementation.utils import (
     get_data_folder_intro,
 )
 from rdagent.core.conf import RD_AGENT_SETTINGS
@@ -29,18 +29,22 @@ def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
     return to_be_finished_task_index
 
 
-def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvovlingItem, former_trace):
+def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvolvingItem, former_trace):
     tasks = []
     for i in to_be_finished_task_index:
         # find corresponding former trace for each task
-        target_factor_task_information = evo.target_factor_tasks[i].get_factor_information()
+        target_factor_task_information = evo.sub_tasks[i].get_factor_information()
         if target_factor_task_information in former_trace:
-            tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information]))
+            tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information]))
 
-    system_prompt = Template(
-        scheduler_prompts["select_implementable_factor_system"],
-    ).render(
-        data_info=get_data_folder_intro(),
+    system_prompt = (
+        Environment(undefined=StrictUndefined)
+        .from_string(
+            scheduler_prompts["select_implementable_factor_system"],
+        )
+        .render(
+            data_info=get_data_folder_intro(),
+        )
     )
 
     session = APIBackend(use_chat_cache=False).build_chat_session(
@@ -48,11 +52,15 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:
     )
 
     while True:
-        user_prompt = Template(
-            scheduler_prompts["select_implementable_factor_user"],
-        ).render(
-            factor_num=implementation_factors_per_round,
-            target_factor_tasks=tasks,
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(
+                scheduler_prompts["select_implementable_factor_user"],
+            )
+            .render(
+                factor_num=implementation_factors_per_round,
+                sub_tasks=tasks,
+            )
         )
         if (
             session.build_chat_completion_message_and_calculate_token(

rdagent/components/task_implementation/factor_implementation/factor.py

@@ -9,27 +9,21 @@ from typing import Tuple, Union
 import pandas as pd
 from filelock import FileLock
 
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
+from rdagent.components.task_implementation.factor_implementation.config import (
     FACTOR_IMPLEMENT_SETTINGS,
 )
-from rdagent.core.evolving_framework import EvolvableSubjects
 from rdagent.core.exception import (
     CodeFormatException,
     NoOutputException,
     RuntimeErrorException,
 )
+from rdagent.core.experiment import Experiment, FBImplementation, Task
 from rdagent.core.log import RDAgentLog
-from rdagent.core.task import (
-    BaseTask,
-    FBTaskImplementation,
-    TaskImplementation,
-    TestCase,
-)
 from rdagent.oai.llm_utils import md5_hash
 
 
-class FactorImplementTask(BaseTask):
-    # TODO:  generalized the attributes into the BaseTask
+class FactorTask(Task):
+    # TODO:  generalized the attributes into the Task
     # - factor_* -> *
     def __init__(
         self,
@@ -53,38 +47,13 @@ variables: {str(self.variables)}"""
 
     @staticmethod
     def from_dict(dict):
-        return FactorImplementTask(**dict)
+        return FactorTask(**dict)
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}[{self.factor_name}]>"
 
 
-class FactorEvovlingItem(EvolvableSubjects):
-    """
-    Intermediate item of factor implementation.
-    """
-
-    def __init__(
-        self,
-        target_factor_tasks: list[FactorImplementTask],
-        corresponding_gt_implementations: list[TaskImplementation] = None,
-    ):
-        super().__init__()
-        self.target_factor_tasks = target_factor_tasks
-        self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks]
-        self.corresponding_selection: list = None
-        if corresponding_gt_implementations is not None and len(
-            corresponding_gt_implementations,
-        ) != len(target_factor_tasks):
-            self.corresponding_gt_implementations = None
-            RDAgentLog().warning(
-                "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
-            )
-        else:
-            self.corresponding_gt_implementations = corresponding_gt_implementations
-
-
-class FileBasedFactorImplementation(FBTaskImplementation):
+class FileBasedFactorImplementation(FBImplementation):
     """
     This class is used to implement a factor by writing the code to a file.
     Input data and output factor value are also written to files.
@@ -100,7 +69,7 @@ class FileBasedFactorImplementation(FBTaskImplementation):
 
     def __init__(
         self,
-        target_task: FactorImplementTask,
+        target_task: FactorTask,
         code,
         executed_factor_value_dataframe=None,
         raise_exception=False,
@@ -243,9 +212,12 @@ class FileBasedFactorImplementation(FBTaskImplementation):
         return self.__str__()
 
     @staticmethod
-    def from_folder(task: FactorImplementTask, path: Union[str, Path], **kwargs):
+    def from_folder(task: FactorTask, path: Union[str, Path], **kwargs):
         path = Path(path)
         factor_path = (path / task.factor_name).with_suffix(".py")
         with factor_path.open("r") as f:
             code = f.read()
         return FileBasedFactorImplementation(task, code=code, **kwargs)
+
+
+FactorExperiment = Experiment

rdagent/components/task_implementation/factor_implementation/prompts.yaml

@@ -45,7 +45,7 @@ evolving_strategy_factor_implementation_v1_system: |-
   2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the factor value. You should analyze the feedback and try to correct the latest code.
   3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code.
 
-  Your must write your code based on your former lastest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
+  Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
   {% if queried_former_failed_knowledge|length != 0 %}
   --------------Your former latest attempt:---------------
   {% for former_failed_knowledge in queried_former_failed_knowledge %}
@@ -199,7 +199,7 @@ select_implementable_factor_system: |-
 
 select_implementable_factor_user: |-
   Number of factor you should pick: {{ factor_num }}
-  {% for factor_info in target_factor_tasks %} 
+  {% for factor_info in sub_tasks %} 
   =============Factor index:{{factor_info[0]}}:=============
   =====Factor name:=====
   {{ factor_info[1].factor_name }}

rdagent/components/task_implementation/factor_implementation/utils.py

@@ -3,12 +3,9 @@ from pathlib import Path
 import pandas as pd
 
 # render it with jinja
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
 
-from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
-    FactorImplementTask,
-)
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
+from rdagent.components.task_implementation.factor_implementation.config import (
     FACTOR_IMPLEMENT_SETTINGS,
 )
 
@@ -19,11 +16,11 @@ TPL = """
 ````
 """
 # Create a Jinja template from the string
-JJ_TPL = Template(TPL)
+JJ_TPL = Environment(undefined=StrictUndefined).from_string(TPL)
 
 
 def get_data_folder_intro():
-    """Direclty get the info of the data folder.
+    """Directly get the info of the data folder.
     It is for preparing prompting message.
     """
     content_l = []
@@ -53,4 +50,4 @@ def get_data_folder_intro():
             raise NotImplementedError(
                 f"file type {p.name} is not supported. Please implement its description function.",
             )
-    return "\n ----------------- file spliter -------------\n".join(content_l)
+    return "\n ----------------- file splitter -------------\n".join(content_l)

rdagent/components/task_implementation/model_implementation/one_shot/__init__.py

@@ -2,13 +2,13 @@ import re
 from pathlib import Path
 from typing import Sequence
 
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
 
 from rdagent.components.task_implementation.model_implementation.task import (
     ModelImplTask,
     ModelTaskImpl,
 )
-from rdagent.core.implementation import TaskGenerator
+from rdagent.core.task_generator import TaskGenerator
 from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_utils import APIBackend
 
@@ -23,8 +23,8 @@ class ModelTaskGen(TaskGenerator):
             mti.prepare()
             pr = Prompts(file_path=DIRNAME / "prompt.yaml")
 
-            user_prompt_tpl = Template(pr["code_implement_user"])
-            sys_prompt_tpl = Template(pr["code_implement_sys"])
+            user_prompt_tpl = Environment(undefined=StrictUndefined).from_string(pr["code_implement_user"])
+            sys_prompt_tpl = Environment(undefined=StrictUndefined).from_string(pr["code_implement_sys"])
 
             user_prompt = user_prompt_tpl.render(
                 name=t.name,

rdagent/components/task_implementation/model_implementation/task.py

@@ -1,26 +1,21 @@
+import json
 import uuid
 from pathlib import Path
 from typing import Dict, Optional, Sequence
 
 import torch
 
+from rdagent.components.loader.task_loader import ModelTaskLoader
 from rdagent.components.task_implementation.model_implementation.conf import (
     MODEL_IMPL_SETTINGS,
 )
 from rdagent.core.exception import CodeFormatException
-from rdagent.core.task import (
-    BaseTask,
-    FBTaskImplementation,
-    ImpLoader,
-    TaskImplementation,
-    TaskLoader,
-)
+from rdagent.core.experiment import FBImplementation, ImpLoader, Task
 from rdagent.utils import get_module_by_module_path
-import json
 
 
-class ModelImplTask(BaseTask):
-    # TODO: it should change when the BaseTask changes.
+class ModelImplTask(Task):
+    # TODO: it should change when the Task changes.
     name: str
     description: str
     formulation: str
@@ -43,6 +38,7 @@ class ModelImplTask(BaseTask):
         self.formulation = formulation
         self.variables = variables
         self.key = key
+
     def get_information(self):
         return f"""name: {self.name}
 description: {self.description}
@@ -50,16 +46,16 @@ formulation: {self.formulation}
 variables: {self.variables}
 key: {self.key}
 """
-    
+
     @staticmethod
     def from_dict(dict):
         return ModelImplTask(**dict)
-    
+
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__} {self.name}>"
 
 
-class ModelTaskLoderJson(TaskLoader):
+class ModelTaskLoaderJson(ModelTaskLoader):
     # def __init__(self, json_uri: str, select_model: Optional[str] = None) -> None:
     #     super().__init__()
     #     self.json_uri = json_uri
@@ -82,7 +78,7 @@ class ModelTaskLoderJson(TaskLoader):
     #         variables=model_data["variables"],
     #         key=model_name
     #     )
-        
+
     #     return [model_impl_task]
 
     def __init__(self, json_uri: str) -> None:
@@ -95,7 +91,7 @@ class ModelTaskLoderJson(TaskLoader):
 
         # FIXME: the model in the json file is not right due to extraction error
         #       We should fix them case by case in the future
-        # 
+        #
         # formula_info = {
         #     "name": "Anti-Symmetric Deep Graph Network (A-DGN)",
         #     "description": "A framework for stable and non-dissipative DGN design. It ensures long-range information preservation between nodes and prevents gradient vanishing or explosion during training.",
@@ -119,12 +115,13 @@ class ModelTaskLoderJson(TaskLoader):
                 description=model_data["description"],
                 formulation=model_data["formulation"],
                 variables=model_data["variables"],
-                key=model_data["key"]
+                key=model_data["key"],
             )
             model_impl_task_list.append(model_impl_task)
         return model_impl_task_list
-    
-class ModelImplementationTaskLoaderFromDict(TaskLoader):
+
+
+class ModelImplementationTaskLoaderFromDict(ModelTaskLoader):
     def load(self, model_dict: dict) -> list:
         """Load data from a dict."""
         task_l = []
@@ -134,13 +131,13 @@ class ModelImplementationTaskLoaderFromDict(TaskLoader):
                 description=model_data["description"],
                 formulation=model_data["formulation"],
                 variables=model_data["variables"],
-                key=model_name
+                key=model_name,
             )
             task_l.append(task)
         return task_l
 
 
-class ModelTaskImpl(FBTaskImplementation):
+class ModelTaskImpl(FBImplementation):
     """
     It is a Pytorch model implementation task;
     All the things are placed in a folder.
@@ -156,11 +153,11 @@ class ModelTaskImpl(FBTaskImplementation):
     We'll import the model in the implementation in file `model.py` after setting the cwd into the directory
     - from model import model_cls
     - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
-    - And then verify the modle.
+    - And then verify the model.
 
     """
 
-    def __init__(self, target_task: BaseTask) -> None:
+    def __init__(self, target_task: Task) -> None:
         super().__init__(target_task)
         self.path = None
 
@@ -202,7 +199,7 @@ The the implemented code will be placed in a file like <uuid>/model.py
 
 We'll import the model in the implementation in file `model.py` after setting the cwd into the directory
 - from model import model_cls (So you must have a variable named `model_cls` in the file)
-  - So your implelemented code could follow the following pattern
+  - So your implemented code could follow the following pattern
     ```Python
     class XXXLayer(torch.nn.Module):
         ...

rdagent/components/task_implementation/model_implementation/task_extraction.py

@@ -1,26 +1,23 @@
 from __future__ import annotations
 
 import json
-import multiprocessing as mp
 import re
 from pathlib import Path
-from typing import Mapping
 
-import numpy as np
-import pandas as pd
-import tiktoken
-from jinja2 import Template
-from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.components.document_reader.document_reader import (
+    load_and_process_pdfs_by_langchain,
+)
+from rdagent.components.loader.task_loader import ModelTaskLoader
+from rdagent.components.task_implementation.model_implementation.task import (
+    ModelImplementationTaskLoaderFromDict,
+)
 from rdagent.core.log import RDAgentLog
 from rdagent.core.prompts import Prompts
-from rdagent.core.task import TaskLoader
-from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain
-from rdagent.components.task_implementation.model_implementation.task import ModelImplementationTaskLoaderFromDict
-from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing
-
+from rdagent.oai.llm_utils import APIBackend
 
 document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
 
+
 def extract_model_from_doc(doc_content: str) -> dict:
     """
     Extract model information from document content.
@@ -67,11 +64,11 @@ def extract_model_from_doc(doc_content: str) -> dict:
             else:
                 break
 
-
     RDAgentLog().info(f"已经完成{len(model_dict)}个模型的提取")
 
     return model_dict
 
+
 def merge_file_to_model_dict_to_model_dict(
     file_to_model_dict: dict[str, dict],
 ) -> dict:
@@ -80,7 +77,7 @@ def merge_file_to_model_dict_to_model_dict(
         for model_name in file_to_model_dict[file_name]:
             model_dict.setdefault(model_name, [])
             model_dict[model_name].append(file_to_model_dict[file_name][model_name])
-    
+
     model_dict_simple_deduplication = {}
     for model_name in model_dict:
         if len(model_dict[model_name]) > 1:
@@ -100,18 +97,24 @@ def extract_model_from_docs(docs_dict):
     return model_dict
 
 
-class ModelImplementationTaskLoaderFromPDFfiles(TaskLoader):
+class ModelImplementationTaskLoaderFromPDFfiles(ModelTaskLoader):
     def load(self, file_or_folder_path: Path) -> dict:
-        docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) # dict{file_path:content}
-        model_dict = extract_model_from_docs(docs_dict) # dict{file_name: dict{model_name: dict{description, formulation, variables}}}
-        model_dict = merge_file_to_model_dict_to_model_dict(model_dict) # dict {model_name: dict{description, formulation, variables}}
+        docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path))  # dict{file_path:content}
+        model_dict = extract_model_from_docs(
+            docs_dict
+        )  # dict{file_name: dict{model_name: dict{description, formulation, variables}}}
+        model_dict = merge_file_to_model_dict_to_model_dict(
+            model_dict
+        )  # dict {model_name: dict{description, formulation, variables}}
         return ModelImplementationTaskLoaderFromDict().load(model_dict)
 
+
 def main(path="../test_doc"):
     doc_dict = load_and_process_pdfs_by_langchain(Path(path))
-    print(doc_dict.keys()) # if you run code like "python -u", the print content will be truncated
+    print(doc_dict.keys())  # if you run code like "python -u", the print content will be truncated
 
 
 import fire
+
 if __name__ == "__main__":
     fire.Fire(main)

rdagent/components/task_loader/__init__.py

@@ -1,9 +0,0 @@
-from rdagent.core.task import TaskLoader
-
-
-class FactorTaskLoader(TaskLoader):
-    pass
-
-
-class ModelTaskLoader(TaskLoader):
-    pass

rdagent/core/evaluation.py

@@ -1,15 +1,15 @@
 from abc import ABC, abstractmethod
 
-from rdagent.core.task import BaseTask, TaskImplementation
+from rdagent.core.experiment import Task, Implementation
 
 
 class Evaluator(ABC):
     @abstractmethod
     def evaluate(
         self,
-        target_task: BaseTask,
-        implementation: TaskImplementation,
-        gt_implementation: TaskImplementation,
+        target_task: Task,
+        implementation: Implementation,
+        gt_implementation: Implementation,
         **kwargs,
     ):
         raise NotImplementedError

rdagent/core/evolving_framework.py

@@ -33,8 +33,7 @@ class EvolvableSubjects:
         return copy.deepcopy(self)
 
 
-class QlibEvolvableSubjects(EvolvableSubjects):
-    ...
+class QlibEvolvableSubjects(EvolvableSubjects): ...
 
 
 @dataclass
@@ -42,7 +41,7 @@ class EvoStep:
     """At a specific step,
     based on
     - previous trace
-    - newly RAG kownledge `QueriedKnowledge`
+    - newly RAG knowledge `QueriedKnowledge`
 
     the EvolvableSubjects is evolved to a new one `EvolvableSubjects`.
 
@@ -73,7 +72,7 @@ class EvolvingStrategy(ABC):
 
 
 class RAGStrategy(ABC):
-    """Retrival Augmentation Generation Strategy"""
+    """Retrieval Augmentation Generation Strategy"""
 
     def __init__(self, knowledgebase: KnowledgeBase) -> None:
         self.knowledgebase = knowledgebase

rdagent/core/experiment.py

@@ -1,25 +1,23 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Generic, Optional, Sequence, Tuple, TypeVar
-
-import pandas as pd
+from typing import Generic, Optional, Sequence, TypeVar
 
 """
-This file contains the all the data class for rdagent task.
+This file contains the all the class about organizing the task in RD-Agent.
 """
 
 
-class BaseTask(ABC):
+class Task:
     # TODO: 把name放在这里作为主键
     # Please refer to rdagent/model_implementation/task.py for the implementation
     # I think the task version applies to the base class.
     pass
 
 
-ASpecificTask = TypeVar("ASpecificTask", bound=BaseTask)
+ASpecificTask = TypeVar("ASpecificTask", bound=Task)
 
 
-class TaskImplementation(ABC, Generic[ASpecificTask]):
+class Implementation(ABC, Generic[ASpecificTask]):
     def __init__(self, target_task: ASpecificTask) -> None:
         self.target_task = target_task
 
@@ -28,7 +26,7 @@ class TaskImplementation(ABC, Generic[ASpecificTask]):
         """
         The execution of the implementation can be dynamic.
 
-        So we may passin the data and config dynamically.
+        So we may pass in the data and config dynamically.
         """
         raise NotImplementedError("execute method is not implemented.")
 
@@ -44,16 +42,16 @@ class TaskImplementation(ABC, Generic[ASpecificTask]):
     # Some evaluators will input the results and output
 
 
-ASpecificTaskImp = TypeVar("ASpecificTaskImp", bound=TaskImplementation)
+ASpecificImp = TypeVar("ASpecificImp", bound=Implementation)
 
 
-class ImpLoader(ABC, Generic[ASpecificTask, ASpecificTaskImp]):
+class ImpLoader(ABC, Generic[ASpecificTask, ASpecificImp]):
     @abstractmethod
-    def load(self, task: ASpecificTask) -> ASpecificTaskImp:
+    def load(self, task: ASpecificTask) -> ASpecificImp:
         raise NotImplementedError("load method is not implemented.")
 
 
-class FBTaskImplementation(TaskImplementation):
+class FBImplementation(Implementation):
     """
     File-based task implementation
 
@@ -63,12 +61,12 @@ class FBTaskImplementation(TaskImplementation):
     - Output
         - After execution, it will generate the final output as file.
 
-    A typical way to run the pipeline of FBTaskImplementation will be
+    A typical way to run the pipeline of FBImplementation will be
     (We didn't add it as a method due to that we may pass arguments into `prepare` or `execute` based on our requirements.)
 
     .. code-block:: python
 
-        def run_pipline(self, **files: str):
+        def run_pipeline(self, **files: str):
             self.prepare()
             self.inject_code(**files)
             self.execute()
@@ -76,9 +74,9 @@ class FBTaskImplementation(TaskImplementation):
     """
 
     # TODO:
-    # FileBasedFactorImplementation should inherient from it.
+    # FileBasedFactorImplementation should inherit from it.
     # Why not directly reuse FileBasedFactorImplementation.
-    #   Because it has too much concerete dependencies.
+    #   Because it has too much concrete dependencies.
     #   e.g.  dataframe, factors
 
     path: Optional[Path]
@@ -116,17 +114,20 @@ class FBTaskImplementation(TaskImplementation):
         return list(self.path.iterdir())
 
 
-class TestCase:
-    def __init__(
-        self,
-        target_task: list[BaseTask] = [],
-        ground_truth: list[TaskImplementation] = [],
-    ):
-        self.ground_truth = ground_truth
-        self.target_task = target_task
+class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]):
+    """
+    The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator.
+    """
+
+    def __init__(self, sub_tasks: Sequence[Task]) -> None:
+        self.sub_tasks = sub_tasks
+        self.sub_implementations: Sequence[ASpecificImp] = [None for _ in self.sub_tasks]
 
 
-class TaskLoader:
+TaskOrExperiment = TypeVar("TaskOrExperiment", Task, Experiment)
+
+
+class Loader(ABC, Generic[TaskOrExperiment]):
     @abstractmethod
-    def load(self, *args, **kwargs) -> Sequence[BaseTask]:
+    def load(self, *args, **kwargs) -> TaskOrExperiment:
         raise NotImplementedError("load method is not implemented.")

rdagent/core/proposal.py

@@ -0,0 +1,101 @@
+"""
+
+"""
+
+from typing import Dict, List, Tuple
+
+from rdagent.core.evolving_framework import Feedback
+from rdagent.core.experiment import Experiment, Implementation, Loader, Task
+
+# class data_ana: XXX
+
+
+class Hypothesis:
+    """
+    TODO: We may have better name for it.
+
+    Name Candidates:
+    - Belief
+    """
+
+    hypothesis: str = None
+    reason: str = None
+
+    # source: data_ana | model_nan = None
+
+
+# Origin(path of repo/data/feedback) => view/summarization => generated Hypothesis
+
+
+class Scenario:
+    def get_repo_path(self):
+        """codebase"""
+
+    def get_data(self):
+        """ "data info"""
+
+    def get_env(self):
+        """env description"""
+
+    def get_scenario_all_desc(self) -> str:
+        """Combine all the description together"""
+
+
+class HypothesisFeedback(Feedback): ...
+
+
+class Trace:
+    scen: Scenario
+    hist: list[Tuple[Hypothesis, Experiment, HypothesisFeedback]]
+
+
+class HypothesisGen:
+    def __init__(self, scen: Scenario):
+        self.scen = scen
+
+    def gen(self, trace: Trace) -> Hypothesis:
+        # def gen(self, scenario_desc: str, ) -> Hypothesis:
+        """
+        Motivation of the variable `scenario_desc`:
+        - Mocking a data-scientist is observing the scenario.
+
+        scenario_desc may conclude:
+        - data observation:
+            - Original or derivative
+        - Task information:
+        """
+
+
+class HypothesisSet:
+    """
+    # drop, append
+
+    hypothesis_imp: list[float] | None  # importance of each hypothesis
+    true_hypothesis or false_hypothesis
+    """
+
+    hypothesis_list: list[Hypothesis]
+    trace: Trace
+
+
+class Hypothesis2Experiment(Loader[Experiment]):
+    """
+    [Abstract description => concrete description] => Code implement
+    """
+
+    def convert(self, bs: HypothesisSet) -> Experiment:
+        """Connect the idea proposal to implementation"""
+        ...
+
+
+# Boolean, Reason, Confidence, etc.
+
+
+class Experiment2Feedback:
+    """ "Generated(summarize) feedback from **Executed** Implementation"""
+
+    def summarize(self, ti: Experiment) -> HypothesisFeedback:
+        """
+        The `ti` should be executed and the results should be included.
+        For example: `mlflow` of Qlib will be included.
+        """

rdagent/core/proposal/__init__.py

@@ -1,95 +0,0 @@
-"""
-
-"""
-
-from typing import Tuple
-
-from rdagent.core.task import BaseTask, TaskLoader
-
-# class data_ana: XXX
-
-
-class Belief:
-    """
-    TODO: We may have better name for it.
-
-    Name Candidates:
-    - Hypothesis
-    """
-
-    # source: data_ana | model_nan = None
-
-
-# Origin(path of repo/data/feedback) => view/summarization => generated Belief
-
-
-class Scenario:
-    def get_repo_path(self):
-        """codebase"""
-
-    def get_data(self):
-        """ "data info"""
-
-    def get_env(self):
-        """env description"""
-
-
-class Trace:
-    scen: Scenario
-    hist: list[Tuple[Belief, Feedback]]
-
-
-class BeliefGen:
-    def __init__(self, scen: Scenario):
-        self.scen = scen
-
-    def gen(self, trace: Trace) -> Belief:
-        # def gen(self, scenario_desc: str, ) -> Belief:
-        """
-        Motivation of the variable `scenario_desc`:
-        - Mocking a data-scientist is observing the scenario.
-
-        scenario_desc may conclude:
-        - data observation:
-            - Original or derivative
-        - Task information:
-        """
-
-
-class BeliefSet:
-    """
-    # drop, append
-
-    belief_imp: list[float] | None  # importance of each belief
-    failed_belief or success belief
-    """
-
-    belief_l: list[Belief]
-    feedbacks: Dict[Tuple[Belief, Scenario], BeliefFeedback]
-
-
-class Belief2Task(TaskLoader):
-    """
-    [Abstract description => conceret description] => Code implement
-    """
-
-    def convert(self, bs: BeliefSet) -> BaseTask:
-        """Connect the idea proposal to implementation"""
-        ...
-
-
-class BeliefFeedback:
-    ...
-
-
-# Boolean, Reason, Confidence, etc.
-
-
-class Imp2Feedback:
-    """ "Generated(summarize) feedback from **Executed** Implemenation"""
-
-    def summarize(self, ti: TaskImplementation) -> BeliefFeedback:
-        """
-        The `ti` should be exectued and the results should be included.
-        For example: `mlflow` of Qlib will be included.
-        """

rdagent/core/task_generator.py

@@ -1,14 +1,16 @@
 from abc import ABC, abstractmethod
-from typing import List, Sequence
+from typing import Generic, List, Sequence, TypeVar
 
-from rdagent.core.task import BaseTask, TaskImplementation
+from rdagent.core.experiment import Experiment
+
+ASpecificExp = TypeVar("ASpecificExp", bound=Experiment)
 
 
-class TaskGenerator(ABC):
+class TaskGenerator(ABC, Generic[ASpecificExp]):
     @abstractmethod
-    def generate(self, task_l: Sequence[BaseTask]) -> Sequence[TaskImplementation]:
+    def generate(self, exp: ASpecificExp) -> ASpecificExp:
         """
-        Task Generator should take in a sequence of tasks.
+        Task Generator should take in an experiment.
 
         Because the schedule of different tasks is crucial for the final performance
         due to it affects the learning process.
@@ -19,7 +21,7 @@ class TaskGenerator(ABC):
     def collect_feedback(self, feedback_obj_l: List[object]):
         """
         When online evaluation.
-        The preivous feedbacks will be collected to support advanced factor generator
+        The previous feedbacks will be collected to support advanced factor generator
 
         Parameters
         ----------

rdagent/scenarios/qlib/experiment/factor_experiment.py

@@ -0,0 +1,5 @@
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorExperiment,
+)
+
+QlibFactorExperiment = FactorExperiment

rdagent/scenarios/qlib/experiment/model_experiment.py

@@ -0,0 +1 @@
+# TODO define QlibModelExperiment here which should be subclass of Experiment

rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py

@@ -1,49 +1,54 @@
 import json
 from pathlib import Path
 
-from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
-    FactorImplementTask,
+from rdagent.components.benchmark.eval_method import TestCase
+from rdagent.components.loader.experiment_loader import FactorExperimentLoader
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorExperiment,
+    FactorTask,
     FileBasedFactorImplementation,
 )
-from rdagent.components.task_loader import FactorTaskLoader
-from rdagent.core.task import TaskLoader, TestCase
+from rdagent.core.experiment import Loader
 
 
-class FactorImplementationTaskLoaderFromDict(FactorTaskLoader):
+class FactorImplementationExperimentLoaderFromDict(FactorExperimentLoader):
     def load(self, factor_dict: dict) -> list:
         """Load data from a dict."""
         task_l = []
         for factor_name, factor_data in factor_dict.items():
-            task = FactorImplementTask(
+            task = FactorTask(
                 factor_name=factor_name,
                 factor_description=factor_data["description"],
                 factor_formulation=factor_data["formulation"],
                 variables=factor_data["variables"],
             )
             task_l.append(task)
-        return task_l
+        exp = FactorExperiment(sub_tasks=task_l)
+        return exp
 
 
-class FactorImplementationTaskLoaderFromJsonFile(FactorTaskLoader):
+class FactorImplementationExperimentLoaderFromJsonFile(FactorExperimentLoader):
     def load(self, json_file_path: Path) -> list:
         with open(json_file_path, "r") as file:
             factor_dict = json.load(file)
-        return FactorImplementationTaskLoaderFromDict().load(factor_dict)
+        return FactorImplementationExperimentLoaderFromDict().load(factor_dict)
 
 
-class FactorImplementationTaskLoaderFromJsonString(FactorTaskLoader):
+class FactorImplementationExperimentLoaderFromJsonString(FactorExperimentLoader):
     def load(self, json_string: str) -> list:
         factor_dict = json.loads(json_string)
-        return FactorImplementationTaskLoaderFromDict().load(factor_dict)
+        return FactorImplementationExperimentLoaderFromDict().load(factor_dict)
 
 
-class FactorTestCaseLoaderFromJsonFile(TaskLoader):
+# TODO loader only supports generic of task or experiment, testcase might cause CI error here
+# class FactorTestCaseLoaderFromJsonFile(Loader[TestCase]):
+class FactorTestCaseLoaderFromJsonFile:
     def load(self, json_file_path: Path) -> list:
         with open(json_file_path, "r") as file:
             factor_dict = json.load(file)
         TestData = TestCase()
         for factor_name, factor_data in factor_dict.items():
-            task = FactorImplementTask(
+            task = FactorTask(
                 factor_name=factor_name,
                 factor_description=factor_data["description"],
                 factor_formulation=factor_data["formulation"],

rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py

@@ -8,8 +8,7 @@ from typing import Mapping
 
 import numpy as np
 import pandas as pd
-import tiktoken
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
 from sklearn.cluster import KMeans
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.preprocessing import normalize
@@ -17,13 +16,13 @@ from sklearn.preprocessing import normalize
 from rdagent.components.document_reader.document_reader import (
     load_and_process_pdfs_by_langchain,
 )
-from rdagent.components.task_loader import FactorTaskLoader
+from rdagent.components.loader.experiment_loader import FactorExperimentLoader
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.log import RDAgentLog
 from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing
-from rdagent.scenarios.qlib.factor_task_loader.json_loader import (
-    FactorImplementationTaskLoaderFromDict,
+from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
+    FactorImplementationExperimentLoaderFromDict,
 )
 
 document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
@@ -160,9 +159,13 @@ def __extract_factors_formulation_from_content(
     )
 
     system_prompt = document_process_prompts["extract_factor_formulation_system"]
-    current_user_prompt = Template(
-        document_process_prompts["extract_factor_formulation_user"],
-    ).render(report_content=content, factor_dict=factor_dict_df.to_string())
+    current_user_prompt = (
+        Environment(undefined=StrictUndefined)
+        .from_string(
+            document_process_prompts["extract_factor_formulation_user"],
+        )
+        .render(report_content=content, factor_dict=factor_dict_df.to_string())
+    )
 
     session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
     factor_to_formulation = {}
@@ -576,7 +579,7 @@ def deduplicate_factors_by_llm(  # noqa: C901, PLR0912
     return llm_deduplicated_factor_dict, final_duplication_names_list
 
 
-class FactorImplementationTaskLoaderFromPDFfiles(FactorTaskLoader):
+class FactorImplementationExperimentLoaderFromPDFfiles(FactorExperimentLoader):
     def load(self, file_or_folder_path: Path) -> dict:
         docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path))
 
@@ -586,4 +589,4 @@ class FactorImplementationTaskLoaderFromPDFfiles(FactorTaskLoader):
 
         factor_viability = check_factor_viability(factor_dict)
         factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability)
-        return FactorImplementationTaskLoaderFromDict().load(factor_dict)
+        return FactorImplementationExperimentLoaderFromDict().load(factor_dict)

rdagent/scenarios/qlib/factor_idea_proposal.py

@@ -0,0 +1,40 @@
+import json
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.idea_proposal.factor_proposal import (
+    FactorHypothesis,
+    FactorHypothesisGen,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import Scenario, Trace
+
+prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+
+QlibFactorHypothesis = FactorHypothesis
+
+
+class QlibFactorHypothesisGen(FactorHypothesisGen):
+    def __init__(self, scen: Scenario):
+        super().__init__(scen)
+        self.gen_json_flag = True
+
+    def prepare_gen_context(self, trace: Trace) -> None:
+        self.gen_context_flag = True
+
+        hypothesis_feedback = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["hypothesis_and_feedback"])
+            .render(trace=trace)
+        )
+        self.gen_context_dict = {
+            "hypothesis_and_feedback": hypothesis_feedback,
+            "RAG": ...,
+            "factor_output_format": prompt_dict["output_format"],
+        }
+
+    def gen_response_to_hypothesis_list(self, response: str) -> FactorHypothesis:
+        response_dict = json.loads(response)
+        hypothesis = QlibFactorHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"])
+        return hypothesis

rdagent/scenarios/qlib/prompts.yaml

@@ -0,0 +1,12 @@
+hypothesis_and_feedback: |-
+  {% for hypothesis, feedback in trace %}
+  Hypothesis {{ loop.index }}: {{ hypothesis }}
+  Feedback: {{ feedback }}
+  {% endfor %}
+
+output_format: |-
+  The output should follow JSON format. The schema is as follows:
+  {
+    "hypothesis": "The new hypothesis generated based on the information provided.",
+    "reason": "The reason why you generate this hypothesis."
+  }

rdagent/scenarios/qlib/task_generator/data.py

@@ -1,7 +1,8 @@
-from rdagent.core.task import FBTaskImplementation
+from rdagent.core.task_generator import TaskGenerator
+from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
 
 
-class QlibDataTaskImplementation(FBTaskImplementation):
+class QlibDataImplementation(TaskGenerator[QlibFactorExperiment]):
     """
     Docker run
     Everything in a folder

rdagent/scenarios/qlib/task_generator/model.py

@@ -1,7 +1,7 @@
-from rdagent.core.task import FBTaskImplementation
+from rdagent.core.task_generator import TaskGenerator
 
 
-class QlibModelTaskImplementation(FBTaskImplementation):
+class QlibModelImplementation(TaskGenerator[QlibModelExperiment]):
     """
     Docker run
     Everything in a folder