reporeformat V2 (#23)

* reformat factor implement process

* move some code to more reasonable place

* fix the bug

* add test function in factor_extract_and_implement.py

* change select factor number to ratio , add some factor implement setting and fix some bug while using knowledgebase

* change evoagent

* add abstract class EvoAgent

* add benchmark workflow

* fix some bug in llm_utils

* run wenjun's code

* fix the knowledgebase instance check

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/factor_implementation/evolving/evaluators.py

@@ -1,30 +1,526 @@
-from __future__ import annotations
-
+import json
 import re
-from typing import List
 
-from pandas.core.api import DataFrame as DataFrame
-from rdagent.core.evolving_framework import Evaluator as EvolvingEvaluator
-from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
+from abc import abstractmethod
+from typing import Tuple
+
+import pandas as pd
+from jinja2 import Template
+
+from rdagent.oai.llm_utils import APIBackend
 from rdagent.core.log import FinCoLog
+from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask, FactorEvovlingItem
+from rdagent.core.task import (
+    TaskImplementation,
+)
+from typing import List, Tuple
+from rdagent.core.evolving_framework import QueriedKnowledge,Feedback
+from rdagent.core.evaluation import Evaluator
+from rdagent.core.prompts import Prompts
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
 from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.evolving.evolvable_subjects import FactorImplementationList
-from rdagent.factor_implementation.share_modules.evaluator import (
-    Evaluator as FactorImplementationEvaluator,
-)
-from rdagent.factor_implementation.share_modules.evaluator import (
-    FactorImplementationCodeEvaluator,
-    FactorImplementationFinalDecisionEvaluator,
-    FactorImplementationValueEvaluator,
-)
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-)
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
-)
+from pathlib import Path
 
+evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+class FactorImplementationEvaluator(Evaluator):
+    # TODO:
+    # I think we should have unified interface for all evaluates, for examples.
+    # So we should adjust the interface of other factors
+    @abstractmethod
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        """You can get the dataframe by
+
+        .. code-block:: python
+
+            _, gt_df = gt.execute()
+            _, gen_df = gen.execute()
+
+        Returns
+        -------
+        Tuple[str, object]
+            - str: the text-based description of the evaluation result
+            - object: a comparable metric (bool, integer, float ...)
+
+        """
+        raise NotImplementedError("Please implement the `evaluator` method")
+
+    def _get_df(self, gt: TaskImplementation, gen: TaskImplementation):
+        _, gt_df = gt.execute()
+        _, gen_df = gen.execute()
+        if isinstance(gen_df, pd.Series):
+            gen_df = gen_df.to_frame("source_factor")
+        if isinstance(gt_df, pd.Series):
+            gt_df = gt_df.to_frame("gt_factor")
+        return gt_df, gen_df
+
+class FactorImplementationCodeEvaluator(Evaluator):
+    def evaluate(
+        self,
+        target_task: FactorImplementTask,
+        implementation: TaskImplementation,
+        execution_feedback: str,
+        factor_value_feedback: str = "",
+        gt_implementation: TaskImplementation = None,
+        **kwargs,
+    ):
+        factor_information = target_task.get_factor_information()
+        code = implementation.code
+
+        system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"]
+
+        execution_feedback_to_render = execution_feedback
+        user_prompt = Template(
+            evaluate_prompts["evaluator_code_feedback_v1_user"],
+        ).render(
+            factor_information=factor_information,
+            code=code,
+            execution_feedback=execution_feedback_to_render,
+            factor_value_feedback=factor_value_feedback,
+            gt_code=gt_implementation.code if gt_implementation else None,
+        )
+        while (
+            APIBackend().build_messages_and_calculate_token(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                former_messages=[],
+            )
+            > FactorImplementSettings().chat_token_limit
+        ):
+            execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            user_prompt = Template(
+                evaluate_prompts["evaluator_code_feedback_v1_user"],
+            ).render(
+                factor_information=factor_information,
+                code=code,
+                execution_feedback=execution_feedback_to_render,
+                factor_value_feedback=factor_value_feedback,
+                gt_code=gt_implementation.code if gt_implementation else None,
+            )
+        critic_response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            json_mode=False,
+        )
+
+        return critic_response
+
+class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        if len(gen_df.columns) == 1 and len(gt_df.columns) == 1:
+            return "Both dataframes have only one column.", True
+        elif len(gen_df.columns) != 1:
+            gen_df = gen_df.iloc(axis=1)[
+                [
+                    0,
+                ]
+            ]
+            return (
+                "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
+                False,
+            )
+        return "", False
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+        idx_name_right = gen_df.index.names == ("datetime", "instrument")
+        if idx_name_right:
+            return (
+                'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
+                True,
+            )
+        else:
+            return (
+                'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.',
+                False,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        if gen_df.shape[0] == gt_df.shape[0]:
+            return "Both dataframes have the same rows count.", True
+        else:
+            return (
+                f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
+                False,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        if gen_df.index.equals(gt_df.index):
+            return "Both dataframes have the same index.", True
+        else:
+            return (
+                "The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
+                False,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
+            return "Both dataframes have the same missing values.", True
+        else:
+            return (
+                f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
+                False,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        try:
+            close_values = gen_df.sub(gt_df).abs().lt(1e-6)
+            result_int = close_values.astype(int)
+            pos_num = result_int.sum().sum()
+            acc_rate = pos_num / close_values.size
+        except:
+            close_values = gen_df
+        if close_values.all().iloc[0]:
+            return (
+                "All values in the dataframes are equal within the tolerance of 1e-6.",
+                acc_rate,
+            )
+        else:
+            return (
+                "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
+                acc_rate,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
+    def __init__(self, hard_check: bool) -> None:
+        self.hard_check = hard_check
+
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        concat_df = pd.concat([gen_df, gt_df], axis=1)
+        concat_df.columns = ["source", "gt"]
+        ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
+        ric = (
+            concat_df.groupby("datetime")
+            .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
+            .dropna()
+            .mean()
+        )
+
+        if self.hard_check:
+            if ic > 0.99 and ric > 0.99:
+                return (
+                    f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
+                    True,
+                )
+            else:
+                return (
+                    f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
+                    False,
+                )
+        else:
+            return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationValEvaluator(FactorImplementationEvaluator):
+    def evaluate(self, gt: TaskImplementation, gen: TaskImplementation):
+        _, gt_df = gt.execute()
+        _, gen_df = gen.execute()
+        # FIXME: refactor the two classes
+        fiv = FactorImplementationValueEvaluator()
+        return fiv.evaluate(source_df=gen_df, gt_df=gt_df)
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationValueEvaluator(Evaluator):
+    # TODO: let's discuss the about the interface of the evaluator
+    def evaluate(
+        self,
+        source_df: pd.DataFrame,
+        gt_df: pd.DataFrame,
+        **kwargs,
+    ) -> Tuple:
+        conclusions = []
+
+        if isinstance(source_df, pd.Series):
+            source_df = source_df.to_frame("source_factor")
+            conclusions.append(
+                "The source dataframe is a series, better convert it to a dataframe.",
+            )
+        if gt_df is not None and isinstance(gt_df, pd.Series):
+            gt_df = gt_df.to_frame("gt_factor")
+            conclusions.append(
+                "The ground truth dataframe is a series, convert it to a dataframe.",
+            )
+
+        # Check if both dataframe has only one columns
+        if len(source_df.columns) == 1:
+            conclusions.append("The source dataframe has only one column which is correct.")
+        else:
+            conclusions.append(
+                "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
+            )
+            source_df = source_df.iloc(axis=1)[
+                [
+                    0,
+                ]
+            ]
+
+        if list(source_df.index.names) != ["datetime", "instrument"]:
+            conclusions.append(
+                rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.",
+            )
+        else:
+            conclusions.append(
+                'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
+            )
+
+        # Check if both dataframe have the same rows count
+        if gt_df is not None:
+            if source_df.shape[0] == gt_df.shape[0]:
+                conclusions.append("Both dataframes have the same rows count.")
+                same_row_count_result = True
+            else:
+                conclusions.append(
+                    f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
+                )
+                same_row_count_result = False
+
+            # Check whether both dataframe has the same index
+            if source_df.index.equals(gt_df.index):
+                conclusions.append("Both dataframes have the same index.")
+                same_index_result = True
+            else:
+                conclusions.append(
+                    "The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
+                )
+                same_index_result = False
+
+            # Check for the same missing values (NaN)
+            if source_df.isna().sum().sum() == gt_df.isna().sum().sum():
+                conclusions.append("Both dataframes have the same missing values.")
+                same_missing_values_result = True
+            else:
+                conclusions.append(
+                    f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
+                )
+                same_missing_values_result = False
+
+            # Check if the values are the same within a small tolerance
+            if not same_index_result:
+                conclusions.append(
+                    "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless",
+                )
+                same_values_result = False
+                high_correlation_result = False
+            else:
+                close_values = source_df.sub(gt_df).abs().lt(1e-6)
+                if close_values.all().iloc[0]:
+                    conclusions.append(
+                        "All values in the dataframes are equal within the tolerance of 1e-6.",
+                    )
+                    same_values_result = True
+                else:
+                    conclusions.append(
+                        "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
+                    )
+                    same_values_result = False
+
+                # Check the ic and rankic between the two dataframes
+                concat_df = pd.concat([source_df, gt_df], axis=1)
+                concat_df.columns = ["source", "gt"]
+                try:
+                    ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
+                    ric = (
+                        concat_df.groupby("datetime")
+                        .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
+                        .dropna()
+                        .mean()
+                    )
+
+                    if ic > 0.99 and ric > 0.99:
+                        conclusions.append(
+                            f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
+                        )
+                        high_correlation_result = True
+                    else:
+                        conclusions.append(
+                            f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
+                        )
+                        high_correlation_result = False
+
+                    # Check for shifted alignments only in the "datetime" index
+                    max_shift_days = 2
+                    for shift in range(-max_shift_days, max_shift_days + 1):
+                        if shift == 0:
+                            continue  # Skip the case where there is no shift
+
+                        shifted_source_df = source_df.groupby(level="instrument").shift(shift)
+                        concat_df = pd.concat([shifted_source_df, gt_df], axis=1)
+                        concat_df.columns = ["source", "gt"]
+                        shifted_ric = (
+                            concat_df.groupby("datetime")
+                            .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
+                            .dropna()
+                            .mean()
+                        )
+                        if shifted_ric > 0.99:
+                            conclusions.append(
+                                f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.",
+                            )
+                            break
+                    else:
+                        conclusions.append(
+                            f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.",
+                        )
+
+                except Exception as e:
+                    FinCoLog().warning(f"Error occurred when calculating the correlation: {str(e)}")
+                    conclusions.append(
+                        f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
+                    )
+                    high_correlation_result = False
+
+        # Combine all conclusions into a single string
+        conclusion_str = "\n".join(conclusions)
+
+        final_result = (same_values_result or high_correlation_result) if gt_df is not None else False
+        return conclusion_str, final_result
+
+
+# TODO:
+def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
+    """When the prompt is too long. We have to shorten it.
+    But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
+    """
+    # TODO: this should replace most of code in
+    # - FactorImplementationFinalDecisionEvaluator.evaluate
+    # - FactorImplementationCodeEvaluator.evaluate
+
+
+class FactorImplementationFinalDecisionEvaluator(Evaluator):
+    def evaluate(
+        self,
+        target_task: FactorImplementTask,
+        execution_feedback: str,
+        value_feedback: str,
+        code_feedback: str,
+        **kwargs,
+    ) -> Tuple:
+        system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")["evaluator_final_decision_v1_system"]
+        execution_feedback_to_render = execution_feedback
+        user_prompt = Template(
+            evaluate_prompts["evaluator_final_decision_v1_user"],
+        ).render(
+            factor_information=target_task.get_factor_information(),
+            execution_feedback=execution_feedback_to_render,
+            code_feedback=code_feedback,
+            factor_value_feedback=(
+                value_feedback
+                if value_feedback is not None
+                else "No Ground Truth Value provided, so no evaluation on value is performed."
+            ),
+        )
+        while (
+            APIBackend().build_messages_and_calculate_token(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                former_messages=[],
+            )
+            > FactorImplementSettings().chat_token_limit
+        ):
+            execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            user_prompt = Template(
+                evaluate_prompts["evaluator_final_decision_v1_user"],
+            ).render(
+                factor_information=target_task.get_factor_information(),
+                execution_feedback=execution_feedback_to_render,
+                code_feedback=code_feedback,
+                factor_value_feedback=(
+                    value_feedback
+                    if value_feedback is not None
+                    else "No Ground Truth Value provided, so no evaluation on value is performed."
+                ),
+            )
+
+        final_evaluation_dict = json.loads(
+            APIBackend().build_messages_and_create_chat_completion(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                json_mode=True,
+            ),
+        )
+        return (
+            final_evaluation_dict["final_decision"],
+            final_evaluation_dict["final_feedback"],
+        )
 
 class FactorImplementationSingleFeedback:
     """This class is a feedback to single implementation which is generated from an evaluator."""
@@ -60,14 +556,12 @@ class FactorImplementationSingleFeedback:
 This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
 """
 
-
 class FactorImplementationsMultiFeedback(
     Feedback,
     List[FactorImplementationSingleFeedback],
 ):
     """Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
 
-
 class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
     """This class is the v1 version of evaluator for a single factor implementation.
     It calls several evaluators in share modules to evaluate the factor implementation.
@@ -80,9 +574,9 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
 
     def evaluate(
         self,
-        target_task: FactorImplementationTask,
-        implementation: FactorImplementation,
-        gt_implementation: FactorImplementation = None,
+        target_task: FactorImplementTask,
+        implementation: TaskImplementation,
+        gt_implementation: TaskImplementation = None,
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> FactorImplementationSingleFeedback:
@@ -170,14 +664,14 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
             return factor_feedback
 
 
-class FactorImplementationsMultiEvaluator(EvolvingEvaluator):
+class FactorImplementationsMultiEvaluator(Evaluator):
     def __init__(self, single_evaluator=FactorImplementationEvaluatorV1()) -> None:
         super().__init__()
         self.single_factor_implementation_evaluator = single_evaluator
 
     def evaluate(
         self,
-        evo: FactorImplementationList,
+        evo: FactorEvovlingItem,
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> FactorImplementationsMultiFeedback:
@@ -228,3 +722,4 @@ class FactorImplementationsMultiEvaluator(EvolvingEvaluator):
         print(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
 
         return multi_implementation_feedback
+

rdagent/factor_implementation/evolving/evolvable_subjects.py

@@ -1,32 +0,0 @@
-from __future__ import annotations
-
-from rdagent.core.evolving_framework import EvolvableSubjects
-from rdagent.core.log import FinCoLog
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-)
-
-
-class FactorImplementationList(EvolvableSubjects):
-    """
-    Factors is a list.
-    """
-
-    def __init__(
-        self,
-        target_factor_tasks: list[FactorImplementationTask],
-        corresponding_gt_implementations: list[FactorImplementation] = None,
-    ):
-        super().__init__()
-        self.target_factor_tasks = target_factor_tasks
-        self.corresponding_implementations: list[FactorImplementation] = []
-        if corresponding_gt_implementations is not None and len(
-            corresponding_gt_implementations,
-        ) != len(target_factor_tasks):
-            self.corresponding_gt_implementations = None
-            FinCoLog.warning(
-                "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
-            )
-        else:
-            self.corresponding_gt_implementations = corresponding_gt_implementations

rdagent/factor_implementation/evolving/evolving_agent.py

@@ -0,0 +1,50 @@
+from rdagent.core.evolving_framework import Feedback, EvolvableSubjects, Evaluator, EvoStep
+from rdagent.core.evolving_framework import EvoAgent
+from tqdm import tqdm
+
+class RAGEvoAgent(EvoAgent):
+    def __init__(self, max_loop, evolving_strategy, rag) -> None:
+        super().__init__(max_loop, evolving_strategy)
+        self.rag = rag
+        self.evolving_trace = []
+
+    def multistep_evolve(
+        self, 
+        evo: EvolvableSubjects,
+        eva: Evaluator | Feedback, 
+        *,
+        with_knowledge: bool = False,
+        with_feedback: bool = True,
+        knowledge_self_gen: bool = False) -> EvolvableSubjects:
+
+        for _ in tqdm(range(self.max_loop), "Implementing factors"):
+            # 1. knowledge self-evolving
+            if knowledge_self_gen and self.rag is not None:
+                self.rag.generate_knowledge(self.evolving_trace)
+            # 2. 检索需要的Knowledge
+            queried_knowledge = None
+            if with_knowledge and self.rag is not None:
+                # TODO: 这里放了evolving_trace实际上没有作用
+                queried_knowledge = self.rag.query(evo, self.evolving_trace)
+
+            # 3. evolve
+            evo = self.evolving_strategy.evolve(
+                    evo=evo,
+                    evolving_trace=self.evolving_trace,
+                    queried_knowledge=queried_knowledge,
+                )
+            
+            # 4. 封装Evolve结果
+            es = EvoStep(evo, queried_knowledge)
+
+            # 5. 环境评测反馈
+            if with_feedback:
+                es.feedback = eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge)
+
+            # 6. 更新trace
+            self.evolving_trace.append(es)
+            for index, feedback in enumerate(es.feedback):
+                if feedback is not None:
+                    evo.evolve_trace[evo.target_factor_tasks[index].factor_name][-1].feedback = feedback
+
+        return evo  

rdagent/factor_implementation/evolving/evolving_strategy.py

@@ -1,58 +1,71 @@
 from __future__ import annotations
 
 import json
-import random
 from abc import abstractmethod
 from copy import deepcopy
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from jinja2 import Template
+
 from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
-from rdagent.core.prompts import Prompts
-from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-    FileBasedFactorImplementation,
-)
+from rdagent.oai.llm_utils import APIBackend
 from rdagent.factor_implementation.share_modules.factor_implementation_config import (
     FactorImplementSettings,
 )
-from rdagent.factor_implementation.share_modules.factor_implementation_utils import (
-    get_data_folder_intro,
+
+from rdagent.core.task import (
+    TaskImplementation,
 )
+from rdagent.core.prompts import Prompts
+
+from pathlib import Path
+
+from rdagent.factor_implementation.evolving.scheduler import (
+    RandomSelect,
+    LLMSelect,
+)
+
+from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
 from rdagent.oai.llm_utils import APIBackend
 
+from rdagent.core.utils import multiprocessing_wrapper
+
+from rdagent.factor_implementation.evolving.factor import (
+    FactorImplementTask,
+    FactorEvovlingItem,
+    FileBasedFactorImplementation,
+)
+
 if TYPE_CHECKING:
-    from factor_implementation.evolving.evolvable_subjects import (
-        FactorImplementationList,
-    )
-    from factor_implementation.evolving.knowledge_management import (
+    from rdagent.factor_implementation.evolving.knowledge_management import (
         FactorImplementationQueriedKnowledge,
         FactorImplementationQueriedKnowledgeV1,
     )
 
+implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
 
 class MultiProcessEvolvingStrategy(EvolvingStrategy):
     @abstractmethod
     def implement_one_factor(
         self,
-        target_task: FactorImplementationTask,
+        target_task: FactorImplementTask,
         queried_knowledge: QueriedKnowledge = None,
-    ) -> FactorImplementation:
+    ) -> TaskImplementation:
         raise NotImplementedError
 
     def evolve(
         self,
         *,
-        evo: FactorImplementationList,
+        evo: FactorEvovlingItem,
         queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
         **kwargs,
-    ) -> FactorImplementationList:
+    ) -> FactorEvovlingItem:
+        self.num_loop += 1
         new_evo = deepcopy(evo)
-        new_evo.corresponding_implementations = [None for _ in new_evo.target_factor_tasks]
 
+        # 1.找出需要evolve的factor
         to_be_finished_task_index = []
         for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
             target_factor_task_desc = target_factor_task.get_factor_information()
@@ -65,11 +78,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
                 and target_factor_task_desc not in queried_knowledge.failed_task_info_set
             ):
                 to_be_finished_task_index.append(index)
-        if FactorImplementSettings().implementation_factors_per_round < len(to_be_finished_task_index):
-            to_be_finished_task_index = random.sample(
-                to_be_finished_task_index,
-                FactorImplementSettings().implementation_factors_per_round,
+
+        # 2. 选择selection方法
+        # if the number of factors to be implemented is larger than the limit, we need to select some of them
+        if FactorImplementSettings().select_ratio < 1:
+            # if the number of loops is equal to the select_loop, we need to select some of them
+            implementation_factors_per_round = int(
+                FactorImplementSettings().select_ratio * len(to_be_finished_task_index)
             )
+            if FactorImplementSettings().select_method == "random":
+                to_be_finished_task_index = RandomSelect(
+                    to_be_finished_task_index,
+                    implementation_factors_per_round,
+                )
+
+            if FactorImplementSettings().select_method == "scheduler":
+                to_be_finished_task_index = LLMSelect(
+                    to_be_finished_task_index,
+                    implementation_factors_per_round,
+                    new_evo,
+                    queried_knowledge.former_traces,
+                )
 
         result = multiprocessing_wrapper(
             [
@@ -81,11 +110,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 
         for index, target_index in enumerate(to_be_finished_task_index):
             new_evo.corresponding_implementations[target_index] = result[index]
+            if result[index].target_task.factor_name in new_evo.evolve_trace:
+                new_evo.evolve_trace[result[index].target_task.factor_name].append(result[index])
+            else:
+                new_evo.evolve_trace[result[index].target_task.factor_name] = [result[index]]
 
-        # for target_index in to_be_finished_task_index:
-        #     new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
-        #         new_evo.target_factor_tasks[target_index], queried_knowledge
-        #     )
+        new_evo.corresponding_selection.append(to_be_finished_task_index)
 
         return new_evo
 
@@ -93,9 +123,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
     def implement_one_factor(
         self,
-        target_task: FactorImplementationTask,
+        target_task: FactorImplementTask,
         queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
-    ) -> FactorImplementation:
+    ) -> TaskImplementation:
         factor_information_str = target_task.get_factor_information()
 
         if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -117,9 +147,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
             system_prompt = Template(
-                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                    "evolving_strategy_factor_implementation_v1_system"
-                ],
+                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
             ).render(
                 data_info=get_data_folder_intro(),
                 queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -132,9 +160,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
             while True:
                 user_prompt = (
                     Template(
-                        Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                            "evolving_strategy_factor_implementation_v1_user"
-                        ],
+                        implement_prompts["evolving_strategy_factor_implementation_v1_user"],
                     )
                     .render(
                         factor_information_str=factor_information_str,
@@ -153,9 +179,6 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
                 elif len(queried_similar_successful_knowledge_to_render) > 1:
                     queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
-            # print(
-            #     f"length of queried_similar_successful_knowledge_to_render: {len(queried_similar_successful_knowledge_to_render)}, length of queried_former_failed_knowledge_to_render: {len(queried_former_failed_knowledge_to_render)}"
-            # )
 
             code = json.loads(
                 session.build_chat_completion(
@@ -173,14 +196,20 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
 
 
 class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
+    def __init__(self) -> None:
+        self.num_loop = 0
+        self.haveSelected = False
+
     def implement_one_factor(
         self,
-        target_task: FactorImplementationTask,
+        target_task: FactorImplementTask,
         queried_knowledge,
-    ) -> FactorImplementation:
+    ) -> TaskImplementation:
         error_summary = FactorImplementSettings().v2_error_summary
+        # 1. 提取因子的背景信息
         target_factor_task_information = target_task.get_factor_information()
 
+        # 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
         if (
             queried_knowledge is not None
             and target_factor_task_information in queried_knowledge.success_task_to_knowledge_dict
@@ -189,6 +218,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
         elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
             return None
         else:
+
+            # 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
             queried_similar_component_knowledge = (
                 queried_knowledge.component_with_success_task[target_factor_task_information]
                 if queried_knowledge is not None
@@ -208,9 +239,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
             system_prompt = Template(
-                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                    "evolving_strategy_factor_implementation_v1_system"
-                ],
+                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
             ).render(
                 data_info=get_data_folder_intro(),
                 queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -223,18 +252,17 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
             queried_similar_component_knowledge_to_render = queried_similar_component_knowledge
             queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
             error_summary_critics = ""
+            # 动态地防止prompt超长
             while True:
+                # 总结error（可选）
                 if (
                     error_summary
                     and len(queried_similar_error_knowledge_to_render) != 0
                     and len(queried_former_failed_knowledge_to_render) != 0
                 ):
+
                     error_summary_system_prompt = (
-                        Template(
-                            Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                                "evolving_strategy_error_summary_v2_system"
-                            ],
-                        )
+                        Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
                         .render(
                             factor_information_str=target_factor_task_information,
                             code_and_feedback=queried_former_failed_knowledge_to_render[
@@ -248,11 +276,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     )
                     while True:
                         error_summary_user_prompt = (
-                            Template(
-                                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                                    "evolving_strategy_error_summary_v2_user"
-                                ],
-                            )
+                            Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
                             .render(
                                 queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
                             )
@@ -269,12 +293,10 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         user_prompt=error_summary_user_prompt,
                         json_mode=False,
                     )
-
+                # 构建user_prompt。开始写代码
                 user_prompt = (
                     Template(
-                        Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                            "evolving_strategy_factor_implementation_v2_user"
-                        ],
+                        implement_prompts["evolving_strategy_factor_implementation_v2_user"],
                     )
                     .render(
                         factor_information_str=target_factor_task_information,
@@ -301,12 +323,6 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                 elif len(queried_similar_error_knowledge_to_render) > 0:
                     queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
 
-            # print(
-            #     len(queried_similar_component_knowledge_to_render),
-            #     len(queried_similar_error_knowledge_to_render),
-            #     len(queried_former_failed_knowledge_to_render),
-            # )
-
             response = session.build_chat_completion(
                 user_prompt=user_prompt,
                 json_mode=True,

rdagent/factor_implementation/evolving/factor.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+from rdagent.factor_implementation.share_modules.factor_implementation_config import (
+    FactorImplementSettings,
+)
+
+from rdagent.core.task import (
+    TaskImplementation,
+    BaseTask,
+    TestCase,
+)
+from rdagent.core.evolving_framework import EvolvableSubjects
+from rdagent.core.log import FinCoLog
+
+from pathlib import Path
+
+from rdagent.oai.llm_utils import md5_hash
+
+from rdagent.core.exception import (
+    CodeFormatException,
+    NoOutputException,
+    RuntimeErrorException,
+)
+
+import pandas as pd
+import uuid
+import pickle
+import subprocess
+from typing import Tuple, Union
+from filelock import FileLock
+
+
+class FactorImplementTask(BaseTask):
+    def __init__(
+        self,
+        factor_name,
+        factor_description,
+        factor_formulation,
+        factor_formulation_description: str = '',
+        variables: dict = {},
+        resource: str = None,
+    ) -> None:
+        self.factor_name = factor_name
+        self.factor_description = factor_description
+        self.factor_formulation = factor_formulation
+        self.factor_formulation_description = factor_formulation_description
+        self.variables = variables
+        self.factor_resources = resource
+
+    def get_factor_information(self):
+        return f"""factor_name: {self.factor_name}
+factor_description: {self.factor_description}
+factor_formulation: {self.factor_formulation}
+factor_formulation_description: {self.factor_formulation_description}"""
+
+    @staticmethod
+    def from_dict(dict):
+        return FactorImplementTask(**dict)
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}[{self.factor_name}]>"
+
+
+class FactorEvovlingItem(EvolvableSubjects):
+    """
+    Intermediate item of factor implementation.
+    """
+
+    def __init__(
+        self,
+        target_factor_tasks: list[FactorImplementTask],
+        corresponding_gt: list[TestCase] = None,
+        corresponding_gt_implementations: list[TaskImplementation] = None,
+    ):
+        super().__init__()
+        self.target_factor_tasks = target_factor_tasks
+        self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks]
+        self.corresponding_selection: list[list] = []
+        self.evolve_trace = {}
+        self.corresponding_gt = corresponding_gt
+        if corresponding_gt_implementations is not None and len(
+            corresponding_gt_implementations,
+        ) != len(target_factor_tasks):
+            self.corresponding_gt_implementations = None
+            FinCoLog.warning(
+                "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
+            )
+        else:
+            self.corresponding_gt_implementations = corresponding_gt_implementations
+
+
+class FileBasedFactorImplementation(TaskImplementation):
+    """
+    This class is used to implement a factor by writing the code to a file.
+    Input data and output factor value are also written to files.
+    """
+
+    # TODO: (Xiao) think raising errors may get better information for processing
+    FB_FROM_CACHE = "The factor value has been executed and stored in the instance variable."
+    FB_EXEC_SUCCESS = "Execution succeeded without error."
+    FB_CODE_NOT_SET = "code is not set."
+    FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
+    FB_OUTPUT_FILE_NOT_FOUND = "\nExpected output file not found."
+    FB_OUTPUT_FILE_FOUND = "\nExpected output file found."
+
+    def __init__(
+        self,
+        target_task: FactorImplementTask,
+        code,
+        executed_factor_value_dataframe=None,
+        raise_exception=False,
+    ) -> None:
+        super().__init__(target_task)
+        self.code = code
+        self.executed_factor_value_dataframe = executed_factor_value_dataframe
+        self.logger = FinCoLog()
+        self.raise_exception = raise_exception
+        self.workspace_path = Path(
+            FactorImplementSettings().file_based_execution_workspace,
+        ) / str(uuid.uuid4())
+
+    @staticmethod
+    def link_data_to_workspace(data_path: Path, workspace_path: Path):
+        data_path = Path(data_path)
+        workspace_path = Path(workspace_path)
+        for data_file_path in data_path.iterdir():
+            workspace_data_file_path = workspace_path / data_file_path.name
+            if workspace_data_file_path.exists():
+                workspace_data_file_path.unlink()
+            subprocess.run(
+                ["ln", "-s", data_file_path, workspace_data_file_path],
+                check=False,
+            )
+
+    def execute(self, store_result: bool = False) -> Tuple[str, pd.DataFrame]:
+        """
+        execute the implementation and get the factor value by the following steps:
+        1. make the directory in workspace path
+        2. write the code to the file in the workspace path
+        3. link all the source data to the workspace path folder
+        4. execute the code
+        5. read the factor value from the output file in the workspace path folder
+        returns the execution feedback as a string and the factor value as a pandas dataframe
+
+        parameters:
+        store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation
+        """
+        if self.code is None:
+            if self.raise_exception:
+                raise CodeFormatException(self.FB_CODE_NOT_SET)
+            else:
+                # TODO: to make the interface compatible with previous code. I kept the original behavior.
+                raise ValueError(self.FB_CODE_NOT_SET)
+        with FileLock(self.workspace_path / "execution.lock"):
+            (Path.cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").mkdir(
+                exist_ok=True, parents=True
+            )
+            if FactorImplementSettings().enable_execution_cache:
+                # NOTE: cache the result for the same code
+                target_file_name = md5_hash(self.code)
+                cache_file_path = (
+                    Path.cwd()
+                    / "git_ignore_folder"
+                    / "factor_implementation_execution_cache"
+                    / f"{target_file_name}.pkl"
+                )
+                if cache_file_path.exists() and not self.raise_exception:
+                    cached_res = pickle.load(open(cache_file_path, "rb"))
+                    if store_result and cached_res[1] is not None:
+                        self.executed_factor_value_dataframe = cached_res[1]
+                    return cached_res
+
+            if self.executed_factor_value_dataframe is not None:
+                return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
+
+            source_data_path = Path(
+                FactorImplementSettings().file_based_execution_data_folder,
+            )
+            self.workspace_path.mkdir(exist_ok=True, parents=True)
+
+            code_path = self.workspace_path / f"{self.target_task.factor_name}.py"
+            code_path.write_text(self.code)
+
+            self.link_data_to_workspace(source_data_path, self.workspace_path)
+
+            execution_feedback = self.FB_EXECUTION_SUCCEEDED
+            try:
+                subprocess.check_output(
+                    f"python {code_path}",
+                    shell=True,
+                    cwd=self.workspace_path,
+                    stderr=subprocess.STDOUT,
+                    timeout=FactorImplementSettings().file_based_execution_timeout,
+                )
+            except subprocess.CalledProcessError as e:
+                import site
+
+                execution_feedback = (
+                    e.output.decode()
+                    .replace(str(code_path.parent.absolute()), r"/path/to")
+                    .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
+                )
+                if len(execution_feedback) > 2000:
+                    execution_feedback = (
+                        execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
+                    )
+                if self.raise_exception:
+                    raise RuntimeErrorException(execution_feedback)
+            except subprocess.TimeoutExpired:
+                execution_feedback += f"Execution timeout error and the timeout is set to {FactorImplementSettings().file_based_execution_timeout} seconds."
+                if self.raise_exception:
+                    raise RuntimeErrorException(execution_feedback)
+
+            workspace_output_file_path = self.workspace_path / "result.h5"
+            if not workspace_output_file_path.exists():
+                execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND
+                executed_factor_value_dataframe = None
+                if self.raise_exception:
+                    raise NoOutputException(execution_feedback)
+            else:
+                try:
+                    executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path)
+                    execution_feedback += self.FB_OUTPUT_FILE_FOUND
+                except Exception as e:
+                    execution_feedback += f"Error found when reading hdf file: {e}"[:1000]
+                    executed_factor_value_dataframe = None
+
+            if store_result and executed_factor_value_dataframe is not None:
+                self.executed_factor_value_dataframe = executed_factor_value_dataframe
+
+        if FactorImplementSettings().enable_execution_cache:
+            pickle.dump(
+                (execution_feedback, executed_factor_value_dataframe),
+                open(cache_file_path, "wb"),
+            )
+        return execution_feedback, executed_factor_value_dataframe
+
+    def __str__(self) -> str:
+        # NOTE:
+        # If the code cache works, the workspace will be None.
+        return f"File Factor[{self.target_task.factor_name}]: {self.workspace_path}"
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    @staticmethod
+    def from_folder(task: FactorImplementTask, path: Union[str, Path], **kwargs):
+        path = Path(path)
+        factor_path = (path / task.factor_name).with_suffix(".py")
+        with factor_path.open("r") as f:
+            code = f.read()
+        return FileBasedFactorImplementation(task, code=code, **kwargs)
+

rdagent/factor_implementation/evolving/factor_implementation_evolving_cli.py

@@ -1,312 +0,0 @@
-import json
-import pickle
-import subprocess
-from pathlib import Path
-
-import pandas as pd
-from fire.core import Fire
-from rdagent.core.evolving_framework import EvoAgent, KnowledgeBase
-from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.evolving.evaluators import (
-    FactorImplementationEvaluatorV1,
-    FactorImplementationsMultiEvaluator,
-)
-from rdagent.factor_implementation.evolving.evolvable_subjects import FactorImplementationList
-from rdagent.factor_implementation.evolving.evolving_strategy import (
-    FactorEvolvingStrategy,
-    FactorEvolvingStrategyWithGraph,
-)
-from rdagent.factor_implementation.evolving.knowledge_management import (
-    FactorImplementationGraphKnowledgeBase,
-    FactorImplementationGraphRAGStrategy,
-    FactorImplementationKnowledgeBaseV1,
-    FactorImplementationRAGStrategyV1,
-)
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementationTask,
-    FileBasedFactorImplementation,
-)
-from tqdm import tqdm
-
-ALPHA101_INIT_COMPONENTS = [
-    "1. abs(): absolute value to certain columns",
-    "2. log(): log value to certain columns",
-    "3. sign(): sign value to certain columns",
-    "4. add_two_columns(): add two columns",
-    "5. minus_two_columns(): minus two columns",
-    "6. times_two_columns(): times two columns",
-    "7. divide_two_columns(): divide two columns",
-    "8. add_value_to_columns(): add value to columns",
-    "9. minus_value_to_columns(): minus value to columns",
-    "10. rank(): cross-sectional rank value to columns",
-    "11. delay(): value of each data d days ago",
-    "12. correlation(): time-serial correlation of column_left and column_right for the past d days",
-    "13. covariance(): time-serial covariance of column_left and column_right for the past d days",
-    "14. scale_to_a(): scale the columns to sum(abs(x)) is a",
-    "15. delta(): today’s value of x minus the value of x d days ago",
-    "16. signedpower(): x^a",
-    "17. decay_linear(): weighted moving average over the past d days with linearly decaying weights d, d – 1, …, 1 (rescaled to sum up to 1)",
-    "18. indneutralize(): x cross-sectionally neutralized against groups g (subindustries, industries, sectors, etc.), i.e., x is cross-sectionally demeaned within each group g",
-    "19. ts_min(): time-series min over the past d days, operator min applied across the time-series for the past d days; non-integer number of days d is converted to floor(d)",
-    "20. ts_max(): time-series max over the past d days, operator max applied across the time-series for the past d days; non-integer number of days d is converted to floor(d)",
-    "21. ts_argmax(): which day ts_max(x, d) occurred on",
-    "22. ts_argmin(): which day ts_min(x, d) occurred on",
-    "23. ts_rank(): time-series rank in the past d days",
-    "24. min(): ts_min(x, d)",
-    "25. max(): ts_max(x, d)",
-    "26. sum(): time-series sum over the past d days",
-    "27. product(): time-series product over the past d days",
-    "28. stddev(): moving time-series standard deviation over the past d days",
-]
-
-
-class FactorImplementationEvolvingCli:
-    # TODO: we should use polymorphism to load knowledge base, strategies instead of evolving_version
-    # TODO: Can we refactor FactorImplementationEvolvingCli into a learning framework to differentiate our learning paradiagm with other ones by iteratively retrying?
-    def __init__(self, evolving_version=2) -> None:
-        self.evolving_version = evolving_version
-        self.knowledge_base = None
-        self.latest_factor_implementations = None
-
-    def run_evolving_framework(
-        self,
-        factor_implementations: FactorImplementationList,
-        factor_knowledge_base: KnowledgeBase,
-        max_loops: int = 20,
-        with_knowledge: bool = True,
-        with_feedback: bool = True,
-        knowledge_self_gen: bool = True,
-    ) -> FactorImplementationList:
-        """
-        Main target: Implement factors.
-        The system also leverages the former knowledge to help implement the factors. Also, new knowledge might be generated during the implementation to help the following implementation.
-        The gt_code and gt_value in the Factor instance is used to evaluate the implementation, and the feedback is used to generate high-quality knowledge which helps the agent to evolve.
-        """
-        es = FactorEvolvingStrategyWithGraph() if self.evolving_version == 2 else FactorEvolvingStrategy()
-        rag = (
-            FactorImplementationGraphRAGStrategy(factor_knowledge_base)
-            if self.evolving_version == 2
-            else FactorImplementationRAGStrategyV1(factor_knowledge_base)
-        )
-        factor_evaluator = FactorImplementationsMultiEvaluator(FactorImplementationEvaluatorV1())
-        ea = EvoAgent(es, rag=rag)
-
-        for _ in tqdm(range(max_loops), "Implementing factors"):
-            factor_implementations = ea.step_evolving(
-                factor_implementations,
-                factor_evaluator,
-                with_knowledge=with_knowledge,
-                with_feedback=with_feedback,
-                knowledge_self_gen=knowledge_self_gen,
-            )
-        return factor_implementations
-
-    def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
-        if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
-            factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
-            if (
-                self.evolving_version == 1
-                and not isinstance(
-                    factor_knowledge_base,
-                    FactorImplementationKnowledgeBaseV1,
-                )
-                or self.evolving_version == 2
-                and not isinstance(
-                    factor_knowledge_base,
-                    FactorImplementationGraphKnowledgeBase,
-                )
-            ):
-                raise ValueError("The former knowledge base is not compatible with the current version")
-        else:
-            factor_knowledge_base = (
-                FactorImplementationGraphKnowledgeBase(
-                    init_component_list=component_init_list,
-                )
-                if self.evolving_version == 2
-                else FactorImplementationKnowledgeBaseV1()
-            )
-        return factor_knowledge_base
-
-    def implement_factors(
-        self,
-        factor_implementations: FactorImplementationList,
-        former_knowledge_base_path: Path = None,
-        new_knowledge_base_path: Path = None,
-        component_init_list: list = [],
-        max_loops: int = 20,
-    ):
-        factor_knowledge_base = self.load_or_init_knowledge_base(
-            former_knowledge_base_path=former_knowledge_base_path,
-            component_init_list=component_init_list,
-        )
-
-        new_factor_implementations = self.run_evolving_framework(
-            factor_implementations=factor_implementations,
-            factor_knowledge_base=factor_knowledge_base,
-            max_loops=max_loops,
-            with_knowledge=True,
-            with_feedback=True,
-            knowledge_self_gen=True,
-        )
-        if new_knowledge_base_path is not None:
-            pickle.dump(factor_knowledge_base, open(new_knowledge_base_path, "wb"))
-        self.knowledge_base = factor_knowledge_base
-        self.latest_factor_implementations = factor_implementations
-        return new_factor_implementations
-
-    def _read_alpha101_factors(
-        self,
-        alpha101_evo_subs_path: Path = None,
-        alpha101_data_path=Path().cwd() / "git_ignore_folder" / "alpha101_related_files",
-        start_index=0,
-        end_index=32,
-        read_gt_factors=True,
-    ) -> FactorImplementationList:
-        """
-        Read the alpha101 factors from the alpha101_related_files folder
-        """
-        if alpha101_evo_subs_path is not None and alpha101_evo_subs_path.exists():
-            factor_implementations = pickle.load(open(alpha101_evo_subs_path, "rb"))
-        else:
-            target_factor_plain_list = json.load(
-                open(alpha101_data_path / "target_factor_task_list.json"),
-            )
-            name_to_code = json.load(open(alpha101_data_path / "name_to_code.json"))
-            gt_df = pd.read_hdf(alpha101_data_path / "gt_filtered.h5")
-
-            # First read the target factor task
-            target_factor_tasks = []
-            for factor_list_item in target_factor_plain_list:
-                target_factor_tasks.append(
-                    FactorImplementationTask(
-                        factor_name=factor_list_item[0],
-                        factor_description=factor_list_item[1],
-                        factor_formulation=factor_list_item[2],
-                        factor_formulation_description=factor_list_item[3],
-                    ),
-                )
-
-            # Second read the gt factor implementations
-            corresponding_gt_implementations = []
-            for factor_task in target_factor_tasks:
-                name = factor_task.factor_name
-                gt_code = name_to_code[name]
-                gt_value = gt_df.loc(axis=1)[[name]]
-                corresponding_gt_implementations.append(
-                    FileBasedFactorImplementation(
-                        code=gt_code,
-                        executed_factor_value_dataframe=gt_value,
-                        target_task=factor_task,
-                    ),
-                )
-
-            # Finally generate the factor implementations as evolvable subjects
-            factor_implementations = FactorImplementationList(
-                target_factor_tasks=target_factor_tasks,
-                corresponding_gt_implementations=(corresponding_gt_implementations if read_gt_factors else None),
-            )
-
-        factor_implementations.target_factor_tasks = factor_implementations.target_factor_tasks[start_index:end_index]
-        factor_implementations.corresponding_gt_implementations = (
-            factor_implementations.corresponding_gt_implementations[start_index:end_index] if read_gt_factors else None
-        )
-
-        return factor_implementations
-
-    def implement_alpha101(
-        self,
-        max_loops=30,
-    ) -> FactorImplementationList:
-        """
-        Implement the alpha101 factors to gather knowledge TODO: implement the code
-        """
-        factor_implementations = self._read_alpha101_factors(
-            alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
-            start_index=0,
-            end_index=64,
-            read_gt_factors=True,
-        )
-        self.implement_factors(
-            factor_implementations,
-            former_knowledge_base_path=Path.cwd()
-            / f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
-            new_knowledge_base_path=Path.cwd()
-            / f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
-            component_init_list=ALPHA101_INIT_COMPONENTS,
-            max_loops=100,
-        )
-
-        factor_implementations = self._read_alpha101_factors(
-            alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
-            start_index=64,
-            end_index=96,
-            read_gt_factors=False,
-        )
-        final_imp = self.implement_factors(
-            factor_implementations,
-            former_knowledge_base_path=Path.cwd()
-            / f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
-            new_knowledge_base_path=Path.cwd()
-            / f"alpha101_knowledge_base_v{self.evolving_version}_self_evolving_project_product.pkl",
-            component_init_list=ALPHA101_INIT_COMPONENTS,
-            max_loops=10,
-        )
-        final_imp.corresponding_gt_implementations = factor_implementations = self._read_alpha101_factors(
-            alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
-            start_index=64,
-            end_index=96,
-            read_gt_factors=True,
-        ).corresponding_gt_implementations
-
-        feedbacks = FactorImplementationsMultiEvaluator().evaluate(final_imp)
-        print([feedback.final_decision if feedback is not None else None for feedback in feedbacks].count(True))
-
-    def implement_amc(
-        self, evo_sub_path_str, former_knowledge_base_path_str, implementation_dump_path_str, slice_index,
-    ):
-        factor_implementations: FactorImplementationList = pickle.load(open(evo_sub_path_str, "rb"))
-        factor_implementations.target_factor_tasks = factor_implementations.target_factor_tasks[
-            slice_index * 16 : slice_index * 16 + 16
-        ]
-        if len(factor_implementations.target_factor_tasks) == 0:
-            return
-        if Path(implementation_dump_path_str).exists():
-            return
-        factor_implementations = self.implement_factors(
-            factor_implementations,
-            former_knowledge_base_path=Path(former_knowledge_base_path_str),
-            component_init_list=ALPHA101_INIT_COMPONENTS,
-            max_loops=10,
-        )
-        pickle.dump(factor_implementations, open(implementation_dump_path_str, "wb"))
-
-    def execute_command(self, command, cwd):
-        print(command, cwd)
-        try:
-            subprocess.check_output(
-                command,
-                shell=True,
-                cwd=cwd,
-            )
-        except subprocess.CalledProcessError as e:
-            print(e.output.decode())
-
-    def multi_inference_amc_factors(self, type):
-        slice_count = {"price_volume": 35, "fundamental": 24, "high_frequency": 16}[type]
-        res = multiprocessing_wrapper(
-            [
-                (
-                    self.execute_command,
-                    (
-                        f"python src/scripts/factor_implementation/baselines/evolving/factor_implementation_evolving_cli.py implement_amc ./{type}_factors.pkl ./knowledge_base_v2_with_alpha101_and_10_factors.pkl ./inference_amc_factors_{type}_{slice}.pkl {slice}",
-                        Path.cwd(),
-                    ),
-                )
-                for slice in range(slice_count)
-            ],
-            n=2,
-        )
-
-
-if __name__ == "__main__":
-    Fire(FactorImplementationEvolvingCli)

rdagent/factor_implementation/evolving/knowledge_management.py

@@ -6,6 +6,7 @@ import random
 import re
 from itertools import combinations
 from pathlib import Path
+from jinja2 import Template
 from typing import Union
 
 from jinja2 import Template
@@ -20,22 +21,23 @@ from rdagent.core.evolving_framework import (
 from rdagent.core.log import FinCoLog
 from rdagent.core.prompts import Prompts
 from rdagent.factor_implementation.evolving.evaluators import FactorImplementationSingleFeedback
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-)
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
+from rdagent.core.task import (
+    TaskImplementation,
 )
+from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask
+from rdagent.core.prompts import Prompts
 from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode
 from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
 
+from rdagent.factor_implementation.share_modules.factor_implementation_config import (
+    FactorImplementSettings,
+)
 
 class FactorImplementationKnowledge(Knowledge):
     def __init__(
         self,
-        target_task: FactorImplementationTask,
-        implementation: FactorImplementation,
+        target_task: FactorImplementTask,
+        implementation: TaskImplementation,
         feedback: FactorImplementationSingleFeedback,
     ) -> None:
         """
@@ -899,3 +901,4 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
                     intersection_node_list_sort_by_freq.append(node)
 
         return intersection_node_list_sort_by_freq
+

rdagent/factor_implementation/evolving/scheduler.py

@@ -0,0 +1,69 @@
+from rdagent.oai.llm_utils import APIBackend
+from jinja2 import Template
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
+import json
+from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
+from rdagent.factor_implementation.evolving.factor import FactorEvovlingItem
+from rdagent.core.prompts import Prompts
+from pathlib import Path
+
+scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
+    import random
+    to_be_finished_task_index = random.sample(
+        to_be_finished_task_index,
+        implementation_factors_per_round,
+    )
+    print("The random selection is:",to_be_finished_task_index)
+    return to_be_finished_task_index
+
+def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:FactorEvovlingItem, former_trace):
+    tasks = []
+    for i in to_be_finished_task_index:
+    # find corresponding former trace for each task
+        target_factor_task_information = evo.target_factor_tasks[i].get_factor_information()
+        if target_factor_task_information in former_trace:
+            tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information]))
+
+    system_prompt = Template(
+        scheduler_prompts["select_implementable_factor_system"],
+    ).render(
+        data_info=get_data_folder_intro(),
+    )
+
+    session = APIBackend(use_chat_cache=False).build_chat_session(
+        session_system_prompt=system_prompt,
+    )
+
+    while True:
+        user_prompt = (
+            Template(
+                scheduler_prompts["select_implementable_factor_user"],
+            )
+            .render(
+                factor_num = implementation_factors_per_round,
+                target_factor_tasks=tasks,
+            )
+        )
+        if (
+            session.build_chat_completion_message_and_calculate_token(
+                user_prompt,
+            )
+            < FactorImplementSettings().chat_token_limit
+        ):
+            break
+
+    response = session.build_chat_completion(
+        user_prompt=user_prompt,
+        json_mode=True,
+    )
+    try:
+        selection = json.loads(response)["selected_factor"]
+        if not isinstance(selection, list):
+            return to_be_finished_task_index
+        selection_index = [x for x in selection if isinstance(x, int)]
+    except:
+        return to_be_finished_task_index
+    
+    return selection_index