reporeformat V2 (#23)

* reformat factor implement process

* move some code to more reasonable place

* fix the bug

* add test function in factor_extract_and_implement.py

* change select factor number to ratio , add some factor implement setting and fix some bug while using knowledgebase

* change evoagent

* add abstract class EvoAgent

* add benchmark workflow

* fix some bug in llm_utils

* run wenjun's code

* fix the knowledgebase instance check

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py

@@ -8,8 +8,13 @@ from rdagent.document_process.document_analysis import (
     deduplicate_factors_by_llm,
     extract_factors_from_report_dict,
     merge_file_to_factor_dict_to_factor_dict,
+    classify_report_from_dict,
 )
 from rdagent.document_process.document_reader import load_and_process_pdfs_by_langchain
+from rdagent.factor_implementation.share_modules.factor_implementation_utils import load_data_from_dict
+from rdagent.factor_implementation.CoSTEER import CoSTEERFG
+import pickle
+from dotenv import load_dotenv
 
 
 def extract_factors_and_implement(report_file_path: str) -> None:
@@ -24,6 +29,15 @@ def extract_factors_and_implement(report_file_path: str) -> None:
 
     factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability)
 
+    factor_tasks = load_data_from_dict(factor_dict)
+
+    factor_generate_method = CoSTEERFG()
+
+    result = factor_generate_method.generate(factor_tasks)
+
+    return result
+
 
 if __name__ == "__main__":
     extract_factors_and_implement("/home/xuyang1/workspace/report.pdf")
+    # test_implement()

rdagent/app/factor_implementation/eval_implement.py

@@ -0,0 +1,30 @@
+from rdagent.core.conf import BenchmarkSettings
+from rdagent.core.utils import import_class
+from rdagent.benchmark.eval_method import FactorImplementEval
+from rdagent.benchmark.data_process import load_eval_data
+
+# 1.read the settings
+bs = BenchmarkSettings()
+
+# 2.read and prepare the eval_data
+test_cases = load_eval_data(bs.bench_version)
+
+# 3.declare the method to be tested and pass the arguments.
+# TODO: Whether it is necessary to define two Eval method classes for two data type?
+method_cls = import_class(bs.bench_method_cls)
+generate_method = method_cls(bs.bench_method_extra_kwargs)
+
+# 4.declare the eval method and pass the arguments.
+eval_method = FactorImplementEval(
+    method=generate_method,
+    test_cases=test_cases,
+    catch_eval_except=True,
+    test_round=bs.bench_test_round,
+)
+
+# 5.run the eval
+eval_method.eval()
+
+# 6.save the result
+eval_method.save(output_path = bs.bench_result_path)
+

rdagent/benchmark/conf.py

@@ -0,0 +1,27 @@
+from dotenv import load_dotenv
+load_dotenv(verbose=True, override=True)
+from dataclasses import field
+from pathlib import Path
+from typing import Literal, Optional, Union
+
+from pydantic_settings import BaseSettings
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+BENCHMARK_VERSION = Literal["paper", "amcV01", "amcV02train", "amcV02test"]
+
+class BenchmarkSettings(BaseSettings):
+
+    ground_truth_dir: Path = DIRNAME / "ground_truth"
+
+    bench_version: Union[BENCHMARK_VERSION, str] = "paper"
+
+    bench_test_round: int = 20 
+    bench_test_case_n: Optional[int] = None  # how many test cases to run; If not given, all test cases will be run
+
+    bench_method_cls: str = "scripts.factor_implementation.baselines.naive.one_shot.OneshotFactorGen"
+    bench_method_extra_kwargs: dict = field(
+        default_factory=dict,
+    )  # extra kwargs for the method to be tested except the task list
+
+    bench_result_path: Path = DIRNAME / "result"

rdagent/benchmark/eval_method.py

@@ -0,0 +1,158 @@
+from pathlib import Path
+from typing import List, Tuple, Union
+
+from tqdm import tqdm
+from collections import defaultdict
+from rdagent.core.conf import RDAgentSettings
+from rdagent.core.exception import ImplementRunException
+from rdagent.core.task import (
+    TaskImplementation,
+    TestCase,
+)
+from rdagent.factor_implementation.evolving.evaluators import (
+    FactorImplementationCorrelationEvaluator,
+    FactorImplementationIndexEvaluator,
+    FactorImplementationIndexFormatEvaluator,
+    FactorImplementationMissingValuesEvaluator,
+    FactorImplementationRowCountEvaluator,
+    FactorImplementationSingleColumnEvaluator,
+    FactorImplementationValuesEvaluator,
+    FactorImplementationEvaluator,
+)
+from rdagent.core.implementation import TaskGenerator
+from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.factor_implementation.evolving.factor import FileBasedFactorImplementation
+
+class BaseEval:
+    """
+    The benchmark benchmark evaluation.
+    """
+    def __init__(
+        self,
+        evaluator_l: List[FactorImplementationEvaluator],
+        test_cases: List[TestCase],
+        generate_method: TaskGenerator,
+        catch_eval_except: bool = True,
+    ):
+        """Parameters
+        ----------
+        test_cases : List[TestCase]
+            cases to be evaluated, ground truth are included in the test cases.
+        evaluator_l : List[FactorImplementationEvaluator]
+            A list of evaluators to evaluate the generated code.
+        catch_eval_except : bool
+            If we want to debug the evaluators, we recommend to set the this parameter to True.
+        """
+        self.evaluator_l = evaluator_l
+        self.test_cases = test_cases
+        self.generate_method = generate_method
+        self.catch_eval_except = catch_eval_except
+
+    def load_cases_to_eval(
+        self,
+        path: Union[Path, str],
+        **kwargs,
+    ) -> List[TaskImplementation]:
+        path = Path(path)
+        fi_l = []
+        for tc in self.test_cases:
+            try:
+                fi = FileBasedFactorImplementation.from_folder(tc.task, path, **kwargs)
+                fi_l.append(fi)
+            except FileNotFoundError:
+                print("Fail to load test case for factor: ", tc.task.factor_name)
+        return fi_l
+
+    def eval_case(
+        self,
+        case_gt: TaskImplementation,
+        case_gen: TaskImplementation,
+    ) -> List[Union[Tuple[FactorImplementationEvaluator, object], Exception]]:
+        """Parameters
+        ----------
+        case_gt : FactorImplementation
+
+        case_gen : FactorImplementation
+
+
+        Returns
+        -------
+        List[Union[Tuple[FactorImplementationEvaluator, object],Exception]]
+            for each item
+                If the evaluation run successfully, return the evaluate results.  Otherwise, return the exception.
+        """
+        eval_res = []
+        for ev in self.evaluator_l:
+            try:
+                eval_res.append((ev, ev.evaluate(case_gt, case_gen)))
+                # if the corr ev is successfully evaluated and achieve the best performance, then break
+            except ImplementRunException as e:
+                return e
+            except Exception as e:
+                # exception when evaluation
+                if self.catch_eval_except:
+                    eval_res.append((ev, e))
+                else:
+                    raise e
+        return eval_res
+    
+class FactorImplementEval(BaseEval):
+    def __init__(
+        self,
+        test_case: TestCase,
+        method: TaskGenerator,
+        test_round: int = 10,
+        *args,
+        **kwargs,
+    ):
+        # evaluator collection for online evaluation
+        online_evaluator_l = [
+                            FactorImplementationCorrelationEvaluator,
+                            FactorImplementationIndexEvaluator,
+                            FactorImplementationIndexFormatEvaluator,
+                            FactorImplementationMissingValuesEvaluator,
+                            FactorImplementationRowCountEvaluator,
+                            FactorImplementationSingleColumnEvaluator,
+                            FactorImplementationValuesEvaluator,
+                         ],
+        super().__init__(online_evaluator_l, test_case, method, *args, **kwargs)
+        self.test_round = test_round
+
+    def eval(self):
+
+        gen_factor_l_all_rounds = []
+        test_cases_all_rounds = []
+        res = defaultdict(list)
+        for _ in tqdm(range(self.test_round), desc="Rounds of Eval"):
+            print("\n========================================================")
+            print(f"Eval {_}-th times...")
+            print("========================================================\n")
+            try:
+                gen_factor_l = self.generate_method.generate(self.test_cases.target_task)
+            except KeyboardInterrupt:
+                # TODO: Why still need to save result after KeyboardInterrupt?
+                print("Manually interrupted the evaluation. Saving existing results")
+                break
+
+            if len(gen_factor_l) != len(self.test_cases):
+                raise ValueError(
+                    "The number of cases to eval should be equal to the number of test cases.",
+                )
+            gen_factor_l_all_rounds.extend(gen_factor_l)
+            test_cases_all_rounds.extend(self.test_cases)
+
+        eval_res_l = []
+
+        eval_res_list = multiprocessing_wrapper(
+            [
+                (self.eval_case, (gt_case.ground_truth, gen_factor))
+                for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds)
+            ],
+            n=RDAgentSettings().evo_multi_proc_n,
+        )
+
+        for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)):
+            res[gt_case.task.factor_name].append((gen_factor, eval_res))
+            eval_res_l.append(eval_res)
+
+        return res

rdagent/core/conf.py

@@ -11,8 +11,10 @@ from pydantic_settings import BaseSettings
 # make sure that env variable is loaded while calling Config()
 load_dotenv(verbose=True, override=True)
 
+from pydantic_settings import BaseSettings
 
-class FincoSettings(BaseSettings):
+
+class RDAgentSettings(BaseSettings):
     use_azure: bool = True
     use_azure_token_provider: bool = False
     max_retry: int = 10
@@ -96,3 +98,4 @@ class FincoSettings(BaseSettings):
     # factor extraction conf
     max_input_duplicate_factor_group: int = 600
     max_output_duplicate_factor_group: int = 20
+

rdagent/core/evaluation.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from rdagent.core.task import (
+    TaskImplementation,
+    BaseTask,
+)
+
+class Evaluator(ABC):
+    @abstractmethod
+    def evaluate(
+        self,
+        target_task: BaseTask,
+        implementation: TaskImplementation,
+        gt_implementation: TaskImplementation,
+        **kwargs,
+    ):
+        raise NotImplementedError

rdagent/core/evolving_framework.py

@@ -91,6 +91,16 @@ class EvolvingStrategy(ABC):
         """
 
 
+class EvoAgent(ABC):
+    def __init__(self, max_loop, evolving_strategy) -> None:
+        self.max_loop = max_loop
+        self.evolving_strategy = evolving_strategy
+    
+    @abstractmethod
+    def multistep_evolve(self, evo: EvolvableSubjects, eva: Evaluator | Feedback, **kwargs: Any) -> EvolvableSubjects:
+        pass
+
+
 class RAGStrategy(ABC):
     """Retrival Augmentation Generation Strategy"""
 
@@ -119,66 +129,3 @@ class RAGStrategy(ABC):
 
         RAGStrategy should maintain the new knowledge all by itself.
         """
-
-
-class EvoAgent:
-    """It is responsible for driving the workflow."""
-
-    evolving_trace: list[EvoStep]
-
-    def __init__(
-        self,
-        evolving_strategy: EvolvingStrategy,
-        rag: RAGStrategy | None = None,
-    ) -> None:
-        self.evolving_trace = []
-        self.evolving_strategy = evolving_strategy
-        self.rag = rag
-
-    def step_evolving(
-        self,
-        evo: EvolvableSubjects,
-        eva: Evaluator | Feedback,
-        *,
-        with_knowledge: bool = False,
-        with_feedback: bool = True,
-        knowledge_self_gen: bool = False,
-    ) -> EvolvableSubjects:
-        """Common evolving mode are supported in this api .
-        - Interactive evolving:
-            - `with_feedback=True` and `eva` is a external Evaluator.
-
-        - Knowledge-driven evolving:
-            - `with_knowledge=True` and related knowledge are
-            queried based on `self.rag`
-
-        - Self-evolving: we have two ways to self-evolve.
-            - 1) self generating knowledge and then evolve
-                - `knowledge_self_gen=True` and `with_knowledge=True`
-            - 2) self evaluate to generate feedback and then evolve
-                - `with_feedback=True` and `eva` is a internal Evaluator.
-        """
-        # knowledge self-evolving
-        if knowledge_self_gen and self.rag is not None:
-            self.rag.generate_knowledge(self.evolving_trace)
-
-        # RAG
-        queried_knowledge = None
-        if with_knowledge and self.rag is not None:
-            queried_knowledge = self.rag.query(evo, self.evolving_trace)
-
-        # Evolve
-        evo = self.evolving_strategy.evolve(
-            evo=evo,
-            evolving_trace=self.evolving_trace,
-            queried_knowledge=queried_knowledge,
-        )
-        es = EvoStep(evo, queried_knowledge)
-
-        # Evaluate
-        if with_feedback:
-            es.feedback = eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge)
-
-        # Update trace
-        self.evolving_trace.append(es)
-        return evo

rdagent/core/implementation.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from rdagent.core.task import (
+    TaskImplementation,
+)
+
+class TaskGenerator(ABC):
+    @abstractmethod
+    def generate(self, *args, **kwargs) -> List[TaskImplementation]:
+        raise NotImplementedError("generate method is not implemented.")
+
+    def collect_feedback(self, feedback_obj_l: List[object]):
+        """
+        When online evaluation.
+        The preivous feedbacks will be collected to support advanced factor generator
+
+        Parameters
+        ----------
+        feedback_obj_l : List[object]
+
+        """
+
+

rdagent/core/task.py

@@ -0,0 +1,28 @@
+from abc import ABC, abstractmethod
+from typing import Tuple
+import pandas as pd
+
+'''
+This file contains the all the data class for rdagent task.
+'''
+class BaseTask(ABC):
+    # 把name放在这里作为主键
+    pass
+
+class TaskImplementation(ABC):
+    def __init__(self, target_task: BaseTask) -> None:
+        self.target_task = target_task
+
+    @abstractmethod
+    def execute(self, *args, **kwargs) -> Tuple[str, pd.DataFrame]:
+        raise NotImplementedError("__call__ method is not implemented.")
+
+class TestCase:
+    def __init__(
+        self,
+        target_task: BaseTask,
+        ground_truth: TaskImplementation,
+    ):
+        self.ground_truth = ground_truth
+        self.target_task = target_task
+

rdagent/document_process/document_analysis.py

@@ -10,7 +10,7 @@ import numpy as np
 import pandas as pd
 import tiktoken
 from jinja2 import Template
-from rdagent.core.conf import FincoSettings as Config
+from rdagent.core.conf import RDAgentSettings as Config
 from rdagent.core.log import FinCoLog
 from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing

rdagent/document_process/document_reader.py

@@ -9,7 +9,7 @@ from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader
 
 if TYPE_CHECKING:
     from langchain_core.documents import Document
-from rdagent.core.conf import FincoSettings as Config
+from rdagent.core.conf import RDAgentSettings as Config
 
 
 def load_documents_by_langchain(path: Path) -> list:

rdagent/factor_implementation/CoSTEER.py

@@ -0,0 +1,86 @@
+import pickle
+from pathlib import Path
+from typing import List
+from rdagent.core.implementation import TaskGenerator
+from rdagent.core.task import TaskImplementation
+from rdagent.factor_implementation.evolving.knowledge_management import FactorImplementationKnowledgeBaseV1
+from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FactorEvovlingItem
+from rdagent.knowledge_management.knowledgebase import FactorImplementationGraphKnowledgeBase, FactorImplementationGraphRAGStrategy
+from rdagent.factor_implementation.evolving.evolving_strategy import FactorEvolvingStrategyWithGraph
+from rdagent.factor_implementation.evolving.evaluators import FactorImplementationsMultiEvaluator, FactorImplementationEvaluatorV1
+from rdagent.factor_implementation.evolving.evolving_agent import RAGEvoAgent
+from rdagent.factor_implementation.share_modules.factor_implementation_config import (
+    FactorImplementSettings,
+)
+
+class CoSTEERFG(TaskGenerator):
+    def __init__(
+        self,
+        with_knowledge: bool = True,
+        with_feedback: bool = True,
+        knowledge_self_gen: bool = True,
+    ) -> None:
+        self.max_loop = FactorImplementSettings().max_loop
+        self.knowledge_base_path = Path(FactorImplementSettings().knowledge_base_path) if FactorImplementSettings().knowledge_base_path is not None else None
+        self.new_knowledge_base_path = Path(FactorImplementSettings().new_knowledge_base_path) if FactorImplementSettings().new_knowledge_base_path is not None else None
+        self.with_knowledge = with_knowledge
+        self.with_feedback = with_feedback
+        self.knowledge_self_gen = knowledge_self_gen
+        self.evolving_strategy = FactorEvolvingStrategyWithGraph()
+        # declare the factor evaluator
+        self.factor_evaluator = FactorImplementationsMultiEvaluator(FactorImplementationEvaluatorV1())
+        self.evolving_version = 2
+
+    def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
+    
+        if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
+            factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
+            if self.evolving_version == 1 and not isinstance(
+                factor_knowledge_base, FactorImplementationKnowledgeBaseV1
+            ):
+                raise ValueError("The former knowledge base is not compatible with the current version")
+            elif self.evolving_version == 2 and not isinstance(
+                factor_knowledge_base,
+                FactorImplementationGraphKnowledgeBase,
+            ):
+                raise ValueError("The former knowledge base is not compatible with the current version")
+        else:
+            factor_knowledge_base = (
+                FactorImplementationGraphKnowledgeBase(
+                    init_component_list=component_init_list,
+                )
+                if self.evolving_version == 2
+                else FactorImplementationKnowledgeBaseV1()
+            )
+        return factor_knowledge_base
+    
+    def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]:
+        # init knowledge base
+        factor_knowledge_base = self.load_or_init_knowledge_base(
+            former_knowledge_base_path=self.knowledge_base_path,
+            component_init_list=[],
+        )
+        # init rag method
+        self.rag = (
+            FactorImplementationGraphRAGStrategy(factor_knowledge_base)
+        )
+
+        # init indermediate items
+        factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks)
+
+        self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
+
+        factor_implementations = self.evolve_agent.multistep_evolve(
+            factor_implementations,
+            self.factor_evaluator,
+            with_knowledge=self.with_knowledge,
+            with_feedback=self.with_feedback,
+            knowledge_self_gen=self.knowledge_self_gen,
+        )
+
+        # save new knowledge base
+        if self.new_knowledge_base_path is not None:
+            pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
+        self.knowledge_base = factor_knowledge_base
+        self.latest_factor_implementations = tasks
+        return factor_implementations       

rdagent/factor_implementation/evolving/evaluators.py

@@ -1,30 +1,526 @@
-from __future__ import annotations
-
+import json
 import re
-from typing import List
 
-from pandas.core.api import DataFrame as DataFrame
-from rdagent.core.evolving_framework import Evaluator as EvolvingEvaluator
-from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
+from abc import abstractmethod
+from typing import Tuple
+
+import pandas as pd
+from jinja2 import Template
+
+from rdagent.oai.llm_utils import APIBackend
 from rdagent.core.log import FinCoLog
+from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask, FactorEvovlingItem
+from rdagent.core.task import (
+    TaskImplementation,
+)
+from typing import List, Tuple
+from rdagent.core.evolving_framework import QueriedKnowledge,Feedback
+from rdagent.core.evaluation import Evaluator
+from rdagent.core.prompts import Prompts
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
 from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.evolving.evolvable_subjects import FactorImplementationList
-from rdagent.factor_implementation.share_modules.evaluator import (
-    Evaluator as FactorImplementationEvaluator,
-)
-from rdagent.factor_implementation.share_modules.evaluator import (
-    FactorImplementationCodeEvaluator,
-    FactorImplementationFinalDecisionEvaluator,
-    FactorImplementationValueEvaluator,
-)
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-)
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
-)
+from pathlib import Path
 
+evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+class FactorImplementationEvaluator(Evaluator):
+    # TODO:
+    # I think we should have unified interface for all evaluates, for examples.
+    # So we should adjust the interface of other factors
+    @abstractmethod
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        """You can get the dataframe by
+
+        .. code-block:: python
+
+            _, gt_df = gt.execute()
+            _, gen_df = gen.execute()
+
+        Returns
+        -------
+        Tuple[str, object]
+            - str: the text-based description of the evaluation result
+            - object: a comparable metric (bool, integer, float ...)
+
+        """
+        raise NotImplementedError("Please implement the `evaluator` method")
+
+    def _get_df(self, gt: TaskImplementation, gen: TaskImplementation):
+        _, gt_df = gt.execute()
+        _, gen_df = gen.execute()
+        if isinstance(gen_df, pd.Series):
+            gen_df = gen_df.to_frame("source_factor")
+        if isinstance(gt_df, pd.Series):
+            gt_df = gt_df.to_frame("gt_factor")
+        return gt_df, gen_df
+
+class FactorImplementationCodeEvaluator(Evaluator):
+    def evaluate(
+        self,
+        target_task: FactorImplementTask,
+        implementation: TaskImplementation,
+        execution_feedback: str,
+        factor_value_feedback: str = "",
+        gt_implementation: TaskImplementation = None,
+        **kwargs,
+    ):
+        factor_information = target_task.get_factor_information()
+        code = implementation.code
+
+        system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"]
+
+        execution_feedback_to_render = execution_feedback
+        user_prompt = Template(
+            evaluate_prompts["evaluator_code_feedback_v1_user"],
+        ).render(
+            factor_information=factor_information,
+            code=code,
+            execution_feedback=execution_feedback_to_render,
+            factor_value_feedback=factor_value_feedback,
+            gt_code=gt_implementation.code if gt_implementation else None,
+        )
+        while (
+            APIBackend().build_messages_and_calculate_token(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                former_messages=[],
+            )
+            > FactorImplementSettings().chat_token_limit
+        ):
+            execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            user_prompt = Template(
+                evaluate_prompts["evaluator_code_feedback_v1_user"],
+            ).render(
+                factor_information=factor_information,
+                code=code,
+                execution_feedback=execution_feedback_to_render,
+                factor_value_feedback=factor_value_feedback,
+                gt_code=gt_implementation.code if gt_implementation else None,
+            )
+        critic_response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            json_mode=False,
+        )
+
+        return critic_response
+
+class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        if len(gen_df.columns) == 1 and len(gt_df.columns) == 1:
+            return "Both dataframes have only one column.", True
+        elif len(gen_df.columns) != 1:
+            gen_df = gen_df.iloc(axis=1)[
+                [
+                    0,
+                ]
+            ]
+            return (
+                "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
+                False,
+            )
+        return "", False
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+        idx_name_right = gen_df.index.names == ("datetime", "instrument")
+        if idx_name_right:
+            return (
+                'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
+                True,
+            )
+        else:
+            return (
+                'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.',
+                False,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        if gen_df.shape[0] == gt_df.shape[0]:
+            return "Both dataframes have the same rows count.", True
+        else:
+            return (
+                f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
+                False,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        if gen_df.index.equals(gt_df.index):
+            return "Both dataframes have the same index.", True
+        else:
+            return (
+                "The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
+                False,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
+            return "Both dataframes have the same missing values.", True
+        else:
+            return (
+                f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
+                False,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        try:
+            close_values = gen_df.sub(gt_df).abs().lt(1e-6)
+            result_int = close_values.astype(int)
+            pos_num = result_int.sum().sum()
+            acc_rate = pos_num / close_values.size
+        except:
+            close_values = gen_df
+        if close_values.all().iloc[0]:
+            return (
+                "All values in the dataframes are equal within the tolerance of 1e-6.",
+                acc_rate,
+            )
+        else:
+            return (
+                "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
+                acc_rate,
+            )
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
+    def __init__(self, hard_check: bool) -> None:
+        self.hard_check = hard_check
+
+    def evaluate(
+        self,
+        gt: TaskImplementation,
+        gen: TaskImplementation,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt, gen)
+
+        concat_df = pd.concat([gen_df, gt_df], axis=1)
+        concat_df.columns = ["source", "gt"]
+        ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
+        ric = (
+            concat_df.groupby("datetime")
+            .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
+            .dropna()
+            .mean()
+        )
+
+        if self.hard_check:
+            if ic > 0.99 and ric > 0.99:
+                return (
+                    f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
+                    True,
+                )
+            else:
+                return (
+                    f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
+                    False,
+                )
+        else:
+            return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationValEvaluator(FactorImplementationEvaluator):
+    def evaluate(self, gt: TaskImplementation, gen: TaskImplementation):
+        _, gt_df = gt.execute()
+        _, gen_df = gen.execute()
+        # FIXME: refactor the two classes
+        fiv = FactorImplementationValueEvaluator()
+        return fiv.evaluate(source_df=gen_df, gt_df=gt_df)
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorImplementationValueEvaluator(Evaluator):
+    # TODO: let's discuss the about the interface of the evaluator
+    def evaluate(
+        self,
+        source_df: pd.DataFrame,
+        gt_df: pd.DataFrame,
+        **kwargs,
+    ) -> Tuple:
+        conclusions = []
+
+        if isinstance(source_df, pd.Series):
+            source_df = source_df.to_frame("source_factor")
+            conclusions.append(
+                "The source dataframe is a series, better convert it to a dataframe.",
+            )
+        if gt_df is not None and isinstance(gt_df, pd.Series):
+            gt_df = gt_df.to_frame("gt_factor")
+            conclusions.append(
+                "The ground truth dataframe is a series, convert it to a dataframe.",
+            )
+
+        # Check if both dataframe has only one columns
+        if len(source_df.columns) == 1:
+            conclusions.append("The source dataframe has only one column which is correct.")
+        else:
+            conclusions.append(
+                "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
+            )
+            source_df = source_df.iloc(axis=1)[
+                [
+                    0,
+                ]
+            ]
+
+        if list(source_df.index.names) != ["datetime", "instrument"]:
+            conclusions.append(
+                rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.",
+            )
+        else:
+            conclusions.append(
+                'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
+            )
+
+        # Check if both dataframe have the same rows count
+        if gt_df is not None:
+            if source_df.shape[0] == gt_df.shape[0]:
+                conclusions.append("Both dataframes have the same rows count.")
+                same_row_count_result = True
+            else:
+                conclusions.append(
+                    f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
+                )
+                same_row_count_result = False
+
+            # Check whether both dataframe has the same index
+            if source_df.index.equals(gt_df.index):
+                conclusions.append("Both dataframes have the same index.")
+                same_index_result = True
+            else:
+                conclusions.append(
+                    "The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
+                )
+                same_index_result = False
+
+            # Check for the same missing values (NaN)
+            if source_df.isna().sum().sum() == gt_df.isna().sum().sum():
+                conclusions.append("Both dataframes have the same missing values.")
+                same_missing_values_result = True
+            else:
+                conclusions.append(
+                    f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
+                )
+                same_missing_values_result = False
+
+            # Check if the values are the same within a small tolerance
+            if not same_index_result:
+                conclusions.append(
+                    "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless",
+                )
+                same_values_result = False
+                high_correlation_result = False
+            else:
+                close_values = source_df.sub(gt_df).abs().lt(1e-6)
+                if close_values.all().iloc[0]:
+                    conclusions.append(
+                        "All values in the dataframes are equal within the tolerance of 1e-6.",
+                    )
+                    same_values_result = True
+                else:
+                    conclusions.append(
+                        "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
+                    )
+                    same_values_result = False
+
+                # Check the ic and rankic between the two dataframes
+                concat_df = pd.concat([source_df, gt_df], axis=1)
+                concat_df.columns = ["source", "gt"]
+                try:
+                    ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
+                    ric = (
+                        concat_df.groupby("datetime")
+                        .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
+                        .dropna()
+                        .mean()
+                    )
+
+                    if ic > 0.99 and ric > 0.99:
+                        conclusions.append(
+                            f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
+                        )
+                        high_correlation_result = True
+                    else:
+                        conclusions.append(
+                            f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
+                        )
+                        high_correlation_result = False
+
+                    # Check for shifted alignments only in the "datetime" index
+                    max_shift_days = 2
+                    for shift in range(-max_shift_days, max_shift_days + 1):
+                        if shift == 0:
+                            continue  # Skip the case where there is no shift
+
+                        shifted_source_df = source_df.groupby(level="instrument").shift(shift)
+                        concat_df = pd.concat([shifted_source_df, gt_df], axis=1)
+                        concat_df.columns = ["source", "gt"]
+                        shifted_ric = (
+                            concat_df.groupby("datetime")
+                            .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
+                            .dropna()
+                            .mean()
+                        )
+                        if shifted_ric > 0.99:
+                            conclusions.append(
+                                f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.",
+                            )
+                            break
+                    else:
+                        conclusions.append(
+                            f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.",
+                        )
+
+                except Exception as e:
+                    FinCoLog().warning(f"Error occurred when calculating the correlation: {str(e)}")
+                    conclusions.append(
+                        f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
+                    )
+                    high_correlation_result = False
+
+        # Combine all conclusions into a single string
+        conclusion_str = "\n".join(conclusions)
+
+        final_result = (same_values_result or high_correlation_result) if gt_df is not None else False
+        return conclusion_str, final_result
+
+
+# TODO:
+def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
+    """When the prompt is too long. We have to shorten it.
+    But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
+    """
+    # TODO: this should replace most of code in
+    # - FactorImplementationFinalDecisionEvaluator.evaluate
+    # - FactorImplementationCodeEvaluator.evaluate
+
+
+class FactorImplementationFinalDecisionEvaluator(Evaluator):
+    def evaluate(
+        self,
+        target_task: FactorImplementTask,
+        execution_feedback: str,
+        value_feedback: str,
+        code_feedback: str,
+        **kwargs,
+    ) -> Tuple:
+        system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")["evaluator_final_decision_v1_system"]
+        execution_feedback_to_render = execution_feedback
+        user_prompt = Template(
+            evaluate_prompts["evaluator_final_decision_v1_user"],
+        ).render(
+            factor_information=target_task.get_factor_information(),
+            execution_feedback=execution_feedback_to_render,
+            code_feedback=code_feedback,
+            factor_value_feedback=(
+                value_feedback
+                if value_feedback is not None
+                else "No Ground Truth Value provided, so no evaluation on value is performed."
+            ),
+        )
+        while (
+            APIBackend().build_messages_and_calculate_token(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                former_messages=[],
+            )
+            > FactorImplementSettings().chat_token_limit
+        ):
+            execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            user_prompt = Template(
+                evaluate_prompts["evaluator_final_decision_v1_user"],
+            ).render(
+                factor_information=target_task.get_factor_information(),
+                execution_feedback=execution_feedback_to_render,
+                code_feedback=code_feedback,
+                factor_value_feedback=(
+                    value_feedback
+                    if value_feedback is not None
+                    else "No Ground Truth Value provided, so no evaluation on value is performed."
+                ),
+            )
+
+        final_evaluation_dict = json.loads(
+            APIBackend().build_messages_and_create_chat_completion(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                json_mode=True,
+            ),
+        )
+        return (
+            final_evaluation_dict["final_decision"],
+            final_evaluation_dict["final_feedback"],
+        )
 
 class FactorImplementationSingleFeedback:
     """This class is a feedback to single implementation which is generated from an evaluator."""
@@ -60,14 +556,12 @@ class FactorImplementationSingleFeedback:
 This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
 """
 
-
 class FactorImplementationsMultiFeedback(
     Feedback,
     List[FactorImplementationSingleFeedback],
 ):
     """Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
 
-
 class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
     """This class is the v1 version of evaluator for a single factor implementation.
     It calls several evaluators in share modules to evaluate the factor implementation.
@@ -80,9 +574,9 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
 
     def evaluate(
         self,
-        target_task: FactorImplementationTask,
-        implementation: FactorImplementation,
-        gt_implementation: FactorImplementation = None,
+        target_task: FactorImplementTask,
+        implementation: TaskImplementation,
+        gt_implementation: TaskImplementation = None,
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> FactorImplementationSingleFeedback:
@@ -170,14 +664,14 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
             return factor_feedback
 
 
-class FactorImplementationsMultiEvaluator(EvolvingEvaluator):
+class FactorImplementationsMultiEvaluator(Evaluator):
     def __init__(self, single_evaluator=FactorImplementationEvaluatorV1()) -> None:
         super().__init__()
         self.single_factor_implementation_evaluator = single_evaluator
 
     def evaluate(
         self,
-        evo: FactorImplementationList,
+        evo: FactorEvovlingItem,
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> FactorImplementationsMultiFeedback:
@@ -228,3 +722,4 @@ class FactorImplementationsMultiEvaluator(EvolvingEvaluator):
         print(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
 
         return multi_implementation_feedback
+

rdagent/factor_implementation/evolving/evolvable_subjects.py

@@ -1,32 +0,0 @@
-from __future__ import annotations
-
-from rdagent.core.evolving_framework import EvolvableSubjects
-from rdagent.core.log import FinCoLog
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-)
-
-
-class FactorImplementationList(EvolvableSubjects):
-    """
-    Factors is a list.
-    """
-
-    def __init__(
-        self,
-        target_factor_tasks: list[FactorImplementationTask],
-        corresponding_gt_implementations: list[FactorImplementation] = None,
-    ):
-        super().__init__()
-        self.target_factor_tasks = target_factor_tasks
-        self.corresponding_implementations: list[FactorImplementation] = []
-        if corresponding_gt_implementations is not None and len(
-            corresponding_gt_implementations,
-        ) != len(target_factor_tasks):
-            self.corresponding_gt_implementations = None
-            FinCoLog.warning(
-                "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
-            )
-        else:
-            self.corresponding_gt_implementations = corresponding_gt_implementations

rdagent/factor_implementation/evolving/evolving_agent.py

@@ -0,0 +1,50 @@
+from rdagent.core.evolving_framework import Feedback, EvolvableSubjects, Evaluator, EvoStep
+from rdagent.core.evolving_framework import EvoAgent
+from tqdm import tqdm
+
+class RAGEvoAgent(EvoAgent):
+    def __init__(self, max_loop, evolving_strategy, rag) -> None:
+        super().__init__(max_loop, evolving_strategy)
+        self.rag = rag
+        self.evolving_trace = []
+
+    def multistep_evolve(
+        self, 
+        evo: EvolvableSubjects,
+        eva: Evaluator | Feedback, 
+        *,
+        with_knowledge: bool = False,
+        with_feedback: bool = True,
+        knowledge_self_gen: bool = False) -> EvolvableSubjects:
+
+        for _ in tqdm(range(self.max_loop), "Implementing factors"):
+            # 1. knowledge self-evolving
+            if knowledge_self_gen and self.rag is not None:
+                self.rag.generate_knowledge(self.evolving_trace)
+            # 2. 检索需要的Knowledge
+            queried_knowledge = None
+            if with_knowledge and self.rag is not None:
+                # TODO: 这里放了evolving_trace实际上没有作用
+                queried_knowledge = self.rag.query(evo, self.evolving_trace)
+
+            # 3. evolve
+            evo = self.evolving_strategy.evolve(
+                    evo=evo,
+                    evolving_trace=self.evolving_trace,
+                    queried_knowledge=queried_knowledge,
+                )
+            
+            # 4. 封装Evolve结果
+            es = EvoStep(evo, queried_knowledge)
+
+            # 5. 环境评测反馈
+            if with_feedback:
+                es.feedback = eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge)
+
+            # 6. 更新trace
+            self.evolving_trace.append(es)
+            for index, feedback in enumerate(es.feedback):
+                if feedback is not None:
+                    evo.evolve_trace[evo.target_factor_tasks[index].factor_name][-1].feedback = feedback
+
+        return evo  

rdagent/factor_implementation/evolving/evolving_strategy.py

@@ -1,58 +1,71 @@
 from __future__ import annotations
 
 import json
-import random
 from abc import abstractmethod
 from copy import deepcopy
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from jinja2 import Template
+
 from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
-from rdagent.core.prompts import Prompts
-from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-    FileBasedFactorImplementation,
-)
+from rdagent.oai.llm_utils import APIBackend
 from rdagent.factor_implementation.share_modules.factor_implementation_config import (
     FactorImplementSettings,
 )
-from rdagent.factor_implementation.share_modules.factor_implementation_utils import (
-    get_data_folder_intro,
+
+from rdagent.core.task import (
+    TaskImplementation,
 )
+from rdagent.core.prompts import Prompts
+
+from pathlib import Path
+
+from rdagent.factor_implementation.evolving.scheduler import (
+    RandomSelect,
+    LLMSelect,
+)
+
+from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
 from rdagent.oai.llm_utils import APIBackend
 
+from rdagent.core.utils import multiprocessing_wrapper
+
+from rdagent.factor_implementation.evolving.factor import (
+    FactorImplementTask,
+    FactorEvovlingItem,
+    FileBasedFactorImplementation,
+)
+
 if TYPE_CHECKING:
-    from factor_implementation.evolving.evolvable_subjects import (
-        FactorImplementationList,
-    )
-    from factor_implementation.evolving.knowledge_management import (
+    from rdagent.factor_implementation.evolving.knowledge_management import (
         FactorImplementationQueriedKnowledge,
         FactorImplementationQueriedKnowledgeV1,
     )
 
+implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
 
 class MultiProcessEvolvingStrategy(EvolvingStrategy):
     @abstractmethod
     def implement_one_factor(
         self,
-        target_task: FactorImplementationTask,
+        target_task: FactorImplementTask,
         queried_knowledge: QueriedKnowledge = None,
-    ) -> FactorImplementation:
+    ) -> TaskImplementation:
         raise NotImplementedError
 
     def evolve(
         self,
         *,
-        evo: FactorImplementationList,
+        evo: FactorEvovlingItem,
         queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
         **kwargs,
-    ) -> FactorImplementationList:
+    ) -> FactorEvovlingItem:
+        self.num_loop += 1
         new_evo = deepcopy(evo)
-        new_evo.corresponding_implementations = [None for _ in new_evo.target_factor_tasks]
 
+        # 1.找出需要evolve的factor
         to_be_finished_task_index = []
         for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
             target_factor_task_desc = target_factor_task.get_factor_information()
@@ -65,11 +78,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
                 and target_factor_task_desc not in queried_knowledge.failed_task_info_set
             ):
                 to_be_finished_task_index.append(index)
-        if FactorImplementSettings().implementation_factors_per_round < len(to_be_finished_task_index):
-            to_be_finished_task_index = random.sample(
-                to_be_finished_task_index,
-                FactorImplementSettings().implementation_factors_per_round,
+
+        # 2. 选择selection方法
+        # if the number of factors to be implemented is larger than the limit, we need to select some of them
+        if FactorImplementSettings().select_ratio < 1:
+            # if the number of loops is equal to the select_loop, we need to select some of them
+            implementation_factors_per_round = int(
+                FactorImplementSettings().select_ratio * len(to_be_finished_task_index)
             )
+            if FactorImplementSettings().select_method == "random":
+                to_be_finished_task_index = RandomSelect(
+                    to_be_finished_task_index,
+                    implementation_factors_per_round,
+                )
+
+            if FactorImplementSettings().select_method == "scheduler":
+                to_be_finished_task_index = LLMSelect(
+                    to_be_finished_task_index,
+                    implementation_factors_per_round,
+                    new_evo,
+                    queried_knowledge.former_traces,
+                )
 
         result = multiprocessing_wrapper(
             [
@@ -81,11 +110,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 
         for index, target_index in enumerate(to_be_finished_task_index):
             new_evo.corresponding_implementations[target_index] = result[index]
+            if result[index].target_task.factor_name in new_evo.evolve_trace:
+                new_evo.evolve_trace[result[index].target_task.factor_name].append(result[index])
+            else:
+                new_evo.evolve_trace[result[index].target_task.factor_name] = [result[index]]
 
-        # for target_index in to_be_finished_task_index:
-        #     new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
-        #         new_evo.target_factor_tasks[target_index], queried_knowledge
-        #     )
+        new_evo.corresponding_selection.append(to_be_finished_task_index)
 
         return new_evo
 
@@ -93,9 +123,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
     def implement_one_factor(
         self,
-        target_task: FactorImplementationTask,
+        target_task: FactorImplementTask,
         queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
-    ) -> FactorImplementation:
+    ) -> TaskImplementation:
         factor_information_str = target_task.get_factor_information()
 
         if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -117,9 +147,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
             system_prompt = Template(
-                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                    "evolving_strategy_factor_implementation_v1_system"
-                ],
+                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
             ).render(
                 data_info=get_data_folder_intro(),
                 queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -132,9 +160,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
             while True:
                 user_prompt = (
                     Template(
-                        Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                            "evolving_strategy_factor_implementation_v1_user"
-                        ],
+                        implement_prompts["evolving_strategy_factor_implementation_v1_user"],
                     )
                     .render(
                         factor_information_str=factor_information_str,
@@ -153,9 +179,6 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
                 elif len(queried_similar_successful_knowledge_to_render) > 1:
                     queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
-            # print(
-            #     f"length of queried_similar_successful_knowledge_to_render: {len(queried_similar_successful_knowledge_to_render)}, length of queried_former_failed_knowledge_to_render: {len(queried_former_failed_knowledge_to_render)}"
-            # )
 
             code = json.loads(
                 session.build_chat_completion(
@@ -173,14 +196,20 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
 
 
 class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
+    def __init__(self) -> None:
+        self.num_loop = 0
+        self.haveSelected = False
+
     def implement_one_factor(
         self,
-        target_task: FactorImplementationTask,
+        target_task: FactorImplementTask,
         queried_knowledge,
-    ) -> FactorImplementation:
+    ) -> TaskImplementation:
         error_summary = FactorImplementSettings().v2_error_summary
+        # 1. 提取因子的背景信息
         target_factor_task_information = target_task.get_factor_information()
 
+        # 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
         if (
             queried_knowledge is not None
             and target_factor_task_information in queried_knowledge.success_task_to_knowledge_dict
@@ -189,6 +218,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
         elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
             return None
         else:
+
+            # 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
             queried_similar_component_knowledge = (
                 queried_knowledge.component_with_success_task[target_factor_task_information]
                 if queried_knowledge is not None
@@ -208,9 +239,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
             system_prompt = Template(
-                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                    "evolving_strategy_factor_implementation_v1_system"
-                ],
+                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
             ).render(
                 data_info=get_data_folder_intro(),
                 queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -223,18 +252,17 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
             queried_similar_component_knowledge_to_render = queried_similar_component_knowledge
             queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
             error_summary_critics = ""
+            # 动态地防止prompt超长
             while True:
+                # 总结error（可选）
                 if (
                     error_summary
                     and len(queried_similar_error_knowledge_to_render) != 0
                     and len(queried_former_failed_knowledge_to_render) != 0
                 ):
+
                     error_summary_system_prompt = (
-                        Template(
-                            Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                                "evolving_strategy_error_summary_v2_system"
-                            ],
-                        )
+                        Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
                         .render(
                             factor_information_str=target_factor_task_information,
                             code_and_feedback=queried_former_failed_knowledge_to_render[
@@ -248,11 +276,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     )
                     while True:
                         error_summary_user_prompt = (
-                            Template(
-                                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                                    "evolving_strategy_error_summary_v2_user"
-                                ],
-                            )
+                            Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
                             .render(
                                 queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
                             )
@@ -269,12 +293,10 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         user_prompt=error_summary_user_prompt,
                         json_mode=False,
                     )
-
+                # 构建user_prompt。开始写代码
                 user_prompt = (
                     Template(
-                        Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                            "evolving_strategy_factor_implementation_v2_user"
-                        ],
+                        implement_prompts["evolving_strategy_factor_implementation_v2_user"],
                     )
                     .render(
                         factor_information_str=target_factor_task_information,
@@ -301,12 +323,6 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                 elif len(queried_similar_error_knowledge_to_render) > 0:
                     queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
 
-            # print(
-            #     len(queried_similar_component_knowledge_to_render),
-            #     len(queried_similar_error_knowledge_to_render),
-            #     len(queried_former_failed_knowledge_to_render),
-            # )
-
             response = session.build_chat_completion(
                 user_prompt=user_prompt,
                 json_mode=True,

rdagent/factor_implementation/evolving/factor.py

@@ -1,41 +1,50 @@
-import pickle
-import subprocess
-import uuid
-from abc import ABC, abstractmethod
-from pathlib import Path
-from typing import Tuple, Union
-
-import pandas as pd
-from filelock import FileLock
-from finco.log import FinCoLog
+from __future__ import annotations
 from rdagent.factor_implementation.share_modules.factor_implementation_config import (
     FactorImplementSettings,
 )
 
-from factor_implementation.share_modules.exception import (
+from rdagent.core.task import (
+    TaskImplementation,
+    BaseTask,
+    TestCase,
+)
+from rdagent.core.evolving_framework import EvolvableSubjects
+from rdagent.core.log import FinCoLog
+
+from pathlib import Path
+
+from rdagent.oai.llm_utils import md5_hash
+
+from rdagent.core.exception import (
     CodeFormatException,
     NoOutputException,
     RuntimeErrorException,
 )
-from oai.llm_utils import md5_hash
+
+import pandas as pd
+import uuid
+import pickle
+import subprocess
+from typing import Tuple, Union
+from filelock import FileLock
 
 
-class FactorImplementationTask:
-    # TODO: remove the factor_ prefix may be better
+class FactorImplementTask(BaseTask):
     def __init__(
         self,
         factor_name,
         factor_description,
         factor_formulation,
-        factor_formulation_description,
+        factor_formulation_description: str = '',
         variables: dict = {},
+        resource: str = None,
     ) -> None:
         self.factor_name = factor_name
         self.factor_description = factor_description
         self.factor_formulation = factor_formulation
         self.factor_formulation_description = factor_formulation_description
-        # TODO: check variables a good candidate
         self.variables = variables
+        self.factor_resources = resource
 
     def get_factor_information(self):
         return f"""factor_name: {self.factor_name}
@@ -45,22 +54,41 @@ factor_formulation_description: {self.factor_formulation_description}"""
 
     @staticmethod
     def from_dict(dict):
-        return FactorImplementationTask(**dict)
+        return FactorImplementTask(**dict)
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}[{self.factor_name}]>"
 
 
-class FactorImplementation(ABC):
-    def __init__(self, target_task: FactorImplementationTask) -> None:
-        self.target_task = target_task
+class FactorEvovlingItem(EvolvableSubjects):
+    """
+    Intermediate item of factor implementation.
+    """
 
-    @abstractmethod
-    def execute(self, *args, **kwargs) -> Tuple[str, pd.DataFrame]:
-        raise NotImplementedError("__call__ method is not implemented.")
+    def __init__(
+        self,
+        target_factor_tasks: list[FactorImplementTask],
+        corresponding_gt: list[TestCase] = None,
+        corresponding_gt_implementations: list[TaskImplementation] = None,
+    ):
+        super().__init__()
+        self.target_factor_tasks = target_factor_tasks
+        self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks]
+        self.corresponding_selection: list[list] = []
+        self.evolve_trace = {}
+        self.corresponding_gt = corresponding_gt
+        if corresponding_gt_implementations is not None and len(
+            corresponding_gt_implementations,
+        ) != len(target_factor_tasks):
+            self.corresponding_gt_implementations = None
+            FinCoLog.warning(
+                "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
+            )
+        else:
+            self.corresponding_gt_implementations = corresponding_gt_implementations
 
 
-class FileBasedFactorImplementation(FactorImplementation):
+class FileBasedFactorImplementation(TaskImplementation):
     """
     This class is used to implement a factor by writing the code to a file.
     Input data and output factor value are also written to files.
@@ -76,7 +104,7 @@ class FileBasedFactorImplementation(FactorImplementation):
 
     def __init__(
         self,
-        target_task: FactorImplementationTask,
+        target_task: FactorImplementTask,
         code,
         executed_factor_value_dataframe=None,
         raise_exception=False,
@@ -124,7 +152,7 @@ class FileBasedFactorImplementation(FactorImplementation):
                 raise ValueError(self.FB_CODE_NOT_SET)
         with FileLock(self.workspace_path / "execution.lock"):
             (Path.cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").mkdir(
-                exist_ok=True, parents=True,
+                exist_ok=True, parents=True
             )
             if FactorImplementSettings().enable_execution_cache:
                 # NOTE: cache the result for the same code
@@ -215,9 +243,10 @@ class FileBasedFactorImplementation(FactorImplementation):
         return self.__str__()
 
     @staticmethod
-    def from_folder(task: FactorImplementationTask, path: Union[str, Path], **kwargs):
+    def from_folder(task: FactorImplementTask, path: Union[str, Path], **kwargs):
         path = Path(path)
         factor_path = (path / task.factor_name).with_suffix(".py")
         with factor_path.open("r") as f:
             code = f.read()
         return FileBasedFactorImplementation(task, code=code, **kwargs)
+

rdagent/factor_implementation/evolving/factor_implementation_evolving_cli.py

@@ -1,312 +0,0 @@
-import json
-import pickle
-import subprocess
-from pathlib import Path
-
-import pandas as pd
-from fire.core import Fire
-from rdagent.core.evolving_framework import EvoAgent, KnowledgeBase
-from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.evolving.evaluators import (
-    FactorImplementationEvaluatorV1,
-    FactorImplementationsMultiEvaluator,
-)
-from rdagent.factor_implementation.evolving.evolvable_subjects import FactorImplementationList
-from rdagent.factor_implementation.evolving.evolving_strategy import (
-    FactorEvolvingStrategy,
-    FactorEvolvingStrategyWithGraph,
-)
-from rdagent.factor_implementation.evolving.knowledge_management import (
-    FactorImplementationGraphKnowledgeBase,
-    FactorImplementationGraphRAGStrategy,
-    FactorImplementationKnowledgeBaseV1,
-    FactorImplementationRAGStrategyV1,
-)
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementationTask,
-    FileBasedFactorImplementation,
-)
-from tqdm import tqdm
-
-ALPHA101_INIT_COMPONENTS = [
-    "1. abs(): absolute value to certain columns",
-    "2. log(): log value to certain columns",
-    "3. sign(): sign value to certain columns",
-    "4. add_two_columns(): add two columns",
-    "5. minus_two_columns(): minus two columns",
-    "6. times_two_columns(): times two columns",
-    "7. divide_two_columns(): divide two columns",
-    "8. add_value_to_columns(): add value to columns",
-    "9. minus_value_to_columns(): minus value to columns",
-    "10. rank(): cross-sectional rank value to columns",
-    "11. delay(): value of each data d days ago",
-    "12. correlation(): time-serial correlation of column_left and column_right for the past d days",
-    "13. covariance(): time-serial covariance of column_left and column_right for the past d days",
-    "14. scale_to_a(): scale the columns to sum(abs(x)) is a",
-    "15. delta(): today’s value of x minus the value of x d days ago",
-    "16. signedpower(): x^a",
-    "17. decay_linear(): weighted moving average over the past d days with linearly decaying weights d, d – 1, …, 1 (rescaled to sum up to 1)",
-    "18. indneutralize(): x cross-sectionally neutralized against groups g (subindustries, industries, sectors, etc.), i.e., x is cross-sectionally demeaned within each group g",
-    "19. ts_min(): time-series min over the past d days, operator min applied across the time-series for the past d days; non-integer number of days d is converted to floor(d)",
-    "20. ts_max(): time-series max over the past d days, operator max applied across the time-series for the past d days; non-integer number of days d is converted to floor(d)",
-    "21. ts_argmax(): which day ts_max(x, d) occurred on",
-    "22. ts_argmin(): which day ts_min(x, d) occurred on",
-    "23. ts_rank(): time-series rank in the past d days",
-    "24. min(): ts_min(x, d)",
-    "25. max(): ts_max(x, d)",
-    "26. sum(): time-series sum over the past d days",
-    "27. product(): time-series product over the past d days",
-    "28. stddev(): moving time-series standard deviation over the past d days",
-]
-
-
-class FactorImplementationEvolvingCli:
-    # TODO: we should use polymorphism to load knowledge base, strategies instead of evolving_version
-    # TODO: Can we refactor FactorImplementationEvolvingCli into a learning framework to differentiate our learning paradiagm with other ones by iteratively retrying?
-    def __init__(self, evolving_version=2) -> None:
-        self.evolving_version = evolving_version
-        self.knowledge_base = None
-        self.latest_factor_implementations = None
-
-    def run_evolving_framework(
-        self,
-        factor_implementations: FactorImplementationList,
-        factor_knowledge_base: KnowledgeBase,
-        max_loops: int = 20,
-        with_knowledge: bool = True,
-        with_feedback: bool = True,
-        knowledge_self_gen: bool = True,
-    ) -> FactorImplementationList:
-        """
-        Main target: Implement factors.
-        The system also leverages the former knowledge to help implement the factors. Also, new knowledge might be generated during the implementation to help the following implementation.
-        The gt_code and gt_value in the Factor instance is used to evaluate the implementation, and the feedback is used to generate high-quality knowledge which helps the agent to evolve.
-        """
-        es = FactorEvolvingStrategyWithGraph() if self.evolving_version == 2 else FactorEvolvingStrategy()
-        rag = (
-            FactorImplementationGraphRAGStrategy(factor_knowledge_base)
-            if self.evolving_version == 2
-            else FactorImplementationRAGStrategyV1(factor_knowledge_base)
-        )
-        factor_evaluator = FactorImplementationsMultiEvaluator(FactorImplementationEvaluatorV1())
-        ea = EvoAgent(es, rag=rag)
-
-        for _ in tqdm(range(max_loops), "Implementing factors"):
-            factor_implementations = ea.step_evolving(
-                factor_implementations,
-                factor_evaluator,
-                with_knowledge=with_knowledge,
-                with_feedback=with_feedback,
-                knowledge_self_gen=knowledge_self_gen,
-            )
-        return factor_implementations
-
-    def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
-        if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
-            factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
-            if (
-                self.evolving_version == 1
-                and not isinstance(
-                    factor_knowledge_base,
-                    FactorImplementationKnowledgeBaseV1,
-                )
-                or self.evolving_version == 2
-                and not isinstance(
-                    factor_knowledge_base,
-                    FactorImplementationGraphKnowledgeBase,
-                )
-            ):
-                raise ValueError("The former knowledge base is not compatible with the current version")
-        else:
-            factor_knowledge_base = (
-                FactorImplementationGraphKnowledgeBase(
-                    init_component_list=component_init_list,
-                )
-                if self.evolving_version == 2
-                else FactorImplementationKnowledgeBaseV1()
-            )
-        return factor_knowledge_base
-
-    def implement_factors(
-        self,
-        factor_implementations: FactorImplementationList,
-        former_knowledge_base_path: Path = None,
-        new_knowledge_base_path: Path = None,
-        component_init_list: list = [],
-        max_loops: int = 20,
-    ):
-        factor_knowledge_base = self.load_or_init_knowledge_base(
-            former_knowledge_base_path=former_knowledge_base_path,
-            component_init_list=component_init_list,
-        )
-
-        new_factor_implementations = self.run_evolving_framework(
-            factor_implementations=factor_implementations,
-            factor_knowledge_base=factor_knowledge_base,
-            max_loops=max_loops,
-            with_knowledge=True,
-            with_feedback=True,
-            knowledge_self_gen=True,
-        )
-        if new_knowledge_base_path is not None:
-            pickle.dump(factor_knowledge_base, open(new_knowledge_base_path, "wb"))
-        self.knowledge_base = factor_knowledge_base
-        self.latest_factor_implementations = factor_implementations
-        return new_factor_implementations
-
-    def _read_alpha101_factors(
-        self,
-        alpha101_evo_subs_path: Path = None,
-        alpha101_data_path=Path().cwd() / "git_ignore_folder" / "alpha101_related_files",
-        start_index=0,
-        end_index=32,
-        read_gt_factors=True,
-    ) -> FactorImplementationList:
-        """
-        Read the alpha101 factors from the alpha101_related_files folder
-        """
-        if alpha101_evo_subs_path is not None and alpha101_evo_subs_path.exists():
-            factor_implementations = pickle.load(open(alpha101_evo_subs_path, "rb"))
-        else:
-            target_factor_plain_list = json.load(
-                open(alpha101_data_path / "target_factor_task_list.json"),
-            )
-            name_to_code = json.load(open(alpha101_data_path / "name_to_code.json"))
-            gt_df = pd.read_hdf(alpha101_data_path / "gt_filtered.h5")
-
-            # First read the target factor task
-            target_factor_tasks = []
-            for factor_list_item in target_factor_plain_list:
-                target_factor_tasks.append(
-                    FactorImplementationTask(
-                        factor_name=factor_list_item[0],
-                        factor_description=factor_list_item[1],
-                        factor_formulation=factor_list_item[2],
-                        factor_formulation_description=factor_list_item[3],
-                    ),
-                )
-
-            # Second read the gt factor implementations
-            corresponding_gt_implementations = []
-            for factor_task in target_factor_tasks:
-                name = factor_task.factor_name
-                gt_code = name_to_code[name]
-                gt_value = gt_df.loc(axis=1)[[name]]
-                corresponding_gt_implementations.append(
-                    FileBasedFactorImplementation(
-                        code=gt_code,
-                        executed_factor_value_dataframe=gt_value,
-                        target_task=factor_task,
-                    ),
-                )
-
-            # Finally generate the factor implementations as evolvable subjects
-            factor_implementations = FactorImplementationList(
-                target_factor_tasks=target_factor_tasks,
-                corresponding_gt_implementations=(corresponding_gt_implementations if read_gt_factors else None),
-            )
-
-        factor_implementations.target_factor_tasks = factor_implementations.target_factor_tasks[start_index:end_index]
-        factor_implementations.corresponding_gt_implementations = (
-            factor_implementations.corresponding_gt_implementations[start_index:end_index] if read_gt_factors else None
-        )
-
-        return factor_implementations
-
-    def implement_alpha101(
-        self,
-        max_loops=30,
-    ) -> FactorImplementationList:
-        """
-        Implement the alpha101 factors to gather knowledge TODO: implement the code
-        """
-        factor_implementations = self._read_alpha101_factors(
-            alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
-            start_index=0,
-            end_index=64,
-            read_gt_factors=True,
-        )
-        self.implement_factors(
-            factor_implementations,
-            former_knowledge_base_path=Path.cwd()
-            / f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
-            new_knowledge_base_path=Path.cwd()
-            / f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
-            component_init_list=ALPHA101_INIT_COMPONENTS,
-            max_loops=100,
-        )
-
-        factor_implementations = self._read_alpha101_factors(
-            alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
-            start_index=64,
-            end_index=96,
-            read_gt_factors=False,
-        )
-        final_imp = self.implement_factors(
-            factor_implementations,
-            former_knowledge_base_path=Path.cwd()
-            / f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
-            new_knowledge_base_path=Path.cwd()
-            / f"alpha101_knowledge_base_v{self.evolving_version}_self_evolving_project_product.pkl",
-            component_init_list=ALPHA101_INIT_COMPONENTS,
-            max_loops=10,
-        )
-        final_imp.corresponding_gt_implementations = factor_implementations = self._read_alpha101_factors(
-            alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
-            start_index=64,
-            end_index=96,
-            read_gt_factors=True,
-        ).corresponding_gt_implementations
-
-        feedbacks = FactorImplementationsMultiEvaluator().evaluate(final_imp)
-        print([feedback.final_decision if feedback is not None else None for feedback in feedbacks].count(True))
-
-    def implement_amc(
-        self, evo_sub_path_str, former_knowledge_base_path_str, implementation_dump_path_str, slice_index,
-    ):
-        factor_implementations: FactorImplementationList = pickle.load(open(evo_sub_path_str, "rb"))
-        factor_implementations.target_factor_tasks = factor_implementations.target_factor_tasks[
-            slice_index * 16 : slice_index * 16 + 16
-        ]
-        if len(factor_implementations.target_factor_tasks) == 0:
-            return
-        if Path(implementation_dump_path_str).exists():
-            return
-        factor_implementations = self.implement_factors(
-            factor_implementations,
-            former_knowledge_base_path=Path(former_knowledge_base_path_str),
-            component_init_list=ALPHA101_INIT_COMPONENTS,
-            max_loops=10,
-        )
-        pickle.dump(factor_implementations, open(implementation_dump_path_str, "wb"))
-
-    def execute_command(self, command, cwd):
-        print(command, cwd)
-        try:
-            subprocess.check_output(
-                command,
-                shell=True,
-                cwd=cwd,
-            )
-        except subprocess.CalledProcessError as e:
-            print(e.output.decode())
-
-    def multi_inference_amc_factors(self, type):
-        slice_count = {"price_volume": 35, "fundamental": 24, "high_frequency": 16}[type]
-        res = multiprocessing_wrapper(
-            [
-                (
-                    self.execute_command,
-                    (
-                        f"python src/scripts/factor_implementation/baselines/evolving/factor_implementation_evolving_cli.py implement_amc ./{type}_factors.pkl ./knowledge_base_v2_with_alpha101_and_10_factors.pkl ./inference_amc_factors_{type}_{slice}.pkl {slice}",
-                        Path.cwd(),
-                    ),
-                )
-                for slice in range(slice_count)
-            ],
-            n=2,
-        )
-
-
-if __name__ == "__main__":
-    Fire(FactorImplementationEvolvingCli)

rdagent/factor_implementation/evolving/knowledge_management.py

@@ -6,6 +6,7 @@ import random
 import re
 from itertools import combinations
 from pathlib import Path
+from jinja2 import Template
 from typing import Union
 
 from jinja2 import Template
@@ -20,22 +21,23 @@ from rdagent.core.evolving_framework import (
 from rdagent.core.log import FinCoLog
 from rdagent.core.prompts import Prompts
 from rdagent.factor_implementation.evolving.evaluators import FactorImplementationSingleFeedback
-from rdagent.factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-)
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
+from rdagent.core.task import (
+    TaskImplementation,
 )
+from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask
+from rdagent.core.prompts import Prompts
 from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode
 from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
 
+from rdagent.factor_implementation.share_modules.factor_implementation_config import (
+    FactorImplementSettings,
+)
 
 class FactorImplementationKnowledge(Knowledge):
     def __init__(
         self,
-        target_task: FactorImplementationTask,
-        implementation: FactorImplementation,
+        target_task: FactorImplementTask,
+        implementation: TaskImplementation,
         feedback: FactorImplementationSingleFeedback,
     ) -> None:
         """
@@ -899,3 +901,4 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
                     intersection_node_list_sort_by_freq.append(node)
 
         return intersection_node_list_sort_by_freq
+

rdagent/factor_implementation/evolving/scheduler.py

@@ -0,0 +1,69 @@
+from rdagent.oai.llm_utils import APIBackend
+from jinja2 import Template
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
+import json
+from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
+from rdagent.factor_implementation.evolving.factor import FactorEvovlingItem
+from rdagent.core.prompts import Prompts
+from pathlib import Path
+
+scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
+    import random
+    to_be_finished_task_index = random.sample(
+        to_be_finished_task_index,
+        implementation_factors_per_round,
+    )
+    print("The random selection is:",to_be_finished_task_index)
+    return to_be_finished_task_index
+
+def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:FactorEvovlingItem, former_trace):
+    tasks = []
+    for i in to_be_finished_task_index:
+    # find corresponding former trace for each task
+        target_factor_task_information = evo.target_factor_tasks[i].get_factor_information()
+        if target_factor_task_information in former_trace:
+            tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information]))
+
+    system_prompt = Template(
+        scheduler_prompts["select_implementable_factor_system"],
+    ).render(
+        data_info=get_data_folder_intro(),
+    )
+
+    session = APIBackend(use_chat_cache=False).build_chat_session(
+        session_system_prompt=system_prompt,
+    )
+
+    while True:
+        user_prompt = (
+            Template(
+                scheduler_prompts["select_implementable_factor_user"],
+            )
+            .render(
+                factor_num = implementation_factors_per_round,
+                target_factor_tasks=tasks,
+            )
+        )
+        if (
+            session.build_chat_completion_message_and_calculate_token(
+                user_prompt,
+            )
+            < FactorImplementSettings().chat_token_limit
+        ):
+            break
+
+    response = session.build_chat_completion(
+        user_prompt=user_prompt,
+        json_mode=True,
+    )
+    try:
+        selection = json.loads(response)["selected_factor"]
+        if not isinstance(selection, list):
+            return to_be_finished_task_index
+        selection_index = [x for x in selection if isinstance(x, int)]
+    except:
+        return to_be_finished_task_index
+    
+    return selection_index

rdagent/factor_implementation/prompts.yaml

@@ -121,19 +121,6 @@ evaluator_final_decision_v1_user: |-
   --------------Factor value feedback:---------------
   {{ factor_value_feedback }}
 
-
-analyze_component_prompt_v1_system: |-
-  User is getting a new task that might consist of the components below (given in component_index: component_description):
-  {{all_component_content}}
-
-  You should find out what components does the new task have, and put their indices in a list.
-  Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
-  {
-      "component_no_list": the list containing indices of components.
-  }
-
-
-
 evolving_strategy_factor_implementation_v2_user: |-
   --------------Target factor information:---------------
   {{ factor_information_str }}
@@ -194,3 +181,39 @@ evolving_strategy_error_summary_v2_user: |-
   {% endfor %}
   {% endif %}
 
+
+select_implementable_factor_system: |-
+  User is trying to implement some factors in quant investment using Python code, You are an assistant who helps the user select the easiest-to-implement factors and some factors may be difficult to implement due to a lack of information or excessive complexity.. 
+  The user will provide the number of factor you should pick and information about the factors, including their descriptions, formulas, and variable explanations.
+
+  At the same time, user will provide with your former attempt to implement the factor and the feedback to the implementation.You need to carefully review your previous attempts. Some factors have been repeatedly tried without success. You should consider discarding these factors.
+
+  Here is the source data that user will use to implement the factors:
+  {{ data_info }}
+
+  Please analyze the difficulties of the each factors and provide the reason and response the indices of selected implementable factor in the json format. Here is an example structure for the JSON output:
+  {
+      "Analysis": "Analyze the difficulties of the each factors and provide the reason why the factor can be implemented or not."
+      "selected_factor": "The indices of selected factor index in the list, like [0, 2, 3].The length should be the number of factor left after filtering.",
+  }
+
+select_implementable_factor_user: |-
+  Number of factor you should pick: {{ factor_num }}
+  {% for factor_info in target_factor_tasks %} 
+  =============Factor index:{{factor_info[0]}}:=============
+  =====Factor name:=====
+  {{ factor_info[1].factor_name }}
+  =====Factor description:=====
+  {{ factor_info[1].factor_description }}
+  =====Factor formulation:=====
+  {{ factor_info[1].factor_formulation }}
+  {% if factor_info[2]|length != 0 %}
+  --------------Your former attempt:---------------
+  {% for former_attempt in factor_info[2] %}
+  =====Code to attempt {{ loop.index }}=====
+  {{ former_attempt.implementation.code }}
+  =====Feedback to attempt {{ loop.index }}=====
+  {{ former_attempt.feedback }}
+  {% endfor %}
+  {% endif %}
+  {% endfor %}

rdagent/factor_implementation/share_modules/evaluator.py

@@ -1,535 +0,0 @@
-import json
-from abc import ABC, abstractmethod
-from typing import Tuple
-
-import pandas as pd
-from finco.log import FinCoLog
-from jinja2 import Template
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
-)
-from rdagent.oai.llm_utils import APIBackend
-
-from factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-)
-from factor_implementation.share_modules.prompt import FactorImplementationPrompts
-
-
-class Evaluator(ABC):
-    @abstractmethod
-    def evaluate(
-        self,
-        target_task: FactorImplementationTask,
-        implementation: FactorImplementation,
-        gt_implementation: FactorImplementation,
-        **kwargs,
-    ):
-        raise NotImplementedError
-
-
-class FactorImplementationCodeEvaluator(Evaluator):
-    def evaluate(
-        self,
-        target_task: FactorImplementationTask,
-        implementation: FactorImplementation,
-        execution_feedback: str,
-        factor_value_feedback: str = "",
-        gt_implementation: FactorImplementation = None,
-        **kwargs,
-    ):
-        factor_information = target_task.get_factor_information()
-        code = implementation.code
-
-        system_prompt = FactorImplementationPrompts()["evaluator_code_feedback_v1_system"]
-
-        execution_feedback_to_render = execution_feedback
-        user_prompt = Template(
-            FactorImplementationPrompts()["evaluator_code_feedback_v1_user"],
-        ).render(
-            factor_information=factor_information,
-            code=code,
-            execution_feedback=execution_feedback_to_render,
-            factor_value_feedback=factor_value_feedback,
-            gt_code=gt_implementation.code if gt_implementation else None,
-        )
-        while (
-            APIBackend().build_messages_and_calculate_token(
-                user_prompt=user_prompt,
-                system_prompt=system_prompt,
-                former_messages=[],
-            )
-            > FactorImplementSettings().chat_token_limit
-        ):
-            execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
-            user_prompt = Template(
-                FactorImplementationPrompts()["evaluator_code_feedback_v1_user"],
-            ).render(
-                factor_information=factor_information,
-                code=code,
-                execution_feedback=execution_feedback_to_render,
-                factor_value_feedback=factor_value_feedback,
-                gt_code=gt_implementation.code if gt_implementation else None,
-            )
-        critic_response = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=user_prompt,
-            system_prompt=system_prompt,
-            json_mode=False,
-        )
-
-        # critic_response = json.loads(critic_response)
-        return critic_response
-
-
-class FactorImplementationEvaluator(Evaluator):
-    # TODO:
-    # I think we should have unified interface for all evaluates, for examples.
-    # So we should adjust the interface of other factors
-    @abstractmethod
-    def evaluate(
-        self,
-        gt: FactorImplementation,
-        gen: FactorImplementation,
-    ) -> Tuple[str, object]:
-        """You can get the dataframe by
-
-        .. code-block:: python
-
-            _, gt_df = gt.execute()
-            _, gen_df = gen.execute()
-
-        Returns
-        -------
-        Tuple[str, object]
-            - str: the text-based description of the evaluation result
-            - object: a comparable metric (bool, integer, float ...)
-
-        """
-        raise NotImplementedError("Please implement the `evaluator` method")
-
-    def _get_df(self, gt: FactorImplementation, gen: FactorImplementation):
-        _, gt_df = gt.execute()
-        _, gen_df = gen.execute()
-        if isinstance(gen_df, pd.Series):
-            gen_df = gen_df.to_frame("source_factor")
-        if isinstance(gt_df, pd.Series):
-            gt_df = gt_df.to_frame("gt_factor")
-        return gt_df, gen_df
-
-
-# NOTE: the following evaluators are splited from FactorImplementationValueEvaluator
-
-
-class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
-    def evaluate(
-        self,
-        gt: FactorImplementation,
-        gen: FactorImplementation,
-    ) -> Tuple[str, object]:
-        gt_df, gen_df = self._get_df(gt, gen)
-
-        if len(gen_df.columns) == 1 and len(gt_df.columns) == 1:
-            return "Both dataframes have only one column.", True
-        elif len(gen_df.columns) != 1:
-            gen_df = gen_df.iloc(axis=1)[
-                [
-                    0,
-                ]
-            ]
-            return (
-                "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
-                False,
-            )
-        return "", False
-
-    def __str__(self) -> str:
-        return self.__class__.__name__
-
-
-class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
-    def evaluate(
-        self,
-        gt: FactorImplementation,
-        gen: FactorImplementation,
-    ) -> Tuple[str, object]:
-        gt_df, gen_df = self._get_df(gt, gen)
-        idx_name_right = gen_df.index.names == ("datetime", "instrument")
-        if idx_name_right:
-            return (
-                'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
-                True,
-            )
-        else:
-            return (
-                'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.',
-                False,
-            )
-
-    def __str__(self) -> str:
-        return self.__class__.__name__
-
-
-class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
-    def evaluate(
-        self,
-        gt: FactorImplementation,
-        gen: FactorImplementation,
-    ) -> Tuple[str, object]:
-        gt_df, gen_df = self._get_df(gt, gen)
-
-        if gen_df.shape[0] == gt_df.shape[0]:
-            return "Both dataframes have the same rows count.", True
-        else:
-            return (
-                f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
-                False,
-            )
-
-    def __str__(self) -> str:
-        return self.__class__.__name__
-
-
-class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
-    def evaluate(
-        self,
-        gt: FactorImplementation,
-        gen: FactorImplementation,
-    ) -> Tuple[str, object]:
-        gt_df, gen_df = self._get_df(gt, gen)
-
-        if gen_df.index.equals(gt_df.index):
-            return "Both dataframes have the same index.", True
-        else:
-            return (
-                "The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
-                False,
-            )
-
-    def __str__(self) -> str:
-        return self.__class__.__name__
-
-
-class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
-    def evaluate(
-        self,
-        gt: FactorImplementation,
-        gen: FactorImplementation,
-    ) -> Tuple[str, object]:
-        gt_df, gen_df = self._get_df(gt, gen)
-
-        if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
-            return "Both dataframes have the same missing values.", True
-        else:
-            return (
-                f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
-                False,
-            )
-
-    def __str__(self) -> str:
-        return self.__class__.__name__
-
-
-class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
-    def evaluate(
-        self,
-        gt: FactorImplementation,
-        gen: FactorImplementation,
-    ) -> Tuple[str, object]:
-        gt_df, gen_df = self._get_df(gt, gen)
-
-        try:
-            close_values = gen_df.sub(gt_df).abs().lt(1e-6)
-            result_int = close_values.astype(int)
-            pos_num = result_int.sum().sum()
-            acc_rate = pos_num / close_values.size
-        except:
-            close_values = gen_df
-        if close_values.all().iloc[0]:
-            return (
-                "All values in the dataframes are equal within the tolerance of 1e-6.",
-                acc_rate,
-            )
-        else:
-            return (
-                "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
-                acc_rate,
-            )
-
-    def __str__(self) -> str:
-        return self.__class__.__name__
-
-
-class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
-    def __init__(self, hard_check: bool) -> None:
-        self.hard_check = hard_check
-
-    def evaluate(
-        self,
-        gt: FactorImplementation,
-        gen: FactorImplementation,
-    ) -> Tuple[str, object]:
-        gt_df, gen_df = self._get_df(gt, gen)
-
-        concat_df = pd.concat([gen_df, gt_df], axis=1)
-        concat_df.columns = ["source", "gt"]
-        ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
-        ric = (
-            concat_df.groupby("datetime")
-            .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
-            .dropna()
-            .mean()
-        )
-
-        if self.hard_check:
-            if ic > 0.99 and ric > 0.99:
-                return (
-                    f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
-                    True,
-                )
-            else:
-                return (
-                    f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
-                    False,
-                )
-        else:
-            return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
-
-    def __str__(self) -> str:
-        return self.__class__.__name__
-
-
-class FactorImplementationValEvaluator(FactorImplementationEvaluator):
-    def evaluate(self, gt: FactorImplementation, gen: FactorImplementation):
-        _, gt_df = gt.execute()
-        _, gen_df = gen.execute()
-        # FIXME: refactor the two classes
-        fiv = FactorImplementationValueEvaluator()
-        return fiv.evaluate(source_df=gen_df, gt_df=gt_df)
-
-    def __str__(self) -> str:
-        return self.__class__.__name__
-
-
-class FactorImplementationValueEvaluator(Evaluator):
-    # TODO: let's discuss the about the interface of the evaluator
-    def evaluate(
-        self,
-        source_df: pd.DataFrame,
-        gt_df: pd.DataFrame,
-        **kwargs,
-    ) -> Tuple:
-        conclusions = []
-
-        if isinstance(source_df, pd.Series):
-            source_df = source_df.to_frame("source_factor")
-            conclusions.append(
-                "The source dataframe is a series, better convert it to a dataframe.",
-            )
-        if gt_df is not None and isinstance(gt_df, pd.Series):
-            gt_df = gt_df.to_frame("gt_factor")
-            conclusions.append(
-                "The ground truth dataframe is a series, convert it to a dataframe.",
-            )
-
-        # Check if both dataframe has only one columns
-        if len(source_df.columns) == 1:
-            conclusions.append("The source dataframe has only one column which is correct.")
-        else:
-            conclusions.append(
-                "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
-            )
-            source_df = source_df.iloc(axis=1)[
-                [
-                    0,
-                ]
-            ]
-
-        if list(source_df.index.names) != ["datetime", "instrument"]:
-            conclusions.append(
-                rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.",
-            )
-        else:
-            conclusions.append(
-                'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
-            )
-
-        # Check if both dataframe have the same rows count
-        if gt_df is not None:
-            if source_df.shape[0] == gt_df.shape[0]:
-                conclusions.append("Both dataframes have the same rows count.")
-                same_row_count_result = True
-            else:
-                conclusions.append(
-                    f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
-                )
-                same_row_count_result = False
-
-            # Check whether both dataframe has the same index
-            if source_df.index.equals(gt_df.index):
-                conclusions.append("Both dataframes have the same index.")
-                same_index_result = True
-            else:
-                conclusions.append(
-                    "The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
-                )
-                same_index_result = False
-
-            # Check for the same missing values (NaN)
-            if source_df.isna().sum().sum() == gt_df.isna().sum().sum():
-                conclusions.append("Both dataframes have the same missing values.")
-                same_missing_values_result = True
-            else:
-                conclusions.append(
-                    f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
-                )
-                same_missing_values_result = False
-
-            # Check if the values are the same within a small tolerance
-            if not same_index_result:
-                conclusions.append(
-                    "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless",
-                )
-                same_values_result = False
-                high_correlation_result = False
-            else:
-                close_values = source_df.sub(gt_df).abs().lt(1e-6)
-                if close_values.all().iloc[0]:
-                    conclusions.append(
-                        "All values in the dataframes are equal within the tolerance of 1e-6.",
-                    )
-                    same_values_result = True
-                else:
-                    conclusions.append(
-                        "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
-                    )
-                    same_values_result = False
-
-                # Check the ic and rankic between the two dataframes
-                concat_df = pd.concat([source_df, gt_df], axis=1)
-                concat_df.columns = ["source", "gt"]
-                try:
-                    ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
-                    ric = (
-                        concat_df.groupby("datetime")
-                        .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
-                        .dropna()
-                        .mean()
-                    )
-
-                    if ic > 0.99 and ric > 0.99:
-                        conclusions.append(
-                            f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
-                        )
-                        high_correlation_result = True
-                    else:
-                        conclusions.append(
-                            f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
-                        )
-                        high_correlation_result = False
-
-                    # Check for shifted alignments only in the "datetime" index
-                    max_shift_days = 2
-                    for shift in range(-max_shift_days, max_shift_days + 1):
-                        if shift == 0:
-                            continue  # Skip the case where there is no shift
-
-                        shifted_source_df = source_df.groupby(level="instrument").shift(shift)
-                        concat_df = pd.concat([shifted_source_df, gt_df], axis=1)
-                        concat_df.columns = ["source", "gt"]
-                        shifted_ric = (
-                            concat_df.groupby("datetime")
-                            .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
-                            .dropna()
-                            .mean()
-                        )
-                        if shifted_ric > 0.99:
-                            conclusions.append(
-                                f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.",
-                            )
-                            break
-                    else:
-                        conclusions.append(
-                            f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.",
-                        )
-
-                except Exception as e:
-                    FinCoLog().warning(f"Error occurred when calculating the correlation: {e!s}")
-                    conclusions.append(
-                        f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
-                    )
-                    high_correlation_result = False
-
-        # Combine all conclusions into a single string
-        conclusion_str = "\n".join(conclusions)
-
-        final_result = (same_values_result or high_correlation_result) if gt_df is not None else False
-        return conclusion_str, final_result
-
-
-# TODO:
-def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
-    """When the prompt is too long. We have to shorten it.
-    But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
-    """
-    # TODO: this should replace most of code in
-    # - FactorImplementationFinalDecisionEvaluator.evaluate
-    # - FactorImplementationCodeEvaluator.evaluate
-
-
-class FactorImplementationFinalDecisionEvaluator(Evaluator):
-    def evaluate(
-        self,
-        target_task: FactorImplementationTask,
-        execution_feedback: str,
-        value_feedback: str,
-        code_feedback: str,
-        **kwargs,
-    ) -> Tuple:
-        system_prompt = FactorImplementationPrompts()["evaluator_final_decision_v1_system"]
-        execution_feedback_to_render = execution_feedback
-        user_prompt = Template(
-            FactorImplementationPrompts()["evaluator_final_decision_v1_user"],
-        ).render(
-            factor_information=target_task.get_factor_information(),
-            execution_feedback=execution_feedback_to_render,
-            code_feedback=code_feedback,
-            factor_value_feedback=(
-                value_feedback
-                if value_feedback is not None
-                else "No Ground Truth Value provided, so no evaluation on value is performed."
-            ),
-        )
-        while (
-            APIBackend().build_messages_and_calculate_token(
-                user_prompt=user_prompt,
-                system_prompt=system_prompt,
-                former_messages=[],
-            )
-            > FactorImplementSettings().chat_token_limit
-        ):
-            execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
-            user_prompt = Template(
-                FactorImplementationPrompts()["evaluator_final_decision_v1_user"],
-            ).render(
-                factor_information=target_task.get_factor_information(),
-                execution_feedback=execution_feedback_to_render,
-                code_feedback=code_feedback,
-                factor_value_feedback=(
-                    value_feedback
-                    if value_feedback is not None
-                    else "No Ground Truth Value provided, so no evaluation on value is performed."
-                ),
-            )
-
-        final_evaluation_dict = json.loads(
-            APIBackend().build_messages_and_create_chat_completion(
-                user_prompt=user_prompt,
-                system_prompt=system_prompt,
-                json_mode=True,
-            ),
-        )
-        return (
-            final_evaluation_dict["final_decision"],
-            final_evaluation_dict["final_feedback"],
-        )

rdagent/factor_implementation/share_modules/factor_gen.py

@@ -1,31 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List
-
-from factor_implementation.share_modules.factor import (
-    FactorImplementation,
-    FactorImplementationTask,
-)
-
-
-class FactorGenerator(ABC):
-    """
-    Because implementing factors will help each other in the process of implementation, we use the interface `List[FactorImplementationTask] -> List[FactorImplementation]` instead of single factor .
-    """
-
-    def __init__(self, target_task_l: List[FactorImplementationTask]) -> None:
-        self.target_task_l = target_task_l
-
-    @abstractmethod
-    def generate(self, *args, **kwargs) -> List[FactorImplementation]:
-        raise NotImplementedError("generate method is not implemented.")
-
-    def collect_feedback(self, feedback_obj_l: List[object]):
-        """
-        When online evaluation.
-        The preivous feedbacks will be collected to support advanced factor generator
-
-        Parameters
-        ----------
-        feedback_obj_l : List[object]
-
-        """

rdagent/factor_implementation/share_modules/factor_implementation_config.py

@@ -1,9 +1,13 @@
 from pathlib import Path
 
-from core.conf import FincoSettings
+from pydantic_settings import BaseSettings
+
+from typing import Literal, Union
+
+SELECT_METHOD = Literal["random", "scheduler"]
 
 
-class FactorImplementSettings(FincoSettings):
+class FactorImplementSettings(BaseSettings):
     file_based_execution_data_folder: str = str(
         (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
     )
@@ -29,10 +33,21 @@ class FactorImplementSettings(FincoSettings):
     v2_error_summary: bool = False
     v2_knowledge_sampler: float = 1.0
 
-    implementation_factors_per_round: int = 100  # how many factors to choose for each round of evolving
     evo_multi_proc_n: int = 16  # how many processes to use for evolving (including eval & generation)
 
     file_based_execution_timeout: int = 120  # seconds for each factor implementation execution
 
+    select_method: SELECT_METHOD = "random"
+    select_ratio: float = 0.5
+
+    max_loop: int = 10
+
+    knowledge_base_path: Union[str, None] = None
+    new_knowledge_base_path: Union[str, None] = None
+
+    chat_token_limit: int = (
+        100000  # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
+    )
+
 
 FIS = FactorImplementSettings()

rdagent/factor_implementation/share_modules/factor_implementation_utils.py

@@ -5,6 +5,8 @@ import pandas as pd
 # render it with jinja
 from jinja2 import Template
 from rdagent.factor_implementation.share_modules.factor_implementation_config import FIS
+from rdagent.factor_implementation.evolving.factor import FactorImplementTask
+
 
 TPL = """
 {{file_name}}
@@ -15,7 +17,6 @@ TPL = """
 # Create a Jinja template from the string
 JJ_TPL = Template(TPL)
 
-
 def get_data_folder_intro():
     """Direclty get the info of the data folder.
     It is for preparing prompting message.
@@ -48,3 +49,17 @@ def get_data_folder_intro():
                 f"file type {p.name} is not supported. Please implement its description function.",
             )
     return "\n ----------------- file spliter -------------\n".join(content_l)
+
+def load_data_from_dict(factor_dict:dict) -> list[FactorImplementTask]:
+    """Load data from a dict.
+    """
+    task_l = []
+    for factor_name, factor_data in factor_dict.items():
+        task = FactorImplementTask(
+            factor_name=factor_name,
+            factor_description=factor_data["description"],
+            factor_formulation=factor_data["formulation"],
+            variables=factor_data["variables"],
+        )
+        task_l.append(task)
+    return task_l

rdagent/knowledge_management/graph.py

@@ -6,8 +6,8 @@ from collections import deque
 from pathlib import Path
 from typing import Any, NoReturn
 
-from finco.llm import APIBackend
-from finco.vector_base import KnowledgeMetaData, PDVectorBase, VectorBase, cosine
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.knowledge_management.vector_base import KnowledgeMetaData, PDVectorBase, VectorBase, cosine
 
 Node = KnowledgeMetaData
 
@@ -124,7 +124,7 @@ class UndirectedGraph(Graph):
         self,
         node: UndirectedNode,
         neighbor: UndirectedNode = None,
-        same_node_threshold: float = 0.95, # noqa: ARG002
+        same_node_threshold: float = 0.95,  # noqa: ARG002
     ) -> None:
         """
         add node and neighbor to the Graph
@@ -189,7 +189,6 @@ class UndirectedGraph(Graph):
     def get_node(self, node_id: str) -> UndirectedNode:
         return self.nodes.get(node_id)
 
-
     def get_node_by_content(self, content: str) -> UndirectedNode | None:
         """
         Get node by semantic distance
@@ -234,7 +233,8 @@ class UndirectedGraph(Graph):
                 result.append(node)
 
                 for neighbor in sorted(
-                    self.get_node(node.id).neighbors, key=lambda x: x.content,
+                    self.get_node(node.id).neighbors,
+                    key=lambda x: x.content,
                 ):  # to make sure the result is deterministic
                     if neighbor not in visited and not (block and neighbor.label not in constraint_labels):
                         queue.append((neighbor, current_steps + 1))
@@ -271,12 +271,16 @@ class UndirectedGraph(Graph):
         for node in nodes:
             if intersection is None:
                 intersection = self.get_nodes_within_steps(
-                    node, steps=steps, constraint_labels=constraint_labels,
+                    node,
+                    steps=steps,
+                    constraint_labels=constraint_labels,
                 )
             intersection = self.intersection(
                 nodes1=intersection,
                 nodes2=self.get_nodes_within_steps(
-                    node, steps=steps, constraint_labels=constraint_labels,
+                    node,
+                    steps=steps,
+                    constraint_labels=constraint_labels,
                 ),
             )
         return intersection
@@ -399,7 +403,9 @@ class UndirectedGraph(Graph):
         res_list = []
         for query in content:
             similar_nodes = self.semantic_search(
-                content=query, topk_k=topk_k, similarity_threshold=similarity_threshold,
+                content=query,
+                topk_k=topk_k,
+                similarity_threshold=similarity_threshold,
             )
 
             connected_nodes = []
@@ -413,11 +419,7 @@ class UndirectedGraph(Graph):
                     block=block,
                 )
                 connected_nodes.extend(
-                    [
-                        node
-                        for node in graph_query_node_res
-                        if node not in connected_nodes
-                    ],
+                    [node for node in graph_query_node_res if node not in connected_nodes],
                 )
                 if len(connected_nodes) >= topk_k:
                     break
@@ -444,7 +446,6 @@ class UndirectedGraph(Graph):
         return [node for node in nodes if node.label in labels]
 
 
-
 def graph_to_edges(graph: dict[str, list[str]]) -> list[tuple[str, str]]:
     edges = []
 
@@ -458,7 +459,9 @@ def graph_to_edges(graph: dict[str, list[str]]) -> list[tuple[str, str]]:
 
 
 def assign_random_coordinate_to_node(
-    nodes: list[str], scope: float = 1.0, origin: tuple[float, float] = (0.0, 0.0),
+    nodes: list[str],
+    scope: float = 1.0,
+    origin: tuple[float, float] = (0.0, 0.0),
 ) -> dict[str, tuple[float, float]]:
     coordinates = {}
     for node in nodes:
@@ -470,7 +473,10 @@ def assign_random_coordinate_to_node(
 
 
 def assign_isometric_coordinate_to_node(
-    nodes: list, x_step: float = 1.0, x_origin: float = 0.0, y_origin: float = 0.0,
+    nodes: list,
+    x_step: float = 1.0,
+    x_origin: float = 0.0,
+    y_origin: float = 0.0,
 ) -> dict:
     coordinates = {}
 
@@ -483,7 +489,9 @@ def assign_isometric_coordinate_to_node(
 
 
 def curly_node_coordinate(
-    coordinates: dict, center_y: float = 1.0, r: float = 1.0,
+    coordinates: dict,
+    center_y: float = 1.0,
+    r: float = 1.0,
 ) -> dict:
     # noto: this method can only curly < 90 degree, and the curl line is circle.
     # the original function is: x**2 + (y-m)**2 = r**2

rdagent/knowledge_management/knowledgebase.py

@@ -0,0 +1,722 @@
+from __future__ import annotations
+
+import re
+import random
+import json
+import copy
+
+from jinja2 import Template
+
+from itertools import combinations
+from pathlib import Path
+from typing import Union
+
+from rdagent.core.evolving_framework import (
+    KnowledgeBase,
+)
+from rdagent.core.evolving_framework import EvoStep, EvolvableSubjects, RAGStrategy, Knowledge, QueriedKnowledge
+from rdagent.factor_implementation.evolving.knowledge_management import FactorImplementationKnowledge, FactorImplementationQueriedGraphKnowledge
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
+
+from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode
+from rdagent.core.prompts import Prompts
+from rdagent.core.log import FinCoLog
+from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
+
+
+class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
+    def __init__(self, init_component_list=None) -> None:
+        """
+        Load knowledge, offer brief information of knowledge and common handle interfaces
+        """
+        self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl")
+        FinCoLog().info(f"Knowledge Graph loaded, size={self.graph.size()}")
+
+        if init_component_list:
+            for component in init_component_list:
+                exist_node = self.graph.get_node_by_content(content=component)
+                node = exist_node if exist_node else UndirectedNode(content=component, label="component")
+                self.graph.add_nodes(node=node, neighbors=[])
+
+        # A dict containing all working trace until they fail or succeed
+        self.working_trace_knowledge = {}
+
+        # A dict containing error analysis each step aligned with working trace
+        self.working_trace_error_analysis = {}
+
+        # Add already success task
+        self.success_task_to_knowledge_dict = {}
+
+        # key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorImplementationKnowledge')
+        self.node_to_implementation_knowledge_dict = {}
+
+        # store the task description to component nodes
+        self.task_to_component_nodes = {}
+
+    def get_all_nodes_by_label(self, label: str) -> list[UndirectedNode]:
+        return self.graph.get_all_nodes_by_label(label)
+
+    def update_success_task(
+        self,
+        success_task_info: str,
+    ):  # Transfer the success tasks' working trace to knowledge storage & graph
+        success_task_trace = self.working_trace_knowledge[success_task_info]
+        success_task_error_analysis_record = (
+            self.working_trace_error_analysis[success_task_info]
+            if success_task_info in self.working_trace_error_analysis
+            else []
+        )
+        task_des_node = UndirectedNode(content=success_task_info, label="task_description")
+        self.graph.add_nodes(
+            node=task_des_node,
+            neighbors=self.task_to_component_nodes[success_task_info],
+        )  # 1st version, we assume that all component nodes are given
+        for index, trace_unit in enumerate(success_task_trace):  # every unit: single_knowledge
+            neighbor_nodes = [task_des_node]
+            if index != len(success_task_trace) - 1:
+                trace_node = UndirectedNode(
+                    content=trace_unit.get_implementation_and_feedback_str(),
+                    label="task_trace",
+                )
+                self.node_to_implementation_knowledge_dict[trace_node.id] = trace_unit
+                for node_index, error_node in enumerate(success_task_error_analysis_record[index]):
+                    if type(error_node).__name__ == "str":
+                        queried_node = self.graph.get_node_by_content(content=error_node)
+                        if queried_node is None:
+                            new_error_node = UndirectedNode(content=error_node, label="error")
+                            self.graph.add_node(node=new_error_node)
+                            success_task_error_analysis_record[index][node_index] = new_error_node
+                        else:
+                            success_task_error_analysis_record[index][node_index] = queried_node
+                neighbor_nodes.extend(success_task_error_analysis_record[index])
+                self.graph.add_nodes(node=trace_node, neighbors=neighbor_nodes)
+            else:
+                success_node = UndirectedNode(
+                    content=trace_unit.get_implementation_and_feedback_str(),
+                    label="task_success_implement",
+                )
+                self.graph.add_nodes(node=success_node, neighbors=neighbor_nodes)
+                self.node_to_implementation_knowledge_dict[success_node.id] = trace_unit
+
+    def query(self):
+        pass
+
+    def graph_get_node_by_content(self, content: str) -> UndirectedNode:
+        return self.graph.get_node_by_content(content=content)
+
+    def graph_query_by_content(
+        self,
+        content: Union[str, list[str]],
+        topk_k: int = 5,
+        step: int = 1,
+        constraint_labels: list[str] = None,
+        constraint_node: UndirectedNode = None,
+        similarity_threshold: float = 0.0,
+        constraint_distance: float = 0,
+        block: bool = False,
+    ) -> list[UndirectedNode]:
+        """
+        search graph by content similarity and connection relationship, return empty list if nodes' chain without node
+        near to constraint_node
+
+        Parameters
+        ----------
+        constraint_distance
+        content
+        topk_k: the upper number of output for each query, if the number of fit nodes is less than topk_k, return all fit nodes's content
+        step
+        constraint_labels
+        constraint_node
+        similarity_threshold
+        block: despite the start node, the search can only flow through the constraint_label type nodes
+
+        Returns
+        -------
+
+        """
+
+        return self.graph.query_by_content(
+            content=content,
+            topk_k=topk_k,
+            step=step,
+            constraint_labels=constraint_labels,
+            constraint_node=constraint_node,
+            similarity_threshold=similarity_threshold,
+            constraint_distance=constraint_distance,
+            block=block,
+        )
+
+    def graph_query_by_node(
+        self,
+        node: UndirectedNode,
+        step: int = 1,
+        constraint_labels: list[str] = None,
+        constraint_node: UndirectedNode = None,
+        constraint_distance: float = 0,
+        block: bool = False,
+    ) -> list[UndirectedNode]:
+        """
+        search graph by connection, return empty list if nodes' chain without node near to constraint_node
+        Parameters
+        ----------
+        node : start node
+        step : the max steps will be searched
+        constraint_labels : the labels of output nodes
+        constraint_node : the node that the output nodes must connect to
+        constraint_distance : the max distance between output nodes and constraint_node
+        block: despite the start node, the search can only flow through the constraint_label type nodes
+
+        Returns
+        -------
+        A list of nodes
+
+        """
+        nodes = self.graph.query_by_node(
+            node=node,
+            step=step,
+            constraint_labels=constraint_labels,
+            constraint_node=constraint_node,
+            constraint_distance=constraint_distance,
+            block=block,
+        )
+        return nodes
+
+    def graph_query_by_intersection(
+        self,
+        nodes: list[UndirectedNode],
+        steps: int = 1,
+        constraint_labels: list[str] = None,
+        output_intersection_origin: bool = False,
+    ) -> list[UndirectedNode] | list[list[list[UndirectedNode], UndirectedNode]]:
+        """
+        search graph by node intersection, node intersected by a higher frequency has a prior order in the list
+        Parameters
+        ----------
+        nodes : node list
+        step : the max steps will be searched
+        constraint_labels : the labels of output nodes
+        output_intersection_origin: output the list that contains the node which form this intersection node
+
+        Returns
+        -------
+        A list of nodes
+
+        """
+        node_count = len(nodes)
+        assert node_count >= 2, "nodes length must >=2"
+        intersection_node_list = []
+        if output_intersection_origin:
+            origin_list = []
+        for k in range(node_count, 1, -1):
+            possible_combinations = combinations(nodes, k)
+            for possible_combination in possible_combinations:
+                node_list = list(possible_combination)
+                intersection_node_list.extend(
+                    self.graph.get_nodes_intersection(node_list, steps=steps, constraint_labels=constraint_labels)
+                )
+                if output_intersection_origin:
+                    for _ in range(len(intersection_node_list)):
+                        origin_list.append(node_list)
+        intersection_node_list_sort_by_freq = []
+        for index, node in enumerate(intersection_node_list):
+            if node not in intersection_node_list_sort_by_freq:
+                if output_intersection_origin:
+                    intersection_node_list_sort_by_freq.append([origin_list[index], node])
+                else:
+                    intersection_node_list_sort_by_freq.append(node)
+
+        return intersection_node_list_sort_by_freq
+
+
+
+class FactorImplementationGraphRAGStrategy(RAGStrategy):
+    def __init__(self, knowledgebase: FactorImplementationGraphKnowledgeBase) -> None:
+        super().__init__(knowledgebase)
+        self.current_generated_trace_count = 0
+        self.prompt = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+
+    def generate_knowledge(
+        self,
+        evolving_trace: list[EvoStep],
+        *,
+        return_knowledge: bool = False,
+    ) -> Knowledge | None:
+        if len(evolving_trace) == self.current_generated_trace_count:
+            return None
+
+        else:
+            for trace_index in range(self.current_generated_trace_count, len(evolving_trace)):
+                evo_step = evolving_trace[trace_index]
+                implementations = evo_step.evolvable_subjects
+                feedback = evo_step.feedback
+                for task_index in range(len(implementations.target_factor_tasks)):
+                    single_feedback = feedback[task_index]
+                    target_task = implementations.target_factor_tasks[task_index]
+                    target_task_information = target_task.get_factor_information()
+                    implementation = implementations.corresponding_implementations[task_index]
+                    single_feedback = feedback[task_index]
+                    if single_feedback is None:
+                        continue
+                    single_knowledge = FactorImplementationKnowledge(
+                        target_task=target_task,
+                        implementation=implementation,
+                        feedback=single_feedback,
+                    )
+                    if (
+                        target_task_information not in self.knowledgebase.success_task_to_knowledge_dict
+                        and implementation is not None
+                    ):
+                        self.knowledgebase.working_trace_knowledge.setdefault(target_task_information, []).append(
+                            single_knowledge,
+                        )  # save to working trace
+                        if single_feedback.final_decision == True:
+                            self.knowledgebase.success_task_to_knowledge_dict.setdefault(
+                                target_task_information,
+                                single_knowledge,
+                            )
+                            # Do summary for the last step and update the knowledge graph
+                            self.knowledgebase.update_success_task(
+                                target_task_information,
+                            )
+                        else:
+                            # generate error node and store into knowledge base
+                            error_analysis_result = []
+                            if not single_feedback.value_generated_flag:
+                                error_analysis_result = self.analyze_error(
+                                    single_feedback.execution_feedback,
+                                    feedback_type="execution",
+                                )
+                            else:
+                                error_analysis_result = self.analyze_error(
+                                    single_feedback.factor_value_feedback,
+                                    feedback_type="value",
+                                )
+                            self.knowledgebase.working_trace_error_analysis.setdefault(
+                                target_task_information,
+                                [],
+                            ).append(
+                                error_analysis_result,
+                            )  # save to working trace error record, for graph update
+
+            self.current_generated_trace_count = len(evolving_trace)
+            return None
+
+    def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None:
+        conf_knowledge_sampler = FactorImplementSettings().v2_knowledge_sampler
+        factor_implementation_queried_graph_knowledge = FactorImplementationQueriedGraphKnowledge(
+            success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict,
+        )
+        
+        factor_implementation_queried_graph_knowledge = self.former_trace_query(
+            evo,
+            factor_implementation_queried_graph_knowledge,
+            FactorImplementSettings().v2_query_former_trace_limit,
+        )
+        factor_implementation_queried_graph_knowledge = self.component_query(
+            evo,
+            factor_implementation_queried_graph_knowledge,
+            FactorImplementSettings().v2_query_component_limit,
+            knowledge_sampler=conf_knowledge_sampler,
+        )
+        factor_implementation_queried_graph_knowledge = self.error_query(
+            evo,
+            factor_implementation_queried_graph_knowledge,
+            FactorImplementSettings().v2_query_error_limit,
+            knowledge_sampler=conf_knowledge_sampler,
+        )
+        return factor_implementation_queried_graph_knowledge
+
+    def analyze_component(
+        self,
+        target_factor_task_information,
+    ) -> list[UndirectedNode]:  # Hardcode: certain component nodes
+        all_component_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["component"])
+        all_component_content = ""
+        for _, component_node in enumerate(all_component_nodes):
+            all_component_content += f"{component_node.content}, \n"
+        analyze_component_system_prompt = Template(self.prompt["analyze_component_prompt_v1_system"]).render(
+            all_component_content=all_component_content,
+        )
+
+        analyze_component_user_prompt = target_factor_task_information
+        try:
+            component_no_list = json.loads(
+                APIBackend().build_messages_and_create_chat_completion(
+                    system_prompt=analyze_component_system_prompt,
+                    user_prompt=analyze_component_user_prompt,
+                    json_mode=True,
+                ),
+            )["component_no_list"]
+            return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))]
+        except:
+            FinCoLog.warning("Error when analyzing components.")
+            analyze_component_user_prompt = "Your response is not a valid component index list."
+
+        return []
+
+    def analyze_error(
+        self,
+        single_feedback,
+        feedback_type="execution",
+    ) -> list[
+        UndirectedNode | str
+    ]:  # Hardcode: Raised errors, existed error nodes + not existed error nodes(here, they are strs)
+        if feedback_type == "execution":
+            match = re.search(
+                r'File "(?P<file>.+)", line (?P<line>\d+), in (?P<function>.+)\n\s+(?P<error_line>.+)\n(?P<error_type>\w+): (?P<error_message>.+)',
+                single_feedback,
+            )
+            if match:
+                error_details = match.groupdict()
+                # last_traceback = f'File "{error_details["file"]}", line {error_details["line"]}, in {error_details["function"]}\n    {error_details["error_line"]}'
+                error_type = error_details["error_type"]
+                error_line = error_details["error_line"]
+                error_contents = [f"ErrorType: {error_type}" + "\n" + f"Error line: {error_line}"]
+            else:
+                error_contents = ["Undefined Error"]
+        elif feedback_type == "value":  # value check error
+            value_check_types = r"The source dataframe and the ground truth dataframe have different rows count.|The source dataframe and the ground truth dataframe have different index.|Some values differ by more than the tolerance of 1e-6.|No sufficient correlation found when shifting up|Something wrong happens when naming the multi indices of the dataframe."
+            error_contents = re.findall(value_check_types, single_feedback)
+        else:
+            error_contents = ["Undefined Error"]
+
+        all_error_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["error"])
+        if not len(all_error_nodes):
+            return error_contents
+        else:
+            error_list = []
+            for error_content in error_contents:
+                for error_node in all_error_nodes:
+                    if error_content == error_node.content:
+                        error_list.append(error_node)
+                    else:
+                        error_list.append(error_content)
+                    if error_list[-1] in error_list[:-1]:
+                        error_list.pop()
+
+            return error_list
+
+    def former_trace_query(
+        self,
+        evo: EvolvableSubjects,
+        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        v2_query_former_trace_limit: int = 5,
+    ) -> Union[QueriedKnowledge, set]:
+        """
+        Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
+        """
+        fail_task_trial_limit = FactorImplementSettings().fail_task_trial_limit
+
+        for target_factor_task in evo.target_factor_tasks:
+            target_factor_task_information = target_factor_task.get_factor_information()
+            if (
+                target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
+                and target_factor_task_information in self.knowledgebase.working_trace_knowledge
+                and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information])
+                >= fail_task_trial_limit
+            ):
+                factor_implementation_queried_graph_knowledge.failed_task_info_set.add(target_factor_task_information)
+
+            if (
+                target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
+                and target_factor_task_information
+                not in factor_implementation_queried_graph_knowledge.failed_task_info_set
+                and target_factor_task_information in self.knowledgebase.working_trace_knowledge
+            ):
+                former_trace_knowledge = copy.copy(
+                    self.knowledgebase.working_trace_knowledge[target_factor_task_information],
+                )
+                # in former trace query we will delete the right trace in the following order:[..., value_generated_flag is True, value_generated_flag is False, ...]
+                # because we think this order means a deterioration of the trial (like a wrong gradient descent)
+                current_index = 1
+                while current_index < len(former_trace_knowledge):
+                    if (
+                        not former_trace_knowledge[current_index].feedback.value_generated_flag
+                        and former_trace_knowledge[current_index - 1].feedback.value_generated_flag
+                    ):
+                        former_trace_knowledge.pop(current_index)
+                    else:
+                        current_index += 1
+
+                factor_implementation_queried_graph_knowledge.former_traces[
+                    target_factor_task_information
+                ] = former_trace_knowledge[-v2_query_former_trace_limit:]
+            else:
+                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
+
+        return factor_implementation_queried_graph_knowledge
+
+    def component_query(
+        self,
+        evo: EvolvableSubjects,
+        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        v2_query_component_limit: int = 5,
+        knowledge_sampler: float = 1.0,
+    ) -> QueriedKnowledge | None:
+        # queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge()
+        for target_factor_task in evo.target_factor_tasks:
+            target_factor_task_information = target_factor_task.get_factor_information()
+            if (
+                target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
+                or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
+            ):
+                factor_implementation_queried_graph_knowledge.component_with_success_task[
+                    target_factor_task_information
+                ] = []
+            else:
+                if target_factor_task_information not in self.knowledgebase.task_to_component_nodes:
+                    self.knowledgebase.task_to_component_nodes[target_factor_task_information] = self.analyze_component(
+                        target_factor_task_information,
+                    )
+
+                component_analysis_result = self.knowledgebase.task_to_component_nodes[target_factor_task_information]
+
+                if len(component_analysis_result) > 1:
+                    task_des_node_list = self.knowledgebase.graph_query_by_intersection(
+                        component_analysis_result,
+                        constraint_labels=["task_description"],
+                    )
+                    single_component_constraint = (v2_query_component_limit // len(component_analysis_result)) + 1
+                else:
+                    task_des_node_list = []
+                    single_component_constraint = v2_query_component_limit
+                factor_implementation_queried_graph_knowledge.component_with_success_task[
+                    target_factor_task_information
+                ] = []
+                for component_node in component_analysis_result:
+                    # Reverse iterate, a trade-off with intersection search
+                    count = 0
+                    for task_des_node in self.knowledgebase.graph_query_by_node(
+                        node=component_node,
+                        step=1,
+                        constraint_labels=["task_description"],
+                        block=True,
+                    )[::-1]:
+                        if task_des_node not in task_des_node_list:
+                            task_des_node_list.append(task_des_node)
+                            count += 1
+                        if count >= single_component_constraint:
+                            break
+
+                for node in task_des_node_list:
+                    for searched_node in self.knowledgebase.graph_query_by_node(
+                        node=node,
+                        step=50,
+                        constraint_labels=[
+                            "task_success_implement",
+                        ],
+                        block=True,
+                    ):
+                        if searched_node.label == "task_success_implement":
+                            target_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[
+                                searched_node.id
+                            ]
+                        if (
+                            target_knowledge
+                            not in factor_implementation_queried_graph_knowledge.component_with_success_task[
+                                target_factor_task_information
+                            ]
+                        ):
+                            factor_implementation_queried_graph_knowledge.component_with_success_task[
+                                target_factor_task_information
+                            ].append(target_knowledge)
+
+                # finally add embedding related knowledge
+                knowledge_base_success_task_list = list(self.knowledgebase.success_task_to_knowledge_dict)
+
+                similarity = calculate_embedding_distance_between_str_list(
+                    [target_factor_task_information],
+                    knowledge_base_success_task_list,
+                )[0]
+                similar_indexes = sorted(
+                    range(len(similarity)),
+                    key=lambda i: similarity[i],
+                    reverse=True,
+                )
+                embedding_similar_successful_knowledge = [
+                    self.knowledgebase.success_task_to_knowledge_dict[knowledge_base_success_task_list[index]]
+                    for index in similar_indexes
+                ]
+                for knowledge in embedding_similar_successful_knowledge:
+                    if (
+                        knowledge
+                        not in factor_implementation_queried_graph_knowledge.component_with_success_task[
+                            target_factor_task_information
+                        ]
+                    ):
+                        factor_implementation_queried_graph_knowledge.component_with_success_task[
+                            target_factor_task_information
+                        ].append(knowledge)
+
+                if knowledge_sampler > 0:
+                    factor_implementation_queried_graph_knowledge.component_with_success_task[
+                        target_factor_task_information
+                    ] = [
+                        knowledge
+                        for knowledge in factor_implementation_queried_graph_knowledge.component_with_success_task[
+                            target_factor_task_information
+                        ]
+                        if random.uniform(0, 1) <= knowledge_sampler
+                    ]
+
+                # Make sure no less than half of the knowledge are from GT
+                queried_knowledge_list = factor_implementation_queried_graph_knowledge.component_with_success_task[
+                    target_factor_task_information
+                ]
+                queried_from_gt_knowledge_list = [
+                    knowledge
+                    for knowledge in queried_knowledge_list
+                    if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == True
+                ]
+                queried_without_gt_knowledge_list = [
+                    knowledge
+                    for knowledge in queried_knowledge_list
+                    if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False
+                ]
+                queried_from_gt_knowledge_count = max(
+                    min(v2_query_component_limit // 2, len(queried_from_gt_knowledge_list)),
+                    v2_query_component_limit - len(queried_without_gt_knowledge_list),
+                )
+                factor_implementation_queried_graph_knowledge.component_with_success_task[
+                    target_factor_task_information
+                ] = (
+                    queried_from_gt_knowledge_list[:queried_from_gt_knowledge_count]
+                    + queried_without_gt_knowledge_list[: v2_query_component_limit - queried_from_gt_knowledge_count]
+                )
+
+        return factor_implementation_queried_graph_knowledge
+
+    def error_query(
+        self,
+        evo: EvolvableSubjects,
+        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        v2_query_error_limit: int = 5,
+        knowledge_sampler: float = 1.0,
+    ) -> QueriedKnowledge | None:
+        # queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge()
+        for task_index, target_factor_task in enumerate(evo.target_factor_tasks):
+            target_factor_task_information = target_factor_task.get_factor_information()
+            factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
+            if (
+                target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
+                or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
+            ):
+                factor_implementation_queried_graph_knowledge.error_with_success_task[
+                    target_factor_task_information
+                ] = []
+            else:
+                factor_implementation_queried_graph_knowledge.error_with_success_task[
+                    target_factor_task_information
+                ] = []
+                if (
+                    target_factor_task_information in self.knowledgebase.working_trace_error_analysis
+                    and len(self.knowledgebase.working_trace_error_analysis[target_factor_task_information]) > 0
+                    and len(factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information])
+                    > 0
+                ):
+                    queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[
+                        target_factor_task_information
+                    ][-1]
+                    target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
+                        queried_last_trace,
+                    )
+                    last_knowledge_error_analysis_result = self.knowledgebase.working_trace_error_analysis[
+                        target_factor_task_information
+                    ][target_index]
+                else:
+                    last_knowledge_error_analysis_result = []
+
+                error_nodes = []
+                for error_node in last_knowledge_error_analysis_result:
+                    if not isinstance(error_node, UndirectedNode):
+                        error_node = self.knowledgebase.graph_get_node_by_content(content=error_node)
+                        if error_node is None:
+                            continue
+                    error_nodes.append(error_node)
+
+                if len(error_nodes) > 1:
+                    task_trace_node_list = self.knowledgebase.graph_query_by_intersection(
+                        error_nodes,
+                        constraint_labels=["task_trace"],
+                        output_intersection_origin=True,
+                    )
+                    single_error_constraint = (v2_query_error_limit // len(error_nodes)) + 1
+                else:
+                    task_trace_node_list = []
+                    single_error_constraint = v2_query_error_limit
+                for error_node in error_nodes:
+                    # Reverse iterate, a trade-off with intersection search
+                    count = 0
+                    for task_trace_node in self.knowledgebase.graph_query_by_node(
+                        node=error_node,
+                        step=1,
+                        constraint_labels=["task_trace"],
+                        block=True,
+                    )[::-1]:
+                        if task_trace_node not in task_trace_node_list:
+                            task_trace_node_list.append([[error_node], task_trace_node])
+                            count += 1
+                        if count >= single_error_constraint:
+                            break
+
+                # for error_node in last_knowledge_error_analysis_result:
+                #     if not isinstance(error_node, UndirectedNode):
+                #         error_node = self.knowledgebase.graph_get_node_by_content(content=error_node)
+                #         if error_node is None:
+                #             continue
+                #     for searched_node in self.knowledgebase.graph_query_by_node(
+                #         node=error_node,
+                #         step=1,
+                #         constraint_labels=["task_trace"],
+                #         block=True,
+                #     ):
+                #         if searched_node not in [node[0] for node in task_trace_node_list]:
+                #             task_trace_node_list.append((searched_node, error_node.content))
+
+                same_error_success_knowledge_pair_list = []
+                same_error_success_node_set = set()
+                for error_node_list, trace_node in task_trace_node_list:
+                    for searched_trace_success_node in self.knowledgebase.graph_query_by_node(
+                        node=trace_node,
+                        step=50,
+                        constraint_labels=[
+                            "task_trace",
+                            "task_success_implement",
+                            "task_description",
+                        ],
+                        block=True,
+                    ):
+                        if (
+                            searched_trace_success_node not in same_error_success_node_set
+                            and searched_trace_success_node.label == "task_success_implement"
+                        ):
+                            same_error_success_node_set.add(searched_trace_success_node)
+
+                            trace_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[trace_node.id]
+                            success_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[
+                                searched_trace_success_node.id
+                            ]
+                            error_content = ""
+                            for index, error_node in enumerate(error_node_list):
+                                error_content += f"{index+1}. {error_node.content}; "
+                            same_error_success_knowledge_pair_list.append(
+                                (
+                                    error_content,
+                                    (trace_knowledge, success_knowledge),
+                                ),
+                            )
+
+                if knowledge_sampler > 0:
+                    same_error_success_knowledge_pair_list = [
+                        knowledge
+                        for knowledge in same_error_success_knowledge_pair_list
+                        if random.uniform(0, 1) <= knowledge_sampler
+                    ]
+
+                same_error_success_knowledge_pair_list = same_error_success_knowledge_pair_list[:v2_query_error_limit]
+                factor_implementation_queried_graph_knowledge.error_with_success_task[
+                    target_factor_task_information
+                ] = same_error_success_knowledge_pair_list
+
+        return factor_implementation_queried_graph_knowledge
+

rdagent/knowledge_management/prompts.yaml

@@ -0,0 +1,9 @@
+analyze_component_prompt_v1_system: |-
+  User is getting a new task that might consist of the components below (given in component_index: component_description):
+  {{all_component_content}}
+
+  You should find out what components does the new task have, and put their indices in a list.
+  Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
+  {
+      "component_no_list": the list containing indices of components.
+  }

rdagent/knowledge_management/vector_base.py

@@ -0,0 +1,208 @@
+import uuid
+from pathlib import Path
+from typing import List, Tuple, Union
+
+import pandas as pd
+from scipy.spatial.distance import cosine
+
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.core.log import FinCoLog
+
+
+class KnowledgeMetaData:
+    def __init__(self, content: str = "", label: str = None, embedding=None, identity=None):
+        self.label = label
+        self.content = content
+        self.id = str(uuid.uuid3(uuid.NAMESPACE_DNS, str(self.content))) if identity is None else identity
+        self.embedding = embedding
+        self.trunks = []
+        self.trunks_embedding = []
+
+    def split_into_trunk(self, size: int = 1000, overlap: int = 0):
+        """
+        split content into trunks and create embedding by trunk
+        Returns
+        -------
+
+        """
+
+        def split_string_into_chunks(string: str, chunk_size: int):
+            chunks = []
+            for i in range(0, len(string), chunk_size):
+                chunk = string[i : i + chunk_size]
+                chunks.append(chunk)
+            return chunks
+
+        self.trunks = split_string_into_chunks(self.content, chunk_size=size)
+        self.trunks_embedding = APIBackend().create_embedding(input_content=self.trunks)
+
+    def create_embedding(self):
+        """
+        create content's embedding
+        Returns
+        -------
+
+        """
+        if self.embedding is None:
+            self.embedding = APIBackend().create_embedding(input_content=self.content)
+
+    def from_dict(self, data: dict):
+        for key, value in data.items():
+            setattr(self, key, value)
+        return self
+
+    def __repr__(self):
+        return f"Document(id={self.id}, label={self.label}, data={self.content})"
+
+
+Document = KnowledgeMetaData
+
+
+def contents_to_documents(contents: List[str], label: str = None) -> List[Document]:
+    # openai create embedding API input's max length is 16
+    size = 16
+    embedding = []
+    for i in range(0, len(contents), size):
+        embedding.extend(APIBackend().create_embedding(input_content=contents[i : i + size]))
+    docs = [Document(content=c, label=label, embedding=e) for c, e in zip(contents, embedding)]
+    return docs
+
+
+class VectorBase:
+    """
+    This class is used for handling vector storage and query
+    """
+
+    def __init__(self, vector_df_path: Union[str, Path] = None, **kwargs):
+        pass
+
+    def add(self, document: Union[Document, List[Document]]):
+        """
+        add new node to vector_df
+        Parameters
+        ----------
+        document
+
+        Returns
+        -------
+
+        """
+        pass
+
+    def search(self, content: str, topk_k: int = 5, similarity_threshold: float = 0) -> List[Document]:
+        """
+        search vector_df by node
+        Parameters
+        ----------
+        similarity_threshold
+        content
+        topk_k: return topk_k nearest vector_df
+
+        Returns
+        -------
+
+        """
+        pass
+
+    def load(self, **kwargs):
+        """load vector_df"""
+
+    def save(self, **kwargs):
+        """save vector_df"""
+
+
+class PDVectorBase(VectorBase):
+    """
+    Implement of VectorBase using Pandas
+    """
+
+    def __init__(self, vector_df_path: Union[str, Path] = None):
+        super().__init__(vector_df_path)
+
+        if vector_df_path:
+            try:
+                self.vector_df = self.load(vector_df_path)
+            except FileNotFoundError:
+                self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"])
+        else:
+            self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"])
+
+        FinCoLog().info(f"VectorBase loaded, shape={self.vector_df.shape}")
+
+    def shape(self):
+        return self.vector_df.shape
+
+    def add(self, document: Union[Document, List[Document]]):
+        """
+        add new node to vector_df
+        Parameters
+        ----------
+        document
+
+        Returns
+        -------
+
+        """
+        if isinstance(document, Document):
+            if document.embedding is None:
+                document.create_embedding()
+            docs = [
+                {
+                    "id": document.id,
+                    "label": document.label,
+                    "content": document.content,
+                    "trunk": document.content,
+                    "embedding": document.embedding,
+                }
+            ]
+            docs.extend(
+                [
+                    {
+                        "id": document.id,
+                        "label": document.label,
+                        "content": document.content,
+                        "trunk": trunk,
+                        "embedding": embedding,
+                    }
+                    for trunk, embedding in zip(document.trunks, document.trunks_embedding)
+                ]
+            )
+            self.vector_df = pd.concat([self.vector_df, pd.DataFrame(docs)], ignore_index=True)
+        else:
+            for doc in document:
+                self.add(document=doc)
+
+    def search(self, content: str, topk_k: int = 5, similarity_threshold: float = 0) -> Tuple[List[Document], List]:
+        """
+        search vector by node
+        Parameters
+        ----------
+        similarity_threshold
+        content
+        topk_k: return topk_k nearest vector
+
+        Returns
+        -------
+
+        """
+        if not self.vector_df.shape[0]:
+            return [], []
+        document = Document(content=content)
+        document.create_embedding()
+        similarities = self.vector_df["embedding"].apply(
+            lambda x: 1 - cosine(x, document.embedding)
+        )  # cosine is cosine distance, 1-similarity
+        searched_similarities = similarities[similarities > similarity_threshold].nlargest(topk_k)
+        most_similar_docs = self.vector_df.loc[searched_similarities.index]
+        docs = []
+        for _, similar_docs in most_similar_docs.iterrows():
+            docs.append(Document().from_dict(similar_docs.to_dict()))
+        return docs, searched_similarities.to_list()
+
+    def load(self, vector_df_path, **kwargs):
+        vector_df = pd.read_pickle(vector_df_path)
+        return vector_df
+
+    def save(self, vector_df_path, **kwargs):
+        self.vector_df.to_pickle(vector_df_path)
+        FinCoLog().info(f"Save vectorBase vector_df to: {vector_df_path}")

rdagent/oai/llm_utils.py

@@ -17,7 +17,8 @@ from typing import Any
 
 import numpy as np
 import tiktoken
-from rdagent.core.conf import FincoSettings as Config
+
+from rdagent.core.conf import RDAgentSettings as Config
 from rdagent.core.log import FinCoLog, LogColors
 from rdagent.core.utils import SingletonBaseClass
 
@@ -202,7 +203,7 @@ class ChatSession:
         this function is to build the session messages
         user prompt should always be provided
         """
-        messages = self.build_chat_completion_message(user_prompt, **kwargs)
+        messages = self.build_chat_completion_message(user_prompt)
 
         response = self.api_backend._try_create_chat_completion_or_embedding( # noqa: SLF001
             messages=messages, chat_completion=True, **kwargs,
@@ -413,7 +414,7 @@ class APIBackend:
     ) -> str:
         if former_messages is None:
             former_messages = []
-        messages = self.build_messages(user_prompt, system_prompt, former_messages, shrink_multiple_break)
+        messages = self.build_messages(user_prompt, system_prompt, former_messages, shrink_multiple_break=shrink_multiple_break)
         return self._try_create_chat_completion_or_embedding(
             messages=messages,
             chat_completion=True,
@@ -690,7 +691,7 @@ class APIBackend:
     ) -> int:
         if former_messages is None:
             former_messages = []
-        messages = self.build_messages(user_prompt, system_prompt, former_messages, shrink_multiple_break)
+        messages = self.build_messages(user_prompt, system_prompt, former_messages, shrink_multiple_break=shrink_multiple_break)
         return self.calculate_token_from_messages(messages)