Refine all the implementation code to higher quality for release (#29)

* refine CI script

* refine all the code to higher quality

* refine the script to factor extraction and implementation

* add task loader interface

* add a task loader interface && move pdf analysis to pdf task loader

* change the name to global variables

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/factor_implementation/CoSTEER.py

@@ -5,13 +5,20 @@ from rdagent.core.implementation import TaskGenerator
 from rdagent.core.task import TaskImplementation
 from rdagent.factor_implementation.evolving.knowledge_management import FactorImplementationKnowledgeBaseV1
 from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FactorEvovlingItem
-from rdagent.knowledge_management.knowledgebase import FactorImplementationGraphKnowledgeBase, FactorImplementationGraphRAGStrategy
-from rdagent.factor_implementation.evolving.evolving_strategy import FactorEvolvingStrategyWithGraph
-from rdagent.factor_implementation.evolving.evaluators import FactorImplementationsMultiEvaluator, FactorImplementationEvaluatorV1
-from rdagent.factor_implementation.evolving.evolving_agent import RAGEvoAgent
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
+from rdagent.factor_implementation.evolving.knowledge_management import (
+    FactorImplementationGraphKnowledgeBase,
+    FactorImplementationGraphRAGStrategy,
 )
+from rdagent.factor_implementation.evolving.evolving_strategy import FactorEvolvingStrategyWithGraph
+from rdagent.factor_implementation.evolving.evaluators import (
+    FactorImplementationsMultiEvaluator,
+    FactorImplementationEvaluatorV1,
+)
+from rdagent.core.evolving_agent import RAGEvoAgent
+from rdagent.factor_implementation.share_modules.factor_implementation_config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
+
 
 class CoSTEERFG(TaskGenerator):
     def __init__(
@@ -20,9 +27,17 @@ class CoSTEERFG(TaskGenerator):
         with_feedback: bool = True,
         knowledge_self_gen: bool = True,
     ) -> None:
-        self.max_loop = FactorImplementSettings().max_loop
-        self.knowledge_base_path = Path(FactorImplementSettings().knowledge_base_path) if FactorImplementSettings().knowledge_base_path is not None else None
-        self.new_knowledge_base_path = Path(FactorImplementSettings().new_knowledge_base_path) if FactorImplementSettings().new_knowledge_base_path is not None else None
+        self.max_loop = FACTOR_IMPLEMENT_SETTINGS.max_loop
+        self.knowledge_base_path = (
+            Path(FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path)
+            if FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path is not None
+            else None
+        )
+        self.new_knowledge_base_path = (
+            Path(FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path)
+            if FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path is not None
+            else None
+        )
         self.with_knowledge = with_knowledge
         self.with_feedback = with_feedback
         self.knowledge_self_gen = knowledge_self_gen
@@ -32,7 +47,7 @@ class CoSTEERFG(TaskGenerator):
         self.evolving_version = 2
 
     def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
-    
+
         if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
             factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
             if self.evolving_version == 1 and not isinstance(
@@ -53,7 +68,7 @@ class CoSTEERFG(TaskGenerator):
                 else FactorImplementationKnowledgeBaseV1()
             )
         return factor_knowledge_base
-    
+
     def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]:
         # init knowledge base
         factor_knowledge_base = self.load_or_init_knowledge_base(
@@ -61,9 +76,7 @@ class CoSTEERFG(TaskGenerator):
             component_init_list=[],
         )
         # init rag method
-        self.rag = (
-            FactorImplementationGraphRAGStrategy(factor_knowledge_base)
-        )
+        self.rag = FactorImplementationGraphRAGStrategy(factor_knowledge_base)
 
         # init indermediate items
         factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks)
@@ -83,4 +96,4 @@ class CoSTEERFG(TaskGenerator):
             pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
         self.knowledge_base = factor_knowledge_base
         self.latest_factor_implementations = tasks
-        return factor_implementations       
+        return factor_implementations

rdagent/factor_implementation/evolving/evaluators.py

@@ -8,21 +8,23 @@ import pandas as pd
 from jinja2 import Template
 
 from rdagent.oai.llm_utils import APIBackend
-from rdagent.core.log import FinCoLog
+from rdagent.core.log import RDAgentLog
 from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask, FactorEvovlingItem
 from rdagent.core.task import (
     TaskImplementation,
 )
 from typing import List, Tuple
-from rdagent.core.evolving_framework import QueriedKnowledge,Feedback
+from rdagent.core.evolving_framework import QueriedKnowledge, Feedback
 from rdagent.core.evaluation import Evaluator
 from rdagent.core.prompts import Prompts
-from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.core.utils import multiprocessing_wrapper
 from pathlib import Path
 
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 
+
 class FactorImplementationEvaluator(Evaluator):
     # TODO:
     # I think we should have unified interface for all evaluates, for examples.
@@ -58,6 +60,7 @@ class FactorImplementationEvaluator(Evaluator):
             gt_df = gt_df.to_frame("gt_factor")
         return gt_df, gen_df
 
+
 class FactorImplementationCodeEvaluator(Evaluator):
     def evaluate(
         self,
@@ -89,7 +92,7 @@ class FactorImplementationCodeEvaluator(Evaluator):
                 system_prompt=system_prompt,
                 former_messages=[],
             )
-            > FactorImplementSettings().chat_token_limit
+            > RD_AGENT_SETTINGS.chat_token_limit
         ):
             execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             user_prompt = Template(
@@ -109,6 +112,7 @@ class FactorImplementationCodeEvaluator(Evaluator):
 
         return critic_response
 
+
 class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
     def evaluate(
         self,
@@ -442,7 +446,7 @@ class FactorImplementationValueEvaluator(Evaluator):
                         )
 
                 except Exception as e:
-                    FinCoLog().warning(f"Error occurred when calculating the correlation: {str(e)}")
+                    RDAgentLog().warning(f"Error occurred when calculating the correlation: {str(e)}")
                     conclusions.append(
                         f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
                     )
@@ -474,7 +478,9 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
         code_feedback: str,
         **kwargs,
     ) -> Tuple:
-        system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")["evaluator_final_decision_v1_system"]
+        system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
+            "evaluator_final_decision_v1_system"
+        ]
         execution_feedback_to_render = execution_feedback
         user_prompt = Template(
             evaluate_prompts["evaluator_final_decision_v1_user"],
@@ -494,7 +500,7 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
                 system_prompt=system_prompt,
                 former_messages=[],
             )
-            > FactorImplementSettings().chat_token_limit
+            > RD_AGENT_SETTINGS.chat_token_limit
         ):
             execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             user_prompt = Template(
@@ -522,6 +528,7 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
             final_evaluation_dict["final_feedback"],
         )
 
+
 class FactorImplementationSingleFeedback:
     """This class is a feedback to single implementation which is generated from an evaluator."""
 
@@ -556,12 +563,14 @@ class FactorImplementationSingleFeedback:
 This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
 """
 
+
 class FactorImplementationsMultiFeedback(
     Feedback,
     List[FactorImplementationSingleFeedback],
 ):
     """Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
 
+
 class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
     """This class is the v1 version of evaluator for a single factor implementation.
     It calls several evaluators in share modules to evaluate the factor implementation.
@@ -633,7 +642,7 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
                         value_decision,
                     ) = self.value_evaluator.evaluate(source_df=source_df, gt_df=gt_df)
                 except Exception as e:
-                    FinCoLog().warning("Value evaluation failed with exception: %s", e)
+                    RDAgentLog().warning("Value evaluation failed with exception: %s", e)
                     factor_feedback.factor_value_feedback = "Value evaluation failed."
                     value_decision = False
 
@@ -713,13 +722,12 @@ class FactorImplementationsMultiEvaluator(Evaluator):
                     ),
                 ),
             )
-        multi_implementation_feedback = multiprocessing_wrapper(calls, n=FactorImplementSettings().evo_multi_proc_n)
+        multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n)
 
         final_decision = [
             None if single_feedback is None else single_feedback.final_decision
             for single_feedback in multi_implementation_feedback
         ]
-        print(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
+        RDAgentLog().info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
 
         return multi_implementation_feedback
-

rdagent/factor_implementation/evolving/evolving_agent.py

@@ -1,50 +0,0 @@
-from rdagent.core.evolving_framework import Feedback, EvolvableSubjects, Evaluator, EvoStep
-from rdagent.core.evolving_framework import EvoAgent
-from tqdm import tqdm
-
-class RAGEvoAgent(EvoAgent):
-    def __init__(self, max_loop, evolving_strategy, rag) -> None:
-        super().__init__(max_loop, evolving_strategy)
-        self.rag = rag
-        self.evolving_trace = []
-
-    def multistep_evolve(
-        self, 
-        evo: EvolvableSubjects,
-        eva: Evaluator | Feedback, 
-        *,
-        with_knowledge: bool = False,
-        with_feedback: bool = True,
-        knowledge_self_gen: bool = False) -> EvolvableSubjects:
-
-        for _ in tqdm(range(self.max_loop), "Implementing factors"):
-            # 1. knowledge self-evolving
-            if knowledge_self_gen and self.rag is not None:
-                self.rag.generate_knowledge(self.evolving_trace)
-            # 2. 检索需要的Knowledge
-            queried_knowledge = None
-            if with_knowledge and self.rag is not None:
-                # TODO: 这里放了evolving_trace实际上没有作用
-                queried_knowledge = self.rag.query(evo, self.evolving_trace)
-
-            # 3. evolve
-            evo = self.evolving_strategy.evolve(
-                    evo=evo,
-                    evolving_trace=self.evolving_trace,
-                    queried_knowledge=queried_knowledge,
-                )
-            
-            # 4. 封装Evolve结果
-            es = EvoStep(evo, queried_knowledge)
-
-            # 5. 环境评测反馈
-            if with_feedback:
-                es.feedback = eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge)
-
-            # 6. 更新trace
-            self.evolving_trace.append(es)
-            for index, feedback in enumerate(es.feedback):
-                if feedback is not None:
-                    evo.evolve_trace[evo.target_factor_tasks[index].factor_name][-1].feedback = feedback
-
-        return evo  

rdagent/factor_implementation/evolving/evolving_strategy.py

@@ -8,10 +8,11 @@ from typing import TYPE_CHECKING
 
 from jinja2 import Template
 
+from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
+    FACTOR_IMPLEMENT_SETTINGS,
 )
 
 from rdagent.core.task import (
@@ -81,18 +82,18 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 
         # 2. 选择selection方法
         # if the number of factors to be implemented is larger than the limit, we need to select some of them
-        if FactorImplementSettings().select_ratio < 1:
+        if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1:
             # if the number of loops is equal to the select_loop, we need to select some of them
             implementation_factors_per_round = int(
-                FactorImplementSettings().select_ratio * len(to_be_finished_task_index)
+                FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index)
             )
-            if FactorImplementSettings().select_method == "random":
+            if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
                 to_be_finished_task_index = RandomSelect(
                     to_be_finished_task_index,
                     implementation_factors_per_round,
                 )
 
-            if FactorImplementSettings().select_method == "scheduler":
+            if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
                 to_be_finished_task_index = LLMSelect(
                     to_be_finished_task_index,
                     implementation_factors_per_round,
@@ -105,17 +106,18 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
                 (self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge))
                 for target_index in to_be_finished_task_index
             ],
-            n=FactorImplementSettings().evo_multi_proc_n,
+            n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
         )
 
         for index, target_index in enumerate(to_be_finished_task_index):
             new_evo.corresponding_implementations[target_index] = result[index]
-            if result[index].target_task.factor_name in new_evo.evolve_trace:
-                new_evo.evolve_trace[result[index].target_task.factor_name].append(result[index])
-            else:
-                new_evo.evolve_trace[result[index].target_task.factor_name] = [result[index]]
 
-        new_evo.corresponding_selection.append(to_be_finished_task_index)
+        # for target_index in to_be_finished_task_index:
+        #     new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
+        #         new_evo.target_factor_tasks[target_index], queried_knowledge
+        #     )
+
+        new_evo.corresponding_selection = to_be_finished_task_index
 
         return new_evo
 
@@ -172,7 +174,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     session.build_chat_completion_message_and_calculate_token(
                         user_prompt,
                     )
-                    < FactorImplementSettings().chat_token_limit
+                    < RD_AGENT_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:
@@ -205,7 +207,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
         target_task: FactorImplementTask,
         queried_knowledge,
     ) -> TaskImplementation:
-        error_summary = FactorImplementSettings().v2_error_summary
+        error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
         # 1. 提取因子的背景信息
         target_factor_task_information = target_task.get_factor_information()
 
@@ -284,7 +286,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         )
                         if (
                             session_summary.build_chat_completion_message_and_calculate_token(error_summary_user_prompt)
-                            < FactorImplementSettings().chat_token_limit
+                            < RD_AGENT_SETTINGS.chat_token_limit
                         ):
                             break
                         elif len(queried_similar_error_knowledge_to_render) > 0:
@@ -311,7 +313,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     session.build_chat_completion_message_and_calculate_token(
                         user_prompt,
                     )
-                    < FactorImplementSettings().chat_token_limit
+                    < RD_AGENT_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:

rdagent/factor_implementation/evolving/factor.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
+    FACTOR_IMPLEMENT_SETTINGS,
 )
 
 from rdagent.core.task import (
@@ -9,7 +9,7 @@ from rdagent.core.task import (
     TestCase,
 )
 from rdagent.core.evolving_framework import EvolvableSubjects
-from rdagent.core.log import FinCoLog
+from rdagent.core.log import RDAgentLog
 
 from pathlib import Path
 
@@ -35,7 +35,7 @@ class FactorImplementTask(BaseTask):
         factor_name,
         factor_description,
         factor_formulation,
-        factor_formulation_description: str = '',
+        factor_formulation_description: str = "",
         variables: dict = {},
         resource: str = None,
     ) -> None:
@@ -68,20 +68,17 @@ class FactorEvovlingItem(EvolvableSubjects):
     def __init__(
         self,
         target_factor_tasks: list[FactorImplementTask],
-        corresponding_gt: list[TestCase] = None,
         corresponding_gt_implementations: list[TaskImplementation] = None,
     ):
         super().__init__()
         self.target_factor_tasks = target_factor_tasks
         self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks]
-        self.corresponding_selection: list[list] = []
-        self.evolve_trace = {}
-        self.corresponding_gt = corresponding_gt
+        self.corresponding_selection: list = None
         if corresponding_gt_implementations is not None and len(
             corresponding_gt_implementations,
         ) != len(target_factor_tasks):
             self.corresponding_gt_implementations = None
-            FinCoLog.warning(
+            RDAgentLog().warning(
                 "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
             )
         else:
@@ -112,10 +109,10 @@ class FileBasedFactorImplementation(TaskImplementation):
         super().__init__(target_task)
         self.code = code
         self.executed_factor_value_dataframe = executed_factor_value_dataframe
-        self.logger = FinCoLog()
+        self.logger = RDAgentLog()
         self.raise_exception = raise_exception
         self.workspace_path = Path(
-            FactorImplementSettings().file_based_execution_workspace,
+            FACTOR_IMPLEMENT_SETTINGS.file_based_execution_workspace,
         ) / str(uuid.uuid4())
 
     @staticmethod
@@ -151,17 +148,14 @@ class FileBasedFactorImplementation(TaskImplementation):
                 # TODO: to make the interface compatible with previous code. I kept the original behavior.
                 raise ValueError(self.FB_CODE_NOT_SET)
         with FileLock(self.workspace_path / "execution.lock"):
-            (Path.cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").mkdir(
-                exist_ok=True, parents=True
-            )
-            if FactorImplementSettings().enable_execution_cache:
+            if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
                 # NOTE: cache the result for the same code
                 target_file_name = md5_hash(self.code)
                 cache_file_path = (
-                    Path.cwd()
-                    / "git_ignore_folder"
-                    / "factor_implementation_execution_cache"
-                    / f"{target_file_name}.pkl"
+                    Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl"
+                )
+                Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location).mkdir(
+                    exist_ok=True, parents=True
                 )
                 if cache_file_path.exists() and not self.raise_exception:
                     cached_res = pickle.load(open(cache_file_path, "rb"))
@@ -173,7 +167,7 @@ class FileBasedFactorImplementation(TaskImplementation):
                 return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
 
             source_data_path = Path(
-                FactorImplementSettings().file_based_execution_data_folder,
+                FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder,
             )
             self.workspace_path.mkdir(exist_ok=True, parents=True)
 
@@ -189,7 +183,7 @@ class FileBasedFactorImplementation(TaskImplementation):
                     shell=True,
                     cwd=self.workspace_path,
                     stderr=subprocess.STDOUT,
-                    timeout=FactorImplementSettings().file_based_execution_timeout,
+                    timeout=FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout,
                 )
             except subprocess.CalledProcessError as e:
                 import site
@@ -206,7 +200,7 @@ class FileBasedFactorImplementation(TaskImplementation):
                 if self.raise_exception:
                     raise RuntimeErrorException(execution_feedback)
             except subprocess.TimeoutExpired:
-                execution_feedback += f"Execution timeout error and the timeout is set to {FactorImplementSettings().file_based_execution_timeout} seconds."
+                execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
                 if self.raise_exception:
                     raise RuntimeErrorException(execution_feedback)
 
@@ -227,7 +221,7 @@ class FileBasedFactorImplementation(TaskImplementation):
             if store_result and executed_factor_value_dataframe is not None:
                 self.executed_factor_value_dataframe = executed_factor_value_dataframe
 
-        if FactorImplementSettings().enable_execution_cache:
+        if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
             pickle.dump(
                 (execution_feedback, executed_factor_value_dataframe),
                 open(cache_file_path, "wb"),
@@ -249,4 +243,3 @@ class FileBasedFactorImplementation(TaskImplementation):
         with factor_path.open("r") as f:
             code = f.read()
         return FileBasedFactorImplementation(task, code=code, **kwargs)
-

rdagent/factor_implementation/evolving/knowledge_management.py

@@ -18,7 +18,7 @@ from rdagent.core.evolving_framework import (
     QueriedKnowledge,
     RAGStrategy,
 )
-from rdagent.core.log import FinCoLog
+from rdagent.core.log import RDAgentLog
 from rdagent.core.prompts import Prompts
 from rdagent.factor_implementation.evolving.evaluators import FactorImplementationSingleFeedback
 from rdagent.core.task import (
@@ -30,9 +30,10 @@ from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode
 from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
 
 from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
+    FACTOR_IMPLEMENT_SETTINGS,
 )
 
+
 class FactorImplementationKnowledge(Knowledge):
     def __init__(
         self,
@@ -138,9 +139,9 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
         evo: EvolvableSubjects,
         evolving_trace: list[EvoStep],
     ) -> QueriedKnowledge | None:
-        v1_query_former_trace_limit = FactorImplementSettings().v1_query_former_trace_limit
-        v1_query_similar_success_limit = FactorImplementSettings().v1_query_similar_success_limit
-        fail_task_trial_limit = FactorImplementSettings().fail_task_trial_limit
+        v1_query_former_trace_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_former_trace_limit
+        v1_query_similar_success_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_similar_success_limit
+        fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
 
         queried_knowledge = FactorImplementationQueriedKnowledgeV1()
         for target_factor_task in evo.target_factor_tasks:
@@ -280,7 +281,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
             return None
 
     def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None:
-        conf_knowledge_sampler = FactorImplementSettings().v2_knowledge_sampler
+        conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler
         factor_implementation_queried_graph_knowledge = FactorImplementationQueriedGraphKnowledge(
             success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict,
         )
@@ -288,18 +289,18 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
         factor_implementation_queried_graph_knowledge = self.former_trace_query(
             evo,
             factor_implementation_queried_graph_knowledge,
-            FactorImplementSettings().v2_query_former_trace_limit,
+            FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit,
         )
         factor_implementation_queried_graph_knowledge = self.component_query(
             evo,
             factor_implementation_queried_graph_knowledge,
-            FactorImplementSettings().v2_query_component_limit,
+            FACTOR_IMPLEMENT_SETTINGS.v2_query_component_limit,
             knowledge_sampler=conf_knowledge_sampler,
         )
         factor_implementation_queried_graph_knowledge = self.error_query(
             evo,
             factor_implementation_queried_graph_knowledge,
-            FactorImplementSettings().v2_query_error_limit,
+            FACTOR_IMPLEMENT_SETTINGS.v2_query_error_limit,
             knowledge_sampler=conf_knowledge_sampler,
         )
         return factor_implementation_queried_graph_knowledge
@@ -327,7 +328,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
             )["component_no_list"]
             return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))]
         except:
-            FinCoLog.warning("Error when analyzing components.")
+            RDAgentLog().warning("Error when analyzing components.")
             analyze_component_user_prompt = "Your response is not a valid component index list."
 
         return []
@@ -383,7 +384,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
         """
         Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
         """
-        fail_task_trial_limit = FactorImplementSettings().fail_task_trial_limit
+        fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
 
         for target_factor_task in evo.target_factor_tasks:
             target_factor_task_information = target_factor_task.get_factor_information()
@@ -705,7 +706,7 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
         Load knowledge, offer brief information of knowledge and common handle interfaces
         """
         self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl")
-        FinCoLog().info(f"Knowledge Graph loaded, size={self.graph.size()}")
+        RDAgentLog().info(f"Knowledge Graph loaded, size={self.graph.size()}")
 
         if init_component_list:
             for component in init_component_list:
@@ -901,4 +902,3 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
                     intersection_node_list_sort_by_freq.append(node)
 
         return intersection_node_list_sort_by_freq
-

rdagent/factor_implementation/evolving/scheduler.py

@@ -1,27 +1,32 @@
 from rdagent.oai.llm_utils import APIBackend
 from jinja2 import Template
-from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
 import json
 from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
 from rdagent.factor_implementation.evolving.factor import FactorEvovlingItem
 from rdagent.core.prompts import Prompts
+from rdagent.core.log import RDAgentLog
+from rdagent.core.conf import RD_AGENT_SETTINGS
 from pathlib import Path
 
 scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 
+
 def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
     import random
+
     to_be_finished_task_index = random.sample(
         to_be_finished_task_index,
         implementation_factors_per_round,
     )
-    print("The random selection is:",to_be_finished_task_index)
+
+    RDAgentLog().info(f"The random selection is: {to_be_finished_task_index}")
     return to_be_finished_task_index
 
-def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:FactorEvovlingItem, former_trace):
+
+def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvovlingItem, former_trace):
     tasks = []
     for i in to_be_finished_task_index:
-    # find corresponding former trace for each task
+        # find corresponding former trace for each task
         target_factor_task_information = evo.target_factor_tasks[i].get_factor_information()
         if target_factor_task_information in former_trace:
             tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information]))
@@ -37,20 +42,17 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:F
     )
 
     while True:
-        user_prompt = (
-            Template(
-                scheduler_prompts["select_implementable_factor_user"],
-            )
-            .render(
-                factor_num = implementation_factors_per_round,
-                target_factor_tasks=tasks,
-            )
+        user_prompt = Template(
+            scheduler_prompts["select_implementable_factor_user"],
+        ).render(
+            factor_num=implementation_factors_per_round,
+            target_factor_tasks=tasks,
         )
         if (
             session.build_chat_completion_message_and_calculate_token(
                 user_prompt,
             )
-            < FactorImplementSettings().chat_token_limit
+            < RD_AGENT_SETTINGS.chat_token_limit
         ):
             break
 
@@ -65,5 +67,5 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:F
         selection_index = [x for x in selection if isinstance(x, int)]
     except:
         return to_be_finished_task_index
-    
+
     return selection_index

rdagent/factor_implementation/prompts.yaml

@@ -217,3 +217,13 @@ select_implementable_factor_user: |-
   {% endfor %}
   {% endif %}
   {% endfor %}
+
+analyze_component_prompt_v1_system: |-
+  User is getting a new task that might consist of the components below (given in component_index: component_description):
+  {{all_component_content}}
+
+  You should find out what components does the new task have, and put their indices in a list.
+  Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
+  {
+      "component_no_list": the list containing indices of components.
+  }

rdagent/factor_implementation/share_modules/factor_implementation_config.py

@@ -9,13 +9,13 @@ SELECT_METHOD = Literal["random", "scheduler"]
 
 class FactorImplementSettings(BaseSettings):
     file_based_execution_data_folder: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
+        (Path().cwd() / "factor_implementation_source_data").absolute(),
     )
     file_based_execution_workspace: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(),
+        (Path().cwd() / "factor_implementation_workspace").absolute(),
     )
     implementation_execution_cache_location: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache.pkl").absolute(),
+        (Path().cwd() / "factor_implementation_execution_cache").absolute(),
     )
     enable_execution_cache: bool = True  # whether to enable the execution cache
 
@@ -45,9 +45,5 @@ class FactorImplementSettings(BaseSettings):
     knowledge_base_path: Union[str, None] = None
     new_knowledge_base_path: Union[str, None] = None
 
-    chat_token_limit: int = (
-        100000  # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
-    )
 
-
-FIS = FactorImplementSettings()
+FACTOR_IMPLEMENT_SETTINGS = FactorImplementSettings()

rdagent/factor_implementation/share_modules/factor_implementation_utils.py

@@ -4,7 +4,7 @@ import pandas as pd
 
 # render it with jinja
 from jinja2 import Template
-from rdagent.factor_implementation.share_modules.factor_implementation_config import FIS
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.factor_implementation.evolving.factor import FactorImplementTask
 
 
@@ -17,12 +17,13 @@ TPL = """
 # Create a Jinja template from the string
 JJ_TPL = Template(TPL)
 
+
 def get_data_folder_intro():
     """Direclty get the info of the data folder.
     It is for preparing prompting message.
     """
     content_l = []
-    for p in Path(FIS.file_based_execution_data_folder).iterdir():
+    for p in Path(FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder).iterdir():
         if p.name.endswith(".h5"):
             df = pd.read_hdf(p)
             # get  df.head() as string with full width
@@ -49,17 +50,3 @@ def get_data_folder_intro():
                 f"file type {p.name} is not supported. Please implement its description function.",
             )
     return "\n ----------------- file spliter -------------\n".join(content_l)
-
-def load_data_from_dict(factor_dict:dict) -> list[FactorImplementTask]:
-    """Load data from a dict.
-    """
-    task_l = []
-    for factor_name, factor_data in factor_dict.items():
-        task = FactorImplementTask(
-            factor_name=factor_name,
-            factor_description=factor_data["description"],
-            factor_formulation=factor_data["formulation"],
-            variables=factor_data["variables"],
-        )
-        task_l.append(task)
-    return task_l

rdagent/factor_implementation/task_loader/json_loader.py

@@ -0,0 +1,31 @@
+import json
+from pathlib import Path
+from rdagent.core.task import TaskLoader
+from rdagent.factor_implementation.evolving.factor import FactorImplementTask
+
+
+class FactorImplementationTaskLoaderFromDict(TaskLoader):
+    def load(self, factor_dict: dict) -> list:
+        """Load data from a dict."""
+        task_l = []
+        for factor_name, factor_data in factor_dict.items():
+            task = FactorImplementTask(
+                factor_name=factor_name,
+                factor_description=factor_data["description"],
+                factor_formulation=factor_data["formulation"],
+                variables=factor_data["variables"],
+            )
+            task_l.append(task)
+        return task_l
+
+
+class FactorImplementationTaskLoaderFromJsonFile(TaskLoader):
+    def load(self, json_file_path: Path) -> list:
+        factor_dict = json.load(json_file_path)
+        return FactorImplementationTaskLoaderFromDict().load(factor_dict)
+
+
+class FactorImplementationTaskLoaderFromJsonString(TaskLoader):
+    def load(self, json_string: str) -> list:
+        factor_dict = json.loads(json_string)
+        return FactorImplementationTaskLoaderFromDict().load(factor_dict)

rdagent/factor_implementation/task_loader/pdf_loader.py

@@ -0,0 +1,584 @@
+from __future__ import annotations
+
+import json
+import multiprocessing as mp
+import re
+from pathlib import Path
+from typing import Mapping
+
+import numpy as np
+import pandas as pd
+import tiktoken
+from jinja2 import Template
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.log import RDAgentLog
+from rdagent.core.prompts import Prompts
+from rdagent.core.task import TaskLoader
+from rdagent.document_reader.document_reader import load_and_process_pdfs_by_langchain
+from rdagent.factor_implementation.task_loader.json_loader import FactorImplementationTaskLoaderFromDict
+from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing
+from sklearn.cluster import KMeans
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.preprocessing import normalize
+
+document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+
+
+def classify_report_from_dict(
+    report_dict: Mapping[str, str],
+    vote_time: int = 1,
+    substrings: tuple[str] = (),
+) -> dict[str, dict[str, str]]:
+    """
+    Parameters:
+    - report_dict (Dict[str, str]):
+      A dictionary where the key is the path of the report (ending with .pdf),
+      and the value is either the report content as a string.
+    - input_max_token (int): Specifying the maximum number of input tokens.
+    - vote_time (int): An integer specifying how many times to vote.
+    - substrings (list(str)): List of hardcode substrings.
+
+    Returns:
+    - Dict[str, Dict[str, str]]: A dictionary where each key is the path of the report,
+      with a single key 'class' and its value being the classification result (0 or 1).
+
+    """
+    # if len(substrings) == 0:
+    #     substrings = (
+    #         "金融工程",
+    #         "金工",
+    #         "回测",
+    #         "因子",
+    #         "机器学习",
+    #         "深度学习",
+    #         "量化",
+    #     )
+
+    res_dict = {}
+    classify_prompt = document_process_prompts["classify_system"]
+
+    for key, value in report_dict.items():
+        if not key.endswith(".pdf"):
+            continue
+        file_name = key
+
+        if isinstance(value, str):
+            content = value
+        else:
+            RDAgentLog().warning(f"输入格式不符合要求: {file_name}")
+            res_dict[file_name] = {"class": 0}
+            continue
+
+        # pre-filter document with key words is not necessary, skip this check for now
+        # if (
+        #     not any(substring in content for substring in substrings) and False
+        # ):
+        #     res_dict[file_name] = {"class": 0}
+        # else:
+        while (
+            APIBackend().build_messages_and_calculate_token(
+                user_prompt=content,
+                system_prompt=classify_prompt,
+            )
+            > RD_AGENT_SETTINGS.chat_token_limit
+        ):
+            content = content[: -(RD_AGENT_SETTINGS.chat_token_limit // 100)]
+
+        vote_list = []
+        for _ in range(vote_time):
+            user_prompt = content
+            system_prompt = classify_prompt
+            res = APIBackend().build_messages_and_create_chat_completion(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                json_mode=True,
+            )
+            try:
+                res = json.loads(res)
+                vote_list.append(int(res["class"]))
+            except json.JSONDecodeError:
+                RDAgentLog().warning(f"返回值无法解析: {file_name}")
+                res_dict[file_name] = {"class": 0}
+            count_0 = vote_list.count(0)
+            count_1 = vote_list.count(1)
+            if max(count_0, count_1) > int(vote_time / 2):
+                break
+
+        result = 1 if count_1 > count_0 else 0
+        res_dict[file_name] = {"class": result}
+
+    return res_dict
+
+
+def __extract_factors_name_and_desc_from_content(
+    content: str,
+) -> dict[str, dict[str, str]]:
+    session = APIBackend().build_chat_session(
+        session_system_prompt=document_process_prompts["extract_factors_system"],
+    )
+
+    extracted_factor_dict = {}
+    current_user_prompt = content
+
+    for _ in range(10):
+        extract_result_resp = session.build_chat_completion(
+            user_prompt=current_user_prompt,
+            json_mode=False,
+        )
+        re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
+        ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
+        try:
+            ret_dict = json.loads(ret_json_str)
+            parse_success = bool(isinstance(ret_dict, dict)) and "factors" in ret_dict
+        except json.JSONDecodeError:
+            parse_success = False
+        if ret_json_str is None or not parse_success:
+            current_user_prompt = "Your response didn't follow the instruction might be wrong json format. Try again."
+        else:
+            factors = ret_dict["factors"]
+            if len(factors) == 0:
+                break
+            for factor_name, factor_description in factors.items():
+                extracted_factor_dict[factor_name] = factor_description
+            current_user_prompt = document_process_prompts["extract_factors_follow_user"]
+
+    return extracted_factor_dict
+
+
+def __extract_factors_formulation_from_content(
+    content: str,
+    factor_dict: dict[str, str],
+) -> dict[str, dict[str, str]]:
+    factor_dict_df = pd.DataFrame(
+        factor_dict.items(),
+        columns=["factor_name", "factor_description"],
+    )
+
+    system_prompt = document_process_prompts["extract_factor_formulation_system"]
+    current_user_prompt = Template(
+        document_process_prompts["extract_factor_formulation_user"],
+    ).render(report_content=content, factor_dict=factor_dict_df.to_string())
+
+    session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
+    factor_to_formulation = {}
+
+    for _ in range(10):
+        extract_result_resp = session.build_chat_completion(
+            user_prompt=current_user_prompt,
+            json_mode=False,
+        )
+        re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
+        ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
+        try:
+            ret_dict = json.loads(ret_json_str)
+            parse_success = bool(isinstance(ret_dict, dict))
+        except json.JSONDecodeError:
+            parse_success = False
+        if ret_json_str is None or not parse_success:
+            current_user_prompt = "Your response didn't follow the instruction might be wrong json format. Try again."
+        else:
+            for name, formulation_and_description in ret_dict.items():
+                if name in factor_dict:
+                    factor_to_formulation[name] = formulation_and_description
+            if len(factor_to_formulation) != len(factor_dict):
+                remain_df = factor_dict_df[~factor_dict_df["factor_name"].isin(factor_to_formulation)]
+                current_user_prompt = (
+                    "Some factors are missing. Please check the following"
+                    " factors and their descriptions and continue extraction.\n"
+                    "==========================Remaining factors"
+                    "==========================\n" + remain_df.to_string()
+                )
+            else:
+                break
+
+    return factor_to_formulation
+
+
+def __extract_factor_and_formulation_from_one_report(
+    content: str,
+) -> dict[str, dict[str, str]]:
+    final_factor_dict_to_one_report = {}
+    factor_dict = __extract_factors_name_and_desc_from_content(content)
+    if len(factor_dict) != 0:
+        factor_to_formulation = __extract_factors_formulation_from_content(
+            content,
+            factor_dict,
+        )
+    for factor_name in factor_dict:
+        if factor_name not in factor_to_formulation:
+            continue
+
+        final_factor_dict_to_one_report.setdefault(factor_name, {})
+        final_factor_dict_to_one_report[factor_name]["description"] = factor_dict[factor_name]
+
+        # use code to correct _ in formulation
+        formulation = factor_to_formulation[factor_name]["formulation"]
+        if factor_name in formulation:
+            target_factor_name = factor_name.replace("_", r"\_")
+            formulation = formulation.replace(factor_name, target_factor_name)
+        for variable in factor_to_formulation[factor_name]["variables"]:
+            if variable in formulation:
+                target_variable = variable.replace("_", r"\_")
+                formulation = formulation.replace(variable, target_variable)
+
+        final_factor_dict_to_one_report[factor_name]["formulation"] = formulation
+        final_factor_dict_to_one_report[factor_name]["variables"] = factor_to_formulation[factor_name]["variables"]
+
+    return final_factor_dict_to_one_report
+
+
+def extract_factors_from_report_dict(
+    report_dict: dict[str, str],
+    useful_no_dict: dict[str, dict[str, str]],
+    n_proc: int = 11,
+) -> dict[str, dict[str, dict[str, str]]]:
+    useful_report_dict = {}
+    for key, value in useful_no_dict.items():
+        if isinstance(value, dict):
+            if int(value.get("class")) == 1:
+                useful_report_dict[key] = report_dict[key]
+        else:
+            RDAgentLog().warning(f"Invalid input format: {key}")
+
+    final_report_factor_dict = {}
+    # for file_name, content in useful_report_dict.items():
+    #     final_report_factor_dict.setdefault(file_name, {})
+    #     final_report_factor_dict[file_name] = __extract_factor_and_formulation_from_one_report(content)
+
+    while len(final_report_factor_dict) != len(useful_report_dict):
+        pool = mp.Pool(n_proc)
+        pool_result_list = []
+        file_names = []
+        for file_name, content in useful_report_dict.items():
+            if file_name in final_report_factor_dict:
+                continue
+            file_names.append(file_name)
+            pool_result_list.append(
+                pool.apply_async(
+                    __extract_factor_and_formulation_from_one_report,
+                    (content,),
+                ),
+            )
+
+        pool.close()
+        pool.join()
+
+        for index, result in enumerate(pool_result_list):
+            if result.get is not None:
+                file_name = file_names[index]
+                final_report_factor_dict.setdefault(file_name, {})
+                final_report_factor_dict[file_name] = result.get()
+        RDAgentLog().info(f"已经完成{len(final_report_factor_dict)}个报告的因子提取")
+
+    return final_report_factor_dict
+
+
+def merge_file_to_factor_dict_to_factor_dict(
+    file_to_factor_dict: dict[str, dict],
+) -> dict:
+    factor_dict = {}
+    for file_name in file_to_factor_dict:
+        for factor_name in file_to_factor_dict[file_name]:
+            factor_dict.setdefault(factor_name, [])
+            factor_dict[factor_name].append(file_to_factor_dict[file_name][factor_name])
+
+    factor_dict_simple_deduplication = {}
+    for factor_name in factor_dict:
+        if len(factor_dict[factor_name]) > 1:
+            factor_dict_simple_deduplication[factor_name] = max(
+                factor_dict[factor_name],
+                key=lambda x: len(x["formulation"]),
+            )
+        else:
+            factor_dict_simple_deduplication[factor_name] = factor_dict[factor_name][0]
+    return factor_dict_simple_deduplication
+
+
+def __check_factor_dict_viability_simulate_json_mode(
+    factor_df_string: str,
+) -> dict[str, dict[str, str]]:
+    session = APIBackend().build_chat_session(
+        session_system_prompt=document_process_prompts["factor_viability_system"],
+    )
+    current_user_prompt = factor_df_string
+
+    for _ in range(10):
+        extract_result_resp = session.build_chat_completion(
+            user_prompt=current_user_prompt,
+            json_mode=False,
+        )
+        re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
+        ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
+        try:
+            ret_dict = json.loads(ret_json_str)
+            parse_success = bool(isinstance(ret_dict, dict))
+        except json.JSONDecodeError:
+            parse_success = False
+        if ret_json_str is None or not parse_success:
+            current_user_prompt = "Your response didn't follow the instruction might be wrong json format. Try again."
+        else:
+            return ret_dict
+    return {}
+
+
+def check_factor_viability(
+    factor_dict: dict[str, dict[str, str]],
+) -> tuple[dict[str, dict[str, str]], dict[str, dict[str, str]]]:
+    factor_viability_dict = {}
+
+    factor_df = pd.DataFrame(factor_dict).T
+    factor_df.index.names = ["factor_name"]
+
+    while factor_df.shape[0] > 0:
+        pool = mp.Pool(8)
+
+        result_list = []
+        for i in range(0, factor_df.shape[0], 50):
+            target_factor_df_string = factor_df.iloc[i : i + 50, :].to_string()
+
+            result_list.append(
+                pool.apply_async(
+                    __check_factor_dict_viability_simulate_json_mode,
+                    (target_factor_df_string,),
+                ),
+            )
+
+        pool.close()
+        pool.join()
+
+        for result in result_list:
+            respond = result.get()
+            for factor_name, viability in respond.items():
+                factor_viability_dict[factor_name] = viability
+
+        factor_df = factor_df[~factor_df.index.isin(factor_viability_dict)]
+
+    # filtered_factor_dict = {
+    #     factor_name: factor_dict[factor_name]
+    #     for factor_name in factor_dict
+    #     if factor_viability_dict[factor_name]["viability"]
+    # }
+
+    return factor_viability_dict
+
+
+def __check_factor_duplication_simulate_json_mode(
+    factor_df: pd.DataFrame,
+) -> list[list[str]]:
+    session = APIBackend().build_chat_session(
+        session_system_prompt=document_process_prompts["factor_duplicate_system"],
+    )
+    current_user_prompt = factor_df.to_string()
+
+    generated_duplicated_groups = []
+    for _ in range(20):
+        extract_result_resp = session.build_chat_completion(
+            user_prompt=current_user_prompt,
+            json_mode=False,
+        )
+        re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
+        ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
+        try:
+            ret_dict = json.loads(ret_json_str)
+            parse_success = bool(isinstance(ret_dict, list))
+        except json.JSONDecodeError:
+            parse_success = False
+        if ret_json_str is None or not parse_success:
+            current_user_prompt = (
+                "Your previous response didn't follow"
+                " the instruction might be wrong json"
+                " format. Try reducing the factors."
+            )
+        elif len(ret_dict) == 0:
+            return generated_duplicated_groups
+        else:
+            generated_duplicated_groups.extend(ret_dict)
+            current_user_prompt = (
+                "Continue to extract duplicated"
+                " groups. If no more duplicated group"
+                " found please respond empty dict."
+            )
+    return generated_duplicated_groups
+
+
+def __kmeans_embeddings(embeddings: np.ndarray, k: int = 20) -> list[list[str]]:
+    x_normalized = normalize(embeddings)
+
+    kmeans = KMeans(
+        n_clusters=k,
+        init="random",
+        max_iter=100,
+        n_init=10,
+        random_state=42,
+    )
+
+    # KMeans algorithm uses Euclidean distance, and we need to customize a function to find the most similar cluster center
+    def find_closest_cluster_cosine_similarity(
+        data: np.ndarray,
+        centroids: np.ndarray,
+    ) -> np.ndarray:
+        similarity = cosine_similarity(data, centroids)
+        return np.argmax(similarity, axis=1)
+
+    # Initializes the cluster center
+    rng = np.random.default_rng()
+    centroids = rng.choice(x_normalized, size=k, replace=False)
+
+    # Iterate until convergence or the maximum number of iterations is reached
+    for _ in range(kmeans.max_iter):
+        # Assign the sample to the nearest cluster center
+        closest_clusters = find_closest_cluster_cosine_similarity(
+            x_normalized,
+            centroids,
+        )
+
+        # update the cluster center
+        new_centroids = np.array(
+            [x_normalized[closest_clusters == i].mean(axis=0) for i in range(k)],
+        )
+        new_centroids = normalize(new_centroids)  # 归一化新的簇中心
+
+        # Check whether the cluster center has changed
+        if np.allclose(centroids, new_centroids):
+            break
+
+        centroids = new_centroids
+
+    clusters = find_closest_cluster_cosine_similarity(x_normalized, centroids)
+    cluster_to_index = {}
+    for index, cluster in enumerate(clusters):
+        cluster_to_index.setdefault(cluster, []).append(index)
+    return sorted(
+        cluster_to_index.values(),
+        key=lambda x: len(x),
+        reverse=True,
+    )
+
+
+def __deduplicate_factor_dict(factor_dict: dict[str, dict[str, str]]) -> list[list[str]]:
+    if len(factor_dict) == 0:
+        return []
+    factor_df = pd.DataFrame(factor_dict).T
+    factor_df.index.names = ["factor_name"]
+
+    factor_names = sorted(factor_dict)
+
+    factor_name_to_full_str = {}
+    for factor_name in factor_dict:
+        description = factor_dict[factor_name]["description"]
+        formulation = factor_dict[factor_name]["formulation"]
+        variables = factor_dict[factor_name]["variables"]
+        factor_name_to_full_str[
+            factor_name
+        ] = f"""Factor name: {factor_name}
+Factor description: {description}
+Factor formulation: {formulation}
+Factor variables: {variables}
+"""
+
+    full_str_list = [factor_name_to_full_str[factor_name] for factor_name in factor_names]
+    embeddings = create_embedding_with_multiprocessing(full_str_list)
+
+    target_k = None
+    if len(full_str_list) < RD_AGENT_SETTINGS.max_input_duplicate_factor_group:
+        kmeans_index_group = [list(range(len(full_str_list)))]
+        target_k = 1
+    else:
+        for k in range(
+            len(full_str_list) // RD_AGENT_SETTINGS.max_input_duplicate_factor_group,
+            30,
+        ):
+            kmeans_index_group = __kmeans_embeddings(embeddings=embeddings, k=k)
+            if len(kmeans_index_group[0]) < RD_AGENT_SETTINGS.max_input_duplicate_factor_group:
+                target_k = k
+                RDAgentLog().info(f"K-means group number: {k}")
+                break
+    factor_name_groups = [[factor_names[index] for index in index_group] for index_group in kmeans_index_group]
+
+    duplication_names_list = []
+
+    pool = mp.Pool(target_k)
+    result_list = []
+    result_list = [
+        pool.apply_async(
+            __check_factor_duplication_simulate_json_mode,
+            (factor_df.loc[factor_name_group, :],),
+        )
+        for factor_name_group in factor_name_groups
+    ]
+
+    pool.close()
+    pool.join()
+
+    for result in result_list:
+        deduplication_factor_names_list = result.get()
+        for deduplication_factor_names in deduplication_factor_names_list:
+            filter_factor_names = [
+                factor_name for factor_name in set(deduplication_factor_names) if factor_name in factor_dict
+            ]
+            if len(filter_factor_names) > 1:
+                duplication_names_list.append(filter_factor_names)
+
+    return duplication_names_list
+
+
+def deduplicate_factors_by_llm(  # noqa: C901, PLR0912
+    factor_dict: dict[str, dict[str, str]],
+    factor_viability_dict: dict[str, dict[str, str]] | None = None,
+) -> list[list[str]]:
+    final_duplication_names_list = []
+    current_round_factor_dict = factor_dict
+    for _ in range(10):
+        duplication_names_list = __deduplicate_factor_dict(current_round_factor_dict)
+
+        new_round_names = []
+        for duplication_names in duplication_names_list:
+            if len(duplication_names) < RD_AGENT_SETTINGS.max_output_duplicate_factor_group:
+                final_duplication_names_list.append(duplication_names)
+            else:
+                new_round_names.extend(duplication_names)
+
+        if len(new_round_names) != 0:
+            current_round_factor_dict = {factor_name: factor_dict[factor_name] for factor_name in new_round_names}
+        else:
+            break
+
+    final_duplication_names_list = sorted(final_duplication_names_list, key=lambda x: len(x), reverse=True)
+
+    to_replace_dict = {}
+    for duplication_names in duplication_names_list:
+        if factor_viability_dict is not None:
+            viability_list = [factor_viability_dict[name]["viability"] for name in duplication_names]
+            if True not in viability_list:
+                continue
+            target_factor_name = duplication_names[viability_list.index(True)]
+        else:
+            target_factor_name = duplication_names[0]
+        for duplication_factor_name in duplication_names:
+            if duplication_factor_name == target_factor_name:
+                continue
+            to_replace_dict[duplication_factor_name] = target_factor_name
+
+    llm_deduplicated_factor_dict = {}
+    added_lower_name_set = set()
+    for factor_name in factor_dict:
+        if factor_name not in to_replace_dict and factor_name.lower() not in added_lower_name_set:
+            if factor_viability_dict is not None and not factor_viability_dict[factor_name]["viability"]:
+                continue
+            added_lower_name_set.add(factor_name.lower())
+            llm_deduplicated_factor_dict[factor_name] = factor_dict[factor_name]
+
+    return llm_deduplicated_factor_dict, final_duplication_names_list
+
+
+class FactorImplementationTaskLoaderFromPDFfiles(TaskLoader):
+    def load(self, file_or_folder_path: Path) -> dict:
+        docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path))
+
+        selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1)
+        file_to_factor_result = extract_factors_from_report_dict(docs_dict, selected_report_dict)
+        factor_dict = merge_file_to_factor_dict_to_factor_dict(file_to_factor_result)
+
+        factor_viability = check_factor_viability(factor_dict)
+        factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability)
+        return FactorImplementationTaskLoaderFromDict().load(factor_dict)

rdagent/factor_implementation/task_loader/prompts.yaml

@@ -0,0 +1,201 @@
+extract_factors_system: |-
+    用户会提供一篇金融工程研报，其中包括了量化因子和模型研究，请按照要求抽取以下信息:
+    1. 概述这篇研报的主要研究思路;
+    2. 抽取出所有的因子，并概述因子的计算过程，请注意有些因子可能存在于表格中，请不要遗漏，因子的名称请使用英文，不能包含空格，可用下划线连接，研报中可能不含有因子，若没有请返回空字典;
+    3. 抽取研报里面的所有模型，并概述模型的计算过程，可以分步骤描述模型搭建或计算的过程，研报中可能不含有模型，若没有请返回空字典;
+
+    user will treat your factor name as key to store the factor, don't put any interaction message in the content. Just response the output without any interaction and explanation.
+    All names should be in English.
+    Respond with your analysis in JSON format. The JSON schema should include:
+    ```json
+    {
+        "summary": "The summary of this report",
+        "factors": {
+            "Name of factor 1": "Description to factor 1",
+            "Name of factor 2": "Description to factor 2"
+        },
+        "models": {
+            "Name of model 1": "Description to model 1",
+            "Name of model 2": "Description to model 2"
+        }
+    }
+    ```
+
+extract_factors_follow_user: |-
+    Please continue extracting the factors. Please ignore factors appeared in former messages. If no factor is found, please return an empty dict.
+    Notice: You should not miss any factor in the report! Some factors might appear several times in the report. You can repeat them to avoid missing other factors.
+    Respond with your analysis in JSON format. The JSON schema should include:
+    ```json
+    {
+        "factors": {
+            "Name of factor 1": "Description to factor 1",
+            "Name of factor 2": "Description to factor 2"
+        }
+    }
+    ```
+
+extract_factor_formulation_system: |-
+    I have a financial engineering research report and a list of factors extracted from it. I need assistance in extracting specific information based on the report and the provided list of factors. The tasks are as follows:
+
+    1. For each factor, I need its calculation formula in LaTeX format. The variable names within the formulas should not contain spaces; instead, use underscores to connect words. Ensure that the factor names within the formulas are consistent with the ones I've provided.
+    2. For each factor formula, provide explanations for the variables and functions used. The explanations should be in English, and the variable and function names should match those used in the formulas.
+
+    Here are the sources of data I have:
+
+    1. Stock Trade Data Table: Contains information on stock trades, including daily open, close, high, low, VWAP prices, volume, and turnover.
+    2. Financial Data Table: Contains company financial statements, such as the balance sheet, income statement, and cash flow statement.
+    3. Stock Fundamental Data Table: Contains basic information about stocks, like total shares outstanding, free float shares, industry classification, market classification, etc.
+    4. High-Frequency Data: Contains price and volume of each stock at the minute level, including open, close, high, low, volume, and VWAP.
+
+    Please expand the formulation to use the source data I have provided. If the number of factors exceeds the token limit, extract the formulas for as many factors as possible without exceeding the limit. Ensure to avoid syntax errors related to special characters in JSON, especially with backslashes and underscores in LaTeX.
+
+    Provide your analysis in JSON format, using the following schema:
+
+    ```json
+    {
+        "factor name 1": {
+            "formulation": "latex formulation of factor 1",
+            "variables": {
+                "variable or function name 1": "description of variable or function 1",
+                "variable or function name 2": "description of variable or function 2"
+            }
+        },
+        "factor name 2": {
+            "formulation": "latex formulation of factor 2",
+            "variables": {
+                "variable or function name 1": "description of variable or function 1",
+                "variable or function name 2": "description of variable or function 2"
+            }
+        }
+        # Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here!
+    }
+    ```
+
+    Please only include the factors you've extracted in your response, without any additional messages or explanations. If you can only provide part of the factors, do not use ellipsis or any filler text that might cause JSON parsing errors.
+
+
+extract_factor_formulation_user: |-
+    ===========================Report content:=============================
+    {{ report_content }}
+    ===========================Factor list in dataframe=============================
+    {{ factor_dict }}
+
+classify_system_chinese: |-
+    你是一个研报分类助手。用户会输入一篇金融研报。请按照要求回答：
+    因子指能够解释资产收益率或价格等的变量；而模型则指机器学习或深度学习模型，利用因子等变量来预测价格或收益率变化。
+
+    请你对研报进行分类，考虑两个条件：
+        1. 是金工量化领域中选股（需与择时，选基等严格区分开）方面的研报;
+        2. 涉及了因子或模型的构成，或者是测试了它们的表现。
+    如果研报同时满足上述两个条件，请输出1；若没有，请输出0。
+
+    请使用json进行回答。json key为：class
+
+classify_system: |-
+    Your job is classify whether the user input document is a quantitative investment research report. The user will input a document and you should classify it based on the following conditions:
+    1. The document is about finance other than other fields like biology, physics, chemistry, etc.
+    2. The document is a research report on stock selection (which needs to be strictly separated from time selection and base selection) in the field of metalworking quantification.
+    3. The document involves the composition of factors or models, or tests their performance.
+
+    If the document meets all the above conditions, please return 1; otherwise, please return 0.
+
+    Please respond with your decision in JSON format. Just respond the output json string without any interaction and explanation.
+    The JSON schema should include:
+    {
+        "class": 1
+    }
+
+factor_viability_system: |-
+    User has designed several factors in quant investment. Please help the user to check the viability of these factors.
+    These factors are used to build a daily frequency strategy in China A-share market.
+
+    User will provide a pandas dataframe like table containing following information:
+    1. The name of the factor;
+    2. The simple description of the factor;
+    3. The formulation of the factor in latex format;
+    4. The description to the variables and functions in the formulation of the factor.
+
+    User has several source data:
+    1. The Stock Trade Data Table containing information about stock trades, such as daily open, close, high, low, vwap prices, volume, and turnover;
+    2. The Financial Data Table containing company financial statements such as the balance sheet, income statement, and cash flow statement;
+    3. The Stock Fundamental Data Table containing basic information about stocks, like total shares outstanding, free float shares, industry classification, market classification, etc;
+    4. The high frequency data containing price and volume of each stock containing open close high low volume vwap in each minute;
+    5. The Consensus Expectations Factor containing the consensus expectations of the analysts about the future performance of the company.
+
+
+    A viable factor should satisfy the following conditions:
+    1. The factor should be able to be calculated in daily frequency;
+    2. The factor should be able to be calculated based on each stock;
+    3. The factor should be able to be calculated based on the source data provided by user.
+
+    You should give decision to each factor provided by the user. You should reject the factor based on very solid reason.
+    Please return true to the viable factor and false to the non-viable factor.
+
+    Notice, you can just return part of the factors due to token limit. Your factor name should be the same as the user's factor name.
+
+    Please respond with your decision in JSON format. Just respond the output json string without any interaction and explanation.
+    The JSON schema should include:
+    ```json
+    {
+        "Name to factor 1":
+        {
+            "viability": true,
+            "reason": "The reason to the viability of this factor"
+        },
+        "Name to factor 2":
+        {
+            "viability": false,
+            "reason": "The reason to the non-viability of this factor"
+        }
+        "Name to factor 3":
+        {
+            "viability": true,
+            "reason": "The reason to the viability of this factor"
+        }
+    }
+    ```
+
+
+factor_duplicate_system: |-
+    User has designed several factors in quant investment. Please help the user to duplicate these factors.
+    These factors are used to build a daily frequency strategy in China A-share market.
+
+    User will provide a pandas dataframe like table containing following information:
+    1. The name of the factor;
+    2. The simple description of the factor;
+    3. The formulation of the factor in latex format;
+    4. The description to the variables and functions in the formulation of the factor.
+
+    User wants to find whether there are duplicated groups. The factors in a duplicate group should satisfy the following conditions:
+    1. They might differ in the name, description, formulation, or the description to the variables and functions in the formulation, some upper or lower case difference is included;
+    2. They should be talking about exactly the same factor;
+    3. If horizon information like 1 day, 5 days, 10 days, etc is provided, the horizon information should be the same.
+
+    To make your response valid, we have some very important constraint for you to follow! Listed here:
+    1. You should be very confident to put duplicated factors into a group;
+    2. A group should contain at least two factors;
+    3. To a factor which has no duplication, don't put them into your response;
+    4. To avoid merging too many similar factor, don't put more than ten factors into a group!
+    You should always follow the above constraints to make your response valid. 
+
+    Your response JSON schema should include:
+    ```json
+    [
+        [
+            "factor name 1",
+            "factor name 2"
+        ],
+        [
+            "factor name 5",
+            "factor name 6"
+        ],
+        [
+            "factor name 7",
+            "factor name 8",
+            "factor name 9"
+        ]
+    ]
+    ```
+    Your response is a list of lists. Each list represents a duplicate group containing all the factor names in this group. 
+    The factor names in the list should be unique and the factor names should be the same as the user's factor name.
+    To avoid reaching token limit, don't respond more than fifty groups in one response. You should respond the output json string without any interaction and explanation.