gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

reporeformat V2 (#23)

Browse Source 
* reformat factor implement process

* move some code to more reasonable place

* fix the bug

* add test function in factor_extract_and_implement.py

* change select factor number to ratio , add some factor implement setting and fix some bug while using knowledgebase

* change evoagent

* add abstract class EvoAgent

* add benchmark workflow

* fix some bug in llm_utils

* run wenjun's code

* fix the knowledgebase instance check

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
USTCKevinF
2024-06-14 12:59:44 +08:00
committed by GitHub
parent 9e82da243b
commit ebb659a018
32 changed files with 2227 additions and 1141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/factor_implementation/CoSTEER.py
+86
View File
@@ -0,0 +1,86 @@
import pickle
from pathlib import Path
from typing import List
from rdagent.core.implementation import TaskGenerator
from rdagent.core.task import TaskImplementation
from rdagent.factor_implementation.evolving.knowledge_management import FactorImplementationKnowledgeBaseV1
from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FactorEvovlingItem
from rdagent.knowledge_management.knowledgebase import FactorImplementationGraphKnowledgeBase, FactorImplementationGraphRAGStrategy
from rdagent.factor_implementation.evolving.evolving_strategy import FactorEvolvingStrategyWithGraph
from rdagent.factor_implementation.evolving.evaluators import FactorImplementationsMultiEvaluator, FactorImplementationEvaluatorV1
from rdagent.factor_implementation.evolving.evolving_agent import RAGEvoAgent
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
class CoSTEERFG(TaskGenerator):
def __init__(
self,
with_knowledge: bool = True,
with_feedback: bool = True,
knowledge_self_gen: bool = True,
) -> None:
self.max_loop = FactorImplementSettings().max_loop
self.knowledge_base_path = Path(FactorImplementSettings().knowledge_base_path) if FactorImplementSettings().knowledge_base_path is not None else None
self.new_knowledge_base_path = Path(FactorImplementSettings().new_knowledge_base_path) if FactorImplementSettings().new_knowledge_base_path is not None else None
self.with_knowledge = with_knowledge
self.with_feedback = with_feedback
self.knowledge_self_gen = knowledge_self_gen
self.evolving_strategy = FactorEvolvingStrategyWithGraph()
# declare the factor evaluator
self.factor_evaluator = FactorImplementationsMultiEvaluator(FactorImplementationEvaluatorV1())
self.evolving_version = 2
def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
if self.evolving_version == 1 and not isinstance(
factor_knowledge_base, FactorImplementationKnowledgeBaseV1
):
raise ValueError("The former knowledge base is not compatible with the current version")
elif self.evolving_version == 2 and not isinstance(
factor_knowledge_base,
FactorImplementationGraphKnowledgeBase,
):
raise ValueError("The former knowledge base is not compatible with the current version")
else:
factor_knowledge_base = (
FactorImplementationGraphKnowledgeBase(
init_component_list=component_init_list,
)
if self.evolving_version == 2
else FactorImplementationKnowledgeBaseV1()
)
return factor_knowledge_base
def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]:
# init knowledge base
factor_knowledge_base = self.load_or_init_knowledge_base(
former_knowledge_base_path=self.knowledge_base_path,
component_init_list=[],
)
# init rag method
self.rag = (
FactorImplementationGraphRAGStrategy(factor_knowledge_base)
)
# init indermediate items
factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks)
self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
factor_implementations = self.evolve_agent.multistep_evolve(
factor_implementations,
self.factor_evaluator,
with_knowledge=self.with_knowledge,
with_feedback=self.with_feedback,
knowledge_self_gen=self.knowledge_self_gen,
)
# save new knowledge base
if self.new_knowledge_base_path is not None:
pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
self.knowledge_base = factor_knowledge_base
self.latest_factor_implementations = tasks
return factor_implementations
rdagent/factor_implementation/evolving/__init__.py
View File
rdagent/factor_implementation/evolving/evaluators.py
+524 -29
View File
@@ -1,30 +1,526 @@
from __future__ import annotations
import json
import re
from typing import List
from pandas.core.api import DataFrame as DataFrame
from rdagent.core.evolving_framework import Evaluator as EvolvingEvaluator
from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
from abc import abstractmethod
from typing import Tuple
import pandas as pd
from jinja2 import Template
from rdagent.oai.llm_utils import APIBackend
from rdagent.core.log import FinCoLog
from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask, FactorEvovlingItem
from rdagent.core.task import (
TaskImplementation,
)
from typing import List, Tuple
from rdagent.core.evolving_framework import QueriedKnowledge,Feedback
from rdagent.core.evaluation import Evaluator
from rdagent.core.prompts import Prompts
from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.factor_implementation.evolving.evolvable_subjects import FactorImplementationList
from rdagent.factor_implementation.share_modules.evaluator import (
Evaluator as FactorImplementationEvaluator,
)
from rdagent.factor_implementation.share_modules.evaluator import (
FactorImplementationCodeEvaluator,
FactorImplementationFinalDecisionEvaluator,
FactorImplementationValueEvaluator,
)
from rdagent.factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
)
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
from pathlib import Path
evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
class FactorImplementationEvaluator(Evaluator):
# TODO:
# I think we should have unified interface for all evaluates, for examples.
# So we should adjust the interface of other factors
@abstractmethod
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
) -> Tuple[str, object]:
"""You can get the dataframe by
.. code-block:: python
_, gt_df = gt.execute()
_, gen_df = gen.execute()
Returns
-------
Tuple[str, object]
- str: the text-based description of the evaluation result
- object: a comparable metric (bool, integer, float ...)
"""
raise NotImplementedError("Please implement the `evaluator` method")
def _get_df(self, gt: TaskImplementation, gen: TaskImplementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
if isinstance(gen_df, pd.Series):
gen_df = gen_df.to_frame("source_factor")
if isinstance(gt_df, pd.Series):
gt_df = gt_df.to_frame("gt_factor")
return gt_df, gen_df
class FactorImplementationCodeEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementTask,
implementation: TaskImplementation,
execution_feedback: str,
factor_value_feedback: str = "",
gt_implementation: TaskImplementation = None,
**kwargs,
):
factor_information = target_task.get_factor_information()
code = implementation.code
system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
evaluate_prompts["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
while (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
former_messages=[],
)
> FactorImplementSettings().chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
evaluate_prompts["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
critic_response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=False,
)
return critic_response
class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if len(gen_df.columns) == 1 and len(gt_df.columns) == 1:
return "Both dataframes have only one column.", True
elif len(gen_df.columns) != 1:
gen_df = gen_df.iloc(axis=1)[
[
0,
]
]
return (
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
False,
)
return "", False
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
idx_name_right = gen_df.index.names == ("datetime", "instrument")
if idx_name_right:
return (
'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
True,
)
else:
return (
'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.',
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.shape[0] == gt_df.shape[0]:
return "Both dataframes have the same rows count.", True
else:
return (
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.index.equals(gt_df.index):
return "Both dataframes have the same index.", True
else:
return (
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
return "Both dataframes have the same missing values.", True
else:
return (
f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
try:
close_values = gen_df.sub(gt_df).abs().lt(1e-6)
result_int = close_values.astype(int)
pos_num = result_int.sum().sum()
acc_rate = pos_num / close_values.size
except:
close_values = gen_df
if close_values.all().iloc[0]:
return (
"All values in the dataframes are equal within the tolerance of 1e-6.",
acc_rate,
)
else:
return (
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
acc_rate,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
def __init__(self, hard_check: bool) -> None:
self.hard_check = hard_check
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
concat_df = pd.concat([gen_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if self.hard_check:
if ic > 0.99 and ric > 0.99:
return (
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
True,
)
else:
return (
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
False,
)
else:
return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValEvaluator(FactorImplementationEvaluator):
def evaluate(self, gt: TaskImplementation, gen: TaskImplementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
# FIXME: refactor the two classes
fiv = FactorImplementationValueEvaluator()
return fiv.evaluate(source_df=gen_df, gt_df=gt_df)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValueEvaluator(Evaluator):
# TODO: let's discuss the about the interface of the evaluator
def evaluate(
self,
source_df: pd.DataFrame,
gt_df: pd.DataFrame,
**kwargs,
) -> Tuple:
conclusions = []
if isinstance(source_df, pd.Series):
source_df = source_df.to_frame("source_factor")
conclusions.append(
"The source dataframe is a series, better convert it to a dataframe.",
)
if gt_df is not None and isinstance(gt_df, pd.Series):
gt_df = gt_df.to_frame("gt_factor")
conclusions.append(
"The ground truth dataframe is a series, convert it to a dataframe.",
)
# Check if both dataframe has only one columns
if len(source_df.columns) == 1:
conclusions.append("The source dataframe has only one column which is correct.")
else:
conclusions.append(
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
)
source_df = source_df.iloc(axis=1)[
[
0,
]
]
if list(source_df.index.names) != ["datetime", "instrument"]:
conclusions.append(
rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.",
)
else:
conclusions.append(
'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
)
# Check if both dataframe have the same rows count
if gt_df is not None:
if source_df.shape[0] == gt_df.shape[0]:
conclusions.append("Both dataframes have the same rows count.")
same_row_count_result = True
else:
conclusions.append(
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
)
same_row_count_result = False
# Check whether both dataframe has the same index
if source_df.index.equals(gt_df.index):
conclusions.append("Both dataframes have the same index.")
same_index_result = True
else:
conclusions.append(
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
)
same_index_result = False
# Check for the same missing values (NaN)
if source_df.isna().sum().sum() == gt_df.isna().sum().sum():
conclusions.append("Both dataframes have the same missing values.")
same_missing_values_result = True
else:
conclusions.append(
f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
)
same_missing_values_result = False
# Check if the values are the same within a small tolerance
if not same_index_result:
conclusions.append(
"The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless",
)
same_values_result = False
high_correlation_result = False
else:
close_values = source_df.sub(gt_df).abs().lt(1e-6)
if close_values.all().iloc[0]:
conclusions.append(
"All values in the dataframes are equal within the tolerance of 1e-6.",
)
same_values_result = True
else:
conclusions.append(
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
)
same_values_result = False
# Check the ic and rankic between the two dataframes
concat_df = pd.concat([source_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
try:
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if ic > 0.99 and ric > 0.99:
conclusions.append(
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
)
high_correlation_result = True
else:
conclusions.append(
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
)
high_correlation_result = False
# Check for shifted alignments only in the "datetime" index
max_shift_days = 2
for shift in range(-max_shift_days, max_shift_days + 1):
if shift == 0:
continue # Skip the case where there is no shift
shifted_source_df = source_df.groupby(level="instrument").shift(shift)
concat_df = pd.concat([shifted_source_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
shifted_ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if shifted_ric > 0.99:
conclusions.append(
f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.",
)
break
else:
conclusions.append(
f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.",
)
except Exception as e:
FinCoLog().warning(f"Error occurred when calculating the correlation: {str(e)}")
conclusions.append(
f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
)
high_correlation_result = False
# Combine all conclusions into a single string
conclusion_str = "\n".join(conclusions)
final_result = (same_values_result or high_correlation_result) if gt_df is not None else False
return conclusion_str, final_result
# TODO:
def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
"""When the prompt is too long. We have to shorten it.
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
"""
# TODO: this should replace most of code in
# - FactorImplementationFinalDecisionEvaluator.evaluate
# - FactorImplementationCodeEvaluator.evaluate
class FactorImplementationFinalDecisionEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementTask,
execution_feedback: str,
value_feedback: str,
code_feedback: str,
**kwargs,
) -> Tuple:
system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")["evaluator_final_decision_v1_system"]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
evaluate_prompts["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
while (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
former_messages=[],
)
> FactorImplementSettings().chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
evaluate_prompts["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
final_evaluation_dict = json.loads(
APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=True,
),
)
return (
final_evaluation_dict["final_decision"],
final_evaluation_dict["final_feedback"],
)
class FactorImplementationSingleFeedback:
"""This class is a feedback to single implementation which is generated from an evaluator."""
@@ -60,14 +556,12 @@ class FactorImplementationSingleFeedback:
This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
"""
class FactorImplementationsMultiFeedback(
Feedback,
List[FactorImplementationSingleFeedback],
):
"""Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
"""This class is the v1 version of evaluator for a single factor implementation.
It calls several evaluators in share modules to evaluate the factor implementation.
@@ -80,9 +574,9 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
def evaluate(
self,
target_task: FactorImplementationTask,
implementation: FactorImplementation,
gt_implementation: FactorImplementation = None,
target_task: FactorImplementTask,
implementation: TaskImplementation,
gt_implementation: TaskImplementation = None,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorImplementationSingleFeedback:
@@ -170,14 +664,14 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
return factor_feedback
class FactorImplementationsMultiEvaluator(EvolvingEvaluator):
class FactorImplementationsMultiEvaluator(Evaluator):
def __init__(self, single_evaluator=FactorImplementationEvaluatorV1()) -> None:
super().__init__()
self.single_factor_implementation_evaluator = single_evaluator
def evaluate(
self,
evo: FactorImplementationList,
evo: FactorEvovlingItem,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorImplementationsMultiFeedback:
@@ -228,3 +722,4 @@ class FactorImplementationsMultiEvaluator(EvolvingEvaluator):
print(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
return multi_implementation_feedback
rdagent/factor_implementation/evolving/evolvable_subjects.py
-32
View File
@@ -1,32 +0,0 @@
from __future__ import annotations
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.core.log import FinCoLog
from rdagent.factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
)
class FactorImplementationList(EvolvableSubjects):
"""
Factors is a list.
"""
def __init__(
self,
target_factor_tasks: list[FactorImplementationTask],
corresponding_gt_implementations: list[FactorImplementation] = None,
):
super().__init__()
self.target_factor_tasks = target_factor_tasks
self.corresponding_implementations: list[FactorImplementation] = []
if corresponding_gt_implementations is not None and len(
corresponding_gt_implementations,
) != len(target_factor_tasks):
self.corresponding_gt_implementations = None
FinCoLog.warning(
"The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
)
else:
self.corresponding_gt_implementations = corresponding_gt_implementations
rdagent/factor_implementation/evolving/evolving_agent.py
+50
View File
@@ -0,0 +1,50 @@
from rdagent.core.evolving_framework import Feedback, EvolvableSubjects, Evaluator, EvoStep
from rdagent.core.evolving_framework import EvoAgent
from tqdm import tqdm
class RAGEvoAgent(EvoAgent):
def __init__(self, max_loop, evolving_strategy, rag) -> None:
super().__init__(max_loop, evolving_strategy)
self.rag = rag
self.evolving_trace = []
def multistep_evolve(
self,
evo: EvolvableSubjects,
eva: Evaluator | Feedback,
*,
with_knowledge: bool = False,
with_feedback: bool = True,
knowledge_self_gen: bool = False) -> EvolvableSubjects:
for _ in tqdm(range(self.max_loop), "Implementing factors"):
# 1. knowledge self-evolving
if knowledge_self_gen and self.rag is not None:
self.rag.generate_knowledge(self.evolving_trace)
# 2. 检索需要的Knowledge
queried_knowledge = None
if with_knowledge and self.rag is not None:
# TODO: 这里放了evolving_trace实际上没有作用
queried_knowledge = self.rag.query(evo, self.evolving_trace)
# 3. evolve
evo = self.evolving_strategy.evolve(
evo=evo,
evolving_trace=self.evolving_trace,
queried_knowledge=queried_knowledge,
)
# 4. 封装Evolve结果
es = EvoStep(evo, queried_knowledge)
# 5. 环境评测反馈
if with_feedback:
es.feedback = eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge)
# 6. 更新trace
self.evolving_trace.append(es)
for index, feedback in enumerate(es.feedback):
if feedback is not None:
evo.evolve_trace[evo.target_factor_tasks[index].factor_name][-1].feedback = feedback
return evo
rdagent/factor_implementation/evolving/evolving_strategy.py
+79 -63
View File
@@ -1,58 +1,71 @@
from __future__ import annotations
import json
import random
from abc import abstractmethod
from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING
from jinja2 import Template
from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
FileBasedFactorImplementation,
)
from rdagent.oai.llm_utils import APIBackend
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
from rdagent.factor_implementation.share_modules.factor_implementation_utils import (
get_data_folder_intro,
from rdagent.core.task import (
TaskImplementation,
)
from rdagent.core.prompts import Prompts
from pathlib import Path
from rdagent.factor_implementation.evolving.scheduler import (
RandomSelect,
LLMSelect,
)
from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
from rdagent.oai.llm_utils import APIBackend
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.factor_implementation.evolving.factor import (
FactorImplementTask,
FactorEvovlingItem,
FileBasedFactorImplementation,
)
if TYPE_CHECKING:
from factor_implementation.evolving.evolvable_subjects import (
FactorImplementationList,
)
from factor_implementation.evolving.knowledge_management import (
from rdagent.factor_implementation.evolving.knowledge_management import (
FactorImplementationQueriedKnowledge,
FactorImplementationQueriedKnowledgeV1,
)
implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
class MultiProcessEvolvingStrategy(EvolvingStrategy):
@abstractmethod
def implement_one_factor(
self,
target_task: FactorImplementationTask,
target_task: FactorImplementTask,
queried_knowledge: QueriedKnowledge = None,
) -> FactorImplementation:
) -> TaskImplementation:
raise NotImplementedError
def evolve(
self,
*,
evo: FactorImplementationList,
evo: FactorEvovlingItem,
queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
**kwargs,
) -> FactorImplementationList:
) -> FactorEvovlingItem:
self.num_loop += 1
new_evo = deepcopy(evo)
new_evo.corresponding_implementations = [None for _ in new_evo.target_factor_tasks]
# 1.找出需要evolve的factor
to_be_finished_task_index = []
for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
target_factor_task_desc = target_factor_task.get_factor_information()
@@ -65,11 +78,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
and target_factor_task_desc not in queried_knowledge.failed_task_info_set
):
to_be_finished_task_index.append(index)
if FactorImplementSettings().implementation_factors_per_round < len(to_be_finished_task_index):
to_be_finished_task_index = random.sample(
to_be_finished_task_index,
FactorImplementSettings().implementation_factors_per_round,
# 2. 选择selection方法
# if the number of factors to be implemented is larger than the limit, we need to select some of them
if FactorImplementSettings().select_ratio < 1:
# if the number of loops is equal to the select_loop, we need to select some of them
implementation_factors_per_round = int(
FactorImplementSettings().select_ratio * len(to_be_finished_task_index)
)
if FactorImplementSettings().select_method == "random":
to_be_finished_task_index = RandomSelect(
to_be_finished_task_index,
implementation_factors_per_round,
)
if FactorImplementSettings().select_method == "scheduler":
to_be_finished_task_index = LLMSelect(
to_be_finished_task_index,
implementation_factors_per_round,
new_evo,
queried_knowledge.former_traces,
)
result = multiprocessing_wrapper(
[
@@ -81,11 +110,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
for index, target_index in enumerate(to_be_finished_task_index):
new_evo.corresponding_implementations[target_index] = result[index]
if result[index].target_task.factor_name in new_evo.evolve_trace:
new_evo.evolve_trace[result[index].target_task.factor_name].append(result[index])
else:
new_evo.evolve_trace[result[index].target_task.factor_name] = [result[index]]
# for target_index in to_be_finished_task_index:
# new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
# new_evo.target_factor_tasks[target_index], queried_knowledge
# )
new_evo.corresponding_selection.append(to_be_finished_task_index)
return new_evo
@@ -93,9 +123,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_factor(
self,
target_task: FactorImplementationTask,
target_task: FactorImplementTask,
queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
) -> FactorImplementation:
) -> TaskImplementation:
factor_information_str = target_task.get_factor_information()
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -117,9 +147,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_factor_implementation_v1_system"
],
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
).render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -132,9 +160,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
while True:
user_prompt = (
Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_factor_implementation_v1_user"
],
implement_prompts["evolving_strategy_factor_implementation_v1_user"],
)
.render(
factor_information_str=factor_information_str,
@@ -153,9 +179,6 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
elif len(queried_similar_successful_knowledge_to_render) > 1:
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
# print(
# f"length of queried_similar_successful_knowledge_to_render: {len(queried_similar_successful_knowledge_to_render)}, length of queried_former_failed_knowledge_to_render: {len(queried_former_failed_knowledge_to_render)}"
# )
code = json.loads(
session.build_chat_completion(
@@ -173,14 +196,20 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
def __init__(self) -> None:
self.num_loop = 0
self.haveSelected = False
def implement_one_factor(
self,
target_task: FactorImplementationTask,
target_task: FactorImplementTask,
queried_knowledge,
) -> FactorImplementation:
) -> TaskImplementation:
error_summary = FactorImplementSettings().v2_error_summary
# 1. 提取因子的背景信息
target_factor_task_information = target_task.get_factor_information()
# 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
if (
queried_knowledge is not None
and target_factor_task_information in queried_knowledge.success_task_to_knowledge_dict
@@ -189,6 +218,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
return None
else:
# 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
queried_similar_component_knowledge = (
queried_knowledge.component_with_success_task[target_factor_task_information]
if queried_knowledge is not None
@@ -208,9 +239,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_factor_implementation_v1_system"
],
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
).render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -223,18 +252,17 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
queried_similar_component_knowledge_to_render = queried_similar_component_knowledge
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
error_summary_critics = ""
# 动态地防止prompt超长
while True:
# 总结error（可选）
if (
error_summary
and len(queried_similar_error_knowledge_to_render) != 0
and len(queried_former_failed_knowledge_to_render) != 0
):
error_summary_system_prompt = (
Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_error_summary_v2_system"
],
)
Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
.render(
factor_information_str=target_factor_task_information,
code_and_feedback=queried_former_failed_knowledge_to_render[
@@ -248,11 +276,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
)
while True:
error_summary_user_prompt = (
Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_error_summary_v2_user"
],
)
Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
.render(
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
)
@@ -269,12 +293,10 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
user_prompt=error_summary_user_prompt,
json_mode=False,
)
# 构建user_prompt。开始写代码
user_prompt = (
Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_factor_implementation_v2_user"
],
implement_prompts["evolving_strategy_factor_implementation_v2_user"],
)
.render(
factor_information_str=target_factor_task_information,
@@ -301,12 +323,6 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
elif len(queried_similar_error_knowledge_to_render) > 0:
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
# print(
# len(queried_similar_component_knowledge_to_render),
# len(queried_similar_error_knowledge_to_render),
# len(queried_former_failed_knowledge_to_render),
# )
response = session.build_chat_completion(
user_prompt=user_prompt,
json_mode=True,
rdagent/factor_implementation/share_modules/factor.py → rdagent/factor_implementation/evolving/factor.py
+56 -27
View File
@@ -1,41 +1,50 @@
import pickle
import subprocess
import uuid
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Tuple, Union
import pandas as pd
from filelock import FileLock
from finco.log import FinCoLog
from __future__ import annotations
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
from factor_implementation.share_modules.exception import (
from rdagent.core.task import (
TaskImplementation,
BaseTask,
TestCase,
)
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.core.log import FinCoLog
from pathlib import Path
from rdagent.oai.llm_utils import md5_hash
from rdagent.core.exception import (
CodeFormatException,
NoOutputException,
RuntimeErrorException,
)
from oai.llm_utils import md5_hash
import pandas as pd
import uuid
import pickle
import subprocess
from typing import Tuple, Union
from filelock import FileLock
class FactorImplementationTask:
# TODO: remove the factor_ prefix may be better
class FactorImplementTask(BaseTask):
def __init__(
self,
factor_name,
factor_description,
factor_formulation,
factor_formulation_description,
factor_formulation_description: str = '',
variables: dict = {},
resource: str = None,
) -> None:
self.factor_name = factor_name
self.factor_description = factor_description
self.factor_formulation = factor_formulation
self.factor_formulation_description = factor_formulation_description
# TODO: check variables a good candidate
self.variables = variables
self.factor_resources = resource
def get_factor_information(self):
return f"""factor_name: {self.factor_name}
@@ -45,22 +54,41 @@ factor_formulation_description: {self.factor_formulation_description}"""
@staticmethod
def from_dict(dict):
return FactorImplementationTask(**dict)
return FactorImplementTask(**dict)
def __repr__(self) -> str:
return f"<{self.__class__.__name__}[{self.factor_name}]>"
class FactorImplementation(ABC):
def __init__(self, target_task: FactorImplementationTask) -> None:
self.target_task = target_task
class FactorEvovlingItem(EvolvableSubjects):
"""
Intermediate item of factor implementation.
"""
@abstractmethod
def execute(self, *args, **kwargs) -> Tuple[str, pd.DataFrame]:
raise NotImplementedError("__call__ method is not implemented.")
def __init__(
self,
target_factor_tasks: list[FactorImplementTask],
corresponding_gt: list[TestCase] = None,
corresponding_gt_implementations: list[TaskImplementation] = None,
):
super().__init__()
self.target_factor_tasks = target_factor_tasks
self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks]
self.corresponding_selection: list[list] = []
self.evolve_trace = {}
self.corresponding_gt = corresponding_gt
if corresponding_gt_implementations is not None and len(
corresponding_gt_implementations,
) != len(target_factor_tasks):
self.corresponding_gt_implementations = None
FinCoLog.warning(
"The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
)
else:
self.corresponding_gt_implementations = corresponding_gt_implementations
class FileBasedFactorImplementation(FactorImplementation):
class FileBasedFactorImplementation(TaskImplementation):
"""
This class is used to implement a factor by writing the code to a file.
Input data and output factor value are also written to files.
@@ -76,7 +104,7 @@ class FileBasedFactorImplementation(FactorImplementation):
def __init__(
self,
target_task: FactorImplementationTask,
target_task: FactorImplementTask,
code,
executed_factor_value_dataframe=None,
raise_exception=False,
@@ -124,7 +152,7 @@ class FileBasedFactorImplementation(FactorImplementation):
raise ValueError(self.FB_CODE_NOT_SET)
with FileLock(self.workspace_path / "execution.lock"):
(Path.cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").mkdir(
exist_ok=True, parents=True,
exist_ok=True, parents=True
)
if FactorImplementSettings().enable_execution_cache:
# NOTE: cache the result for the same code
@@ -215,9 +243,10 @@ class FileBasedFactorImplementation(FactorImplementation):
return self.__str__()
@staticmethod
def from_folder(task: FactorImplementationTask, path: Union[str, Path], **kwargs):
def from_folder(task: FactorImplementTask, path: Union[str, Path], **kwargs):
path = Path(path)
factor_path = (path / task.factor_name).with_suffix(".py")
with factor_path.open("r") as f:
code = f.read()
return FileBasedFactorImplementation(task, code=code, **kwargs)
rdagent/factor_implementation/evolving/factor_implementation_evolving_cli.py
-312
View File
@@ -1,312 +0,0 @@
import json
import pickle
import subprocess
from pathlib import Path
import pandas as pd
from fire.core import Fire
from rdagent.core.evolving_framework import EvoAgent, KnowledgeBase
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.factor_implementation.evolving.evaluators import (
FactorImplementationEvaluatorV1,
FactorImplementationsMultiEvaluator,
)
from rdagent.factor_implementation.evolving.evolvable_subjects import FactorImplementationList
from rdagent.factor_implementation.evolving.evolving_strategy import (
FactorEvolvingStrategy,
FactorEvolvingStrategyWithGraph,
)
from rdagent.factor_implementation.evolving.knowledge_management import (
FactorImplementationGraphKnowledgeBase,
FactorImplementationGraphRAGStrategy,
FactorImplementationKnowledgeBaseV1,
FactorImplementationRAGStrategyV1,
)
from rdagent.factor_implementation.share_modules.factor import (
FactorImplementationTask,
FileBasedFactorImplementation,
)
from tqdm import tqdm
ALPHA101_INIT_COMPONENTS = [
"1. abs(): absolute value to certain columns",
"2. log(): log value to certain columns",
"3. sign(): sign value to certain columns",
"4. add_two_columns(): add two columns",
"5. minus_two_columns(): minus two columns",
"6. times_two_columns(): times two columns",
"7. divide_two_columns(): divide two columns",
"8. add_value_to_columns(): add value to columns",
"9. minus_value_to_columns(): minus value to columns",
"10. rank(): cross-sectional rank value to columns",
"11. delay(): value of each data d days ago",
"12. correlation(): time-serial correlation of column_left and column_right for the past d days",
"13. covariance(): time-serial covariance of column_left and column_right for the past d days",
"14. scale_to_a(): scale the columns to sum(abs(x)) is a",
"15. delta(): today’s value of x minus the value of x d days ago",
"16. signedpower(): x^a",
"17. decay_linear(): weighted moving average over the past d days with linearly decaying weights d, d – 1, …, 1 (rescaled to sum up to 1)",
"18. indneutralize(): x cross-sectionally neutralized against groups g (subindustries, industries, sectors, etc.), i.e., x is cross-sectionally demeaned within each group g",
"19. ts_min(): time-series min over the past d days, operator min applied across the time-series for the past d days; non-integer number of days d is converted to floor(d)",
"20. ts_max(): time-series max over the past d days, operator max applied across the time-series for the past d days; non-integer number of days d is converted to floor(d)",
"21. ts_argmax(): which day ts_max(x, d) occurred on",
"22. ts_argmin(): which day ts_min(x, d) occurred on",
"23. ts_rank(): time-series rank in the past d days",
"24. min(): ts_min(x, d)",
"25. max(): ts_max(x, d)",
"26. sum(): time-series sum over the past d days",
"27. product(): time-series product over the past d days",
"28. stddev(): moving time-series standard deviation over the past d days",
]
class FactorImplementationEvolvingCli:
# TODO: we should use polymorphism to load knowledge base, strategies instead of evolving_version
# TODO: Can we refactor FactorImplementationEvolvingCli into a learning framework to differentiate our learning paradiagm with other ones by iteratively retrying?
def __init__(self, evolving_version=2) -> None:
self.evolving_version = evolving_version
self.knowledge_base = None
self.latest_factor_implementations = None
def run_evolving_framework(
self,
factor_implementations: FactorImplementationList,
factor_knowledge_base: KnowledgeBase,
max_loops: int = 20,
with_knowledge: bool = True,
with_feedback: bool = True,
knowledge_self_gen: bool = True,
) -> FactorImplementationList:
"""
Main target: Implement factors.
The system also leverages the former knowledge to help implement the factors. Also, new knowledge might be generated during the implementation to help the following implementation.
The gt_code and gt_value in the Factor instance is used to evaluate the implementation, and the feedback is used to generate high-quality knowledge which helps the agent to evolve.
"""
es = FactorEvolvingStrategyWithGraph() if self.evolving_version == 2 else FactorEvolvingStrategy()
rag = (
FactorImplementationGraphRAGStrategy(factor_knowledge_base)
if self.evolving_version == 2
else FactorImplementationRAGStrategyV1(factor_knowledge_base)
)
factor_evaluator = FactorImplementationsMultiEvaluator(FactorImplementationEvaluatorV1())
ea = EvoAgent(es, rag=rag)
for _ in tqdm(range(max_loops), "Implementing factors"):
factor_implementations = ea.step_evolving(
factor_implementations,
factor_evaluator,
with_knowledge=with_knowledge,
with_feedback=with_feedback,
knowledge_self_gen=knowledge_self_gen,
)
return factor_implementations
def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
if (
self.evolving_version == 1
and not isinstance(
factor_knowledge_base,
FactorImplementationKnowledgeBaseV1,
)
or self.evolving_version == 2
and not isinstance(
factor_knowledge_base,
FactorImplementationGraphKnowledgeBase,
)
):
raise ValueError("The former knowledge base is not compatible with the current version")
else:
factor_knowledge_base = (
FactorImplementationGraphKnowledgeBase(
init_component_list=component_init_list,
)
if self.evolving_version == 2
else FactorImplementationKnowledgeBaseV1()
)
return factor_knowledge_base
def implement_factors(
self,
factor_implementations: FactorImplementationList,
former_knowledge_base_path: Path = None,
new_knowledge_base_path: Path = None,
component_init_list: list = [],
max_loops: int = 20,
):
factor_knowledge_base = self.load_or_init_knowledge_base(
former_knowledge_base_path=former_knowledge_base_path,
component_init_list=component_init_list,
)
new_factor_implementations = self.run_evolving_framework(
factor_implementations=factor_implementations,
factor_knowledge_base=factor_knowledge_base,
max_loops=max_loops,
with_knowledge=True,
with_feedback=True,
knowledge_self_gen=True,
)
if new_knowledge_base_path is not None:
pickle.dump(factor_knowledge_base, open(new_knowledge_base_path, "wb"))
self.knowledge_base = factor_knowledge_base
self.latest_factor_implementations = factor_implementations
return new_factor_implementations
def _read_alpha101_factors(
self,
alpha101_evo_subs_path: Path = None,
alpha101_data_path=Path().cwd() / "git_ignore_folder" / "alpha101_related_files",
start_index=0,
end_index=32,
read_gt_factors=True,
) -> FactorImplementationList:
"""
Read the alpha101 factors from the alpha101_related_files folder
"""
if alpha101_evo_subs_path is not None and alpha101_evo_subs_path.exists():
factor_implementations = pickle.load(open(alpha101_evo_subs_path, "rb"))
else:
target_factor_plain_list = json.load(
open(alpha101_data_path / "target_factor_task_list.json"),
)
name_to_code = json.load(open(alpha101_data_path / "name_to_code.json"))
gt_df = pd.read_hdf(alpha101_data_path / "gt_filtered.h5")
# First read the target factor task
target_factor_tasks = []
for factor_list_item in target_factor_plain_list:
target_factor_tasks.append(
FactorImplementationTask(
factor_name=factor_list_item[0],
factor_description=factor_list_item[1],
factor_formulation=factor_list_item[2],
factor_formulation_description=factor_list_item[3],
),
)
# Second read the gt factor implementations
corresponding_gt_implementations = []
for factor_task in target_factor_tasks:
name = factor_task.factor_name
gt_code = name_to_code[name]
gt_value = gt_df.loc(axis=1)[[name]]
corresponding_gt_implementations.append(
FileBasedFactorImplementation(
code=gt_code,
executed_factor_value_dataframe=gt_value,
target_task=factor_task,
),
)
# Finally generate the factor implementations as evolvable subjects
factor_implementations = FactorImplementationList(
target_factor_tasks=target_factor_tasks,
corresponding_gt_implementations=(corresponding_gt_implementations if read_gt_factors else None),
)
factor_implementations.target_factor_tasks = factor_implementations.target_factor_tasks[start_index:end_index]
factor_implementations.corresponding_gt_implementations = (
factor_implementations.corresponding_gt_implementations[start_index:end_index] if read_gt_factors else None
)
return factor_implementations
def implement_alpha101(
self,
max_loops=30,
) -> FactorImplementationList:
"""
Implement the alpha101 factors to gather knowledge TODO: implement the code
"""
factor_implementations = self._read_alpha101_factors(
alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
start_index=0,
end_index=64,
read_gt_factors=True,
)
self.implement_factors(
factor_implementations,
former_knowledge_base_path=Path.cwd()
/ f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
new_knowledge_base_path=Path.cwd()
/ f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
component_init_list=ALPHA101_INIT_COMPONENTS,
max_loops=100,
)
factor_implementations = self._read_alpha101_factors(
alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
start_index=64,
end_index=96,
read_gt_factors=False,
)
final_imp = self.implement_factors(
factor_implementations,
former_knowledge_base_path=Path.cwd()
/ f"alpha101_knowledge_base_v{self.evolving_version}_project_product.pkl",
new_knowledge_base_path=Path.cwd()
/ f"alpha101_knowledge_base_v{self.evolving_version}_self_evolving_project_product.pkl",
component_init_list=ALPHA101_INIT_COMPONENTS,
max_loops=10,
)
final_imp.corresponding_gt_implementations = factor_implementations = self._read_alpha101_factors(
alpha101_evo_subs_path=Path.cwd() / "alpha101_evo_subs.pkl",
start_index=64,
end_index=96,
read_gt_factors=True,
).corresponding_gt_implementations
feedbacks = FactorImplementationsMultiEvaluator().evaluate(final_imp)
print([feedback.final_decision if feedback is not None else None for feedback in feedbacks].count(True))
def implement_amc(
self, evo_sub_path_str, former_knowledge_base_path_str, implementation_dump_path_str, slice_index,
):
factor_implementations: FactorImplementationList = pickle.load(open(evo_sub_path_str, "rb"))
factor_implementations.target_factor_tasks = factor_implementations.target_factor_tasks[
slice_index * 16 : slice_index * 16 + 16
]
if len(factor_implementations.target_factor_tasks) == 0:
return
if Path(implementation_dump_path_str).exists():
return
factor_implementations = self.implement_factors(
factor_implementations,
former_knowledge_base_path=Path(former_knowledge_base_path_str),
component_init_list=ALPHA101_INIT_COMPONENTS,
max_loops=10,
)
pickle.dump(factor_implementations, open(implementation_dump_path_str, "wb"))
def execute_command(self, command, cwd):
print(command, cwd)
try:
subprocess.check_output(
command,
shell=True,
cwd=cwd,
)
except subprocess.CalledProcessError as e:
print(e.output.decode())
def multi_inference_amc_factors(self, type):
slice_count = {"price_volume": 35, "fundamental": 24, "high_frequency": 16}[type]
res = multiprocessing_wrapper(
[
(
self.execute_command,
(
f"python src/scripts/factor_implementation/baselines/evolving/factor_implementation_evolving_cli.py implement_amc ./{type}_factors.pkl ./knowledge_base_v2_with_alpha101_and_10_factors.pkl ./inference_amc_factors_{type}_{slice}.pkl {slice}",
Path.cwd(),
),
)
for slice in range(slice_count)
],
n=2,
)
if __name__ == "__main__":
Fire(FactorImplementationEvolvingCli)
rdagent/factor_implementation/evolving/knowledge_management.py
+11 -8
View File
@@ -6,6 +6,7 @@ import random
import re
from itertools import combinations
from pathlib import Path
from jinja2 import Template
from typing import Union
from jinja2 import Template
@@ -20,22 +21,23 @@ from rdagent.core.evolving_framework import (
from rdagent.core.log import FinCoLog
from rdagent.core.prompts import Prompts
from rdagent.factor_implementation.evolving.evaluators import FactorImplementationSingleFeedback
from rdagent.factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
)
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
from rdagent.core.task import (
TaskImplementation,
)
from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask
from rdagent.core.prompts import Prompts
from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode
from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
class FactorImplementationKnowledge(Knowledge):
def __init__(
self,
target_task: FactorImplementationTask,
implementation: FactorImplementation,
target_task: FactorImplementTask,
implementation: TaskImplementation,
feedback: FactorImplementationSingleFeedback,
) -> None:
"""
@@ -899,3 +901,4 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
intersection_node_list_sort_by_freq.append(node)
return intersection_node_list_sort_by_freq
rdagent/factor_implementation/evolving/scheduler.py
+69
View File
@@ -0,0 +1,69 @@
from rdagent.oai.llm_utils import APIBackend
from jinja2 import Template
from rdagent.factor_implementation.share_modules.factor_implementation_config import FactorImplementSettings
import json
from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
from rdagent.factor_implementation.evolving.factor import FactorEvovlingItem
from rdagent.core.prompts import Prompts
from pathlib import Path
scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
import random
to_be_finished_task_index = random.sample(
to_be_finished_task_index,
implementation_factors_per_round,
)
print("The random selection is:",to_be_finished_task_index)
return to_be_finished_task_index
def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:FactorEvovlingItem, former_trace):
tasks = []
for i in to_be_finished_task_index:
# find corresponding former trace for each task
target_factor_task_information = evo.target_factor_tasks[i].get_factor_information()
if target_factor_task_information in former_trace:
tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information]))
system_prompt = Template(
scheduler_prompts["select_implementable_factor_system"],
).render(
data_info=get_data_folder_intro(),
)
session = APIBackend(use_chat_cache=False).build_chat_session(
session_system_prompt=system_prompt,
)
while True:
user_prompt = (
Template(
scheduler_prompts["select_implementable_factor_user"],
)
.render(
factor_num = implementation_factors_per_round,
target_factor_tasks=tasks,
)
)
if (
session.build_chat_completion_message_and_calculate_token(
user_prompt,
)
< FactorImplementSettings().chat_token_limit
):
break
response = session.build_chat_completion(
user_prompt=user_prompt,
json_mode=True,
)
try:
selection = json.loads(response)["selected_factor"]
if not isinstance(selection, list):
return to_be_finished_task_index
selection_index = [x for x in selection if isinstance(x, int)]
except:
return to_be_finished_task_index
return selection_index
rdagent/factor_implementation/prompts.yaml
+36 -13
View File
@@ -121,19 +121,6 @@ evaluator_final_decision_v1_user: |-
--------------Factor value feedback:---------------
{{ factor_value_feedback }}
analyze_component_prompt_v1_system: |-
User is getting a new task that might consist of the components below (given in component_index: component_description):
{{all_component_content}}
You should find out what components does the new task have, and put their indices in a list.
Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
{
"component_no_list": the list containing indices of components.
}
evolving_strategy_factor_implementation_v2_user: |-
--------------Target factor information:---------------
{{ factor_information_str }}
@@ -194,3 +181,39 @@ evolving_strategy_error_summary_v2_user: |-
{% endfor %}
{% endif %}
select_implementable_factor_system: |-
User is trying to implement some factors in quant investment using Python code, You are an assistant who helps the user select the easiest-to-implement factors and some factors may be difficult to implement due to a lack of information or excessive complexity..
The user will provide the number of factor you should pick and information about the factors, including their descriptions, formulas, and variable explanations.
At the same time, user will provide with your former attempt to implement the factor and the feedback to the implementation.You need to carefully review your previous attempts. Some factors have been repeatedly tried without success. You should consider discarding these factors.
Here is the source data that user will use to implement the factors:
{{ data_info }}
Please analyze the difficulties of the each factors and provide the reason and response the indices of selected implementable factor in the json format. Here is an example structure for the JSON output:
{
"Analysis": "Analyze the difficulties of the each factors and provide the reason why the factor can be implemented or not."
"selected_factor": "The indices of selected factor index in the list, like [0, 2, 3].The length should be the number of factor left after filtering.",
}
select_implementable_factor_user: |-
Number of factor you should pick: {{ factor_num }}
{% for factor_info in target_factor_tasks %}
=============Factor index:{{factor_info[0]}}:=============
=====Factor name:=====
{{ factor_info[1].factor_name }}
=====Factor description:=====
{{ factor_info[1].factor_description }}
=====Factor formulation:=====
{{ factor_info[1].factor_formulation }}
{% if factor_info[2]|length != 0 %}
--------------Your former attempt:---------------
{% for former_attempt in factor_info[2] %}
=====Code to attempt {{ loop.index }}=====
{{ former_attempt.implementation.code }}
=====Feedback to attempt {{ loop.index }}=====
{{ former_attempt.feedback }}
{% endfor %}
{% endif %}
{% endfor %}
rdagent/factor_implementation/share_modules/evaluator.py
-535
View File
@@ -1,535 +0,0 @@
import json
from abc import ABC, abstractmethod
from typing import Tuple
import pandas as pd
from finco.log import FinCoLog
from jinja2 import Template
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
from rdagent.oai.llm_utils import APIBackend
from factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
)
from factor_implementation.share_modules.prompt import FactorImplementationPrompts
class Evaluator(ABC):
@abstractmethod
def evaluate(
self,
target_task: FactorImplementationTask,
implementation: FactorImplementation,
gt_implementation: FactorImplementation,
**kwargs,
):
raise NotImplementedError
class FactorImplementationCodeEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementationTask,
implementation: FactorImplementation,
execution_feedback: str,
factor_value_feedback: str = "",
gt_implementation: FactorImplementation = None,
**kwargs,
):
factor_information = target_task.get_factor_information()
code = implementation.code
system_prompt = FactorImplementationPrompts()["evaluator_code_feedback_v1_system"]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
FactorImplementationPrompts()["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
while (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
former_messages=[],
)
> FactorImplementSettings().chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
FactorImplementationPrompts()["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
critic_response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=False,
)
# critic_response = json.loads(critic_response)
return critic_response
class FactorImplementationEvaluator(Evaluator):
# TODO:
# I think we should have unified interface for all evaluates, for examples.
# So we should adjust the interface of other factors
@abstractmethod
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
"""You can get the dataframe by
.. code-block:: python
_, gt_df = gt.execute()
_, gen_df = gen.execute()
Returns
-------
Tuple[str, object]
- str: the text-based description of the evaluation result
- object: a comparable metric (bool, integer, float ...)
"""
raise NotImplementedError("Please implement the `evaluator` method")
def _get_df(self, gt: FactorImplementation, gen: FactorImplementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
if isinstance(gen_df, pd.Series):
gen_df = gen_df.to_frame("source_factor")
if isinstance(gt_df, pd.Series):
gt_df = gt_df.to_frame("gt_factor")
return gt_df, gen_df
# NOTE: the following evaluators are splited from FactorImplementationValueEvaluator
class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if len(gen_df.columns) == 1 and len(gt_df.columns) == 1:
return "Both dataframes have only one column.", True
elif len(gen_df.columns) != 1:
gen_df = gen_df.iloc(axis=1)[
[
0,
]
]
return (
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
False,
)
return "", False
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
idx_name_right = gen_df.index.names == ("datetime", "instrument")
if idx_name_right:
return (
'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
True,
)
else:
return (
'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.',
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.shape[0] == gt_df.shape[0]:
return "Both dataframes have the same rows count.", True
else:
return (
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.index.equals(gt_df.index):
return "Both dataframes have the same index.", True
else:
return (
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
return "Both dataframes have the same missing values.", True
else:
return (
f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
try:
close_values = gen_df.sub(gt_df).abs().lt(1e-6)
result_int = close_values.astype(int)
pos_num = result_int.sum().sum()
acc_rate = pos_num / close_values.size
except:
close_values = gen_df
if close_values.all().iloc[0]:
return (
"All values in the dataframes are equal within the tolerance of 1e-6.",
acc_rate,
)
else:
return (
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
acc_rate,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
def __init__(self, hard_check: bool) -> None:
self.hard_check = hard_check
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
concat_df = pd.concat([gen_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if self.hard_check:
if ic > 0.99 and ric > 0.99:
return (
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
True,
)
else:
return (
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
False,
)
else:
return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValEvaluator(FactorImplementationEvaluator):
def evaluate(self, gt: FactorImplementation, gen: FactorImplementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
# FIXME: refactor the two classes
fiv = FactorImplementationValueEvaluator()
return fiv.evaluate(source_df=gen_df, gt_df=gt_df)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValueEvaluator(Evaluator):
# TODO: let's discuss the about the interface of the evaluator
def evaluate(
self,
source_df: pd.DataFrame,
gt_df: pd.DataFrame,
**kwargs,
) -> Tuple:
conclusions = []
if isinstance(source_df, pd.Series):
source_df = source_df.to_frame("source_factor")
conclusions.append(
"The source dataframe is a series, better convert it to a dataframe.",
)
if gt_df is not None and isinstance(gt_df, pd.Series):
gt_df = gt_df.to_frame("gt_factor")
conclusions.append(
"The ground truth dataframe is a series, convert it to a dataframe.",
)
# Check if both dataframe has only one columns
if len(source_df.columns) == 1:
conclusions.append("The source dataframe has only one column which is correct.")
else:
conclusions.append(
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
)
source_df = source_df.iloc(axis=1)[
[
0,
]
]
if list(source_df.index.names) != ["datetime", "instrument"]:
conclusions.append(
rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.",
)
else:
conclusions.append(
'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
)
# Check if both dataframe have the same rows count
if gt_df is not None:
if source_df.shape[0] == gt_df.shape[0]:
conclusions.append("Both dataframes have the same rows count.")
same_row_count_result = True
else:
conclusions.append(
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
)
same_row_count_result = False
# Check whether both dataframe has the same index
if source_df.index.equals(gt_df.index):
conclusions.append("Both dataframes have the same index.")
same_index_result = True
else:
conclusions.append(
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
)
same_index_result = False
# Check for the same missing values (NaN)
if source_df.isna().sum().sum() == gt_df.isna().sum().sum():
conclusions.append("Both dataframes have the same missing values.")
same_missing_values_result = True
else:
conclusions.append(
f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
)
same_missing_values_result = False
# Check if the values are the same within a small tolerance
if not same_index_result:
conclusions.append(
"The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless",
)
same_values_result = False
high_correlation_result = False
else:
close_values = source_df.sub(gt_df).abs().lt(1e-6)
if close_values.all().iloc[0]:
conclusions.append(
"All values in the dataframes are equal within the tolerance of 1e-6.",
)
same_values_result = True
else:
conclusions.append(
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
)
same_values_result = False
# Check the ic and rankic between the two dataframes
concat_df = pd.concat([source_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
try:
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if ic > 0.99 and ric > 0.99:
conclusions.append(
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
)
high_correlation_result = True
else:
conclusions.append(
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
)
high_correlation_result = False
# Check for shifted alignments only in the "datetime" index
max_shift_days = 2
for shift in range(-max_shift_days, max_shift_days + 1):
if shift == 0:
continue # Skip the case where there is no shift
shifted_source_df = source_df.groupby(level="instrument").shift(shift)
concat_df = pd.concat([shifted_source_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
shifted_ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if shifted_ric > 0.99:
conclusions.append(
f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.",
)
break
else:
conclusions.append(
f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.",
)
except Exception as e:
FinCoLog().warning(f"Error occurred when calculating the correlation: {e!s}")
conclusions.append(
f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
)
high_correlation_result = False
# Combine all conclusions into a single string
conclusion_str = "\n".join(conclusions)
final_result = (same_values_result or high_correlation_result) if gt_df is not None else False
return conclusion_str, final_result
# TODO:
def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
"""When the prompt is too long. We have to shorten it.
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
"""
# TODO: this should replace most of code in
# - FactorImplementationFinalDecisionEvaluator.evaluate
# - FactorImplementationCodeEvaluator.evaluate
class FactorImplementationFinalDecisionEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementationTask,
execution_feedback: str,
value_feedback: str,
code_feedback: str,
**kwargs,
) -> Tuple:
system_prompt = FactorImplementationPrompts()["evaluator_final_decision_v1_system"]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
FactorImplementationPrompts()["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
while (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
former_messages=[],
)
> FactorImplementSettings().chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
FactorImplementationPrompts()["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
final_evaluation_dict = json.loads(
APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=True,
),
)
return (
final_evaluation_dict["final_decision"],
final_evaluation_dict["final_feedback"],
)
rdagent/factor_implementation/share_modules/exception.py
-26
View File
@@ -1,26 +0,0 @@
class ImplementRunException(Exception):
"""
Exceptions raised when Implementing and running code.
- start: FactorImplementationTask => FactorGenerator
- end: Get dataframe after execution
The more detailed evaluation in dataframe values are managed by the evaluator.
"""
class CodeFormatException(ImplementRunException):
"""
The generated code is not found due format error.
"""
class RuntimeErrorException(ImplementRunException):
"""
The generated code fail to execute the script.
"""
class NoOutputException(ImplementRunException):
"""
The code fail to generate output file.
"""
rdagent/factor_implementation/share_modules/factor_gen.py
-31
View File
@@ -1,31 +0,0 @@
from abc import ABC, abstractmethod
from typing import List
from factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
)
class FactorGenerator(ABC):
"""
Because implementing factors will help each other in the process of implementation, we use the interface `List[FactorImplementationTask] -> List[FactorImplementation]` instead of single factor .
"""
def __init__(self, target_task_l: List[FactorImplementationTask]) -> None:
self.target_task_l = target_task_l
@abstractmethod
def generate(self, *args, **kwargs) -> List[FactorImplementation]:
raise NotImplementedError("generate method is not implemented.")
def collect_feedback(self, feedback_obj_l: List[object]):
"""
When online evaluation.
The preivous feedbacks will be collected to support advanced factor generator
Parameters
----------
feedback_obj_l : List[object]
"""
rdagent/factor_implementation/share_modules/factor_implementation_config.py
+18 -3
View File
@@ -1,9 +1,13 @@
from pathlib import Path
from core.conf import FincoSettings
from pydantic_settings import BaseSettings
from typing import Literal, Union
SELECT_METHOD = Literal["random", "scheduler"]
class FactorImplementSettings(FincoSettings):
class FactorImplementSettings(BaseSettings):
file_based_execution_data_folder: str = str(
(Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
)
@@ -29,10 +33,21 @@ class FactorImplementSettings(FincoSettings):
v2_error_summary: bool = False
v2_knowledge_sampler: float = 1.0
implementation_factors_per_round: int = 100 # how many factors to choose for each round of evolving
evo_multi_proc_n: int = 16 # how many processes to use for evolving (including eval & generation)
file_based_execution_timeout: int = 120 # seconds for each factor implementation execution
select_method: SELECT_METHOD = "random"
select_ratio: float = 0.5
max_loop: int = 10
knowledge_base_path: Union[str, None] = None
new_knowledge_base_path: Union[str, None] = None
chat_token_limit: int = (
100000 # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
)
FIS = FactorImplementSettings()
rdagent/factor_implementation/share_modules/factor_implementation_utils.py
+16 -1
View File
@@ -5,6 +5,8 @@ import pandas as pd
# render it with jinja
from jinja2 import Template
from rdagent.factor_implementation.share_modules.factor_implementation_config import FIS
from rdagent.factor_implementation.evolving.factor import FactorImplementTask
TPL = """
{{file_name}}
@@ -15,7 +17,6 @@ TPL = """
# Create a Jinja template from the string
JJ_TPL = Template(TPL)
def get_data_folder_intro():
"""Direclty get the info of the data folder.
It is for preparing prompting message.
@@ -48,3 +49,17 @@ def get_data_folder_intro():
f"file type {p.name} is not supported. Please implement its description function.",
)
return "\n ----------------- file spliter -------------\n".join(content_l)
def load_data_from_dict(factor_dict:dict) -> list[FactorImplementTask]:
"""Load data from a dict.
"""
task_l = []
for factor_name, factor_data in factor_dict.items():
task = FactorImplementTask(
factor_name=factor_name,
factor_description=factor_data["description"],
factor_formulation=factor_data["formulation"],
variables=factor_data["variables"],
)
task_l.append(task)
return task_l