gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-29 00:17:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

reporeformat V2 (#23)

Browse Source 
* reformat factor implement process

* move some code to more reasonable place

* fix the bug

* add test function in factor_extract_and_implement.py

* change select factor number to ratio , add some factor implement setting and fix some bug while using knowledgebase

* change evoagent

* add abstract class EvoAgent

* add benchmark workflow

* fix some bug in llm_utils

* run wenjun's code

* fix the knowledgebase instance check

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
USTCKevinF
2024-06-14 12:59:44 +08:00
committed by GitHub
parent 9e82da243b
commit ebb659a018
32 changed files with 2227 additions and 1141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/factor_implementation/share_modules/evaluator.py
-535
View File
@@ -1,535 +0,0 @@
import json
from abc import ABC, abstractmethod
from typing import Tuple
import pandas as pd
from finco.log import FinCoLog
from jinja2 import Template
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
from rdagent.oai.llm_utils import APIBackend
from factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
)
from factor_implementation.share_modules.prompt import FactorImplementationPrompts
class Evaluator(ABC):
@abstractmethod
def evaluate(
self,
target_task: FactorImplementationTask,
implementation: FactorImplementation,
gt_implementation: FactorImplementation,
**kwargs,
):
raise NotImplementedError
class FactorImplementationCodeEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementationTask,
implementation: FactorImplementation,
execution_feedback: str,
factor_value_feedback: str = "",
gt_implementation: FactorImplementation = None,
**kwargs,
):
factor_information = target_task.get_factor_information()
code = implementation.code
system_prompt = FactorImplementationPrompts()["evaluator_code_feedback_v1_system"]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
FactorImplementationPrompts()["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
while (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
former_messages=[],
)
> FactorImplementSettings().chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
FactorImplementationPrompts()["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
critic_response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=False,
)
# critic_response = json.loads(critic_response)
return critic_response
class FactorImplementationEvaluator(Evaluator):
# TODO:
# I think we should have unified interface for all evaluates, for examples.
# So we should adjust the interface of other factors
@abstractmethod
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
"""You can get the dataframe by
.. code-block:: python
_, gt_df = gt.execute()
_, gen_df = gen.execute()
Returns
-------
Tuple[str, object]
- str: the text-based description of the evaluation result
- object: a comparable metric (bool, integer, float ...)
"""
raise NotImplementedError("Please implement the `evaluator` method")
def _get_df(self, gt: FactorImplementation, gen: FactorImplementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
if isinstance(gen_df, pd.Series):
gen_df = gen_df.to_frame("source_factor")
if isinstance(gt_df, pd.Series):
gt_df = gt_df.to_frame("gt_factor")
return gt_df, gen_df
# NOTE: the following evaluators are splited from FactorImplementationValueEvaluator
class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if len(gen_df.columns) == 1 and len(gt_df.columns) == 1:
return "Both dataframes have only one column.", True
elif len(gen_df.columns) != 1:
gen_df = gen_df.iloc(axis=1)[
[
0,
]
]
return (
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
False,
)
return "", False
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
idx_name_right = gen_df.index.names == ("datetime", "instrument")
if idx_name_right:
return (
'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
True,
)
else:
return (
'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.',
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.shape[0] == gt_df.shape[0]:
return "Both dataframes have the same rows count.", True
else:
return (
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.index.equals(gt_df.index):
return "Both dataframes have the same index.", True
else:
return (
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
return "Both dataframes have the same missing values.", True
else:
return (
f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
False,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
try:
close_values = gen_df.sub(gt_df).abs().lt(1e-6)
result_int = close_values.astype(int)
pos_num = result_int.sum().sum()
acc_rate = pos_num / close_values.size
except:
close_values = gen_df
if close_values.all().iloc[0]:
return (
"All values in the dataframes are equal within the tolerance of 1e-6.",
acc_rate,
)
else:
return (
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
acc_rate,
)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
def __init__(self, hard_check: bool) -> None:
self.hard_check = hard_check
def evaluate(
self,
gt: FactorImplementation,
gen: FactorImplementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
concat_df = pd.concat([gen_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if self.hard_check:
if ic > 0.99 and ric > 0.99:
return (
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
True,
)
else:
return (
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
False,
)
else:
return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValEvaluator(FactorImplementationEvaluator):
def evaluate(self, gt: FactorImplementation, gen: FactorImplementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
# FIXME: refactor the two classes
fiv = FactorImplementationValueEvaluator()
return fiv.evaluate(source_df=gen_df, gt_df=gt_df)
def __str__(self) -> str:
return self.__class__.__name__
class FactorImplementationValueEvaluator(Evaluator):
# TODO: let's discuss the about the interface of the evaluator
def evaluate(
self,
source_df: pd.DataFrame,
gt_df: pd.DataFrame,
**kwargs,
) -> Tuple:
conclusions = []
if isinstance(source_df, pd.Series):
source_df = source_df.to_frame("source_factor")
conclusions.append(
"The source dataframe is a series, better convert it to a dataframe.",
)
if gt_df is not None and isinstance(gt_df, pd.Series):
gt_df = gt_df.to_frame("gt_factor")
conclusions.append(
"The ground truth dataframe is a series, convert it to a dataframe.",
)
# Check if both dataframe has only one columns
if len(source_df.columns) == 1:
conclusions.append("The source dataframe has only one column which is correct.")
else:
conclusions.append(
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
)
source_df = source_df.iloc(axis=1)[
[
0,
]
]
if list(source_df.index.names) != ["datetime", "instrument"]:
conclusions.append(
rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.",
)
else:
conclusions.append(
'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
)
# Check if both dataframe have the same rows count
if gt_df is not None:
if source_df.shape[0] == gt_df.shape[0]:
conclusions.append("Both dataframes have the same rows count.")
same_row_count_result = True
else:
conclusions.append(
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
)
same_row_count_result = False
# Check whether both dataframe has the same index
if source_df.index.equals(gt_df.index):
conclusions.append("Both dataframes have the same index.")
same_index_result = True
else:
conclusions.append(
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
)
same_index_result = False
# Check for the same missing values (NaN)
if source_df.isna().sum().sum() == gt_df.isna().sum().sum():
conclusions.append("Both dataframes have the same missing values.")
same_missing_values_result = True
else:
conclusions.append(
f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
)
same_missing_values_result = False
# Check if the values are the same within a small tolerance
if not same_index_result:
conclusions.append(
"The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless",
)
same_values_result = False
high_correlation_result = False
else:
close_values = source_df.sub(gt_df).abs().lt(1e-6)
if close_values.all().iloc[0]:
conclusions.append(
"All values in the dataframes are equal within the tolerance of 1e-6.",
)
same_values_result = True
else:
conclusions.append(
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
)
same_values_result = False
# Check the ic and rankic between the two dataframes
concat_df = pd.concat([source_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
try:
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if ic > 0.99 and ric > 0.99:
conclusions.append(
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
)
high_correlation_result = True
else:
conclusions.append(
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
)
high_correlation_result = False
# Check for shifted alignments only in the "datetime" index
max_shift_days = 2
for shift in range(-max_shift_days, max_shift_days + 1):
if shift == 0:
continue # Skip the case where there is no shift
shifted_source_df = source_df.groupby(level="instrument").shift(shift)
concat_df = pd.concat([shifted_source_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
shifted_ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if shifted_ric > 0.99:
conclusions.append(
f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.",
)
break
else:
conclusions.append(
f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.",
)
except Exception as e:
FinCoLog().warning(f"Error occurred when calculating the correlation: {e!s}")
conclusions.append(
f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
)
high_correlation_result = False
# Combine all conclusions into a single string
conclusion_str = "\n".join(conclusions)
final_result = (same_values_result or high_correlation_result) if gt_df is not None else False
return conclusion_str, final_result
# TODO:
def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
"""When the prompt is too long. We have to shorten it.
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
"""
# TODO: this should replace most of code in
# - FactorImplementationFinalDecisionEvaluator.evaluate
# - FactorImplementationCodeEvaluator.evaluate
class FactorImplementationFinalDecisionEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementationTask,
execution_feedback: str,
value_feedback: str,
code_feedback: str,
**kwargs,
) -> Tuple:
system_prompt = FactorImplementationPrompts()["evaluator_final_decision_v1_system"]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
FactorImplementationPrompts()["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
while (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
former_messages=[],
)
> FactorImplementSettings().chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
FactorImplementationPrompts()["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
final_evaluation_dict = json.loads(
APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=True,
),
)
return (
final_evaluation_dict["final_decision"],
final_evaluation_dict["final_feedback"],
)
rdagent/factor_implementation/share_modules/exception.py
-26
View File
@@ -1,26 +0,0 @@
class ImplementRunException(Exception):
"""
Exceptions raised when Implementing and running code.
- start: FactorImplementationTask => FactorGenerator
- end: Get dataframe after execution
The more detailed evaluation in dataframe values are managed by the evaluator.
"""
class CodeFormatException(ImplementRunException):
"""
The generated code is not found due format error.
"""
class RuntimeErrorException(ImplementRunException):
"""
The generated code fail to execute the script.
"""
class NoOutputException(ImplementRunException):
"""
The code fail to generate output file.
"""
rdagent/factor_implementation/share_modules/factor.py
-223
View File
@@ -1,223 +0,0 @@
import pickle
import subprocess
import uuid
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Tuple, Union
import pandas as pd
from filelock import FileLock
from finco.log import FinCoLog
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
from factor_implementation.share_modules.exception import (
CodeFormatException,
NoOutputException,
RuntimeErrorException,
)
from oai.llm_utils import md5_hash
class FactorImplementationTask:
# TODO: remove the factor_ prefix may be better
def __init__(
self,
factor_name,
factor_description,
factor_formulation,
factor_formulation_description,
variables: dict = {},
) -> None:
self.factor_name = factor_name
self.factor_description = factor_description
self.factor_formulation = factor_formulation
self.factor_formulation_description = factor_formulation_description
# TODO: check variables a good candidate
self.variables = variables
def get_factor_information(self):
return f"""factor_name: {self.factor_name}
factor_description: {self.factor_description}
factor_formulation: {self.factor_formulation}
factor_formulation_description: {self.factor_formulation_description}"""
@staticmethod
def from_dict(dict):
return FactorImplementationTask(**dict)
def __repr__(self) -> str:
return f"<{self.__class__.__name__}[{self.factor_name}]>"
class FactorImplementation(ABC):
def __init__(self, target_task: FactorImplementationTask) -> None:
self.target_task = target_task
@abstractmethod
def execute(self, *args, **kwargs) -> Tuple[str, pd.DataFrame]:
raise NotImplementedError("__call__ method is not implemented.")
class FileBasedFactorImplementation(FactorImplementation):
"""
This class is used to implement a factor by writing the code to a file.
Input data and output factor value are also written to files.
"""
# TODO: (Xiao) think raising errors may get better information for processing
FB_FROM_CACHE = "The factor value has been executed and stored in the instance variable."
FB_EXEC_SUCCESS = "Execution succeeded without error."
FB_CODE_NOT_SET = "code is not set."
FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
FB_OUTPUT_FILE_NOT_FOUND = "\nExpected output file not found."
FB_OUTPUT_FILE_FOUND = "\nExpected output file found."
def __init__(
self,
target_task: FactorImplementationTask,
code,
executed_factor_value_dataframe=None,
raise_exception=False,
) -> None:
super().__init__(target_task)
self.code = code
self.executed_factor_value_dataframe = executed_factor_value_dataframe
self.logger = FinCoLog()
self.raise_exception = raise_exception
self.workspace_path = Path(
FactorImplementSettings().file_based_execution_workspace,
) / str(uuid.uuid4())
@staticmethod
def link_data_to_workspace(data_path: Path, workspace_path: Path):
data_path = Path(data_path)
workspace_path = Path(workspace_path)
for data_file_path in data_path.iterdir():
workspace_data_file_path = workspace_path / data_file_path.name
if workspace_data_file_path.exists():
workspace_data_file_path.unlink()
subprocess.run(
["ln", "-s", data_file_path, workspace_data_file_path],
check=False,
)
def execute(self, store_result: bool = False) -> Tuple[str, pd.DataFrame]:
"""
execute the implementation and get the factor value by the following steps:
1. make the directory in workspace path
2. write the code to the file in the workspace path
3. link all the source data to the workspace path folder
4. execute the code
5. read the factor value from the output file in the workspace path folder
returns the execution feedback as a string and the factor value as a pandas dataframe
parameters:
store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation
"""
if self.code is None:
if self.raise_exception:
raise CodeFormatException(self.FB_CODE_NOT_SET)
else:
# TODO: to make the interface compatible with previous code. I kept the original behavior.
raise ValueError(self.FB_CODE_NOT_SET)
with FileLock(self.workspace_path / "execution.lock"):
(Path.cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").mkdir(
exist_ok=True, parents=True,
)
if FactorImplementSettings().enable_execution_cache:
# NOTE: cache the result for the same code
target_file_name = md5_hash(self.code)
cache_file_path = (
Path.cwd()
/ "git_ignore_folder"
/ "factor_implementation_execution_cache"
/ f"{target_file_name}.pkl"
)
if cache_file_path.exists() and not self.raise_exception:
cached_res = pickle.load(open(cache_file_path, "rb"))
if store_result and cached_res[1] is not None:
self.executed_factor_value_dataframe = cached_res[1]
return cached_res
if self.executed_factor_value_dataframe is not None:
return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
source_data_path = Path(
FactorImplementSettings().file_based_execution_data_folder,
)
self.workspace_path.mkdir(exist_ok=True, parents=True)
code_path = self.workspace_path / f"{self.target_task.factor_name}.py"
code_path.write_text(self.code)
self.link_data_to_workspace(source_data_path, self.workspace_path)
execution_feedback = self.FB_EXECUTION_SUCCEEDED
try:
subprocess.check_output(
f"python {code_path}",
shell=True,
cwd=self.workspace_path,
stderr=subprocess.STDOUT,
timeout=FactorImplementSettings().file_based_execution_timeout,
)
except subprocess.CalledProcessError as e:
import site
execution_feedback = (
e.output.decode()
.replace(str(code_path.parent.absolute()), r"/path/to")
.replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
)
if len(execution_feedback) > 2000:
execution_feedback = (
execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
)
if self.raise_exception:
raise RuntimeErrorException(execution_feedback)
except subprocess.TimeoutExpired:
execution_feedback += f"Execution timeout error and the timeout is set to {FactorImplementSettings().file_based_execution_timeout} seconds."
if self.raise_exception:
raise RuntimeErrorException(execution_feedback)
workspace_output_file_path = self.workspace_path / "result.h5"
if not workspace_output_file_path.exists():
execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND
executed_factor_value_dataframe = None
if self.raise_exception:
raise NoOutputException(execution_feedback)
else:
try:
executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path)
execution_feedback += self.FB_OUTPUT_FILE_FOUND
except Exception as e:
execution_feedback += f"Error found when reading hdf file: {e}"[:1000]
executed_factor_value_dataframe = None
if store_result and executed_factor_value_dataframe is not None:
self.executed_factor_value_dataframe = executed_factor_value_dataframe
if FactorImplementSettings().enable_execution_cache:
pickle.dump(
(execution_feedback, executed_factor_value_dataframe),
open(cache_file_path, "wb"),
)
return execution_feedback, executed_factor_value_dataframe
def __str__(self) -> str:
# NOTE:
# If the code cache works, the workspace will be None.
return f"File Factor[{self.target_task.factor_name}]: {self.workspace_path}"
def __repr__(self) -> str:
return self.__str__()
@staticmethod
def from_folder(task: FactorImplementationTask, path: Union[str, Path], **kwargs):
path = Path(path)
factor_path = (path / task.factor_name).with_suffix(".py")
with factor_path.open("r") as f:
code = f.read()
return FileBasedFactorImplementation(task, code=code, **kwargs)
rdagent/factor_implementation/share_modules/factor_gen.py
-31
View File
@@ -1,31 +0,0 @@
from abc import ABC, abstractmethod
from typing import List
from factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
)
class FactorGenerator(ABC):
"""
Because implementing factors will help each other in the process of implementation, we use the interface `List[FactorImplementationTask] -> List[FactorImplementation]` instead of single factor .
"""
def __init__(self, target_task_l: List[FactorImplementationTask]) -> None:
self.target_task_l = target_task_l
@abstractmethod
def generate(self, *args, **kwargs) -> List[FactorImplementation]:
raise NotImplementedError("generate method is not implemented.")
def collect_feedback(self, feedback_obj_l: List[object]):
"""
When online evaluation.
The preivous feedbacks will be collected to support advanced factor generator
Parameters
----------
feedback_obj_l : List[object]
"""
rdagent/factor_implementation/share_modules/factor_implementation_config.py
+18 -3
View File
@@ -1,9 +1,13 @@
from pathlib import Path
from core.conf import FincoSettings
from pydantic_settings import BaseSettings
from typing import Literal, Union
SELECT_METHOD = Literal["random", "scheduler"]
class FactorImplementSettings(FincoSettings):
class FactorImplementSettings(BaseSettings):
file_based_execution_data_folder: str = str(
(Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
)
@@ -29,10 +33,21 @@ class FactorImplementSettings(FincoSettings):
v2_error_summary: bool = False
v2_knowledge_sampler: float = 1.0
implementation_factors_per_round: int = 100 # how many factors to choose for each round of evolving
evo_multi_proc_n: int = 16 # how many processes to use for evolving (including eval & generation)
file_based_execution_timeout: int = 120 # seconds for each factor implementation execution
select_method: SELECT_METHOD = "random"
select_ratio: float = 0.5
max_loop: int = 10
knowledge_base_path: Union[str, None] = None
new_knowledge_base_path: Union[str, None] = None
chat_token_limit: int = (
100000 # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
)
FIS = FactorImplementSettings()
rdagent/factor_implementation/share_modules/factor_implementation_utils.py
+16 -1
View File
@@ -5,6 +5,8 @@ import pandas as pd
# render it with jinja
from jinja2 import Template
from rdagent.factor_implementation.share_modules.factor_implementation_config import FIS
from rdagent.factor_implementation.evolving.factor import FactorImplementTask
TPL = """
{{file_name}}
@@ -15,7 +17,6 @@ TPL = """
# Create a Jinja template from the string
JJ_TPL = Template(TPL)
def get_data_folder_intro():
"""Direclty get the info of the data folder.
It is for preparing prompting message.
@@ -48,3 +49,17 @@ def get_data_folder_intro():
f"file type {p.name} is not supported. Please implement its description function.",
)
return "\n ----------------- file spliter -------------\n".join(content_l)
def load_data_from_dict(factor_dict:dict) -> list[FactorImplementTask]:
"""Load data from a dict.
"""
task_l = []
for factor_name, factor_data in factor_dict.items():
task = FactorImplementTask(
factor_name=factor_name,
factor_description=factor_data["description"],
factor_formulation=factor_data["formulation"],
variables=factor_data["variables"],
)
task_l.append(task)
return task_l