gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-28 16:07:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

refine class design and inheritance first version code (#41)

Browse Source 
* refine class design and inheritance first version code

* fix all typos

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
2024-07-02 17:58:37 +08:00
committed by GitHub
parent 0faf232b81
commit 1b31bcd2c2
39 changed files with 717 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py
+98 -82
View File
@@ -5,21 +5,23 @@ from pathlib import Path
from typing import List, Tuple
import pandas as pd
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
FactorEvovlingItem,
FactorImplementTask,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
FactorTask,
)
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evaluation import Evaluator
from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
from rdagent.core.experiment import Implementation
from rdagent.core.log import RDAgentLog
from rdagent.core.prompts import Prompts
from rdagent.core.task import TaskImplementation
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.oai.llm_utils import APIBackend
@@ -33,8 +35,8 @@ class FactorImplementationEvaluator(Evaluator):
@abstractmethod
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
"""You can get the dataframe by
@@ -52,7 +54,7 @@ class FactorImplementationEvaluator(Evaluator):
"""
raise NotImplementedError("Please implement the `evaluator` method")
def _get_df(self, gt: TaskImplementation, gen: TaskImplementation):
def _get_df(self, gt: Implementation, gen: Implementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
if isinstance(gen_df, pd.Series):
@@ -65,11 +67,11 @@ class FactorImplementationEvaluator(Evaluator):
class FactorImplementationCodeEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementTask,
implementation: TaskImplementation,
target_task: FactorTask,
implementation: Implementation,
execution_feedback: str,
factor_value_feedback: str = "",
gt_implementation: TaskImplementation = None,
gt_implementation: Implementation = None,
**kwargs,
):
factor_information = target_task.get_factor_information()
@@ -78,14 +80,18 @@ class FactorImplementationCodeEvaluator(Evaluator):
system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
evaluate_prompts["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_code_feedback_v1_user"],
)
.render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
)
while (
APIBackend().build_messages_and_calculate_token(
@@ -96,14 +102,18 @@ class FactorImplementationCodeEvaluator(Evaluator):
> RD_AGENT_SETTINGS.chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
evaluate_prompts["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_code_feedback_v1_user"],
)
.render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
)
critic_response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
@@ -117,8 +127,8 @@ class FactorImplementationCodeEvaluator(Evaluator):
class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -143,8 +153,8 @@ class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
idx_name_right = gen_df.index.names == ("datetime", "instrument")
@@ -166,8 +176,8 @@ class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -186,8 +196,8 @@ class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -206,8 +216,8 @@ class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -226,8 +236,8 @@ class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -259,8 +269,8 @@ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -293,7 +303,7 @@ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
class FactorImplementationValEvaluator(FactorImplementationEvaluator):
def evaluate(self, gt: TaskImplementation, gen: TaskImplementation):
def evaluate(self, gt: Implementation, gen: Implementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
# FIXME: refactor the two classes
@@ -473,7 +483,7 @@ def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: i
class FactorImplementationFinalDecisionEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
execution_feedback: str,
value_feedback: str,
code_feedback: str,
@@ -483,17 +493,21 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
"evaluator_final_decision_v1_system"
]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
evaluate_prompts["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_final_decision_v1_user"],
)
.render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
)
while (
APIBackend().build_messages_and_calculate_token(
@@ -504,17 +518,21 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
> RD_AGENT_SETTINGS.chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
evaluate_prompts["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_final_decision_v1_user"],
)
.render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
)
final_evaluation_dict = json.loads(
@@ -584,9 +602,9 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
def evaluate(
self,
target_task: FactorImplementTask,
implementation: TaskImplementation,
gt_implementation: TaskImplementation = None,
target_task: FactorTask,
implementation: Implementation,
gt_implementation: Implementation = None,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorImplementationSingleFeedback:
@@ -681,23 +699,23 @@ class FactorImplementationsMultiEvaluator(Evaluator):
def evaluate(
self,
evo: FactorEvovlingItem,
evo: FactorEvolvingItem,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorImplementationsMultiFeedback:
multi_implementation_feedback = FactorImplementationsMultiFeedback()
# for index in range(len(evo.target_factor_tasks)):
# corresponding_implementation = evo.corresponding_implementations[index]
# for index in range(len(evo.sub_tasks)):
# corresponding_implementation = evo.sub_implementations[index]
# corresponding_gt_implementation = (
# evo.corresponding_gt_implementations[index]
# if evo.corresponding_gt_implementations is not None
# evo.sub_gt_implementations[index]
# if evo.sub_gt_implementations is not None
# else None
# )
# multi_implementation_feedback.append(
# self.single_factor_implementation_evaluator.evaluate(
# target_task=evo.target_factor_tasks[index],
# target_task=evo.sub_tasks[index],
# implementation=corresponding_implementation,
# gt_implementation=corresponding_gt_implementation,
# queried_knowledge=queried_knowledge,
@@ -705,18 +723,16 @@ class FactorImplementationsMultiEvaluator(Evaluator):
# )
calls = []
for index in range(len(evo.target_factor_tasks)):
corresponding_implementation = evo.corresponding_implementations[index]
for index in range(len(evo.sub_tasks)):
corresponding_implementation = evo.sub_implementations[index]
corresponding_gt_implementation = (
evo.corresponding_gt_implementations[index]
if evo.corresponding_gt_implementations is not None
else None
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
)
calls.append(
(
self.single_factor_implementation_evaluator.evaluate,
(
evo.target_factor_tasks[index],
evo.sub_tasks[index],
corresponding_implementation,
corresponding_gt_implementation,
queried_knowledge,
rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py
+30
View File
@@ -0,0 +1,30 @@
from rdagent.components.task_implementation.factor_implementation.factor import (
FactorExperiment,
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.core.log import RDAgentLog
class FactorEvolvingItem(FactorExperiment[FactorTask, FileBasedFactorImplementation], EvolvableSubjects):
"""
Intermediate item of factor implementation.
"""
def __init__(
self,
sub_tasks: list[FactorTask],
sub_gt_implementations: list[FileBasedFactorImplementation] = None,
):
FactorExperiment.__init__(self, sub_tasks=sub_tasks)
self.corresponding_selection: list = None
if sub_gt_implementations is not None and len(
sub_gt_implementations,
) != len(self.sub_tasks):
self.sub_gt_implementations = None
RDAgentLog().warning(
"The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
)
else:
self.sub_gt_implementations = sub_gt_implementations
rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py
+51 -37
View File
@@ -6,27 +6,29 @@ from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
FactorEvovlingItem,
FactorImplementTask,
FileBasedFactorImplementation,
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.task_implementation.factor_implementation.evolving.scheduler import (
LLMSelect,
RandomSelect,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
FACTOR_IMPLEMENT_SETTINGS,
from rdagent.components.task_implementation.factor_implementation.factor import (
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import (
from rdagent.components.task_implementation.factor_implementation.utils import (
get_data_folder_intro,
)
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
from rdagent.core.experiment import Implementation
from rdagent.core.prompts import Prompts
from rdagent.core.task import TaskImplementation
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.oai.llm_utils import APIBackend
@@ -43,27 +45,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
@abstractmethod
def implement_one_factor(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
queried_knowledge: QueriedKnowledge = None,
) -> TaskImplementation:
) -> Implementation:
raise NotImplementedError
def evolve(
self,
*,
evo: FactorEvovlingItem,
evo: FactorEvolvingItem,
queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
**kwargs,
) -> FactorEvovlingItem:
) -> FactorEvolvingItem:
self.num_loop += 1
new_evo = deepcopy(evo)
# 1.找出需要evolve的factor
to_be_finished_task_index = []
for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
for index, target_factor_task in enumerate(new_evo.sub_tasks):
target_factor_task_desc = target_factor_task.get_factor_information()
if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
new_evo.corresponding_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
target_factor_task_desc
].implementation
elif (
@@ -95,18 +97,18 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
result = multiprocessing_wrapper(
[
(self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge))
(self.implement_one_factor, (new_evo.sub_tasks[target_index], queried_knowledge))
for target_index in to_be_finished_task_index
],
n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
)
for index, target_index in enumerate(to_be_finished_task_index):
new_evo.corresponding_implementations[target_index] = result[index]
new_evo.sub_implementations[target_index] = result[index]
# for target_index in to_be_finished_task_index:
# new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
# new_evo.target_factor_tasks[target_index], queried_knowledge
# new_evo.sub_implementations[target_index] = self.implement_one_factor(
# new_evo.sub_tasks[target_index], queried_knowledge
# )
new_evo.corresponding_selection = to_be_finished_task_index
@@ -117,9 +119,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_factor(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
) -> TaskImplementation:
) -> Implementation:
factor_information_str = target_task.get_factor_information()
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -140,11 +142,15 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = Template(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
).render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
)
.render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
session_system_prompt=system_prompt,
@@ -153,7 +159,8 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
while True:
user_prompt = (
Template(
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_user"],
)
.render(
@@ -196,9 +203,9 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
def implement_one_factor(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
queried_knowledge,
) -> TaskImplementation:
) -> Implementation:
error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
# 1. 提取因子的背景信息
target_factor_task_information = target_task.get_factor_information()
@@ -231,11 +238,15 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = Template(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
).render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
)
.render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
@@ -254,7 +265,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
and len(queried_former_failed_knowledge_to_render) != 0
):
error_summary_system_prompt = (
Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
Environment(undefined=StrictUndefined)
.from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
.render(
factor_information_str=target_factor_task_information,
code_and_feedback=queried_former_failed_knowledge_to_render[
@@ -268,7 +280,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
)
while True:
error_summary_user_prompt = (
Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
Environment(undefined=StrictUndefined)
.from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
.render(
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
)
@@ -287,7 +300,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
)
# 构建user_prompt。开始写代码
user_prompt = (
Template(
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v2_user"],
)
.render(
rdagent/components/task_implementation/factor_implementation/evolving/factor.py
-251
View File
@@ -1,251 +0,0 @@
from __future__ import annotations
import pickle
import subprocess
import uuid
from pathlib import Path
from typing import Tuple, Union
import pandas as pd
from filelock import FileLock
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.core.exception import (
CodeFormatException,
NoOutputException,
RuntimeErrorException,
)
from rdagent.core.log import RDAgentLog
from rdagent.core.task import (
BaseTask,
FBTaskImplementation,
TaskImplementation,
TestCase,
)
from rdagent.oai.llm_utils import md5_hash
class FactorImplementTask(BaseTask):
# TODO: generalized the attributes into the BaseTask
# - factor_* -> *
def __init__(
self,
factor_name,
factor_description,
factor_formulation,
variables: dict = {},
resource: str = None,
) -> None:
self.factor_name = factor_name
self.factor_description = factor_description
self.factor_formulation = factor_formulation
self.variables = variables
self.factor_resources = resource
def get_factor_information(self):
return f"""factor_name: {self.factor_name}
factor_description: {self.factor_description}
factor_formulation: {self.factor_formulation}
variables: {str(self.variables)}"""
@staticmethod
def from_dict(dict):
return FactorImplementTask(**dict)
def __repr__(self) -> str:
return f"<{self.__class__.__name__}[{self.factor_name}]>"
class FactorEvovlingItem(EvolvableSubjects):
"""
Intermediate item of factor implementation.
"""
def __init__(
self,
target_factor_tasks: list[FactorImplementTask],
corresponding_gt_implementations: list[TaskImplementation] = None,
):
super().__init__()
self.target_factor_tasks = target_factor_tasks
self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks]
self.corresponding_selection: list = None
if corresponding_gt_implementations is not None and len(
corresponding_gt_implementations,
) != len(target_factor_tasks):
self.corresponding_gt_implementations = None
RDAgentLog().warning(
"The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
)
else:
self.corresponding_gt_implementations = corresponding_gt_implementations
class FileBasedFactorImplementation(FBTaskImplementation):
"""
This class is used to implement a factor by writing the code to a file.
Input data and output factor value are also written to files.
"""
# TODO: (Xiao) think raising errors may get better information for processing
FB_FROM_CACHE = "The factor value has been executed and stored in the instance variable."
FB_EXEC_SUCCESS = "Execution succeeded without error."
FB_CODE_NOT_SET = "code is not set."
FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
FB_OUTPUT_FILE_NOT_FOUND = "\nExpected output file not found."
FB_OUTPUT_FILE_FOUND = "\nExpected output file found."
def __init__(
self,
target_task: FactorImplementTask,
code,
executed_factor_value_dataframe=None,
raise_exception=False,
) -> None:
super().__init__(target_task)
self.code = code
self.executed_factor_value_dataframe = executed_factor_value_dataframe
self.logger = RDAgentLog()
self.raise_exception = raise_exception
self.workspace_path = Path(
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_workspace,
) / str(uuid.uuid4())
@staticmethod
def link_data_to_workspace(data_path: Path, workspace_path: Path):
data_path = Path(data_path)
workspace_path = Path(workspace_path)
for data_file_path in data_path.iterdir():
workspace_data_file_path = workspace_path / data_file_path.name
if workspace_data_file_path.exists():
workspace_data_file_path.unlink()
subprocess.run(
["ln", "-s", data_file_path, workspace_data_file_path],
check=False,
)
def execute_desc(self):
raise NotImplementedError
def prepare(self, *args, **kwargs):
# TODO move the prepare part code in execute into here
return super().prepare(*args, **kwargs)
def execute(self, store_result: bool = False) -> Tuple[str, pd.DataFrame]:
"""
execute the implementation and get the factor value by the following steps:
1. make the directory in workspace path
2. write the code to the file in the workspace path
3. link all the source data to the workspace path folder
4. execute the code
5. read the factor value from the output file in the workspace path folder
returns the execution feedback as a string and the factor value as a pandas dataframe
parameters:
store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation
"""
if self.code is None:
if self.raise_exception:
raise CodeFormatException(self.FB_CODE_NOT_SET)
else:
# TODO: to make the interface compatible with previous code. I kept the original behavior.
raise ValueError(self.FB_CODE_NOT_SET)
with FileLock(self.workspace_path / "execution.lock"):
if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
# NOTE: cache the result for the same code
target_file_name = md5_hash(self.code)
cache_file_path = (
Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl"
)
Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location).mkdir(
exist_ok=True, parents=True
)
if cache_file_path.exists() and not self.raise_exception:
cached_res = pickle.load(open(cache_file_path, "rb"))
if store_result and cached_res[1] is not None:
self.executed_factor_value_dataframe = cached_res[1]
return cached_res
if self.executed_factor_value_dataframe is not None:
return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
source_data_path = Path(
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder,
)
self.workspace_path.mkdir(exist_ok=True, parents=True)
source_data_path.mkdir(exist_ok=True, parents=True)
code_path = self.workspace_path / f"{self.target_task.factor_name}.py"
code_path.write_text(self.code)
self.link_data_to_workspace(source_data_path, self.workspace_path)
execution_feedback = self.FB_EXECUTION_SUCCEEDED
try:
subprocess.check_output(
f"python {code_path}",
shell=True,
cwd=self.workspace_path,
stderr=subprocess.STDOUT,
timeout=FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout,
)
except subprocess.CalledProcessError as e:
import site
execution_feedback = (
e.output.decode()
.replace(str(code_path.parent.absolute()), r"/path/to")
.replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
)
if len(execution_feedback) > 2000:
execution_feedback = (
execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
)
if self.raise_exception:
raise RuntimeErrorException(execution_feedback)
except subprocess.TimeoutExpired:
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
if self.raise_exception:
raise RuntimeErrorException(execution_feedback)
workspace_output_file_path = self.workspace_path / "result.h5"
if not workspace_output_file_path.exists():
execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND
executed_factor_value_dataframe = None
if self.raise_exception:
raise NoOutputException(execution_feedback)
else:
try:
executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path)
execution_feedback += self.FB_OUTPUT_FILE_FOUND
except Exception as e:
execution_feedback += f"Error found when reading hdf file: {e}"[:1000]
executed_factor_value_dataframe = None
if store_result and executed_factor_value_dataframe is not None:
self.executed_factor_value_dataframe = executed_factor_value_dataframe
if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
pickle.dump(
(execution_feedback, executed_factor_value_dataframe),
open(cache_file_path, "wb"),
)
return execution_feedback, executed_factor_value_dataframe
def __str__(self) -> str:
# NOTE:
# If the code cache works, the workspace will be None.
return f"File Factor[{self.target_task.factor_name}]: {self.workspace_path}"
def __repr__(self) -> str:
return self.__str__()
@staticmethod
def from_folder(task: FactorImplementTask, path: Union[str, Path], **kwargs):
path = Path(path)
factor_path = (path / task.factor_name).with_suffix(".py")
with factor_path.open("r") as f:
code = f.read()
return FileBasedFactorImplementation(task, code=code, **kwargs)
rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py
+39 -37
View File
@@ -8,20 +8,20 @@ from itertools import combinations
from pathlib import Path
from typing import Union
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.knowledge_management.graph import (
UndirectedGraph,
UndirectedNode,
)
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
FactorImplementationSingleFeedback,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
FactorImplementTask,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
FACTOR_IMPLEMENT_SETTINGS,
FactorTask,
)
from rdagent.core.evolving_framework import (
EvolvableSubjects,
@@ -31,9 +31,9 @@ from rdagent.core.evolving_framework import (
QueriedKnowledge,
RAGStrategy,
)
from rdagent.core.experiment import Implementation
from rdagent.core.log import RDAgentLog
from rdagent.core.prompts import Prompts
from rdagent.core.task import TaskImplementation
from rdagent.oai.llm_utils import (
APIBackend,
calculate_embedding_distance_between_str_list,
@@ -43,8 +43,8 @@ from rdagent.oai.llm_utils import (
class FactorImplementationKnowledge(Knowledge):
def __init__(
self,
target_task: FactorImplementTask,
implementation: TaskImplementation,
target_task: FactorTask,
implementation: Implementation,
feedback: FactorImplementationSingleFeedback,
) -> None:
"""
@@ -116,10 +116,10 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
evo_step = evolving_trace[trace_index]
implementations = evo_step.evolvable_subjects
feedback = evo_step.feedback
for task_index in range(len(implementations.target_factor_tasks)):
target_task = implementations.target_factor_tasks[task_index]
for task_index in range(len(implementations.sub_tasks)):
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_factor_information()
implementation = implementations.corresponding_implementations[task_index]
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
@@ -150,12 +150,12 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
queried_knowledge = FactorImplementationQueriedKnowledgeV1()
for target_factor_task in evo.target_factor_tasks:
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if target_factor_task_information in self.knowledgebase.success_task_info_set:
queried_knowledge.success_task_to_knowledge_dict[
target_factor_task_information
] = self.knowledgebase.implementation_trace[target_factor_task_information][-1]
queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace[target_factor_task_information][-1]
)
elif (
len(
self.knowledgebase.implementation_trace.setdefault(
@@ -167,14 +167,12 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
):
queried_knowledge.failed_task_info_set.add(target_factor_task_information)
else:
queried_knowledge.working_task_to_former_failed_knowledge_dict[
target_factor_task_information
] = self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[
-v1_query_former_trace_limit:
]
queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[-v1_query_former_trace_limit:]
)
knowledge_base_success_task_list = list(
self.knowledgebase.success_task_info_set,
@@ -195,9 +193,9 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
)[-1]
for index in similar_indexes
]
queried_knowledge.working_task_to_similar_successful_knowledge_dict[
target_factor_task_information
] = similar_successful_knowledge
queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = (
similar_successful_knowledge
)
return queried_knowledge
@@ -236,11 +234,11 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
evo_step = evolving_trace[trace_index]
implementations = evo_step.evolvable_subjects
feedback = evo_step.feedback
for task_index in range(len(implementations.target_factor_tasks)):
for task_index in range(len(implementations.sub_tasks)):
single_feedback = feedback[task_index]
target_task = implementations.target_factor_tasks[task_index]
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_factor_information()
implementation = implementations.corresponding_implementations[task_index]
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
@@ -323,8 +321,12 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
all_component_content = ""
for _, component_node in enumerate(all_component_nodes):
all_component_content += f"{component_node.content}, \n"
analyze_component_system_prompt = Template(self.prompt["analyze_component_prompt_v1_system"]).render(
all_component_content=all_component_content,
analyze_component_system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(self.prompt["analyze_component_prompt_v1_system"])
.render(
all_component_content=all_component_content,
)
)
analyze_component_user_prompt = target_factor_task_information
@@ -396,7 +398,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
"""
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
for target_factor_task in evo.target_factor_tasks:
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if (
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
@@ -427,9 +429,9 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
else:
current_index += 1
factor_implementation_queried_graph_knowledge.former_traces[
target_factor_task_information
] = former_trace_knowledge[-v2_query_former_trace_limit:]
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
former_trace_knowledge[-v2_query_former_trace_limit:]
)
else:
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
@@ -443,7 +445,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
knowledge_sampler: float = 1.0,
) -> QueriedKnowledge | None:
# queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge()
for target_factor_task in evo.target_factor_tasks:
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if (
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
@@ -583,7 +585,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
knowledge_sampler: float = 1.0,
) -> QueriedKnowledge | None:
# queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge()
for task_index, target_factor_task in enumerate(evo.target_factor_tasks):
for task_index, target_factor_task in enumerate(evo.sub_tasks):
target_factor_task_information = target_factor_task.get_factor_information()
factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
if (
rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py
+24 -16
View File
@@ -1,12 +1,12 @@
import json
from pathlib import Path
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
FactorEvovlingItem,
from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import (
from rdagent.components.task_implementation.factor_implementation.utils import (
get_data_folder_intro,
)
from rdagent.core.conf import RD_AGENT_SETTINGS
@@ -29,18 +29,22 @@ def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
return to_be_finished_task_index
def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvovlingItem, former_trace):
def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvolvingItem, former_trace):
tasks = []
for i in to_be_finished_task_index:
# find corresponding former trace for each task
target_factor_task_information = evo.target_factor_tasks[i].get_factor_information()
target_factor_task_information = evo.sub_tasks[i].get_factor_information()
if target_factor_task_information in former_trace:
tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information]))
tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information]))
system_prompt = Template(
scheduler_prompts["select_implementable_factor_system"],
).render(
data_info=get_data_folder_intro(),
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
scheduler_prompts["select_implementable_factor_system"],
)
.render(
data_info=get_data_folder_intro(),
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
@@ -48,11 +52,15 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:
)
while True:
user_prompt = Template(
scheduler_prompts["select_implementable_factor_user"],
).render(
factor_num=implementation_factors_per_round,
target_factor_tasks=tasks,
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
scheduler_prompts["select_implementable_factor_user"],
)
.render(
factor_num=implementation_factors_per_round,
sub_tasks=tasks,
)
)
if (
session.build_chat_completion_message_and_calculate_token(