gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-29 00:17:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

refine class design and inheritance first version code (#41)

Browse Source 
* refine class design and inheritance first version code

* fix all typos

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
2024-07-02 17:58:37 +08:00
committed by GitHub
parent 0faf232b81
commit 1b31bcd2c2
39 changed files with 717 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/task_implementation/factor_implementation/CoSTEER.py
+15 -13
View File
@@ -2,31 +2,33 @@ import pickle
from pathlib import Path
from typing import List
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
FactorImplementationEvaluatorV1,
FactorImplementationsMultiEvaluator,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
FactorEvolvingStrategyWithGraph,
)
from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
FactorEvovlingItem,
FactorImplementTask,
)
from rdagent.components.task_implementation.factor_implementation.evolving.knowledge_management import (
FactorImplementationGraphKnowledgeBase,
FactorImplementationGraphRAGStrategy,
FactorImplementationKnowledgeBaseV1,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
FACTOR_IMPLEMENT_SETTINGS,
from rdagent.components.task_implementation.factor_implementation.factor import (
FactorExperiment,
)
from rdagent.core.evolving_agent import RAGEvoAgent
from rdagent.core.implementation import TaskGenerator
from rdagent.core.task import TaskImplementation
from rdagent.core.experiment import Experiment
from rdagent.core.task_generator import TaskGenerator
class CoSTEERFG(TaskGenerator):
class CoSTEERFG(TaskGenerator[FactorExperiment]):
def __init__(
self,
with_knowledge: bool = True,
@@ -74,7 +76,7 @@ class CoSTEERFG(TaskGenerator):
)
return factor_knowledge_base
def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]:
def generate(self, exp: FactorExperiment) -> FactorExperiment:
# init knowledge base
factor_knowledge_base = self.load_or_init_knowledge_base(
former_knowledge_base_path=self.knowledge_base_path,
@@ -83,8 +85,8 @@ class CoSTEERFG(TaskGenerator):
# init rag method
self.rag = FactorImplementationGraphRAGStrategy(factor_knowledge_base)
# init indermediate items
factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks)
# init intermediate items
factor_implementations = FactorEvolvingItem(sub_tasks=exp.sub_tasks)
self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
@@ -100,5 +102,5 @@ class CoSTEERFG(TaskGenerator):
if self.new_knowledge_base_path is not None:
pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
self.knowledge_base = factor_knowledge_base
self.latest_factor_implementations = tasks
self.latest_factor_implementations = exp.sub_tasks
return factor_implementations
rdagent/components/task_implementation/factor_implementation/share_modules/factor_implementation_config.py → rdagent/components/task_implementation/factor_implementation/config.py
View File
rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py
+98 -82
View File
@@ -5,21 +5,23 @@ from pathlib import Path
from typing import List, Tuple
import pandas as pd
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
FactorEvovlingItem,
FactorImplementTask,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
FactorTask,
)
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evaluation import Evaluator
from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
from rdagent.core.experiment import Implementation
from rdagent.core.log import RDAgentLog
from rdagent.core.prompts import Prompts
from rdagent.core.task import TaskImplementation
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.oai.llm_utils import APIBackend
@@ -33,8 +35,8 @@ class FactorImplementationEvaluator(Evaluator):
@abstractmethod
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
"""You can get the dataframe by
@@ -52,7 +54,7 @@ class FactorImplementationEvaluator(Evaluator):
"""
raise NotImplementedError("Please implement the `evaluator` method")
def _get_df(self, gt: TaskImplementation, gen: TaskImplementation):
def _get_df(self, gt: Implementation, gen: Implementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
if isinstance(gen_df, pd.Series):
@@ -65,11 +67,11 @@ class FactorImplementationEvaluator(Evaluator):
class FactorImplementationCodeEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementTask,
implementation: TaskImplementation,
target_task: FactorTask,
implementation: Implementation,
execution_feedback: str,
factor_value_feedback: str = "",
gt_implementation: TaskImplementation = None,
gt_implementation: Implementation = None,
**kwargs,
):
factor_information = target_task.get_factor_information()
@@ -78,14 +80,18 @@ class FactorImplementationCodeEvaluator(Evaluator):
system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
evaluate_prompts["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_code_feedback_v1_user"],
)
.render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
)
while (
APIBackend().build_messages_and_calculate_token(
@@ -96,14 +102,18 @@ class FactorImplementationCodeEvaluator(Evaluator):
> RD_AGENT_SETTINGS.chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
evaluate_prompts["evaluator_code_feedback_v1_user"],
).render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_code_feedback_v1_user"],
)
.render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
)
critic_response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
@@ -117,8 +127,8 @@ class FactorImplementationCodeEvaluator(Evaluator):
class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -143,8 +153,8 @@ class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
idx_name_right = gen_df.index.names == ("datetime", "instrument")
@@ -166,8 +176,8 @@ class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -186,8 +196,8 @@ class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -206,8 +216,8 @@ class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -226,8 +236,8 @@ class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -259,8 +269,8 @@ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
def evaluate(
self,
gt: TaskImplementation,
gen: TaskImplementation,
gt: Implementation,
gen: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt, gen)
@@ -293,7 +303,7 @@ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
class FactorImplementationValEvaluator(FactorImplementationEvaluator):
def evaluate(self, gt: TaskImplementation, gen: TaskImplementation):
def evaluate(self, gt: Implementation, gen: Implementation):
_, gt_df = gt.execute()
_, gen_df = gen.execute()
# FIXME: refactor the two classes
@@ -473,7 +483,7 @@ def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: i
class FactorImplementationFinalDecisionEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
execution_feedback: str,
value_feedback: str,
code_feedback: str,
@@ -483,17 +493,21 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
"evaluator_final_decision_v1_system"
]
execution_feedback_to_render = execution_feedback
user_prompt = Template(
evaluate_prompts["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_final_decision_v1_user"],
)
.render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
)
while (
APIBackend().build_messages_and_calculate_token(
@@ -504,17 +518,21 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator):
> RD_AGENT_SETTINGS.chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
user_prompt = Template(
evaluate_prompts["evaluator_final_decision_v1_user"],
).render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_final_decision_v1_user"],
)
.render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
)
final_evaluation_dict = json.loads(
@@ -584,9 +602,9 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
def evaluate(
self,
target_task: FactorImplementTask,
implementation: TaskImplementation,
gt_implementation: TaskImplementation = None,
target_task: FactorTask,
implementation: Implementation,
gt_implementation: Implementation = None,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorImplementationSingleFeedback:
@@ -681,23 +699,23 @@ class FactorImplementationsMultiEvaluator(Evaluator):
def evaluate(
self,
evo: FactorEvovlingItem,
evo: FactorEvolvingItem,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorImplementationsMultiFeedback:
multi_implementation_feedback = FactorImplementationsMultiFeedback()
# for index in range(len(evo.target_factor_tasks)):
# corresponding_implementation = evo.corresponding_implementations[index]
# for index in range(len(evo.sub_tasks)):
# corresponding_implementation = evo.sub_implementations[index]
# corresponding_gt_implementation = (
# evo.corresponding_gt_implementations[index]
# if evo.corresponding_gt_implementations is not None
# evo.sub_gt_implementations[index]
# if evo.sub_gt_implementations is not None
# else None
# )
# multi_implementation_feedback.append(
# self.single_factor_implementation_evaluator.evaluate(
# target_task=evo.target_factor_tasks[index],
# target_task=evo.sub_tasks[index],
# implementation=corresponding_implementation,
# gt_implementation=corresponding_gt_implementation,
# queried_knowledge=queried_knowledge,
@@ -705,18 +723,16 @@ class FactorImplementationsMultiEvaluator(Evaluator):
# )
calls = []
for index in range(len(evo.target_factor_tasks)):
corresponding_implementation = evo.corresponding_implementations[index]
for index in range(len(evo.sub_tasks)):
corresponding_implementation = evo.sub_implementations[index]
corresponding_gt_implementation = (
evo.corresponding_gt_implementations[index]
if evo.corresponding_gt_implementations is not None
else None
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
)
calls.append(
(
self.single_factor_implementation_evaluator.evaluate,
(
evo.target_factor_tasks[index],
evo.sub_tasks[index],
corresponding_implementation,
corresponding_gt_implementation,
queried_knowledge,
rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py
+30
View File
@@ -0,0 +1,30 @@
from rdagent.components.task_implementation.factor_implementation.factor import (
FactorExperiment,
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.core.log import RDAgentLog
class FactorEvolvingItem(FactorExperiment[FactorTask, FileBasedFactorImplementation], EvolvableSubjects):
"""
Intermediate item of factor implementation.
"""
def __init__(
self,
sub_tasks: list[FactorTask],
sub_gt_implementations: list[FileBasedFactorImplementation] = None,
):
FactorExperiment.__init__(self, sub_tasks=sub_tasks)
self.corresponding_selection: list = None
if sub_gt_implementations is not None and len(
sub_gt_implementations,
) != len(self.sub_tasks):
self.sub_gt_implementations = None
RDAgentLog().warning(
"The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
)
else:
self.sub_gt_implementations = sub_gt_implementations
rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py
+51 -37
View File
@@ -6,27 +6,29 @@ from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
FactorEvovlingItem,
FactorImplementTask,
FileBasedFactorImplementation,
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.task_implementation.factor_implementation.evolving.scheduler import (
LLMSelect,
RandomSelect,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
FACTOR_IMPLEMENT_SETTINGS,
from rdagent.components.task_implementation.factor_implementation.factor import (
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import (
from rdagent.components.task_implementation.factor_implementation.utils import (
get_data_folder_intro,
)
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
from rdagent.core.experiment import Implementation
from rdagent.core.prompts import Prompts
from rdagent.core.task import TaskImplementation
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.oai.llm_utils import APIBackend
@@ -43,27 +45,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
@abstractmethod
def implement_one_factor(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
queried_knowledge: QueriedKnowledge = None,
) -> TaskImplementation:
) -> Implementation:
raise NotImplementedError
def evolve(
self,
*,
evo: FactorEvovlingItem,
evo: FactorEvolvingItem,
queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
**kwargs,
) -> FactorEvovlingItem:
) -> FactorEvolvingItem:
self.num_loop += 1
new_evo = deepcopy(evo)
# 1.找出需要evolve的factor
to_be_finished_task_index = []
for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
for index, target_factor_task in enumerate(new_evo.sub_tasks):
target_factor_task_desc = target_factor_task.get_factor_information()
if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
new_evo.corresponding_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
target_factor_task_desc
].implementation
elif (
@@ -95,18 +97,18 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
result = multiprocessing_wrapper(
[
(self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge))
(self.implement_one_factor, (new_evo.sub_tasks[target_index], queried_knowledge))
for target_index in to_be_finished_task_index
],
n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
)
for index, target_index in enumerate(to_be_finished_task_index):
new_evo.corresponding_implementations[target_index] = result[index]
new_evo.sub_implementations[target_index] = result[index]
# for target_index in to_be_finished_task_index:
# new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
# new_evo.target_factor_tasks[target_index], queried_knowledge
# new_evo.sub_implementations[target_index] = self.implement_one_factor(
# new_evo.sub_tasks[target_index], queried_knowledge
# )
new_evo.corresponding_selection = to_be_finished_task_index
@@ -117,9 +119,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_factor(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
) -> TaskImplementation:
) -> Implementation:
factor_information_str = target_task.get_factor_information()
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -140,11 +142,15 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = Template(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
).render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
)
.render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
session_system_prompt=system_prompt,
@@ -153,7 +159,8 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
while True:
user_prompt = (
Template(
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_user"],
)
.render(
@@ -196,9 +203,9 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
def implement_one_factor(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
queried_knowledge,
) -> TaskImplementation:
) -> Implementation:
error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
# 1. 提取因子的背景信息
target_factor_task_information = target_task.get_factor_information()
@@ -231,11 +238,15 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = Template(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
).render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
)
.render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
@@ -254,7 +265,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
and len(queried_former_failed_knowledge_to_render) != 0
):
error_summary_system_prompt = (
Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
Environment(undefined=StrictUndefined)
.from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
.render(
factor_information_str=target_factor_task_information,
code_and_feedback=queried_former_failed_knowledge_to_render[
@@ -268,7 +280,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
)
while True:
error_summary_user_prompt = (
Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
Environment(undefined=StrictUndefined)
.from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
.render(
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
)
@@ -287,7 +300,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
)
# 构建user_prompt。开始写代码
user_prompt = (
Template(
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v2_user"],
)
.render(
rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py
+39 -37
View File
@@ -8,20 +8,20 @@ from itertools import combinations
from pathlib import Path
from typing import Union
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.knowledge_management.graph import (
UndirectedGraph,
UndirectedNode,
)
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
FactorImplementationSingleFeedback,
)
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
FactorImplementTask,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
FACTOR_IMPLEMENT_SETTINGS,
FactorTask,
)
from rdagent.core.evolving_framework import (
EvolvableSubjects,
@@ -31,9 +31,9 @@ from rdagent.core.evolving_framework import (
QueriedKnowledge,
RAGStrategy,
)
from rdagent.core.experiment import Implementation
from rdagent.core.log import RDAgentLog
from rdagent.core.prompts import Prompts
from rdagent.core.task import TaskImplementation
from rdagent.oai.llm_utils import (
APIBackend,
calculate_embedding_distance_between_str_list,
@@ -43,8 +43,8 @@ from rdagent.oai.llm_utils import (
class FactorImplementationKnowledge(Knowledge):
def __init__(
self,
target_task: FactorImplementTask,
implementation: TaskImplementation,
target_task: FactorTask,
implementation: Implementation,
feedback: FactorImplementationSingleFeedback,
) -> None:
"""
@@ -116,10 +116,10 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
evo_step = evolving_trace[trace_index]
implementations = evo_step.evolvable_subjects
feedback = evo_step.feedback
for task_index in range(len(implementations.target_factor_tasks)):
target_task = implementations.target_factor_tasks[task_index]
for task_index in range(len(implementations.sub_tasks)):
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_factor_information()
implementation = implementations.corresponding_implementations[task_index]
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
@@ -150,12 +150,12 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
queried_knowledge = FactorImplementationQueriedKnowledgeV1()
for target_factor_task in evo.target_factor_tasks:
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if target_factor_task_information in self.knowledgebase.success_task_info_set:
queried_knowledge.success_task_to_knowledge_dict[
target_factor_task_information
] = self.knowledgebase.implementation_trace[target_factor_task_information][-1]
queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace[target_factor_task_information][-1]
)
elif (
len(
self.knowledgebase.implementation_trace.setdefault(
@@ -167,14 +167,12 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
):
queried_knowledge.failed_task_info_set.add(target_factor_task_information)
else:
queried_knowledge.working_task_to_former_failed_knowledge_dict[
target_factor_task_information
] = self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[
-v1_query_former_trace_limit:
]
queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[-v1_query_former_trace_limit:]
)
knowledge_base_success_task_list = list(
self.knowledgebase.success_task_info_set,
@@ -195,9 +193,9 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
)[-1]
for index in similar_indexes
]
queried_knowledge.working_task_to_similar_successful_knowledge_dict[
target_factor_task_information
] = similar_successful_knowledge
queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = (
similar_successful_knowledge
)
return queried_knowledge
@@ -236,11 +234,11 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
evo_step = evolving_trace[trace_index]
implementations = evo_step.evolvable_subjects
feedback = evo_step.feedback
for task_index in range(len(implementations.target_factor_tasks)):
for task_index in range(len(implementations.sub_tasks)):
single_feedback = feedback[task_index]
target_task = implementations.target_factor_tasks[task_index]
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_factor_information()
implementation = implementations.corresponding_implementations[task_index]
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
@@ -323,8 +321,12 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
all_component_content = ""
for _, component_node in enumerate(all_component_nodes):
all_component_content += f"{component_node.content}, \n"
analyze_component_system_prompt = Template(self.prompt["analyze_component_prompt_v1_system"]).render(
all_component_content=all_component_content,
analyze_component_system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(self.prompt["analyze_component_prompt_v1_system"])
.render(
all_component_content=all_component_content,
)
)
analyze_component_user_prompt = target_factor_task_information
@@ -396,7 +398,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
"""
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
for target_factor_task in evo.target_factor_tasks:
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if (
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
@@ -427,9 +429,9 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
else:
current_index += 1
factor_implementation_queried_graph_knowledge.former_traces[
target_factor_task_information
] = former_trace_knowledge[-v2_query_former_trace_limit:]
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
former_trace_knowledge[-v2_query_former_trace_limit:]
)
else:
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
@@ -443,7 +445,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
knowledge_sampler: float = 1.0,
) -> QueriedKnowledge | None:
# queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge()
for target_factor_task in evo.target_factor_tasks:
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if (
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
@@ -583,7 +585,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
knowledge_sampler: float = 1.0,
) -> QueriedKnowledge | None:
# queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge()
for task_index, target_factor_task in enumerate(evo.target_factor_tasks):
for task_index, target_factor_task in enumerate(evo.sub_tasks):
target_factor_task_information = target_factor_task.get_factor_information()
factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
if (
rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py
+24 -16
View File
@@ -1,12 +1,12 @@
import json
from pathlib import Path
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
FactorEvovlingItem,
from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import (
from rdagent.components.task_implementation.factor_implementation.utils import (
get_data_folder_intro,
)
from rdagent.core.conf import RD_AGENT_SETTINGS
@@ -29,18 +29,22 @@ def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
return to_be_finished_task_index
def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvovlingItem, former_trace):
def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvolvingItem, former_trace):
tasks = []
for i in to_be_finished_task_index:
# find corresponding former trace for each task
target_factor_task_information = evo.target_factor_tasks[i].get_factor_information()
target_factor_task_information = evo.sub_tasks[i].get_factor_information()
if target_factor_task_information in former_trace:
tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information]))
tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information]))
system_prompt = Template(
scheduler_prompts["select_implementable_factor_system"],
).render(
data_info=get_data_folder_intro(),
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
scheduler_prompts["select_implementable_factor_system"],
)
.render(
data_info=get_data_folder_intro(),
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
@@ -48,11 +52,15 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo:
)
while True:
user_prompt = Template(
scheduler_prompts["select_implementable_factor_user"],
).render(
factor_num=implementation_factors_per_round,
target_factor_tasks=tasks,
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
scheduler_prompts["select_implementable_factor_user"],
)
.render(
factor_num=implementation_factors_per_round,
sub_tasks=tasks,
)
)
if (
session.build_chat_completion_message_and_calculate_token(
rdagent/components/task_implementation/factor_implementation/evolving/factor.py → rdagent/components/task_implementation/factor_implementation/factor.py
+11 -39
View File
@@ -9,27 +9,21 @@ from typing import Tuple, Union
import pandas as pd
from filelock import FileLock
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.core.exception import (
CodeFormatException,
NoOutputException,
RuntimeErrorException,
)
from rdagent.core.experiment import Experiment, FBImplementation, Task
from rdagent.core.log import RDAgentLog
from rdagent.core.task import (
BaseTask,
FBTaskImplementation,
TaskImplementation,
TestCase,
)
from rdagent.oai.llm_utils import md5_hash
class FactorImplementTask(BaseTask):
# TODO: generalized the attributes into the BaseTask
class FactorTask(Task):
# TODO: generalized the attributes into the Task
# - factor_* -> *
def __init__(
self,
@@ -53,38 +47,13 @@ variables: {str(self.variables)}"""
@staticmethod
def from_dict(dict):
return FactorImplementTask(**dict)
return FactorTask(**dict)
def __repr__(self) -> str:
return f"<{self.__class__.__name__}[{self.factor_name}]>"
class FactorEvovlingItem(EvolvableSubjects):
"""
Intermediate item of factor implementation.
"""
def __init__(
self,
target_factor_tasks: list[FactorImplementTask],
corresponding_gt_implementations: list[TaskImplementation] = None,
):
super().__init__()
self.target_factor_tasks = target_factor_tasks
self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks]
self.corresponding_selection: list = None
if corresponding_gt_implementations is not None and len(
corresponding_gt_implementations,
) != len(target_factor_tasks):
self.corresponding_gt_implementations = None
RDAgentLog().warning(
"The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
)
else:
self.corresponding_gt_implementations = corresponding_gt_implementations
class FileBasedFactorImplementation(FBTaskImplementation):
class FileBasedFactorImplementation(FBImplementation):
"""
This class is used to implement a factor by writing the code to a file.
Input data and output factor value are also written to files.
@@ -100,7 +69,7 @@ class FileBasedFactorImplementation(FBTaskImplementation):
def __init__(
self,
target_task: FactorImplementTask,
target_task: FactorTask,
code,
executed_factor_value_dataframe=None,
raise_exception=False,
@@ -243,9 +212,12 @@ class FileBasedFactorImplementation(FBTaskImplementation):
return self.__str__()
@staticmethod
def from_folder(task: FactorImplementTask, path: Union[str, Path], **kwargs):
def from_folder(task: FactorTask, path: Union[str, Path], **kwargs):
path = Path(path)
factor_path = (path / task.factor_name).with_suffix(".py")
with factor_path.open("r") as f:
code = f.read()
return FileBasedFactorImplementation(task, code=code, **kwargs)
FactorExperiment = Experiment
rdagent/components/task_implementation/factor_implementation/prompts.yaml
+2 -2
View File
@@ -45,7 +45,7 @@ evolving_strategy_factor_implementation_v1_system: |-
2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the factor value. You should analyze the feedback and try to correct the latest code.
3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code.
Your must write your code based on your former lastest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
{% if queried_former_failed_knowledge|length != 0 %}
--------------Your former latest attempt:---------------
{% for former_failed_knowledge in queried_former_failed_knowledge %}
@@ -199,7 +199,7 @@ select_implementable_factor_system: |-
select_implementable_factor_user: |-
Number of factor you should pick: {{ factor_num }}
{% for factor_info in target_factor_tasks %}
{% for factor_info in sub_tasks %}
=============Factor index:{{factor_info[0]}}:=============
=====Factor name:=====
{{ factor_info[1].factor_name }}
rdagent/components/task_implementation/factor_implementation/share_modules/factor_implementation_utils.py → rdagent/components/task_implementation/factor_implementation/utils.py
+5 -8
View File
@@ -3,12 +3,9 @@ from pathlib import Path
import pandas as pd
# render it with jinja
from jinja2 import Template
from jinja2 import Environment, StrictUndefined
from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
FactorImplementTask,
)
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
from rdagent.components.task_implementation.factor_implementation.config import (
FACTOR_IMPLEMENT_SETTINGS,
)
@@ -19,11 +16,11 @@ TPL = """
````
"""
# Create a Jinja template from the string
JJ_TPL = Template(TPL)
JJ_TPL = Environment(undefined=StrictUndefined).from_string(TPL)
def get_data_folder_intro():
"""Direclty get the info of the data folder.
"""Directly get the info of the data folder.
It is for preparing prompting message.
"""
content_l = []
@@ -53,4 +50,4 @@ def get_data_folder_intro():
raise NotImplementedError(
f"file type {p.name} is not supported. Please implement its description function.",
)
return "\n ----------------- file spliter -------------\n".join(content_l)
return "\n ----------------- file splitter -------------\n".join(content_l)