mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 23:47:46 +00:00
feat: a unified CoSTEER to fit more scenarios (#491)
* Use ExtendedBaseSettings to replace BaseSettings * update a more general way to pass the default setting * update all code * fix CI * fix CI * fix qlib scenario * fix CI * fix CI * fix CI & add data science interfaces * remove redundant code * abandon costeer knowledge base v1 --------- Co-authored-by: Xu Yang <xuyang1@microsoft.com> Co-authored-by: XianBW <36835909+XianBW@users.noreply.github.com>
This commit is contained in:
Xu Yang
@@ -1,113 +0,0 @@
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
|
||||
FactorEvaluatorForCoder,
|
||||
FactorMultiEvaluator,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
|
||||
FactorEvolvingItem,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evolving_agent import (
|
||||
FactorRAGEvoAgent,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evolving_strategy import (
|
||||
FactorEvolvingStrategyWithGraph,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import (
|
||||
FactorGraphKnowledgeBase,
|
||||
FactorGraphRAGStrategy,
|
||||
FactorKnowledgeBaseV1,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.factor import FactorExperiment
|
||||
from rdagent.core.developer import Developer
|
||||
from rdagent.core.evolving_agent import RAGEvoAgent
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
|
||||
|
||||
class FactorCoSTEER(Developer[FactorExperiment]):
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
with_knowledge: bool = True,
|
||||
with_feedback: bool = True,
|
||||
knowledge_self_gen: bool = True,
|
||||
filter_final_evo: bool = True,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.max_loop = FACTOR_IMPLEMENT_SETTINGS.max_loop
|
||||
self.knowledge_base_path = (
|
||||
Path(FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path)
|
||||
if FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path is not None
|
||||
else None
|
||||
)
|
||||
self.new_knowledge_base_path = (
|
||||
Path(FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path)
|
||||
if FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path is not None
|
||||
else None
|
||||
)
|
||||
self.with_knowledge = with_knowledge
|
||||
self.with_feedback = with_feedback
|
||||
self.knowledge_self_gen = knowledge_self_gen
|
||||
self.filter_final_evo = filter_final_evo
|
||||
self.evolving_strategy = FactorEvolvingStrategyWithGraph(scen=self.scen)
|
||||
# declare the factor evaluator
|
||||
self.factor_evaluator = FactorMultiEvaluator(FactorEvaluatorForCoder(scen=self.scen), scen=self.scen)
|
||||
self.evolving_version = 2
|
||||
|
||||
def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
|
||||
if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
|
||||
factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
|
||||
if self.evolving_version == 1 and not isinstance(factor_knowledge_base, FactorKnowledgeBaseV1):
|
||||
raise ValueError("The former knowledge base is not compatible with the current version")
|
||||
elif self.evolving_version == 2 and not isinstance(
|
||||
factor_knowledge_base,
|
||||
FactorGraphKnowledgeBase,
|
||||
):
|
||||
raise ValueError("The former knowledge base is not compatible with the current version")
|
||||
else:
|
||||
factor_knowledge_base = (
|
||||
FactorGraphKnowledgeBase(
|
||||
init_component_list=component_init_list,
|
||||
)
|
||||
if self.evolving_version == 2
|
||||
else FactorKnowledgeBaseV1()
|
||||
)
|
||||
return factor_knowledge_base
|
||||
|
||||
def develop(self, exp: FactorExperiment) -> FactorExperiment:
|
||||
# init knowledge base
|
||||
factor_knowledge_base = self.load_or_init_knowledge_base(
|
||||
former_knowledge_base_path=self.knowledge_base_path,
|
||||
component_init_list=[],
|
||||
)
|
||||
# init rag method
|
||||
self.rag = FactorGraphRAGStrategy(factor_knowledge_base)
|
||||
|
||||
# init intermediate items
|
||||
factor_experiment = FactorEvolvingItem.from_experiment(exp)
|
||||
|
||||
self.evolve_agent = FactorRAGEvoAgent(
|
||||
max_loop=self.max_loop,
|
||||
evolving_strategy=self.evolving_strategy,
|
||||
rag=self.rag,
|
||||
with_knowledge=self.with_knowledge,
|
||||
with_feedback=self.with_feedback,
|
||||
knowledge_self_gen=self.knowledge_self_gen,
|
||||
)
|
||||
|
||||
factor_experiment = self.evolve_agent.multistep_evolve(
|
||||
factor_experiment,
|
||||
self.factor_evaluator,
|
||||
filter_final_evo=self.filter_final_evo,
|
||||
)
|
||||
|
||||
# save new knowledge base
|
||||
if self.new_knowledge_base_path is not None:
|
||||
pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
|
||||
logger.info(f"New knowledge base saved to {self.new_knowledge_base_path}")
|
||||
exp.sub_workspace_list = factor_experiment.sub_workspace_list
|
||||
return exp
|
||||
@@ -1,37 +0,0 @@
|
||||
from rdagent.components.coder.factor_coder.factor import (
|
||||
FactorExperiment,
|
||||
FactorFBWorkspace,
|
||||
FactorTask,
|
||||
)
|
||||
from rdagent.core.evolving_framework import EvolvableSubjects
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
|
||||
|
||||
class FactorEvolvingItem(FactorExperiment, EvolvableSubjects):
|
||||
"""
|
||||
Intermediate item of factor implementation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sub_tasks: list[FactorTask],
|
||||
sub_gt_implementations: list[FactorFBWorkspace] = None,
|
||||
):
|
||||
FactorExperiment.__init__(self, sub_tasks=sub_tasks)
|
||||
self.corresponding_selection: list = None
|
||||
if sub_gt_implementations is not None and len(
|
||||
sub_gt_implementations,
|
||||
) != len(self.sub_tasks):
|
||||
self.sub_gt_implementations = None
|
||||
logger.warning(
|
||||
"The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
|
||||
)
|
||||
else:
|
||||
self.sub_gt_implementations = sub_gt_implementations
|
||||
|
||||
@classmethod
|
||||
def from_experiment(cls, exp: FactorExperiment) -> "FactorExperiment":
|
||||
ei = cls(sub_tasks=exp.sub_tasks)
|
||||
ei.based_experiments = exp.based_experiments
|
||||
ei.experiment_workspace = exp.experiment_workspace
|
||||
return ei
|
||||
@@ -1,19 +0,0 @@
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evaluators import FactorMultiFeedback
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
|
||||
FactorEvolvingItem,
|
||||
)
|
||||
from rdagent.core.evaluation import Feedback
|
||||
from rdagent.core.evolving_agent import RAGEvoAgent
|
||||
from rdagent.core.evolving_framework import EvolvableSubjects
|
||||
|
||||
|
||||
class FactorRAGEvoAgent(RAGEvoAgent):
|
||||
def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects:
|
||||
assert isinstance(evo, FactorEvolvingItem)
|
||||
assert isinstance(feedback, list)
|
||||
assert len(evo.sub_workspace_list) == len(feedback)
|
||||
|
||||
for index in range(len(evo.sub_workspace_list)):
|
||||
if feedback[index] and not feedback[index].final_decision:
|
||||
evo.sub_workspace_list[index].clear()
|
||||
return evo
|
||||
@@ -1,331 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from abc import abstractmethod
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
|
||||
FactorEvolvingItem,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.scheduler import (
|
||||
LLMSelect,
|
||||
RandomSelect,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
|
||||
from rdagent.core.conf import RD_AGENT_SETTINGS
|
||||
from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
|
||||
from rdagent.core.experiment import Workspace
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.utils import multiprocessing_wrapper
|
||||
from rdagent.oai.llm_conf import LLM_SETTINGS
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import (
|
||||
FactorQueriedKnowledge,
|
||||
FactorQueriedKnowledgeV1,
|
||||
)
|
||||
|
||||
implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
|
||||
|
||||
class MultiProcessEvolvingStrategy(EvolvingStrategy):
|
||||
@abstractmethod
|
||||
def implement_one_factor(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
) -> Workspace:
|
||||
raise NotImplementedError
|
||||
|
||||
def evolve(
|
||||
self,
|
||||
*,
|
||||
evo: FactorEvolvingItem,
|
||||
queried_knowledge: FactorQueriedKnowledge | None = None,
|
||||
**kwargs,
|
||||
) -> FactorEvolvingItem:
|
||||
# 1.找出需要evolve的factor
|
||||
to_be_finished_task_index = []
|
||||
for index, target_factor_task in enumerate(evo.sub_tasks):
|
||||
target_factor_task_desc = target_factor_task.get_task_information()
|
||||
if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
|
||||
evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[
|
||||
target_factor_task_desc
|
||||
].implementation
|
||||
elif (
|
||||
target_factor_task_desc not in queried_knowledge.success_task_to_knowledge_dict
|
||||
and target_factor_task_desc not in queried_knowledge.failed_task_info_set
|
||||
):
|
||||
to_be_finished_task_index.append(index)
|
||||
|
||||
# 2. 选择selection方法
|
||||
# if the number of factors to be implemented is larger than the limit, we need to select some of them
|
||||
|
||||
if FACTOR_IMPLEMENT_SETTINGS.select_threshold < len(to_be_finished_task_index):
|
||||
# Select a fixed number of factors if the total exceeds the threshold
|
||||
if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
|
||||
to_be_finished_task_index = RandomSelect(
|
||||
to_be_finished_task_index,
|
||||
FACTOR_IMPLEMENT_SETTINGS.select_threshold,
|
||||
)
|
||||
|
||||
if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
|
||||
to_be_finished_task_index = LLMSelect(
|
||||
to_be_finished_task_index,
|
||||
FACTOR_IMPLEMENT_SETTINGS.select_threshold,
|
||||
evo,
|
||||
queried_knowledge.former_traces,
|
||||
self.scen,
|
||||
)
|
||||
|
||||
result = multiprocessing_wrapper(
|
||||
[
|
||||
(self.implement_one_factor, (evo.sub_tasks[target_index], queried_knowledge))
|
||||
for target_index in to_be_finished_task_index
|
||||
],
|
||||
n=RD_AGENT_SETTINGS.multi_proc_n,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
|
||||
|
||||
for index, target_index in enumerate(to_be_finished_task_index):
|
||||
if evo.sub_workspace_list[target_index] is None:
|
||||
evo.sub_workspace_list[target_index] = FactorFBWorkspace(target_task=evo.sub_tasks[target_index])
|
||||
evo.sub_workspace_list[target_index].inject_code(**{"factor.py": result[index]})
|
||||
|
||||
evo.corresponding_selection = to_be_finished_task_index
|
||||
|
||||
return evo
|
||||
|
||||
|
||||
class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
def implement_one_factor(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
queried_knowledge: FactorQueriedKnowledgeV1 = None,
|
||||
) -> str:
|
||||
factor_information_str = target_task.get_task_information()
|
||||
|
||||
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
|
||||
return queried_knowledge.success_task_to_knowledge_dict[factor_information_str].implementation
|
||||
elif queried_knowledge is not None and factor_information_str in queried_knowledge.failed_task_info_set:
|
||||
return None
|
||||
else:
|
||||
queried_similar_successful_knowledge = (
|
||||
queried_knowledge.working_task_to_similar_successful_knowledge_dict[factor_information_str]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
)
|
||||
queried_former_failed_knowledge = (
|
||||
queried_knowledge.working_task_to_former_failed_knowledge_dict[factor_information_str]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
)
|
||||
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
|
||||
|
||||
system_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
|
||||
)
|
||||
.render(
|
||||
scenario=self.scen.get_scenario_all_desc(target_task),
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
|
||||
)
|
||||
)
|
||||
session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session(
|
||||
session_system_prompt=system_prompt,
|
||||
)
|
||||
|
||||
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
|
||||
for _ in range(10): # max attempt to reduce the length of user_prompt
|
||||
user_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v1_user"],
|
||||
)
|
||||
.render(
|
||||
factor_information_str=factor_information_str,
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
if (
|
||||
session.build_chat_completion_message_and_calculate_token(
|
||||
user_prompt,
|
||||
)
|
||||
< LLM_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_former_failed_knowledge_to_render) > 1:
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
|
||||
elif len(queried_similar_successful_knowledge_to_render) > 1:
|
||||
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
|
||||
|
||||
code = json.loads(
|
||||
session.build_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
json_mode=True,
|
||||
),
|
||||
)["code"]
|
||||
|
||||
return code
|
||||
|
||||
|
||||
class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.num_loop = 0
|
||||
self.haveSelected = False
|
||||
|
||||
def implement_one_factor(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
queried_knowledge,
|
||||
) -> str:
|
||||
error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
|
||||
# 1. 提取因子的背景信息
|
||||
target_factor_task_information = target_task.get_task_information()
|
||||
|
||||
# 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_factor_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
):
|
||||
return queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information].implementation
|
||||
elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
|
||||
return None
|
||||
else:
|
||||
# 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
|
||||
queried_similar_component_knowledge = (
|
||||
queried_knowledge.component_with_success_task[target_factor_task_information]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
) # A list, [success task implement knowledge]
|
||||
|
||||
queried_similar_error_knowledge = (
|
||||
queried_knowledge.error_with_success_task[target_factor_task_information]
|
||||
if queried_knowledge is not None
|
||||
else {}
|
||||
) # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
|
||||
|
||||
queried_former_failed_knowledge = (
|
||||
queried_knowledge.former_traces[target_factor_task_information][0]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
)
|
||||
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
|
||||
|
||||
latest_attempt_to_latest_successful_execution = queried_knowledge.former_traces[
|
||||
target_factor_task_information
|
||||
][1]
|
||||
|
||||
system_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
|
||||
)
|
||||
.render(
|
||||
scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
|
||||
)
|
||||
)
|
||||
|
||||
session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session(
|
||||
session_system_prompt=system_prompt,
|
||||
)
|
||||
|
||||
queried_similar_component_knowledge_to_render = queried_similar_component_knowledge
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
|
||||
error_summary_critics = ""
|
||||
# 动态地防止prompt超长
|
||||
for _ in range(10): # max attempt to reduce the length of user_prompt
|
||||
# 总结error（可选）
|
||||
if (
|
||||
error_summary
|
||||
and len(queried_similar_error_knowledge_to_render) != 0
|
||||
and len(queried_former_failed_knowledge_to_render) != 0
|
||||
):
|
||||
error_summary_system_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
|
||||
.render(
|
||||
scenario=self.scen.get_scenario_all_desc(target_task),
|
||||
factor_information_str=target_factor_task_information,
|
||||
code_and_feedback=queried_former_failed_knowledge_to_render[
|
||||
-1
|
||||
].get_implementation_and_feedback_str(),
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
session_summary = APIBackend(
|
||||
use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache
|
||||
).build_chat_session(
|
||||
session_system_prompt=error_summary_system_prompt,
|
||||
)
|
||||
for _ in range(10): # max attempt to reduce the length of error_summary_user_prompt
|
||||
error_summary_user_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
|
||||
.render(
|
||||
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
if (
|
||||
session_summary.build_chat_completion_message_and_calculate_token(error_summary_user_prompt)
|
||||
< LLM_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_similar_error_knowledge_to_render) > 0:
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
|
||||
error_summary_critics = session_summary.build_chat_completion(
|
||||
user_prompt=error_summary_user_prompt,
|
||||
json_mode=False,
|
||||
)
|
||||
# 构建user_prompt。开始写代码
|
||||
user_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v2_user"],
|
||||
)
|
||||
.render(
|
||||
factor_information_str=target_factor_task_information,
|
||||
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
|
||||
queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
|
||||
error_summary=error_summary,
|
||||
error_summary_critics=error_summary_critics,
|
||||
latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
if (
|
||||
session.build_chat_completion_message_and_calculate_token(
|
||||
user_prompt,
|
||||
)
|
||||
< LLM_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_former_failed_knowledge_to_render) > 1:
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
|
||||
elif len(queried_similar_component_knowledge_to_render) > len(
|
||||
queried_similar_error_knowledge_to_render,
|
||||
):
|
||||
queried_similar_component_knowledge_to_render = queried_similar_component_knowledge_to_render[:-1]
|
||||
elif len(queried_similar_error_knowledge_to_render) > 0:
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
|
||||
|
||||
response = session.build_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
json_mode=True,
|
||||
)
|
||||
code = json.loads(response)["code"]
|
||||
return code
|
||||
@@ -1,930 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from itertools import combinations
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
|
||||
FactorSingleFeedback,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.factor import FactorTask
|
||||
from rdagent.components.knowledge_management.graph import (
|
||||
UndirectedGraph,
|
||||
UndirectedNode,
|
||||
)
|
||||
from rdagent.core.evolving_framework import (
|
||||
EvolvableSubjects,
|
||||
EvolvingKnowledgeBase,
|
||||
EvoStep,
|
||||
Knowledge,
|
||||
QueriedKnowledge,
|
||||
RAGStrategy,
|
||||
)
|
||||
from rdagent.core.experiment import Workspace
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_utils import (
|
||||
APIBackend,
|
||||
calculate_embedding_distance_between_str_list,
|
||||
)
|
||||
|
||||
|
||||
class FactorKnowledge(Knowledge):
|
||||
def __init__(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
implementation: Workspace,
|
||||
feedback: FactorSingleFeedback,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize a FactorKnowledge object. The FactorKnowledge object is used to store a factor implementation without the ground truth code and value.
|
||||
|
||||
Args:
|
||||
factor (Factor): The factor object associated with the KnowledgeManagement.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
self.target_task = target_task
|
||||
self.implementation = implementation.copy()
|
||||
self.feedback = feedback
|
||||
|
||||
def get_implementation_and_feedback_str(self) -> str:
|
||||
return f"""------------------Factor implementation code:------------------
|
||||
{self.implementation.code}
|
||||
------------------Factor implementation feedback:------------------
|
||||
{self.feedback!s}
|
||||
"""
|
||||
|
||||
|
||||
class FactorQueriedKnowledge(QueriedKnowledge):
|
||||
def __init__(self, success_task_to_knowledge_dict: dict = {}, failed_task_info_set: set = set()) -> None:
|
||||
self.success_task_to_knowledge_dict = success_task_to_knowledge_dict
|
||||
self.failed_task_info_set = failed_task_info_set
|
||||
|
||||
|
||||
class FactorKnowledgeBaseV1(EvolvingKnowledgeBase):
|
||||
def __init__(self, path: str | Path = None) -> None:
|
||||
self.implementation_trace: dict[str, FactorKnowledge] = dict()
|
||||
self.success_task_info_set: set[str] = set()
|
||||
|
||||
self.task_to_embedding = dict()
|
||||
super().__init__(path)
|
||||
|
||||
def query(self) -> QueriedKnowledge | None:
|
||||
"""
|
||||
Query the knowledge base to get the queried knowledge. So far is handled in RAG strategy.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FactorQueriedKnowledgeV1(FactorQueriedKnowledge):
|
||||
def __init__(self) -> None:
|
||||
self.working_task_to_former_failed_knowledge_dict = dict()
|
||||
self.working_task_to_similar_successful_knowledge_dict = dict()
|
||||
super().__init__()
|
||||
|
||||
|
||||
class FactorRAGStrategyV1(RAGStrategy):
|
||||
def __init__(self, knowledgebase: FactorKnowledgeBaseV1) -> None:
|
||||
super().__init__(knowledgebase)
|
||||
self.current_generated_trace_count = 0
|
||||
|
||||
def generate_knowledge(
|
||||
self,
|
||||
evolving_trace: list[EvoStep],
|
||||
*,
|
||||
return_knowledge: bool = False,
|
||||
) -> Knowledge | None:
|
||||
if len(evolving_trace) == self.current_generated_trace_count:
|
||||
return
|
||||
else:
|
||||
for trace_index in range(
|
||||
self.current_generated_trace_count,
|
||||
len(evolving_trace),
|
||||
):
|
||||
evo_step = evolving_trace[trace_index]
|
||||
implementations = evo_step.evolvable_subjects
|
||||
feedback = evo_step.feedback
|
||||
for task_index in range(len(implementations.sub_tasks)):
|
||||
target_task = implementations.sub_tasks[task_index]
|
||||
target_task_information = target_task.get_task_information()
|
||||
implementation = implementations.sub_workspace_list[task_index]
|
||||
single_feedback = feedback[task_index]
|
||||
if single_feedback is None:
|
||||
continue
|
||||
single_knowledge = FactorKnowledge(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
feedback=single_feedback,
|
||||
)
|
||||
if target_task_information not in self.knowledgebase.success_task_info_set:
|
||||
self.knowledgebase.implementation_trace.setdefault(
|
||||
target_task_information,
|
||||
[],
|
||||
).append(single_knowledge)
|
||||
|
||||
if single_feedback.final_decision == True:
|
||||
self.knowledgebase.success_task_info_set.add(
|
||||
target_task_information,
|
||||
)
|
||||
self.current_generated_trace_count = len(evolving_trace)
|
||||
|
||||
def query(
|
||||
self,
|
||||
evo: EvolvableSubjects,
|
||||
evolving_trace: list[EvoStep],
|
||||
) -> QueriedKnowledge | None:
|
||||
v1_query_former_trace_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_former_trace_limit
|
||||
v1_query_similar_success_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_similar_success_limit
|
||||
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
|
||||
|
||||
queried_knowledge = FactorQueriedKnowledgeV1()
|
||||
for target_factor_task in evo.sub_tasks:
|
||||
target_factor_task_information = target_factor_task.get_task_information()
|
||||
if target_factor_task_information in self.knowledgebase.success_task_info_set:
|
||||
queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
|
||||
self.knowledgebase.implementation_trace[target_factor_task_information][-1]
|
||||
)
|
||||
elif (
|
||||
len(
|
||||
self.knowledgebase.implementation_trace.setdefault(
|
||||
target_factor_task_information,
|
||||
[],
|
||||
),
|
||||
)
|
||||
>= fail_task_trial_limit
|
||||
):
|
||||
queried_knowledge.failed_task_info_set.add(target_factor_task_information)
|
||||
else:
|
||||
queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
|
||||
self.knowledgebase.implementation_trace.setdefault(
|
||||
target_factor_task_information,
|
||||
[],
|
||||
)[-v1_query_former_trace_limit:]
|
||||
)
|
||||
|
||||
knowledge_base_success_task_list = list(
|
||||
self.knowledgebase.success_task_info_set,
|
||||
)
|
||||
similarity = calculate_embedding_distance_between_str_list(
|
||||
[target_factor_task_information],
|
||||
knowledge_base_success_task_list,
|
||||
)[0]
|
||||
similar_indexes = sorted(
|
||||
range(len(similarity)),
|
||||
key=lambda i: similarity[i],
|
||||
reverse=True,
|
||||
)[:v1_query_similar_success_limit]
|
||||
similar_successful_knowledge = [
|
||||
self.knowledgebase.implementation_trace.setdefault(
|
||||
knowledge_base_success_task_list[index],
|
||||
[],
|
||||
)[-1]
|
||||
for index in similar_indexes
|
||||
]
|
||||
queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = (
|
||||
similar_successful_knowledge
|
||||
)
|
||||
return queried_knowledge
|
||||
|
||||
|
||||
class FactorQueriedGraphKnowledge(FactorQueriedKnowledge):
|
||||
# Aggregation of knowledge
|
||||
def __init__(
|
||||
self,
|
||||
former_traces: dict = {},
|
||||
component_with_success_task: dict = {},
|
||||
error_with_success_task: dict = {},
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.former_traces = former_traces
|
||||
self.component_with_success_task = component_with_success_task
|
||||
self.error_with_success_task = error_with_success_task
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class FactorGraphRAGStrategy(RAGStrategy):
|
||||
prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
|
||||
def __init__(self, knowledgebase: FactorGraphKnowledgeBase) -> None:
|
||||
super().__init__(knowledgebase)
|
||||
self.current_generated_trace_count = 0
|
||||
|
||||
def generate_knowledge(
|
||||
self,
|
||||
evolving_trace: list[EvoStep],
|
||||
*,
|
||||
return_knowledge: bool = False,
|
||||
) -> Knowledge | None:
|
||||
if len(evolving_trace) == self.current_generated_trace_count:
|
||||
return None
|
||||
|
||||
else:
|
||||
for trace_index in range(self.current_generated_trace_count, len(evolving_trace)):
|
||||
evo_step = evolving_trace[trace_index]
|
||||
implementations = evo_step.evolvable_subjects
|
||||
feedback = evo_step.feedback
|
||||
for task_index in range(len(implementations.sub_tasks)):
|
||||
single_feedback = feedback[task_index]
|
||||
target_task = implementations.sub_tasks[task_index]
|
||||
target_task_information = target_task.get_task_information()
|
||||
implementation = implementations.sub_workspace_list[task_index]
|
||||
single_feedback = feedback[task_index]
|
||||
if single_feedback is None:
|
||||
continue
|
||||
single_knowledge = FactorKnowledge(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
feedback=single_feedback,
|
||||
)
|
||||
if (
|
||||
target_task_information not in self.knowledgebase.success_task_to_knowledge_dict
|
||||
and implementation is not None
|
||||
):
|
||||
self.knowledgebase.working_trace_knowledge.setdefault(target_task_information, []).append(
|
||||
single_knowledge,
|
||||
) # save to working trace
|
||||
if single_feedback.final_decision == True:
|
||||
self.knowledgebase.success_task_to_knowledge_dict.setdefault(
|
||||
target_task_information,
|
||||
single_knowledge,
|
||||
)
|
||||
# Do summary for the last step and update the knowledge graph
|
||||
self.knowledgebase.update_success_task(
|
||||
target_task_information,
|
||||
)
|
||||
else:
|
||||
# generate error node and store into knowledge base
|
||||
error_analysis_result = []
|
||||
if not single_feedback.value_generated_flag:
|
||||
error_analysis_result = self.analyze_error(
|
||||
single_feedback.execution_feedback,
|
||||
feedback_type="execution",
|
||||
)
|
||||
else:
|
||||
error_analysis_result = self.analyze_error(
|
||||
single_feedback.factor_value_feedback,
|
||||
feedback_type="value",
|
||||
)
|
||||
self.knowledgebase.working_trace_error_analysis.setdefault(
|
||||
target_task_information,
|
||||
[],
|
||||
).append(
|
||||
error_analysis_result,
|
||||
) # save to working trace error record, for graph update
|
||||
|
||||
self.current_generated_trace_count = len(evolving_trace)
|
||||
return None
|
||||
|
||||
def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None:
|
||||
conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler
|
||||
factor_implementation_queried_graph_knowledge = FactorQueriedGraphKnowledge(
|
||||
success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict,
|
||||
)
|
||||
|
||||
factor_implementation_queried_graph_knowledge = self.former_trace_query(
|
||||
evo,
|
||||
factor_implementation_queried_graph_knowledge,
|
||||
FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit,
|
||||
FACTOR_IMPLEMENT_SETTINGS.v2_add_fail_attempt_to_latest_successful_execution,
|
||||
)
|
||||
factor_implementation_queried_graph_knowledge = self.component_query(
|
||||
evo,
|
||||
factor_implementation_queried_graph_knowledge,
|
||||
FACTOR_IMPLEMENT_SETTINGS.v2_query_component_limit,
|
||||
knowledge_sampler=conf_knowledge_sampler,
|
||||
)
|
||||
factor_implementation_queried_graph_knowledge = self.error_query(
|
||||
evo,
|
||||
factor_implementation_queried_graph_knowledge,
|
||||
FACTOR_IMPLEMENT_SETTINGS.v2_query_error_limit,
|
||||
knowledge_sampler=conf_knowledge_sampler,
|
||||
)
|
||||
return factor_implementation_queried_graph_knowledge
|
||||
|
||||
def analyze_component(
|
||||
self,
|
||||
target_factor_task_information,
|
||||
) -> list[UndirectedNode]: # Hardcode: certain component nodes
|
||||
all_component_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["component"])
|
||||
if not len(all_component_nodes):
|
||||
return []
|
||||
all_component_content = ""
|
||||
for _, component_node in enumerate(all_component_nodes):
|
||||
all_component_content += f"{component_node.content}, \n"
|
||||
analyze_component_system_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(self.prompt["analyze_component_prompt_v1_system"])
|
||||
.render(
|
||||
all_component_content=all_component_content,
|
||||
)
|
||||
)
|
||||
|
||||
analyze_component_user_prompt = target_factor_task_information
|
||||
try:
|
||||
component_no_list = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
system_prompt=analyze_component_system_prompt,
|
||||
user_prompt=analyze_component_user_prompt,
|
||||
json_mode=True,
|
||||
),
|
||||
)["component_no_list"]
|
||||
return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))]
|
||||
except:
|
||||
logger.warning("Error when analyzing components.")
|
||||
analyze_component_user_prompt = "Your response is not a valid component index list."
|
||||
|
||||
return []
|
||||
|
||||
def analyze_error(
|
||||
self,
|
||||
single_feedback,
|
||||
feedback_type="execution",
|
||||
) -> list[
|
||||
UndirectedNode | str
|
||||
]: # Hardcode: Raised errors, existed error nodes + not existed error nodes(here, they are strs)
|
||||
if feedback_type == "execution":
|
||||
match = re.search(
|
||||
r'File "(?P<file>.+)", line (?P<line>\d+), in (?P<function>.+)\n\s+(?P<error_line>.+)\n(?P<error_type>\w+): (?P<error_message>.+)',
|
||||
single_feedback,
|
||||
)
|
||||
if match:
|
||||
error_details = match.groupdict()
|
||||
# last_traceback = f'File "{error_details["file"]}", line {error_details["line"]}, in {error_details["function"]}\n {error_details["error_line"]}'
|
||||
error_type = error_details["error_type"]
|
||||
error_line = error_details["error_line"]
|
||||
error_contents = [f"ErrorType: {error_type}" + "\n" + f"Error line: {error_line}"]
|
||||
else:
|
||||
error_contents = ["Undefined Error"]
|
||||
elif feedback_type == "value": # value check error
|
||||
value_check_types = r"The source dataframe and the ground truth dataframe have different rows count.|The source dataframe and the ground truth dataframe have different index.|Some values differ by more than the tolerance of 1e-6.|No sufficient correlation found when shifting up|Something wrong happens when naming the multi indices of the dataframe."
|
||||
error_contents = re.findall(value_check_types, single_feedback)
|
||||
else:
|
||||
error_contents = ["Undefined Error"]
|
||||
|
||||
all_error_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["error"])
|
||||
if not len(all_error_nodes):
|
||||
return error_contents
|
||||
else:
|
||||
error_list = []
|
||||
for error_content in error_contents:
|
||||
for error_node in all_error_nodes:
|
||||
if error_content == error_node.content:
|
||||
error_list.append(error_node)
|
||||
else:
|
||||
error_list.append(error_content)
|
||||
if error_list[-1] in error_list[:-1]:
|
||||
error_list.pop()
|
||||
|
||||
return error_list
|
||||
|
||||
def former_trace_query(
|
||||
self,
|
||||
evo: EvolvableSubjects,
|
||||
factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
|
||||
v2_query_former_trace_limit: int = 5,
|
||||
v2_add_fail_attempt_to_latest_successful_execution: bool = False,
|
||||
) -> Union[QueriedKnowledge, set]:
|
||||
"""
|
||||
Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
|
||||
"""
|
||||
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
|
||||
|
||||
for target_factor_task in evo.sub_tasks:
|
||||
target_factor_task_information = target_factor_task.get_task_information()
|
||||
if (
|
||||
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
|
||||
and target_factor_task_information in self.knowledgebase.working_trace_knowledge
|
||||
and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information])
|
||||
>= fail_task_trial_limit
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.failed_task_info_set.add(target_factor_task_information)
|
||||
|
||||
if (
|
||||
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
|
||||
and target_factor_task_information
|
||||
not in factor_implementation_queried_graph_knowledge.failed_task_info_set
|
||||
and target_factor_task_information in self.knowledgebase.working_trace_knowledge
|
||||
):
|
||||
former_trace_knowledge = copy.copy(
|
||||
self.knowledgebase.working_trace_knowledge[target_factor_task_information],
|
||||
)
|
||||
# in former trace query we will delete the right trace in the following order:[..., value_generated_flag is True, value_generated_flag is False, ...]
|
||||
# because we think this order means a deterioration of the trial (like a wrong gradient descent)
|
||||
current_index = 1
|
||||
while current_index < len(former_trace_knowledge):
|
||||
if (
|
||||
not former_trace_knowledge[current_index].feedback.value_generated_flag
|
||||
and former_trace_knowledge[current_index - 1].feedback.value_generated_flag
|
||||
):
|
||||
former_trace_knowledge.pop(current_index)
|
||||
else:
|
||||
current_index += 1
|
||||
|
||||
latest_attempt = None
|
||||
if v2_add_fail_attempt_to_latest_successful_execution:
|
||||
# When the last successful execution is not the last one in the working trace, it means we have tried to correct it. We should tell the agent this fail trial to avoid endless loop in the future.
|
||||
if (
|
||||
len(former_trace_knowledge) > 0
|
||||
and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) > 1
|
||||
and self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
|
||||
former_trace_knowledge[-1]
|
||||
)
|
||||
< len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) - 1
|
||||
):
|
||||
latest_attempt = self.knowledgebase.working_trace_knowledge[target_factor_task_information][-1]
|
||||
|
||||
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
|
||||
former_trace_knowledge[-v2_query_former_trace_limit:],
|
||||
latest_attempt,
|
||||
)
|
||||
else:
|
||||
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ([], None)
|
||||
|
||||
return factor_implementation_queried_graph_knowledge
|
||||
|
||||
def component_query(
|
||||
self,
|
||||
evo: EvolvableSubjects,
|
||||
factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
|
||||
v2_query_component_limit: int = 5,
|
||||
knowledge_sampler: float = 1.0,
|
||||
) -> QueriedKnowledge | None:
|
||||
# queried_component_knowledge = FactorQueriedGraphComponentKnowledge()
|
||||
for target_factor_task in evo.sub_tasks:
|
||||
target_factor_task_information = target_factor_task.get_task_information()
|
||||
if (
|
||||
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
|
||||
or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
] = []
|
||||
else:
|
||||
if target_factor_task_information not in self.knowledgebase.task_to_component_nodes:
|
||||
self.knowledgebase.task_to_component_nodes[target_factor_task_information] = self.analyze_component(
|
||||
target_factor_task_information,
|
||||
)
|
||||
|
||||
component_analysis_result = self.knowledgebase.task_to_component_nodes[target_factor_task_information]
|
||||
|
||||
if len(component_analysis_result) > 1:
|
||||
task_des_node_list = self.knowledgebase.graph_query_by_intersection(
|
||||
component_analysis_result,
|
||||
constraint_labels=["task_description"],
|
||||
)
|
||||
single_component_constraint = (v2_query_component_limit // len(component_analysis_result)) + 1
|
||||
else:
|
||||
task_des_node_list = []
|
||||
single_component_constraint = v2_query_component_limit
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
] = []
|
||||
for component_node in component_analysis_result:
|
||||
# Reverse iterate, a trade-off with intersection search
|
||||
count = 0
|
||||
for task_des_node in self.knowledgebase.graph_query_by_node(
|
||||
node=component_node,
|
||||
step=1,
|
||||
constraint_labels=["task_description"],
|
||||
block=True,
|
||||
)[::-1]:
|
||||
if task_des_node not in task_des_node_list:
|
||||
task_des_node_list.append(task_des_node)
|
||||
count += 1
|
||||
if count >= single_component_constraint:
|
||||
break
|
||||
|
||||
for node in task_des_node_list:
|
||||
for searched_node in self.knowledgebase.graph_query_by_node(
|
||||
node=node,
|
||||
step=50,
|
||||
constraint_labels=[
|
||||
"task_success_implement",
|
||||
],
|
||||
block=True,
|
||||
):
|
||||
if searched_node.label == "task_success_implement":
|
||||
target_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[
|
||||
searched_node.id
|
||||
]
|
||||
if (
|
||||
target_knowledge
|
||||
not in factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
]
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
].append(target_knowledge)
|
||||
|
||||
# finally add embedding related knowledge
|
||||
knowledge_base_success_task_list = list(self.knowledgebase.success_task_to_knowledge_dict)
|
||||
|
||||
similarity = calculate_embedding_distance_between_str_list(
|
||||
[target_factor_task_information],
|
||||
knowledge_base_success_task_list,
|
||||
)[0]
|
||||
similar_indexes = sorted(
|
||||
range(len(similarity)),
|
||||
key=lambda i: similarity[i],
|
||||
reverse=True,
|
||||
)
|
||||
embedding_similar_successful_knowledge = [
|
||||
self.knowledgebase.success_task_to_knowledge_dict[knowledge_base_success_task_list[index]]
|
||||
for index in similar_indexes
|
||||
]
|
||||
for knowledge in embedding_similar_successful_knowledge:
|
||||
if (
|
||||
knowledge
|
||||
not in factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
]
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
].append(knowledge)
|
||||
|
||||
if knowledge_sampler > 0:
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
] = [
|
||||
knowledge
|
||||
for knowledge in factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
]
|
||||
if random.uniform(0, 1) <= knowledge_sampler
|
||||
]
|
||||
|
||||
# Make sure no less than half of the knowledge are from GT
|
||||
queried_knowledge_list = factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
]
|
||||
queried_from_gt_knowledge_list = [
|
||||
knowledge
|
||||
for knowledge in queried_knowledge_list
|
||||
if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == True
|
||||
]
|
||||
queried_without_gt_knowledge_list = [
|
||||
knowledge
|
||||
for knowledge in queried_knowledge_list
|
||||
if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False
|
||||
]
|
||||
queried_from_gt_knowledge_count = max(
|
||||
min((v2_query_component_limit // 2 + 1), len(queried_from_gt_knowledge_list)),
|
||||
v2_query_component_limit - len(queried_without_gt_knowledge_list),
|
||||
)
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
] = (
|
||||
queried_from_gt_knowledge_list[:queried_from_gt_knowledge_count]
|
||||
+ queried_without_gt_knowledge_list[: v2_query_component_limit - queried_from_gt_knowledge_count]
|
||||
)
|
||||
|
||||
return factor_implementation_queried_graph_knowledge
|
||||
|
||||
def error_query(
|
||||
self,
|
||||
evo: EvolvableSubjects,
|
||||
factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
|
||||
v2_query_error_limit: int = 5,
|
||||
knowledge_sampler: float = 1.0,
|
||||
) -> QueriedKnowledge | None:
|
||||
# queried_error_knowledge = FactorQueriedGraphErrorKnowledge()
|
||||
for task_index, target_factor_task in enumerate(evo.sub_tasks):
|
||||
target_factor_task_information = target_factor_task.get_task_information()
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
|
||||
if (
|
||||
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
|
||||
or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[
|
||||
target_factor_task_information
|
||||
] = []
|
||||
else:
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[
|
||||
target_factor_task_information
|
||||
] = []
|
||||
if (
|
||||
target_factor_task_information in self.knowledgebase.working_trace_error_analysis
|
||||
and len(self.knowledgebase.working_trace_error_analysis[target_factor_task_information]) > 0
|
||||
and len(factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information])
|
||||
> 0
|
||||
):
|
||||
queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[
|
||||
target_factor_task_information
|
||||
][0][-1]
|
||||
target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
|
||||
queried_last_trace,
|
||||
)
|
||||
last_knowledge_error_analysis_result = self.knowledgebase.working_trace_error_analysis[
|
||||
target_factor_task_information
|
||||
][target_index]
|
||||
else:
|
||||
last_knowledge_error_analysis_result = []
|
||||
|
||||
error_nodes = []
|
||||
for error_node in last_knowledge_error_analysis_result:
|
||||
if not isinstance(error_node, UndirectedNode):
|
||||
error_node = self.knowledgebase.graph_get_node_by_content(content=error_node)
|
||||
if error_node is None:
|
||||
continue
|
||||
error_nodes.append(error_node)
|
||||
|
||||
if len(error_nodes) > 1:
|
||||
task_trace_node_list = self.knowledgebase.graph_query_by_intersection(
|
||||
error_nodes,
|
||||
constraint_labels=["task_trace"],
|
||||
output_intersection_origin=True,
|
||||
)
|
||||
single_error_constraint = (v2_query_error_limit // len(error_nodes)) + 1
|
||||
else:
|
||||
task_trace_node_list = []
|
||||
single_error_constraint = v2_query_error_limit
|
||||
for error_node in error_nodes:
|
||||
# Reverse iterate, a trade-off with intersection search
|
||||
count = 0
|
||||
for task_trace_node in self.knowledgebase.graph_query_by_node(
|
||||
node=error_node,
|
||||
step=1,
|
||||
constraint_labels=["task_trace"],
|
||||
block=True,
|
||||
)[::-1]:
|
||||
if task_trace_node not in task_trace_node_list:
|
||||
task_trace_node_list.append([[error_node], task_trace_node])
|
||||
count += 1
|
||||
if count >= single_error_constraint:
|
||||
break
|
||||
|
||||
# for error_node in last_knowledge_error_analysis_result:
|
||||
# if not isinstance(error_node, UndirectedNode):
|
||||
# error_node = self.knowledgebase.graph_get_node_by_content(content=error_node)
|
||||
# if error_node is None:
|
||||
# continue
|
||||
# for searched_node in self.knowledgebase.graph_query_by_node(
|
||||
# node=error_node,
|
||||
# step=1,
|
||||
# constraint_labels=["task_trace"],
|
||||
# block=True,
|
||||
# ):
|
||||
# if searched_node not in [node[0] for node in task_trace_node_list]:
|
||||
# task_trace_node_list.append((searched_node, error_node.content))
|
||||
|
||||
same_error_success_knowledge_pair_list = []
|
||||
same_error_success_node_set = set()
|
||||
for error_node_list, trace_node in task_trace_node_list:
|
||||
for searched_trace_success_node in self.knowledgebase.graph_query_by_node(
|
||||
node=trace_node,
|
||||
step=50,
|
||||
constraint_labels=[
|
||||
"task_trace",
|
||||
"task_success_implement",
|
||||
"task_description",
|
||||
],
|
||||
block=True,
|
||||
):
|
||||
if (
|
||||
searched_trace_success_node not in same_error_success_node_set
|
||||
and searched_trace_success_node.label == "task_success_implement"
|
||||
):
|
||||
same_error_success_node_set.add(searched_trace_success_node)
|
||||
|
||||
trace_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[trace_node.id]
|
||||
success_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[
|
||||
searched_trace_success_node.id
|
||||
]
|
||||
error_content = ""
|
||||
for index, error_node in enumerate(error_node_list):
|
||||
error_content += f"{index+1}. {error_node.content}; "
|
||||
same_error_success_knowledge_pair_list.append(
|
||||
(
|
||||
error_content,
|
||||
(trace_knowledge, success_knowledge),
|
||||
),
|
||||
)
|
||||
|
||||
if knowledge_sampler > 0:
|
||||
same_error_success_knowledge_pair_list = [
|
||||
knowledge
|
||||
for knowledge in same_error_success_knowledge_pair_list
|
||||
if random.uniform(0, 1) <= knowledge_sampler
|
||||
]
|
||||
|
||||
same_error_success_knowledge_pair_list = same_error_success_knowledge_pair_list[:v2_query_error_limit]
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[
|
||||
target_factor_task_information
|
||||
] = same_error_success_knowledge_pair_list
|
||||
|
||||
return factor_implementation_queried_graph_knowledge
|
||||
|
||||
|
||||
class FactorGraphKnowledgeBase(EvolvingKnowledgeBase):
|
||||
def __init__(self, init_component_list=None, path: str | Path = None) -> None:
|
||||
"""
|
||||
Load knowledge, offer brief information of knowledge and common handle interfaces
|
||||
"""
|
||||
self.graph: UndirectedGraph = UndirectedGraph(Path.cwd() / "graph.pkl")
|
||||
logger.info(f"Knowledge Graph loaded, size={self.graph.size()}")
|
||||
|
||||
if init_component_list:
|
||||
for component in init_component_list:
|
||||
exist_node = self.graph.get_node_by_content(content=component)
|
||||
node = exist_node if exist_node else UndirectedNode(content=component, label="component")
|
||||
self.graph.add_nodes(node=node, neighbors=[])
|
||||
|
||||
# A dict containing all working trace until they fail or succeed
|
||||
self.working_trace_knowledge = {}
|
||||
|
||||
# A dict containing error analysis each step aligned with working trace
|
||||
self.working_trace_error_analysis = {}
|
||||
|
||||
# Add already success task
|
||||
self.success_task_to_knowledge_dict = {}
|
||||
|
||||
# key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorKnowledge')
|
||||
self.node_to_implementation_knowledge_dict = {}
|
||||
|
||||
# store the task description to component nodes
|
||||
self.task_to_component_nodes = {}
|
||||
|
||||
def get_all_nodes_by_label(self, label: str) -> list[UndirectedNode]:
|
||||
return self.graph.get_all_nodes_by_label(label)
|
||||
|
||||
def update_success_task(
|
||||
self,
|
||||
success_task_info: str,
|
||||
): # Transfer the success tasks' working trace to knowledge storage & graph
|
||||
success_task_trace = self.working_trace_knowledge[success_task_info]
|
||||
success_task_error_analysis_record = (
|
||||
self.working_trace_error_analysis[success_task_info]
|
||||
if success_task_info in self.working_trace_error_analysis
|
||||
else []
|
||||
)
|
||||
task_des_node = UndirectedNode(content=success_task_info, label="task_description")
|
||||
self.graph.add_nodes(
|
||||
node=task_des_node,
|
||||
neighbors=self.task_to_component_nodes[success_task_info],
|
||||
) # 1st version, we assume that all component nodes are given
|
||||
for index, trace_unit in enumerate(success_task_trace): # every unit: single_knowledge
|
||||
neighbor_nodes = [task_des_node]
|
||||
if index != len(success_task_trace) - 1:
|
||||
trace_node = UndirectedNode(
|
||||
content=trace_unit.get_implementation_and_feedback_str(),
|
||||
label="task_trace",
|
||||
)
|
||||
self.node_to_implementation_knowledge_dict[trace_node.id] = trace_unit
|
||||
for node_index, error_node in enumerate(success_task_error_analysis_record[index]):
|
||||
if type(error_node).__name__ == "str":
|
||||
queried_node = self.graph.get_node_by_content(content=error_node)
|
||||
if queried_node is None:
|
||||
new_error_node = UndirectedNode(content=error_node, label="error")
|
||||
self.graph.add_node(node=new_error_node)
|
||||
success_task_error_analysis_record[index][node_index] = new_error_node
|
||||
else:
|
||||
success_task_error_analysis_record[index][node_index] = queried_node
|
||||
neighbor_nodes.extend(success_task_error_analysis_record[index])
|
||||
self.graph.add_nodes(node=trace_node, neighbors=neighbor_nodes)
|
||||
else:
|
||||
success_node = UndirectedNode(
|
||||
content=trace_unit.get_implementation_and_feedback_str(),
|
||||
label="task_success_implement",
|
||||
)
|
||||
self.graph.add_nodes(node=success_node, neighbors=neighbor_nodes)
|
||||
self.node_to_implementation_knowledge_dict[success_node.id] = trace_unit
|
||||
|
||||
def query(self):
|
||||
pass
|
||||
|
||||
def graph_get_node_by_content(self, content: str) -> UndirectedNode:
|
||||
return self.graph.get_node_by_content(content=content)
|
||||
|
||||
def graph_query_by_content(
|
||||
self,
|
||||
content: Union[str, list[str]],
|
||||
topk_k: int = 5,
|
||||
step: int = 1,
|
||||
constraint_labels: list[str] = None,
|
||||
constraint_node: UndirectedNode = None,
|
||||
similarity_threshold: float = 0.0,
|
||||
constraint_distance: float = 0,
|
||||
block: bool = False,
|
||||
) -> list[UndirectedNode]:
|
||||
"""
|
||||
search graph by content similarity and connection relationship, return empty list if nodes' chain without node
|
||||
near to constraint_node
|
||||
|
||||
Parameters
|
||||
----------
|
||||
constraint_distance
|
||||
content
|
||||
topk_k: the upper number of output for each query, if the number of fit nodes is less than topk_k, return all fit nodes's content
|
||||
step
|
||||
constraint_labels
|
||||
constraint_node
|
||||
similarity_threshold
|
||||
block: despite the start node, the search can only flow through the constraint_label type nodes
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
|
||||
return self.graph.query_by_content(
|
||||
content=content,
|
||||
topk_k=topk_k,
|
||||
step=step,
|
||||
constraint_labels=constraint_labels,
|
||||
constraint_node=constraint_node,
|
||||
similarity_threshold=similarity_threshold,
|
||||
constraint_distance=constraint_distance,
|
||||
block=block,
|
||||
)
|
||||
|
||||
def graph_query_by_node(
|
||||
self,
|
||||
node: UndirectedNode,
|
||||
step: int = 1,
|
||||
constraint_labels: list[str] = None,
|
||||
constraint_node: UndirectedNode = None,
|
||||
constraint_distance: float = 0,
|
||||
block: bool = False,
|
||||
) -> list[UndirectedNode]:
|
||||
"""
|
||||
search graph by connection, return empty list if nodes' chain without node near to constraint_node
|
||||
Parameters
|
||||
----------
|
||||
node : start node
|
||||
step : the max steps will be searched
|
||||
constraint_labels : the labels of output nodes
|
||||
constraint_node : the node that the output nodes must connect to
|
||||
constraint_distance : the max distance between output nodes and constraint_node
|
||||
block: despite the start node, the search can only flow through the constraint_label type nodes
|
||||
|
||||
Returns
|
||||
-------
|
||||
A list of nodes
|
||||
|
||||
"""
|
||||
nodes = self.graph.query_by_node(
|
||||
node=node,
|
||||
step=step,
|
||||
constraint_labels=constraint_labels,
|
||||
constraint_node=constraint_node,
|
||||
constraint_distance=constraint_distance,
|
||||
block=block,
|
||||
)
|
||||
return nodes
|
||||
|
||||
def graph_query_by_intersection(
|
||||
self,
|
||||
nodes: list[UndirectedNode],
|
||||
steps: int = 1,
|
||||
constraint_labels: list[str] = None,
|
||||
output_intersection_origin: bool = False,
|
||||
) -> list[UndirectedNode] | list[list[list[UndirectedNode], UndirectedNode]]:
|
||||
"""
|
||||
search graph by node intersection, node intersected by a higher frequency has a prior order in the list
|
||||
Parameters
|
||||
----------
|
||||
nodes : node list
|
||||
step : the max steps will be searched
|
||||
constraint_labels : the labels of output nodes
|
||||
output_intersection_origin: output the list that contains the node which form this intersection node
|
||||
|
||||
Returns
|
||||
-------
|
||||
A list of nodes
|
||||
|
||||
"""
|
||||
node_count = len(nodes)
|
||||
assert node_count >= 2, "nodes length must >=2"
|
||||
intersection_node_list = []
|
||||
if output_intersection_origin:
|
||||
origin_list = []
|
||||
for k in range(node_count, 1, -1):
|
||||
possible_combinations = combinations(nodes, k)
|
||||
for possible_combination in possible_combinations:
|
||||
node_list = list(possible_combination)
|
||||
intersection_node_list.extend(
|
||||
self.graph.get_nodes_intersection(node_list, steps=steps, constraint_labels=constraint_labels),
|
||||
)
|
||||
if output_intersection_origin:
|
||||
for _ in range(len(intersection_node_list)):
|
||||
origin_list.append(node_list)
|
||||
intersection_node_list_sort_by_freq = []
|
||||
for index, node in enumerate(intersection_node_list):
|
||||
if node not in intersection_node_list_sort_by_freq:
|
||||
if output_intersection_origin:
|
||||
intersection_node_list_sort_by_freq.append([origin_list[index], node])
|
||||
else:
|
||||
intersection_node_list_sort_by_freq.append(node)
|
||||
|
||||
return intersection_node_list_sort_by_freq
|
||||
@@ -1,88 +0,0 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
|
||||
FactorEvolvingItem,
|
||||
)
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_conf import LLM_SETTINGS
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
|
||||
|
||||
def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
|
||||
import random
|
||||
|
||||
to_be_finished_task_index = random.sample(
|
||||
to_be_finished_task_index,
|
||||
implementation_factors_per_round,
|
||||
)
|
||||
|
||||
logger.info(f"The random selection is: {to_be_finished_task_index}")
|
||||
return to_be_finished_task_index
|
||||
|
||||
|
||||
def LLMSelect(
|
||||
to_be_finished_task_index,
|
||||
implementation_factors_per_round,
|
||||
evo: FactorEvolvingItem,
|
||||
former_trace: Dict,
|
||||
scen: Scenario,
|
||||
):
|
||||
tasks = []
|
||||
for i in to_be_finished_task_index:
|
||||
# find corresponding former trace for each task
|
||||
target_factor_task_information = evo.sub_tasks[i].get_task_information()
|
||||
if target_factor_task_information in former_trace:
|
||||
tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information][0]))
|
||||
|
||||
system_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(
|
||||
scheduler_prompts["select_implementable_factor_system"],
|
||||
)
|
||||
.render(
|
||||
scenario=scen.get_scenario_all_desc(),
|
||||
)
|
||||
)
|
||||
|
||||
for _ in range(10): # max attempt to reduce the length of user_prompt
|
||||
user_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(
|
||||
scheduler_prompts["select_implementable_factor_user"],
|
||||
)
|
||||
.render(
|
||||
factor_num=implementation_factors_per_round,
|
||||
sub_tasks=tasks,
|
||||
)
|
||||
)
|
||||
if (
|
||||
APIBackend().build_messages_and_calculate_token(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
)
|
||||
< LLM_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
)
|
||||
try:
|
||||
selection = json.loads(response)["selected_factor"]
|
||||
if not isinstance(selection, list):
|
||||
return to_be_finished_task_index
|
||||
selection_index = [x for x in selection if isinstance(x, int)]
|
||||
except:
|
||||
return to_be_finished_task_index
|
||||
|
||||
return selection_index
|
||||
@@ -0,0 +1,22 @@
|
||||
from rdagent.components.coder.CoSTEER import CoSTEER
|
||||
from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.evaluators import FactorEvaluatorForCoder
|
||||
from rdagent.components.coder.factor_coder.evolving_strategy import (
|
||||
FactorMultiProcessEvolvingStrategy,
|
||||
)
|
||||
from rdagent.core.scenario import Scenario
|
||||
|
||||
|
||||
class FactorCoSTEER(CoSTEER):
|
||||
def __init__(
|
||||
self,
|
||||
scen: Scenario,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
setting = FACTOR_COSTEER_SETTINGS
|
||||
eva = CoSTEERMultiEvaluator(FactorEvaluatorForCoder(scen=scen), scen=scen)
|
||||
es = FactorMultiProcessEvolvingStrategy(scen=scen, settings=FACTOR_COSTEER_SETTINGS)
|
||||
|
||||
super().__init__(*args, settings=setting, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs)
|
||||
@@ -1,18 +1,9 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, Union
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
SELECT_METHOD = Literal["random", "scheduler"]
|
||||
from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
|
||||
from rdagent.core.conf import ExtendedSettingsConfigDict
|
||||
|
||||
|
||||
class FactorImplementSettings(BaseSettings):
|
||||
class Config:
|
||||
env_prefix = "FACTOR_CODER_"
|
||||
"""Use `FACTOR_CODER_` as prefix for environment variables"""
|
||||
|
||||
coder_use_cache: bool = False
|
||||
"""Indicates whether to use cache for the coder"""
|
||||
class FactorCoSTEERSettings(CoSTEERSettings):
|
||||
model_config = ExtendedSettingsConfigDict(env_prefix="FACTOR_CoSTEER_")
|
||||
|
||||
data_folder: str = "git_ignore_folder/factor_implementation_source_data"
|
||||
"""Path to the folder containing financial data (default is fundamental data in Qlib)"""
|
||||
@@ -20,21 +11,6 @@ class FactorImplementSettings(BaseSettings):
|
||||
data_folder_debug: str = "git_ignore_folder/factor_implementation_source_data_debug"
|
||||
"""Path to the folder containing partial financial data (for debugging)"""
|
||||
|
||||
# TODO: the factor implement specific settings should not appear in this settings
|
||||
# Evolving should have a method specific settings
|
||||
# evolving related config
|
||||
fail_task_trial_limit: int = 20
|
||||
|
||||
v1_query_former_trace_limit: int = 5
|
||||
v1_query_similar_success_limit: int = 5
|
||||
|
||||
v2_query_component_limit: int = 1
|
||||
v2_query_error_limit: int = 1
|
||||
v2_query_former_trace_limit: int = 1
|
||||
v2_add_fail_attempt_to_latest_successful_execution: bool = False
|
||||
v2_error_summary: bool = False
|
||||
v2_knowledge_sampler: float = 1.0
|
||||
|
||||
simple_background: bool = False
|
||||
"""Whether to use simple background information for code feedback"""
|
||||
|
||||
@@ -44,20 +20,8 @@ class FactorImplementSettings(BaseSettings):
|
||||
select_method: str = "random"
|
||||
"""Method for the selection of factors implementation"""
|
||||
|
||||
select_threshold: int = 10
|
||||
"""Threshold for the number of factor selections"""
|
||||
|
||||
max_loop: int = 10
|
||||
"""Maximum number of task implementation loops"""
|
||||
|
||||
knowledge_base_path: Union[str, None] = None
|
||||
"""Path to the knowledge base"""
|
||||
|
||||
new_knowledge_base_path: Union[str, None] = None
|
||||
"""Path to the new knowledge base"""
|
||||
|
||||
python_bin: str = "python"
|
||||
"""Path to the Python binary"""
|
||||
|
||||
|
||||
FACTOR_IMPLEMENT_SETTINGS = FactorImplementSettings()
|
||||
FACTOR_COSTEER_SETTINGS = FactorCoSTEERSettings()
|
||||
|
||||
+14
-216
@@ -1,35 +1,28 @@
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
from abc import abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
from typing import Tuple
|
||||
|
||||
import pandas as pd
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
|
||||
FactorEvolvingItem,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.factor import FactorTask
|
||||
from rdagent.core.conf import RD_AGENT_SETTINGS
|
||||
from rdagent.core.evaluation import Evaluator, Feedback
|
||||
from rdagent.core.evolving_framework import QueriedKnowledge
|
||||
from rdagent.core.experiment import Task, Workspace
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.utils import multiprocessing_wrapper
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_conf import LLM_SETTINGS
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
evaluate_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
|
||||
|
||||
|
||||
class FactorEvaluator(Evaluator):
|
||||
# TODO:
|
||||
# I think we should have unified interface for all evaluates, for examples.
|
||||
# So we should adjust the interface of other factors
|
||||
class FactorEvaluator:
|
||||
"""Although the init method is same to Evaluator, but we want to emphasize they are different"""
|
||||
|
||||
def __init__(self, scen=None) -> None:
|
||||
self.scen = scen
|
||||
|
||||
@abstractmethod
|
||||
def evaluate(
|
||||
self,
|
||||
@@ -81,7 +74,7 @@ class FactorCodeEvaluator(FactorEvaluator):
|
||||
target_task: FactorTask,
|
||||
implementation: Workspace,
|
||||
execution_feedback: str,
|
||||
factor_value_feedback: str = "",
|
||||
value_feedback: str = "",
|
||||
gt_implementation: Workspace = None,
|
||||
**kwargs,
|
||||
):
|
||||
@@ -96,7 +89,7 @@ class FactorCodeEvaluator(FactorEvaluator):
|
||||
self.scen.get_scenario_all_desc(
|
||||
target_task,
|
||||
filtered_tag="feature",
|
||||
simple_background=FACTOR_IMPLEMENT_SETTINGS.simple_background,
|
||||
simple_background=FACTOR_COSTEER_SETTINGS.simple_background,
|
||||
)
|
||||
if self.scen is not None
|
||||
else "No scenario description."
|
||||
@@ -115,7 +108,7 @@ class FactorCodeEvaluator(FactorEvaluator):
|
||||
factor_information=factor_information,
|
||||
code=code,
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
factor_value_feedback=factor_value_feedback,
|
||||
value_feedback=value_feedback,
|
||||
gt_code=gt_implementation.code if gt_implementation else None,
|
||||
)
|
||||
)
|
||||
@@ -503,7 +496,7 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
return conclusion_str, decision_from_value_check
|
||||
|
||||
|
||||
class FactorFinalDecisionEvaluator(Evaluator):
|
||||
class FactorFinalDecisionEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
@@ -535,7 +528,7 @@ class FactorFinalDecisionEvaluator(Evaluator):
|
||||
factor_information=target_task.get_task_information(),
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
code_feedback=code_feedback,
|
||||
factor_value_feedback=(
|
||||
value_feedback=(
|
||||
value_feedback
|
||||
if value_feedback is not None
|
||||
else "No Ground Truth Value provided, so no evaluation on value is performed."
|
||||
@@ -585,198 +578,3 @@ class FactorFinalDecisionEvaluator(Evaluator):
|
||||
) from e
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
class FactorSingleFeedback:
|
||||
"""This class is a feedback to single implementation which is generated from an evaluator."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
execution_feedback: str = None,
|
||||
value_generated_flag: bool = False,
|
||||
code_feedback: str = None,
|
||||
factor_value_feedback: str = None,
|
||||
final_decision: bool = None,
|
||||
final_feedback: str = None,
|
||||
final_decision_based_on_gt: bool = None,
|
||||
) -> None:
|
||||
self.execution_feedback = execution_feedback
|
||||
self.value_generated_flag = value_generated_flag
|
||||
self.code_feedback = code_feedback
|
||||
self.factor_value_feedback = factor_value_feedback
|
||||
self.final_decision = final_decision
|
||||
self.final_feedback = final_feedback
|
||||
self.final_decision_based_on_gt = final_decision_based_on_gt
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"""------------------Factor Execution Feedback------------------
|
||||
{self.execution_feedback}
|
||||
------------------Factor Code Feedback------------------
|
||||
{self.code_feedback}
|
||||
------------------Factor Value Feedback------------------
|
||||
{self.factor_value_feedback}
|
||||
------------------Factor Final Feedback------------------
|
||||
{self.final_feedback}
|
||||
------------------Factor Final Decision------------------
|
||||
This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
|
||||
"""
|
||||
|
||||
|
||||
class FactorMultiFeedback(
|
||||
Feedback,
|
||||
List[FactorSingleFeedback],
|
||||
):
|
||||
"""Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
|
||||
|
||||
|
||||
class FactorEvaluatorForCoder(FactorEvaluator):
|
||||
"""This class is the v1 version of evaluator for a single factor implementation.
|
||||
It calls several evaluators in share modules to evaluate the factor implementation.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.value_evaluator = FactorValueEvaluator(self.scen)
|
||||
self.code_evaluator = FactorCodeEvaluator(self.scen)
|
||||
self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen)
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace = None,
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
**kwargs,
|
||||
) -> FactorSingleFeedback:
|
||||
if implementation is None:
|
||||
return None
|
||||
|
||||
target_task_information = target_task.get_task_information()
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
):
|
||||
return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
|
||||
elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
|
||||
return FactorSingleFeedback(
|
||||
execution_feedback="This task has failed too many times, skip implementation.",
|
||||
value_generated_flag=False,
|
||||
code_feedback="This task has failed too many times, skip code evaluation.",
|
||||
factor_value_feedback="This task has failed too many times, skip value evaluation.",
|
||||
final_decision=False,
|
||||
final_feedback="This task has failed too many times, skip final decision evaluation.",
|
||||
final_decision_based_on_gt=False,
|
||||
)
|
||||
else:
|
||||
factor_feedback = FactorSingleFeedback()
|
||||
|
||||
# 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback
|
||||
(
|
||||
execution_feedback,
|
||||
gen_df,
|
||||
) = implementation.execute()
|
||||
|
||||
execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback)
|
||||
factor_feedback.execution_feedback = "\n".join(
|
||||
[line for line in execution_feedback.split("\n") if "warning" not in line.lower()]
|
||||
)
|
||||
|
||||
# 2. Get factor value feedback
|
||||
if gen_df is None:
|
||||
factor_feedback.factor_value_feedback = "No factor value generated, skip value evaluation."
|
||||
factor_feedback.value_generated_flag = False
|
||||
decision_from_value_check = None
|
||||
else:
|
||||
factor_feedback.value_generated_flag = True
|
||||
(
|
||||
factor_feedback.factor_value_feedback,
|
||||
decision_from_value_check,
|
||||
) = self.value_evaluator.evaluate(
|
||||
implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
|
||||
)
|
||||
|
||||
factor_feedback.final_decision_based_on_gt = gt_implementation is not None
|
||||
|
||||
if decision_from_value_check is not None and decision_from_value_check is True:
|
||||
# To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback
|
||||
factor_feedback.code_feedback = "Final decision is True and there are no code critics."
|
||||
factor_feedback.final_decision = decision_from_value_check
|
||||
factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
|
||||
elif decision_from_value_check is not None and decision_from_value_check is False:
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
factor_value_feedback=factor_feedback.factor_value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
factor_feedback.final_decision = decision_from_value_check
|
||||
factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
|
||||
else:
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
factor_value_feedback=factor_feedback.factor_value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
(
|
||||
factor_feedback.final_decision,
|
||||
factor_feedback.final_feedback,
|
||||
) = self.final_decision_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.factor_value_feedback,
|
||||
code_feedback=factor_feedback.code_feedback,
|
||||
)
|
||||
return factor_feedback
|
||||
|
||||
|
||||
class FactorMultiEvaluator(Evaluator):
|
||||
def __init__(self, single_evaluator, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.single_factor_implementation_evaluator = single_evaluator
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
evo: FactorEvolvingItem,
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
**kwargs,
|
||||
) -> FactorMultiFeedback:
|
||||
multi_implementation_feedback = multiprocessing_wrapper(
|
||||
[
|
||||
(
|
||||
self.single_factor_implementation_evaluator.evaluate,
|
||||
(
|
||||
evo.sub_tasks[index],
|
||||
evo.sub_workspace_list[index],
|
||||
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
|
||||
queried_knowledge,
|
||||
),
|
||||
)
|
||||
for index in range(len(evo.sub_tasks))
|
||||
],
|
||||
n=RD_AGENT_SETTINGS.multi_proc_n,
|
||||
)
|
||||
|
||||
final_decision = [
|
||||
None if single_feedback is None else single_feedback.final_decision
|
||||
for single_feedback in multi_implementation_feedback
|
||||
]
|
||||
logger.info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
|
||||
|
||||
for index in range(len(evo.sub_tasks)):
|
||||
if final_decision[index]:
|
||||
evo.sub_tasks[index].factor_implementation = True
|
||||
|
||||
return multi_implementation_feedback
|
||||
|
||||
|
||||
# TODO:
|
||||
def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
|
||||
"""When the prompt is too long. We have to shorten it.
|
||||
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
|
||||
"""
|
||||
# TODO: this should replace most of code in
|
||||
# - FactorFinalDecisionEvaluator.evaluate
|
||||
# - FactorCodeEvaluator.evaluate
|
||||
@@ -0,0 +1,131 @@
|
||||
import re
|
||||
|
||||
from rdagent.components.coder.CoSTEER.evaluators import (
|
||||
CoSTEEREvaluator,
|
||||
CoSTEERMultiFeedback,
|
||||
CoSTEERSingleFeedback,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.eva_utils import (
|
||||
FactorCodeEvaluator,
|
||||
FactorFinalDecisionEvaluator,
|
||||
FactorValueEvaluator,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.factor import FactorTask
|
||||
from rdagent.core.evolving_framework import QueriedKnowledge
|
||||
from rdagent.core.experiment import Workspace
|
||||
|
||||
FactorSingleFeedback = CoSTEERSingleFeedback
|
||||
FactorMultiFeedback = CoSTEERMultiFeedback
|
||||
|
||||
|
||||
class FactorEvaluatorForCoder(CoSTEEREvaluator):
|
||||
"""This class is the v1 version of evaluator for a single factor implementation.
|
||||
It calls several evaluators in share modules to evaluate the factor implementation.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.value_evaluator = FactorValueEvaluator(self.scen)
|
||||
self.code_evaluator = FactorCodeEvaluator(self.scen)
|
||||
self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen)
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace = None,
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
**kwargs,
|
||||
) -> FactorSingleFeedback:
|
||||
if implementation is None:
|
||||
return None
|
||||
|
||||
target_task_information = target_task.get_task_information()
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
):
|
||||
return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
|
||||
elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
|
||||
return FactorSingleFeedback(
|
||||
execution_feedback="This task has failed too many times, skip implementation.",
|
||||
value_generated_flag=False,
|
||||
code_feedback="This task has failed too many times, skip code evaluation.",
|
||||
value_feedback="This task has failed too many times, skip value evaluation.",
|
||||
final_decision=False,
|
||||
final_feedback="This task has failed too many times, skip final decision evaluation.",
|
||||
final_decision_based_on_gt=False,
|
||||
)
|
||||
else:
|
||||
factor_feedback = FactorSingleFeedback()
|
||||
|
||||
# 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback
|
||||
(
|
||||
execution_feedback,
|
||||
gen_df,
|
||||
) = implementation.execute()
|
||||
|
||||
execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback)
|
||||
factor_feedback.execution_feedback = "\n".join(
|
||||
[line for line in execution_feedback.split("\n") if "warning" not in line.lower()]
|
||||
)
|
||||
|
||||
# 2. Get factor value feedback
|
||||
if gen_df is None:
|
||||
factor_feedback.value_feedback = "No factor value generated, skip value evaluation."
|
||||
factor_feedback.value_generated_flag = False
|
||||
decision_from_value_check = None
|
||||
else:
|
||||
factor_feedback.value_generated_flag = True
|
||||
(
|
||||
factor_feedback.value_feedback,
|
||||
decision_from_value_check,
|
||||
) = self.value_evaluator.evaluate(
|
||||
implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
|
||||
)
|
||||
|
||||
factor_feedback.final_decision_based_on_gt = gt_implementation is not None
|
||||
|
||||
if decision_from_value_check is not None and decision_from_value_check is True:
|
||||
# To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback
|
||||
factor_feedback.code_feedback = "Final decision is True and there are no code critics."
|
||||
factor_feedback.final_decision = decision_from_value_check
|
||||
factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
|
||||
elif decision_from_value_check is not None and decision_from_value_check is False:
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
factor_feedback.final_decision = decision_from_value_check
|
||||
factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
|
||||
else:
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
(
|
||||
factor_feedback.final_decision,
|
||||
factor_feedback.final_feedback,
|
||||
) = self.final_decision_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
code_feedback=factor_feedback.code_feedback,
|
||||
)
|
||||
return factor_feedback
|
||||
|
||||
|
||||
# TODO:
|
||||
def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
|
||||
"""When the prompt is too long. We have to shorten it.
|
||||
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
|
||||
"""
|
||||
# TODO: this should replace most of code in
|
||||
# - FactorFinalDecisionEvaluator.evaluate
|
||||
# - FactorCodeEvaluator.evaluate
|
||||
@@ -0,0 +1,177 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
from rdagent.components.coder.CoSTEER.evolving_strategy import (
|
||||
MultiProcessEvolvingStrategy,
|
||||
)
|
||||
from rdagent.components.coder.CoSTEER.knowledge_management import (
|
||||
CoSTEERQueriedKnowledge,
|
||||
CoSTEERQueriedKnowledgeV2,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.oai.llm_conf import LLM_SETTINGS
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
implement_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
|
||||
|
||||
|
||||
class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.num_loop = 0
|
||||
self.haveSelected = False
|
||||
|
||||
def error_summary(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
queried_former_failed_knowledge_to_render: list,
|
||||
queried_similar_error_knowledge_to_render: list,
|
||||
) -> str:
|
||||
error_summary_system_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
|
||||
.render(
|
||||
scenario=self.scen.get_scenario_all_desc(target_task),
|
||||
factor_information_str=target_task.get_task_information(),
|
||||
code_and_feedback=queried_former_failed_knowledge_to_render[-1].get_implementation_and_feedback_str(),
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
for _ in range(10): # max attempt to reduce the length of error_summary_user_prompt
|
||||
error_summary_user_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
|
||||
.render(
|
||||
queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
if (
|
||||
APIBackend().build_messages_and_calculate_token(
|
||||
user_prompt=error_summary_user_prompt, system_prompt=error_summary_system_prompt
|
||||
)
|
||||
< LLM_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_similar_error_knowledge_to_render) > 0:
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
|
||||
error_summary_critics = APIBackend(
|
||||
use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
|
||||
).build_messages_and_create_chat_completion(
|
||||
user_prompt=error_summary_user_prompt, system_prompt=error_summary_system_prompt, json_mode=False
|
||||
)
|
||||
return error_summary_critics
|
||||
|
||||
def implement_one_task(
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
queried_knowledge: CoSTEERQueriedKnowledge,
|
||||
) -> str:
|
||||
target_factor_task_information = target_task.get_task_information()
|
||||
|
||||
queried_similar_successful_knowledge = (
|
||||
queried_knowledge.task_to_similar_task_successful_knowledge[target_factor_task_information]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
) # A list, [success task implement knowledge]
|
||||
|
||||
if isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2):
|
||||
queried_similar_error_knowledge = (
|
||||
queried_knowledge.task_to_similar_error_successful_knowledge[target_factor_task_information]
|
||||
if queried_knowledge is not None
|
||||
else {}
|
||||
) # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
|
||||
else:
|
||||
queried_similar_error_knowledge = {}
|
||||
|
||||
queried_former_failed_knowledge = (
|
||||
queried_knowledge.task_to_former_failed_traces[target_factor_task_information][0]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
)
|
||||
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
|
||||
|
||||
latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[
|
||||
target_factor_task_information
|
||||
][1]
|
||||
|
||||
system_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
|
||||
)
|
||||
.render(
|
||||
scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
|
||||
)
|
||||
)
|
||||
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
|
||||
# 动态地防止prompt超长
|
||||
for _ in range(10): # max attempt to reduce the length of user_prompt
|
||||
# 总结error（可选）
|
||||
if (
|
||||
isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2)
|
||||
and FACTOR_COSTEER_SETTINGS.v2_error_summary
|
||||
and len(queried_similar_error_knowledge_to_render) != 0
|
||||
and len(queried_former_failed_knowledge_to_render) != 0
|
||||
):
|
||||
error_summary_critics = self.error_summary(
|
||||
target_task,
|
||||
queried_former_failed_knowledge_to_render,
|
||||
queried_similar_error_knowledge_to_render,
|
||||
)
|
||||
else:
|
||||
error_summary_critics = None
|
||||
# 构建user_prompt。开始写代码
|
||||
user_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v2_user"],
|
||||
)
|
||||
.render(
|
||||
factor_information_str=target_factor_task_information,
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
|
||||
queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
|
||||
error_summary_critics=error_summary_critics,
|
||||
latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
if (
|
||||
APIBackend().build_messages_and_calculate_token(user_prompt=user_prompt, system_prompt=system_prompt)
|
||||
< LLM_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_former_failed_knowledge_to_render) > 1:
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
|
||||
elif len(queried_similar_successful_knowledge_to_render) > len(
|
||||
queried_similar_error_knowledge_to_render,
|
||||
):
|
||||
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[:-1]
|
||||
elif len(queried_similar_error_knowledge_to_render) > 0:
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
|
||||
code = json.loads(
|
||||
APIBackend(
|
||||
use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
|
||||
).build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True
|
||||
)
|
||||
)["code"]
|
||||
return code
|
||||
|
||||
def assign_code_list_to_evo(self, code_list, evo):
|
||||
for index in range(len(evo.sub_tasks)):
|
||||
if code_list[index] is None:
|
||||
continue
|
||||
if evo.sub_workspace_list[index] is None:
|
||||
evo.sub_workspace_list[index] = FactorFBWorkspace(target_task=evo.sub_tasks[index])
|
||||
evo.sub_workspace_list[index].inject_code(**{"factor.py": code_list[index]})
|
||||
return evo
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pickle
|
||||
import subprocess
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
@@ -10,15 +9,15 @@ import pandas as pd
|
||||
from filelock import FileLock
|
||||
|
||||
from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
|
||||
from rdagent.components.coder.CoSTEER.task import CoSTEERTask
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
|
||||
from rdagent.core.experiment import Experiment, FBWorkspace, Task
|
||||
from rdagent.core.experiment import Experiment, FBWorkspace
|
||||
from rdagent.core.utils import cache_with_pickle
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_utils import md5_hash
|
||||
|
||||
|
||||
class FactorTask(Task):
|
||||
class FactorTask(CoSTEERTask):
|
||||
# TODO: generalized the attributes into the Task
|
||||
# - factor_* -> *
|
||||
def __init__(
|
||||
@@ -124,11 +123,11 @@ class FactorFBWorkspace(FBWorkspace):
|
||||
if self.target_task.version == 1:
|
||||
source_data_path = (
|
||||
Path(
|
||||
FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
|
||||
FACTOR_COSTEER_SETTINGS.data_folder_debug,
|
||||
)
|
||||
if data_type == "Debug" # FIXME: (yx) don't think we should use a debug tag for this.
|
||||
else Path(
|
||||
FACTOR_IMPLEMENT_SETTINGS.data_folder,
|
||||
FACTOR_COSTEER_SETTINGS.data_folder,
|
||||
)
|
||||
)
|
||||
elif self.target_task.version == 2:
|
||||
@@ -152,11 +151,11 @@ class FactorFBWorkspace(FBWorkspace):
|
||||
|
||||
try:
|
||||
subprocess.check_output(
|
||||
f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {execution_code_path}",
|
||||
f"{FACTOR_COSTEER_SETTINGS.python_bin} {execution_code_path}",
|
||||
shell=True,
|
||||
cwd=self.workspace_path,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout,
|
||||
timeout=FACTOR_COSTEER_SETTINGS.file_based_execution_timeout,
|
||||
)
|
||||
execution_success = True
|
||||
except subprocess.CalledProcessError as e:
|
||||
@@ -176,7 +175,7 @@ class FactorFBWorkspace(FBWorkspace):
|
||||
else:
|
||||
execution_error = CustomRuntimeError(execution_feedback)
|
||||
except subprocess.TimeoutExpired:
|
||||
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
|
||||
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_COSTEER_SETTINGS.file_based_execution_timeout} seconds."
|
||||
if self.raise_exception:
|
||||
raise CustomRuntimeError(execution_feedback)
|
||||
else:
|
||||
|
||||
@@ -28,9 +28,9 @@ evaluator_code_feedback_v1_user: |-
|
||||
{{ code }}
|
||||
--------------Execution feedback:---------------
|
||||
{{ execution_feedback }}
|
||||
{% if factor_value_feedback is not none %}
|
||||
{% if value_feedback is not none %}
|
||||
--------------Factor value feedback:---------------
|
||||
{{ factor_value_feedback }}
|
||||
{{ value_feedback }}
|
||||
{% endif %}
|
||||
{% if gt_code is not none %}
|
||||
--------------Ground truth Python code:---------------
|
||||
@@ -62,36 +62,12 @@ evolving_strategy_factor_implementation_v1_system: |-
|
||||
"code": "The Python code as a string."
|
||||
}
|
||||
|
||||
evolving_strategy_factor_implementation_v1_user: |-
|
||||
--------------Target factor information:---------------
|
||||
{{ factor_information_str }}
|
||||
|
||||
{% if queried_similar_successful_knowledge|length != 0 %}
|
||||
--------------Correct code to similar factors:---------------
|
||||
{% for similar_successful_knowledge in queried_similar_successful_knowledge %}
|
||||
=====Factor {{loop.index}}:=====
|
||||
{{ similar_successful_knowledge.target_task.get_task_information() }}
|
||||
=====Code:=====
|
||||
{{ similar_successful_knowledge.implementation.code }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if queried_former_failed_knowledge|length != 0 %}
|
||||
--------------Former failed code:---------------
|
||||
{% for former_failed_knowledge in queried_former_failed_knowledge %}
|
||||
=====Code to implementation {{ loop.index }}=====
|
||||
{{ former_failed_knowledge.implementation.code }}
|
||||
=====Feedback to implementation {{ loop.index }}=====
|
||||
{{ former_failed_knowledge.feedback }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
evolving_strategy_factor_implementation_v2_user: |-
|
||||
--------------Target factor information:---------------
|
||||
{{ factor_information_str }}
|
||||
|
||||
{% if queried_similar_error_knowledge|length != 0 %}
|
||||
{% if not error_summary %}
|
||||
{% if error_summary_critics is none %}
|
||||
Recall your last failure, your implementation met some errors.
|
||||
When doing other tasks, you met some similar errors but you finally solve them. Here are some examples:
|
||||
{% for error_content, similar_error_knowledge in queried_similar_error_knowledge %}
|
||||
@@ -108,14 +84,14 @@ evolving_strategy_factor_implementation_v2_user: |-
|
||||
{{error_summary_critics}}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if queried_similar_component_knowledge|length != 0 %}
|
||||
{% if queried_similar_successful_knowledge|length != 0 %}
|
||||
Here are some success implements of similar component tasks, take them as references:
|
||||
--------------Correct code to similar factors:---------------
|
||||
{% for similar_component_knowledge in queried_similar_component_knowledge %}
|
||||
{% for similar_successful_knowledge in queried_similar_successful_knowledge %}
|
||||
=====Factor {{loop.index}}:=====
|
||||
{{ similar_component_knowledge.target_task.get_task_information() }}
|
||||
{{ similar_successful_knowledge.target_task.get_task_information() }}
|
||||
=====Code:=====
|
||||
{{ similar_component_knowledge.implementation.code }}
|
||||
{{ similar_successful_knowledge.implementation.code }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if latest_attempt_to_latest_successful_execution is not none %}
|
||||
@@ -189,16 +165,6 @@ select_implementable_factor_user: |-
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
analyze_component_prompt_v1_system: |-
|
||||
User is getting a new task that might consist of the components below (given in component_index: component_description):
|
||||
{{all_component_content}}
|
||||
|
||||
You should find out what components does the new task have, and put their indices in a list.
|
||||
Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
|
||||
{
|
||||
"component_no_list": the list containing indices of components.
|
||||
}
|
||||
|
||||
evaluator_output_format_system: |-
|
||||
User is trying to implement some factors in the following scenario:
|
||||
{{ scenario }}
|
||||
@@ -235,4 +201,4 @@ evaluator_final_decision_v1_user: |-
|
||||
--------------Code feedback:---------------
|
||||
{{ code_feedback }}
|
||||
--------------Factor value feedback:---------------
|
||||
{{ factor_value_feedback }}
|
||||
{{ value_feedback }}
|
||||
|
||||
Reference in New Issue
Block a user