gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-02 18:07:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Align factor coder into new framework (#47)

Browse Source 
* use CoSTEER as component name

* rename factorimplementation to avoid confusion

* rename modelimplementation

* align benchmark and evolving evaluators

* add scenario to evaluator init function

* rename all factorimplementationknowledge in CoSTEER

* remove all scenario related information in component

* remove useless code

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
2024-07-05 17:42:00 +08:00
committed by GitHub
parent f61453fbb1
commit 1d9b4cd2ec
57 changed files with 974 additions and 1093 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/factor_coder/CoSTEER/__init__.py
+102
View File
@@ -0,0 +1,102 @@
import pickle
from pathlib import Path
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
FactorEvaluatorForCoder,
FactorMultiEvaluator,
)
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.coder.factor_coder.CoSTEER.evolving_strategy import (
FactorEvolvingStrategyWithGraph,
)
from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import (
FactorGraphKnowledgeBase,
FactorGraphRAGStrategy,
FactorKnowledgeBaseV1,
)
from rdagent.components.coder.factor_coder.factor import FactorExperiment
from rdagent.core.evolving_agent import RAGEvoAgent
from rdagent.core.scenario import Scenario
from rdagent.core.task_generator import TaskGenerator
class FactorCoSTEER(TaskGenerator[FactorExperiment]):
def __init__(
self,
*args,
with_knowledge: bool = True,
with_feedback: bool = True,
knowledge_self_gen: bool = True,
**kwargs,
) -> None:
super().__init__(*args, **kwargs)
self.max_loop = FACTOR_IMPLEMENT_SETTINGS.max_loop
self.knowledge_base_path = (
Path(FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path)
if FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path is not None
else None
)
self.new_knowledge_base_path = (
Path(FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path)
if FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path is not None
else None
)
self.with_knowledge = with_knowledge
self.with_feedback = with_feedback
self.knowledge_self_gen = knowledge_self_gen
self.evolving_strategy = FactorEvolvingStrategyWithGraph(scen=self.scen)
# declare the factor evaluator
self.factor_evaluator = FactorMultiEvaluator(FactorEvaluatorForCoder(scen=self.scen), scen=self.scen)
self.evolving_version = 2
def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
if self.evolving_version == 1 and not isinstance(factor_knowledge_base, FactorKnowledgeBaseV1):
raise ValueError("The former knowledge base is not compatible with the current version")
elif self.evolving_version == 2 and not isinstance(
factor_knowledge_base,
FactorGraphKnowledgeBase,
):
raise ValueError("The former knowledge base is not compatible with the current version")
else:
factor_knowledge_base = (
FactorGraphKnowledgeBase(
init_component_list=component_init_list,
)
if self.evolving_version == 2
else FactorKnowledgeBaseV1()
)
return factor_knowledge_base
def generate(self, exp: FactorExperiment) -> FactorExperiment:
# init knowledge base
factor_knowledge_base = self.load_or_init_knowledge_base(
former_knowledge_base_path=self.knowledge_base_path,
component_init_list=[],
)
# init rag method
self.rag = FactorGraphRAGStrategy(factor_knowledge_base)
# init intermediate items
factor_experiment = FactorEvolvingItem(sub_tasks=exp.sub_tasks)
self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
factor_experiment = self.evolve_agent.multistep_evolve(
factor_experiment,
self.factor_evaluator,
with_knowledge=self.with_knowledge,
with_feedback=self.with_feedback,
knowledge_self_gen=self.knowledge_self_gen,
)
# save new knowledge base
if self.new_knowledge_base_path is not None:
pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
self.knowledge_base = factor_knowledge_base
self.latest_factor_implementations = exp.sub_tasks
return factor_experiment
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+610
View File
@@ -0,0 +1,610 @@
import io
import json
import re
from abc import abstractmethod
from pathlib import Path
from typing import List, Tuple
import pandas as pd
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.coder.factor_coder.factor import FactorTask
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evaluation import Evaluator
from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
from rdagent.core.experiment import Implementation, Task
from rdagent.core.log import RDAgentLog
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.oai.llm_utils import APIBackend
evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
class FactorEvaluator(Evaluator):
# TODO:
# I think we should have unified interface for all evaluates, for examples.
# So we should adjust the interface of other factors
@abstractmethod
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
**kwargs,
) -> Tuple[str, object]:
"""You can get the dataframe by
.. code-block:: python
_, gen_df = implementation.execute()
_, gt_df = gt_implementation.execute()
Returns
-------
Tuple[str, object]
- str: the text-based description of the evaluation result
- object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result
"""
raise NotImplementedError("Please implement the `evaluator` method")
def _get_df(self, gt_implementation: Implementation, implementation: Implementation):
if gt_implementation is not None:
_, gt_df = gt_implementation.execute()
if isinstance(gt_df, pd.Series):
gt_df = gt_df.to_frame("gt_factor")
if isinstance(gt_df, pd.DataFrame):
gt_df = gt_df.sort_index()
else:
gt_df = None
_, gen_df = implementation.execute()
if isinstance(gen_df, pd.Series):
gen_df = gen_df.to_frame("source_factor")
if isinstance(gen_df, pd.DataFrame):
gen_df = gen_df.sort_index()
return gt_df, gen_df
def __str__(self) -> str:
return self.__class__.__name__
class FactorCodeEvaluator(FactorEvaluator):
def evaluate(
self,
target_task: FactorTask,
implementation: Implementation,
execution_feedback: str,
factor_value_feedback: str = "",
gt_implementation: Implementation = None,
**kwargs,
):
factor_information = target_task.get_factor_information()
code = implementation.code
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(evaluate_prompts["evaluator_code_feedback_v1_system"])
.render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
)
execution_feedback_to_render = execution_feedback
for _ in range(10): # 10 times to split the content is enough
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_code_feedback_v1_user"],
)
.render(
factor_information=factor_information,
code=code,
execution_feedback=execution_feedback_to_render,
factor_value_feedback=factor_value_feedback,
gt_code=gt_implementation.code if gt_implementation else None,
)
)
if (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
)
> RD_AGENT_SETTINGS.chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
else:
break
critic_response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=False,
)
return critic_response, None
class FactorSingleColumnEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
) -> Tuple[str, object]:
_, gen_df = self._get_df(gt_implementation, implementation)
if len(gen_df.columns) == 1:
return "The source dataframe has only one column which is correct.", True
else:
return (
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
False,
)
class FactorOutputFormatEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
if gen_df is None:
return (
"The source dataframe is None. Skip the evaluation of the output format.",
False,
)
buffer = io.StringIO()
gen_df.info(buf=buffer)
gen_df_info_str = buffer.getvalue()
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_output_format_system"],
)
.render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
)
resp = APIBackend().build_messages_and_create_chat_completion(
user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True
)
resp_dict = json.loads(resp)
if isinstance(resp_dict["output_format_decision"], str) and resp_dict["output_format_decision"].lower() in (
"true",
"false",
):
resp_dict["output_format_decision"] = bool(resp_dict["output_format_decision"])
return (
resp_dict["output_format_feedback"],
resp_dict["output_format_decision"],
)
class FactorRowCountEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
if gen_df.shape[0] == gt_df.shape[0]:
return "Both dataframes have the same rows count.", True
else:
return (
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
False,
)
class FactorIndexEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
if gen_df.index.equals(gt_df.index):
return "Both dataframes have the same index.", True
else:
return (
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
False,
)
class FactorMissingValuesEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
return "Both dataframes have the same missing values.", True
else:
return (
f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
False,
)
class FactorEqualValueCountEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
try:
close_values = gen_df.sub(gt_df).abs().lt(1e-6)
result_int = close_values.astype(int)
pos_num = result_int.sum().sum()
acc_rate = pos_num / close_values.size
except:
close_values = gen_df
if close_values.all().iloc[0]:
return (
"All values in the dataframes are equal within the tolerance of 1e-6.",
acc_rate,
)
else:
return (
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
acc_rate,
)
class FactorCorrelationEvaluator(FactorEvaluator):
def __init__(self, hard_check: bool, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.hard_check = hard_check
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
concat_df = pd.concat([gen_df, gt_df], axis=1)
concat_df.columns = ["source", "gt"]
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
ric = (
concat_df.groupby("datetime")
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
.dropna()
.mean()
)
if self.hard_check:
if ic > 0.99 and ric > 0.99:
return (
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
True,
)
else:
return (
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
False,
)
else:
return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
class FactorValueEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
**kwargs,
) -> Tuple:
conclusions = []
# Check if both dataframe has only one columns
feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
# Check if the index of the dataframe is ("datetime", "instrument")
feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
# Check if both dataframe have the same rows count
if gt_implementation is not None:
feedback_str, _ = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
conclusions.append(feedback_str)
feedback_str, _ = FactorMissingValuesEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
conclusions.append(feedback_str)
if same_index_result:
feedback_str, high_correlation_result = FactorCorrelationEvaluator(
hard_check=True, scen=self.scen
).evaluate(implementation, gt_implementation)
else:
high_correlation_result = False
feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless"
conclusions.append(feedback_str)
# Combine all conclusions into a single string
conclusion_str = "\n".join(conclusions)
same_value_or_high_correlation = (
((equal_value_ratio_result > 0.99) or high_correlation_result) if gt_implementation is not None else False
)
return conclusion_str, same_value_or_high_correlation
class FactorFinalDecisionEvaluator(Evaluator):
def evaluate(
self,
target_task: FactorTask,
execution_feedback: str,
value_feedback: str,
code_feedback: str,
**kwargs,
) -> Tuple:
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(evaluate_prompts["evaluator_final_decision_v1_system"])
.render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
)
execution_feedback_to_render = execution_feedback
for _ in range(10): # 10 times to split the content is enough
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
evaluate_prompts["evaluator_final_decision_v1_user"],
)
.render(
factor_information=target_task.get_factor_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
value_feedback
if value_feedback is not None
else "No Ground Truth Value provided, so no evaluation on value is performed."
),
)
)
if (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
)
> RD_AGENT_SETTINGS.chat_token_limit
):
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
else:
break
final_evaluation_dict = json.loads(
APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=True,
),
)
if isinstance(final_evaluation_dict["final_decision"], str) and final_evaluation_dict[
"final_decision"
].lower() in ("true", "false"):
final_evaluation_dict["final_decision"] = bool(final_evaluation_dict["final_decision"])
return (
final_evaluation_dict["final_decision"],
final_evaluation_dict["final_feedback"],
)
class FactorSingleFeedback:
"""This class is a feedback to single implementation which is generated from an evaluator."""
def __init__(
self,
execution_feedback: str = None,
value_generated_flag: bool = False,
code_feedback: str = None,
factor_value_feedback: str = None,
final_decision: bool = None,
final_feedback: str = None,
final_decision_based_on_gt: bool = None,
) -> None:
self.execution_feedback = execution_feedback
self.value_generated_flag = value_generated_flag
self.code_feedback = code_feedback
self.factor_value_feedback = factor_value_feedback
self.final_decision = final_decision
self.final_feedback = final_feedback
self.final_decision_based_on_gt = final_decision_based_on_gt
def __str__(self) -> str:
return f"""------------------Factor Execution Feedback------------------
{self.execution_feedback}
------------------Factor Code Feedback------------------
{self.code_feedback}
------------------Factor Value Feedback------------------
{self.factor_value_feedback}
------------------Factor Final Feedback------------------
{self.final_feedback}
------------------Factor Final Decision------------------
This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
"""
class FactorMultiFeedback(
Feedback,
List[FactorSingleFeedback],
):
"""Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
class FactorEvaluatorForCoder(FactorEvaluator):
"""This class is the v1 version of evaluator for a single factor implementation.
It calls several evaluators in share modules to evaluate the factor implementation.
"""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.value_evaluator = FactorValueEvaluator(self.scen)
self.code_evaluator = FactorCodeEvaluator(self.scen)
self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen)
def evaluate(
self,
target_task: FactorTask,
implementation: Implementation,
gt_implementation: Implementation = None,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorSingleFeedback:
if implementation is None:
return None
target_task_information = target_task.get_factor_information()
if (
queried_knowledge is not None
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
):
return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
return FactorSingleFeedback(
execution_feedback="This task has failed too many times, skip implementation.",
value_generated_flag=False,
code_feedback="This task has failed too many times, skip code evaluation.",
factor_value_feedback="This task has failed too many times, skip value evaluation.",
final_decision=False,
final_feedback="This task has failed too many times, skip final decision evaluation.",
final_decision_based_on_gt=False,
)
else:
factor_feedback = FactorSingleFeedback()
# 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback
(
execution_feedback,
gen_df,
) = implementation.execute()
execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback)
factor_feedback.execution_feedback = "\n".join(
[line for line in execution_feedback.split("\n") if "warning" not in line.lower()]
)
# 2. Get factor value feedback
if gen_df is None:
factor_feedback.factor_value_feedback = "No factor value generated, skip value evaluation."
factor_feedback.value_generated_flag = False
same_value_or_high_correlation = None
else:
factor_feedback.value_generated_flag = True
(
factor_feedback.factor_value_feedback,
same_value_or_high_correlation,
) = self.value_evaluator.evaluate(implementation=implementation, gt_implementation=gt_implementation)
factor_feedback.final_decision_based_on_gt = gt_implementation is not None
if same_value_or_high_correlation is not None and same_value_or_high_correlation is True:
# To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback
factor_feedback.code_feedback = "Final decision is True and there are no code critics."
factor_feedback.final_decision = same_value_or_high_correlation
factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
else:
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
target_task=target_task,
implementation=implementation,
execution_feedback=factor_feedback.execution_feedback,
value_feedback=factor_feedback.factor_value_feedback,
gt_implementation=gt_implementation,
)
(
factor_feedback.final_decision,
factor_feedback.final_feedback,
) = self.final_decision_evaluator.evaluate(
target_task=target_task,
execution_feedback=factor_feedback.execution_feedback,
value_feedback=factor_feedback.factor_value_feedback,
code_feedback=factor_feedback.code_feedback,
)
return factor_feedback
class FactorMultiEvaluator(Evaluator):
def __init__(self, single_evaluator, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.single_factor_implementation_evaluator = single_evaluator
def evaluate(
self,
evo: FactorEvolvingItem,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorMultiFeedback:
multi_implementation_feedback = FactorMultiFeedback()
# for index in range(len(evo.sub_tasks)):
# corresponding_implementation = evo.sub_implementations[index]
# corresponding_gt_implementation = (
# evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
# )
# multi_implementation_feedback.append(
# self.single_factor_implementation_evaluator.evaluate(
# target_task=evo.sub_tasks[index],
# implementation=corresponding_implementation,
# gt_implementation=corresponding_gt_implementation,
# queried_knowledge=queried_knowledge,
# )
# )
calls = []
for index in range(len(evo.sub_tasks)):
corresponding_implementation = evo.sub_implementations[index]
corresponding_gt_implementation = (
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
)
calls.append(
(
self.single_factor_implementation_evaluator.evaluate,
(
evo.sub_tasks[index],
corresponding_implementation,
corresponding_gt_implementation,
queried_knowledge,
),
),
)
multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n)
final_decision = [
None if single_feedback is None else single_feedback.final_decision
for single_feedback in multi_implementation_feedback
]
RDAgentLog().info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
return multi_implementation_feedback
# TODO:
def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
"""When the prompt is too long. We have to shorten it.
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
"""
# TODO: this should replace most of code in
# - FactorFinalDecisionEvaluator.evaluate
# - FactorCodeEvaluator.evaluate
rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py
+30
View File
@@ -0,0 +1,30 @@
from rdagent.components.coder.factor_coder.factor import (
FactorExperiment,
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.core.log import RDAgentLog
class FactorEvolvingItem(FactorExperiment, EvolvableSubjects):
"""
Intermediate item of factor implementation.
"""
def __init__(
self,
sub_tasks: list[FactorTask],
sub_gt_implementations: list[FileBasedFactorImplementation] = None,
):
FactorExperiment.__init__(self, sub_tasks=sub_tasks)
self.corresponding_selection: list = None
if sub_gt_implementations is not None and len(
sub_gt_implementations,
) != len(self.sub_tasks):
self.sub_gt_implementations = None
RDAgentLog().warning(
"The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
)
else:
self.sub_gt_implementations = sub_gt_implementations
rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py
+336
View File
@@ -0,0 +1,336 @@
from __future__ import annotations
import json
from abc import abstractmethod
from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.coder.factor_coder.CoSTEER.scheduler import (
LLMSelect,
RandomSelect,
)
from rdagent.components.coder.factor_coder.factor import (
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.components.coder.factor_coder.utils import get_data_folder_intro
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
from rdagent.core.experiment import Implementation
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.oai.llm_utils import APIBackend
if TYPE_CHECKING:
from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import (
FactorQueriedKnowledge,
FactorQueriedKnowledgeV1,
)
implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
class MultiProcessEvolvingStrategy(EvolvingStrategy):
@abstractmethod
def implement_one_factor(
self,
target_task: FactorTask,
queried_knowledge: QueriedKnowledge = None,
) -> Implementation:
raise NotImplementedError
def evolve(
self,
*,
evo: FactorEvolvingItem,
queried_knowledge: FactorQueriedKnowledge | None = None,
**kwargs,
) -> FactorEvolvingItem:
self.num_loop += 1
new_evo = deepcopy(evo)
# 1.找出需要evolve的factor
to_be_finished_task_index = []
for index, target_factor_task in enumerate(new_evo.sub_tasks):
target_factor_task_desc = target_factor_task.get_factor_information()
if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
target_factor_task_desc
].implementation
elif (
target_factor_task_desc not in queried_knowledge.success_task_to_knowledge_dict
and target_factor_task_desc not in queried_knowledge.failed_task_info_set
):
to_be_finished_task_index.append(index)
# 2. 选择selection方法
# if the number of factors to be implemented is larger than the limit, we need to select some of them
if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1:
# if the number of loops is equal to the select_loop, we need to select some of them
implementation_factors_per_round = int(
FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index)
)
if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
to_be_finished_task_index = RandomSelect(
to_be_finished_task_index,
implementation_factors_per_round,
)
if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
to_be_finished_task_index = LLMSelect(
to_be_finished_task_index,
implementation_factors_per_round,
new_evo,
queried_knowledge.former_traces,
self.scen,
)
result = multiprocessing_wrapper(
[
(self.implement_one_factor, (new_evo.sub_tasks[target_index], queried_knowledge))
for target_index in to_be_finished_task_index
],
n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
)
for index, target_index in enumerate(to_be_finished_task_index):
new_evo.sub_implementations[target_index] = result[index]
# for target_index in to_be_finished_task_index:
# new_evo.sub_implementations[target_index] = self.implement_one_factor(
# new_evo.sub_tasks[target_index], queried_knowledge
# )
new_evo.corresponding_selection = to_be_finished_task_index
return new_evo
class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_factor(
self,
target_task: FactorTask,
queried_knowledge: FactorQueriedKnowledgeV1 = None,
) -> Implementation:
factor_information_str = target_task.get_factor_information()
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
return queried_knowledge.success_task_to_knowledge_dict[factor_information_str].implementation
elif queried_knowledge is not None and factor_information_str in queried_knowledge.failed_task_info_set:
return None
else:
queried_similar_successful_knowledge = (
queried_knowledge.working_task_to_similar_successful_knowledge_dict[factor_information_str]
if queried_knowledge is not None
else []
)
queried_former_failed_knowledge = (
queried_knowledge.working_task_to_former_failed_knowledge_dict[factor_information_str]
if queried_knowledge is not None
else []
)
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
)
.render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
session_system_prompt=system_prompt,
)
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
while True:
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_user"],
)
.render(
factor_information_str=factor_information_str,
queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
)
.strip("\n")
)
if (
session.build_chat_completion_message_and_calculate_token(
user_prompt,
)
< RD_AGENT_SETTINGS.chat_token_limit
):
break
elif len(queried_former_failed_knowledge_to_render) > 1:
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
elif len(queried_similar_successful_knowledge_to_render) > 1:
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
code = json.loads(
session.build_chat_completion(
user_prompt=user_prompt,
json_mode=True,
),
)["code"]
# ast.parse(code)
factor_implementation = FileBasedFactorImplementation(
target_task,
code,
)
return factor_implementation
class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.num_loop = 0
self.haveSelected = False
def implement_one_factor(
self,
target_task: FactorTask,
queried_knowledge,
) -> Implementation:
error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
# 1. 提取因子的背景信息
target_factor_task_information = target_task.get_factor_information()
# 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
if (
queried_knowledge is not None
and target_factor_task_information in queried_knowledge.success_task_to_knowledge_dict
):
return queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information].implementation
elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
return None
else:
# 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
queried_similar_component_knowledge = (
queried_knowledge.component_with_success_task[target_factor_task_information]
if queried_knowledge is not None
else []
) # A list, [success task implement knowledge]
queried_similar_error_knowledge = (
queried_knowledge.error_with_success_task[target_factor_task_information]
if queried_knowledge is not None
else {}
) # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
queried_former_failed_knowledge = (
queried_knowledge.former_traces[target_factor_task_information] if queried_knowledge is not None else []
)
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
)
.render(
scenario=self.scen.get_scenario_all_desc(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
session_system_prompt=system_prompt,
)
queried_similar_component_knowledge_to_render = queried_similar_component_knowledge
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
error_summary_critics = ""
# 动态地防止prompt超长
while True:
# 总结error（可选）
if (
error_summary
and len(queried_similar_error_knowledge_to_render) != 0
and len(queried_former_failed_knowledge_to_render) != 0
):
error_summary_system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
.render(
factor_information_str=target_factor_task_information,
code_and_feedback=queried_former_failed_knowledge_to_render[
-1
].get_implementation_and_feedback_str(),
)
.strip("\n")
)
session_summary = APIBackend(use_chat_cache=False).build_chat_session(
session_system_prompt=error_summary_system_prompt,
)
while True:
error_summary_user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
.render(
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
)
.strip("\n")
)
if (
session_summary.build_chat_completion_message_and_calculate_token(error_summary_user_prompt)
< RD_AGENT_SETTINGS.chat_token_limit
):
break
elif len(queried_similar_error_knowledge_to_render) > 0:
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
error_summary_critics = session_summary.build_chat_completion(
user_prompt=error_summary_user_prompt,
json_mode=False,
)
# 构建user_prompt。开始写代码
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
implement_prompts["evolving_strategy_factor_implementation_v2_user"],
)
.render(
factor_information_str=target_factor_task_information,
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
error_summary=error_summary,
error_summary_critics=error_summary_critics,
)
.strip("\n")
)
if (
session.build_chat_completion_message_and_calculate_token(
user_prompt,
)
< RD_AGENT_SETTINGS.chat_token_limit
):
break
elif len(queried_former_failed_knowledge_to_render) > 1:
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
elif len(queried_similar_component_knowledge_to_render) > len(
queried_similar_error_knowledge_to_render,
):
queried_similar_component_knowledge_to_render = queried_similar_component_knowledge_to_render[:-1]
elif len(queried_similar_error_knowledge_to_render) > 0:
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
response = session.build_chat_completion(
user_prompt=user_prompt,
json_mode=True,
)
code = json.loads(response)["code"]
factor_implementation = FileBasedFactorImplementation(target_task, code)
return factor_implementation
rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py
+912
View File
@@ -0,0 +1,912 @@
from __future__ import annotations
import copy
import json
import random
import re
from itertools import combinations
from pathlib import Path
from typing import Union
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
FactorSingleFeedback,
)
from rdagent.components.coder.factor_coder.factor import FactorTask
from rdagent.components.knowledge_management.graph import (
UndirectedGraph,
UndirectedNode,
)
from rdagent.core.evolving_framework import (
EvolvableSubjects,
EvoStep,
Knowledge,
KnowledgeBase,
QueriedKnowledge,
RAGStrategy,
)
from rdagent.core.experiment import Implementation
from rdagent.core.log import RDAgentLog
from rdagent.core.prompts import Prompts
from rdagent.oai.llm_utils import (
APIBackend,
calculate_embedding_distance_between_str_list,
)
class FactorKnowledge(Knowledge):
def __init__(
self,
target_task: FactorTask,
implementation: Implementation,
feedback: FactorSingleFeedback,
) -> None:
"""
Initialize a FactorKnowledge object. The FactorKnowledge object is used to store a factor implementation without the ground truth code and value.
Args:
factor (Factor): The factor object associated with the KnowledgeManagement.
Returns:
None
"""
self.target_task = target_task
self.implementation = implementation
self.feedback = feedback
def get_implementation_and_feedback_str(self) -> str:
return f"""------------------Factor implementation code:------------------
{self.implementation.code}
------------------Factor implementation feedback:------------------
{self.feedback!s}
"""
class FactorQueriedKnowledge(QueriedKnowledge):
def __init__(self, success_task_to_knowledge_dict: dict = {}, failed_task_info_set: set = set()) -> None:
self.success_task_to_knowledge_dict = success_task_to_knowledge_dict
self.failed_task_info_set = failed_task_info_set
class FactorKnowledgeBaseV1(KnowledgeBase):
def __init__(self) -> None:
self.implementation_trace: dict[str, FactorKnowledge] = dict()
self.success_task_info_set: set[str] = set()
self.task_to_embedding = dict()
def query(self) -> QueriedKnowledge | None:
"""
Query the knowledge base to get the queried knowledge. So far is handled in RAG strategy.
"""
raise NotImplementedError
class FactorQueriedKnowledgeV1(FactorQueriedKnowledge):
def __init__(self) -> None:
self.working_task_to_former_failed_knowledge_dict = dict()
self.working_task_to_similar_successful_knowledge_dict = dict()
super().__init__()
class FactorRAGStrategyV1(RAGStrategy):
def __init__(self, knowledgebase: FactorKnowledgeBaseV1) -> None:
super().__init__(knowledgebase)
self.current_generated_trace_count = 0
def generate_knowledge(
self,
evolving_trace: list[EvoStep],
*,
return_knowledge: bool = False,
) -> Knowledge | None:
if len(evolving_trace) == self.current_generated_trace_count:
return
else:
for trace_index in range(
self.current_generated_trace_count,
len(evolving_trace),
):
evo_step = evolving_trace[trace_index]
implementations = evo_step.evolvable_subjects
feedback = evo_step.feedback
for task_index in range(len(implementations.sub_tasks)):
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_factor_information()
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
single_knowledge = FactorKnowledge(
target_task=target_task,
implementation=implementation,
feedback=single_feedback,
)
if target_task_information not in self.knowledgebase.success_task_info_set:
self.knowledgebase.implementation_trace.setdefault(
target_task_information,
[],
).append(single_knowledge)
if single_feedback.final_decision == True:
self.knowledgebase.success_task_info_set.add(
target_task_information,
)
self.current_generated_trace_count = len(evolving_trace)
def query(
self,
evo: EvolvableSubjects,
evolving_trace: list[EvoStep],
) -> QueriedKnowledge | None:
v1_query_former_trace_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_former_trace_limit
v1_query_similar_success_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_similar_success_limit
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
queried_knowledge = FactorQueriedKnowledgeV1()
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if target_factor_task_information in self.knowledgebase.success_task_info_set:
queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace[target_factor_task_information][-1]
)
elif (
len(
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
),
)
>= fail_task_trial_limit
):
queried_knowledge.failed_task_info_set.add(target_factor_task_information)
else:
queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[-v1_query_former_trace_limit:]
)
knowledge_base_success_task_list = list(
self.knowledgebase.success_task_info_set,
)
similarity = calculate_embedding_distance_between_str_list(
[target_factor_task_information],
knowledge_base_success_task_list,
)[0]
similar_indexes = sorted(
range(len(similarity)),
key=lambda i: similarity[i],
reverse=True,
)[:v1_query_similar_success_limit]
similar_successful_knowledge = [
self.knowledgebase.implementation_trace.setdefault(
knowledge_base_success_task_list[index],
[],
)[-1]
for index in similar_indexes
]
queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = (
similar_successful_knowledge
)
return queried_knowledge
class FactorQueriedGraphKnowledge(FactorQueriedKnowledge):
# Aggregation of knowledge
def __init__(
self,
former_traces: dict = {},
component_with_success_task: dict = {},
error_with_success_task: dict = {},
**kwargs,
) -> None:
self.former_traces = former_traces
self.component_with_success_task = component_with_success_task
self.error_with_success_task = error_with_success_task
super().__init__(**kwargs)
class FactorGraphRAGStrategy(RAGStrategy):
def __init__(self, knowledgebase: FactorGraphKnowledgeBase) -> None:
super().__init__(knowledgebase)
self.current_generated_trace_count = 0
self.prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
def generate_knowledge(
self,
evolving_trace: list[EvoStep],
*,
return_knowledge: bool = False,
) -> Knowledge | None:
if len(evolving_trace) == self.current_generated_trace_count:
return None
else:
for trace_index in range(self.current_generated_trace_count, len(evolving_trace)):
evo_step = evolving_trace[trace_index]
implementations = evo_step.evolvable_subjects
feedback = evo_step.feedback
for task_index in range(len(implementations.sub_tasks)):
single_feedback = feedback[task_index]
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_factor_information()
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
single_knowledge = FactorKnowledge(
target_task=target_task,
implementation=implementation,
feedback=single_feedback,
)
if (
target_task_information not in self.knowledgebase.success_task_to_knowledge_dict
and implementation is not None
):
self.knowledgebase.working_trace_knowledge.setdefault(target_task_information, []).append(
single_knowledge,
) # save to working trace
if single_feedback.final_decision == True:
self.knowledgebase.success_task_to_knowledge_dict.setdefault(
target_task_information,
single_knowledge,
)
# Do summary for the last step and update the knowledge graph
self.knowledgebase.update_success_task(
target_task_information,
)
else:
# generate error node and store into knowledge base
error_analysis_result = []
if not single_feedback.value_generated_flag:
error_analysis_result = self.analyze_error(
single_feedback.execution_feedback,
feedback_type="execution",
)
else:
error_analysis_result = self.analyze_error(
single_feedback.factor_value_feedback,
feedback_type="value",
)
self.knowledgebase.working_trace_error_analysis.setdefault(
target_task_information,
[],
).append(
error_analysis_result,
) # save to working trace error record, for graph update
self.current_generated_trace_count = len(evolving_trace)
return None
def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None:
conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler
factor_implementation_queried_graph_knowledge = FactorQueriedGraphKnowledge(
success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict,
)
factor_implementation_queried_graph_knowledge = self.former_trace_query(
evo,
factor_implementation_queried_graph_knowledge,
FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit,
)
factor_implementation_queried_graph_knowledge = self.component_query(
evo,
factor_implementation_queried_graph_knowledge,
FACTOR_IMPLEMENT_SETTINGS.v2_query_component_limit,
knowledge_sampler=conf_knowledge_sampler,
)
factor_implementation_queried_graph_knowledge = self.error_query(
evo,
factor_implementation_queried_graph_knowledge,
FACTOR_IMPLEMENT_SETTINGS.v2_query_error_limit,
knowledge_sampler=conf_knowledge_sampler,
)
return factor_implementation_queried_graph_knowledge
def analyze_component(
self,
target_factor_task_information,
) -> list[UndirectedNode]: # Hardcode: certain component nodes
all_component_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["component"])
if not len(all_component_nodes):
return []
all_component_content = ""
for _, component_node in enumerate(all_component_nodes):
all_component_content += f"{component_node.content}, \n"
analyze_component_system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(self.prompt["analyze_component_prompt_v1_system"])
.render(
all_component_content=all_component_content,
)
)
analyze_component_user_prompt = target_factor_task_information
try:
component_no_list = json.loads(
APIBackend().build_messages_and_create_chat_completion(
system_prompt=analyze_component_system_prompt,
user_prompt=analyze_component_user_prompt,
json_mode=True,
),
)["component_no_list"]
return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))]
except:
RDAgentLog().warning("Error when analyzing components.")
analyze_component_user_prompt = "Your response is not a valid component index list."
return []
def analyze_error(
self,
single_feedback,
feedback_type="execution",
) -> list[
UndirectedNode | str
]: # Hardcode: Raised errors, existed error nodes + not existed error nodes(here, they are strs)
if feedback_type == "execution":
match = re.search(
r'File "(?P<file>.+)", line (?P<line>\d+), in (?P<function>.+)\n\s+(?P<error_line>.+)\n(?P<error_type>\w+): (?P<error_message>.+)',
single_feedback,
)
if match:
error_details = match.groupdict()
# last_traceback = f'File "{error_details["file"]}", line {error_details["line"]}, in {error_details["function"]}\n {error_details["error_line"]}'
error_type = error_details["error_type"]
error_line = error_details["error_line"]
error_contents = [f"ErrorType: {error_type}" + "\n" + f"Error line: {error_line}"]
else:
error_contents = ["Undefined Error"]
elif feedback_type == "value": # value check error
value_check_types = r"The source dataframe and the ground truth dataframe have different rows count.|The source dataframe and the ground truth dataframe have different index.|Some values differ by more than the tolerance of 1e-6.|No sufficient correlation found when shifting up|Something wrong happens when naming the multi indices of the dataframe."
error_contents = re.findall(value_check_types, single_feedback)
else:
error_contents = ["Undefined Error"]
all_error_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["error"])
if not len(all_error_nodes):
return error_contents
else:
error_list = []
for error_content in error_contents:
for error_node in all_error_nodes:
if error_content == error_node.content:
error_list.append(error_node)
else:
error_list.append(error_content)
if error_list[-1] in error_list[:-1]:
error_list.pop()
return error_list
def former_trace_query(
self,
evo: EvolvableSubjects,
factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
v2_query_former_trace_limit: int = 5,
) -> Union[QueriedKnowledge, set]:
"""
Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
"""
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if (
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
and target_factor_task_information in self.knowledgebase.working_trace_knowledge
and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information])
>= fail_task_trial_limit
):
factor_implementation_queried_graph_knowledge.failed_task_info_set.add(target_factor_task_information)
if (
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
and target_factor_task_information
not in factor_implementation_queried_graph_knowledge.failed_task_info_set
and target_factor_task_information in self.knowledgebase.working_trace_knowledge
):
former_trace_knowledge = copy.copy(
self.knowledgebase.working_trace_knowledge[target_factor_task_information],
)
# in former trace query we will delete the right trace in the following order:[..., value_generated_flag is True, value_generated_flag is False, ...]
# because we think this order means a deterioration of the trial (like a wrong gradient descent)
current_index = 1
while current_index < len(former_trace_knowledge):
if (
not former_trace_knowledge[current_index].feedback.value_generated_flag
and former_trace_knowledge[current_index - 1].feedback.value_generated_flag
):
former_trace_knowledge.pop(current_index)
else:
current_index += 1
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
former_trace_knowledge[-v2_query_former_trace_limit:]
)
else:
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
return factor_implementation_queried_graph_knowledge
def component_query(
self,
evo: EvolvableSubjects,
factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
v2_query_component_limit: int = 5,
knowledge_sampler: float = 1.0,
) -> QueriedKnowledge | None:
# queried_component_knowledge = FactorQueriedGraphComponentKnowledge()
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
if (
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
):
factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
] = []
else:
if target_factor_task_information not in self.knowledgebase.task_to_component_nodes:
self.knowledgebase.task_to_component_nodes[target_factor_task_information] = self.analyze_component(
target_factor_task_information,
)
component_analysis_result = self.knowledgebase.task_to_component_nodes[target_factor_task_information]
if len(component_analysis_result) > 1:
task_des_node_list = self.knowledgebase.graph_query_by_intersection(
component_analysis_result,
constraint_labels=["task_description"],
)
single_component_constraint = (v2_query_component_limit // len(component_analysis_result)) + 1
else:
task_des_node_list = []
single_component_constraint = v2_query_component_limit
factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
] = []
for component_node in component_analysis_result:
# Reverse iterate, a trade-off with intersection search
count = 0
for task_des_node in self.knowledgebase.graph_query_by_node(
node=component_node,
step=1,
constraint_labels=["task_description"],
block=True,
)[::-1]:
if task_des_node not in task_des_node_list:
task_des_node_list.append(task_des_node)
count += 1
if count >= single_component_constraint:
break
for node in task_des_node_list:
for searched_node in self.knowledgebase.graph_query_by_node(
node=node,
step=50,
constraint_labels=[
"task_success_implement",
],
block=True,
):
if searched_node.label == "task_success_implement":
target_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[
searched_node.id
]
if (
target_knowledge
not in factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
]
):
factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
].append(target_knowledge)
# finally add embedding related knowledge
knowledge_base_success_task_list = list(self.knowledgebase.success_task_to_knowledge_dict)
similarity = calculate_embedding_distance_between_str_list(
[target_factor_task_information],
knowledge_base_success_task_list,
)[0]
similar_indexes = sorted(
range(len(similarity)),
key=lambda i: similarity[i],
reverse=True,
)
embedding_similar_successful_knowledge = [
self.knowledgebase.success_task_to_knowledge_dict[knowledge_base_success_task_list[index]]
for index in similar_indexes
]
for knowledge in embedding_similar_successful_knowledge:
if (
knowledge
not in factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
]
):
factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
].append(knowledge)
if knowledge_sampler > 0:
factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
] = [
knowledge
for knowledge in factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
]
if random.uniform(0, 1) <= knowledge_sampler
]
# Make sure no less than half of the knowledge are from GT
queried_knowledge_list = factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
]
queried_from_gt_knowledge_list = [
knowledge
for knowledge in queried_knowledge_list
if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == True
]
queried_without_gt_knowledge_list = [
knowledge
for knowledge in queried_knowledge_list
if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False
]
queried_from_gt_knowledge_count = max(
min(v2_query_component_limit // 2, len(queried_from_gt_knowledge_list)),
v2_query_component_limit - len(queried_without_gt_knowledge_list),
)
factor_implementation_queried_graph_knowledge.component_with_success_task[
target_factor_task_information
] = (
queried_from_gt_knowledge_list[:queried_from_gt_knowledge_count]
+ queried_without_gt_knowledge_list[: v2_query_component_limit - queried_from_gt_knowledge_count]
)
return factor_implementation_queried_graph_knowledge
def error_query(
self,
evo: EvolvableSubjects,
factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
v2_query_error_limit: int = 5,
knowledge_sampler: float = 1.0,
) -> QueriedKnowledge | None:
# queried_error_knowledge = FactorQueriedGraphErrorKnowledge()
for task_index, target_factor_task in enumerate(evo.sub_tasks):
target_factor_task_information = target_factor_task.get_factor_information()
factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
if (
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
):
factor_implementation_queried_graph_knowledge.error_with_success_task[
target_factor_task_information
] = []
else:
factor_implementation_queried_graph_knowledge.error_with_success_task[
target_factor_task_information
] = []
if (
target_factor_task_information in self.knowledgebase.working_trace_error_analysis
and len(self.knowledgebase.working_trace_error_analysis[target_factor_task_information]) > 0
and len(factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information])
> 0
):
queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[
target_factor_task_information
][-1]
target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
queried_last_trace,
)
last_knowledge_error_analysis_result = self.knowledgebase.working_trace_error_analysis[
target_factor_task_information
][target_index]
else:
last_knowledge_error_analysis_result = []
error_nodes = []
for error_node in last_knowledge_error_analysis_result:
if not isinstance(error_node, UndirectedNode):
error_node = self.knowledgebase.graph_get_node_by_content(content=error_node)
if error_node is None:
continue
error_nodes.append(error_node)
if len(error_nodes) > 1:
task_trace_node_list = self.knowledgebase.graph_query_by_intersection(
error_nodes,
constraint_labels=["task_trace"],
output_intersection_origin=True,
)
single_error_constraint = (v2_query_error_limit // len(error_nodes)) + 1
else:
task_trace_node_list = []
single_error_constraint = v2_query_error_limit
for error_node in error_nodes:
# Reverse iterate, a trade-off with intersection search
count = 0
for task_trace_node in self.knowledgebase.graph_query_by_node(
node=error_node,
step=1,
constraint_labels=["task_trace"],
block=True,
)[::-1]:
if task_trace_node not in task_trace_node_list:
task_trace_node_list.append([[error_node], task_trace_node])
count += 1
if count >= single_error_constraint:
break
# for error_node in last_knowledge_error_analysis_result:
# if not isinstance(error_node, UndirectedNode):
# error_node = self.knowledgebase.graph_get_node_by_content(content=error_node)
# if error_node is None:
# continue
# for searched_node in self.knowledgebase.graph_query_by_node(
# node=error_node,
# step=1,
# constraint_labels=["task_trace"],
# block=True,
# ):
# if searched_node not in [node[0] for node in task_trace_node_list]:
# task_trace_node_list.append((searched_node, error_node.content))
same_error_success_knowledge_pair_list = []
same_error_success_node_set = set()
for error_node_list, trace_node in task_trace_node_list:
for searched_trace_success_node in self.knowledgebase.graph_query_by_node(
node=trace_node,
step=50,
constraint_labels=[
"task_trace",
"task_success_implement",
"task_description",
],
block=True,
):
if (
searched_trace_success_node not in same_error_success_node_set
and searched_trace_success_node.label == "task_success_implement"
):
same_error_success_node_set.add(searched_trace_success_node)
trace_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[trace_node.id]
success_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[
searched_trace_success_node.id
]
error_content = ""
for index, error_node in enumerate(error_node_list):
error_content += f"{index+1}. {error_node.content}; "
same_error_success_knowledge_pair_list.append(
(
error_content,
(trace_knowledge, success_knowledge),
),
)
if knowledge_sampler > 0:
same_error_success_knowledge_pair_list = [
knowledge
for knowledge in same_error_success_knowledge_pair_list
if random.uniform(0, 1) <= knowledge_sampler
]
same_error_success_knowledge_pair_list = same_error_success_knowledge_pair_list[:v2_query_error_limit]
factor_implementation_queried_graph_knowledge.error_with_success_task[
target_factor_task_information
] = same_error_success_knowledge_pair_list
return factor_implementation_queried_graph_knowledge
class FactorGraphKnowledgeBase(KnowledgeBase):
def __init__(self, init_component_list=None) -> None:
"""
Load knowledge, offer brief information of knowledge and common handle interfaces
"""
self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl")
RDAgentLog().info(f"Knowledge Graph loaded, size={self.graph.size()}")
if init_component_list:
for component in init_component_list:
exist_node = self.graph.get_node_by_content(content=component)
node = exist_node if exist_node else UndirectedNode(content=component, label="component")
self.graph.add_nodes(node=node, neighbors=[])
# A dict containing all working trace until they fail or succeed
self.working_trace_knowledge = {}
# A dict containing error analysis each step aligned with working trace
self.working_trace_error_analysis = {}
# Add already success task
self.success_task_to_knowledge_dict = {}
# key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorKnowledge')
self.node_to_implementation_knowledge_dict = {}
# store the task description to component nodes
self.task_to_component_nodes = {}
def get_all_nodes_by_label(self, label: str) -> list[UndirectedNode]:
return self.graph.get_all_nodes_by_label(label)
def update_success_task(
self,
success_task_info: str,
): # Transfer the success tasks' working trace to knowledge storage & graph
success_task_trace = self.working_trace_knowledge[success_task_info]
success_task_error_analysis_record = (
self.working_trace_error_analysis[success_task_info]
if success_task_info in self.working_trace_error_analysis
else []
)
task_des_node = UndirectedNode(content=success_task_info, label="task_description")
self.graph.add_nodes(
node=task_des_node,
neighbors=self.task_to_component_nodes[success_task_info],
) # 1st version, we assume that all component nodes are given
for index, trace_unit in enumerate(success_task_trace): # every unit: single_knowledge
neighbor_nodes = [task_des_node]
if index != len(success_task_trace) - 1:
trace_node = UndirectedNode(
content=trace_unit.get_implementation_and_feedback_str(),
label="task_trace",
)
self.node_to_implementation_knowledge_dict[trace_node.id] = trace_unit
for node_index, error_node in enumerate(success_task_error_analysis_record[index]):
if type(error_node).__name__ == "str":
queried_node = self.graph.get_node_by_content(content=error_node)
if queried_node is None:
new_error_node = UndirectedNode(content=error_node, label="error")
self.graph.add_node(node=new_error_node)
success_task_error_analysis_record[index][node_index] = new_error_node
else:
success_task_error_analysis_record[index][node_index] = queried_node
neighbor_nodes.extend(success_task_error_analysis_record[index])
self.graph.add_nodes(node=trace_node, neighbors=neighbor_nodes)
else:
success_node = UndirectedNode(
content=trace_unit.get_implementation_and_feedback_str(),
label="task_success_implement",
)
self.graph.add_nodes(node=success_node, neighbors=neighbor_nodes)
self.node_to_implementation_knowledge_dict[success_node.id] = trace_unit
def query(self):
pass
def graph_get_node_by_content(self, content: str) -> UndirectedNode:
return self.graph.get_node_by_content(content=content)
def graph_query_by_content(
self,
content: Union[str, list[str]],
topk_k: int = 5,
step: int = 1,
constraint_labels: list[str] = None,
constraint_node: UndirectedNode = None,
similarity_threshold: float = 0.0,
constraint_distance: float = 0,
block: bool = False,
) -> list[UndirectedNode]:
"""
search graph by content similarity and connection relationship, return empty list if nodes' chain without node
near to constraint_node
Parameters
----------
constraint_distance
content
topk_k: the upper number of output for each query, if the number of fit nodes is less than topk_k, return all fit nodes's content
step
constraint_labels
constraint_node
similarity_threshold
block: despite the start node, the search can only flow through the constraint_label type nodes
Returns
-------
"""
return self.graph.query_by_content(
content=content,
topk_k=topk_k,
step=step,
constraint_labels=constraint_labels,
constraint_node=constraint_node,
similarity_threshold=similarity_threshold,
constraint_distance=constraint_distance,
block=block,
)
def graph_query_by_node(
self,
node: UndirectedNode,
step: int = 1,
constraint_labels: list[str] = None,
constraint_node: UndirectedNode = None,
constraint_distance: float = 0,
block: bool = False,
) -> list[UndirectedNode]:
"""
search graph by connection, return empty list if nodes' chain without node near to constraint_node
Parameters
----------
node : start node
step : the max steps will be searched
constraint_labels : the labels of output nodes
constraint_node : the node that the output nodes must connect to
constraint_distance : the max distance between output nodes and constraint_node
block: despite the start node, the search can only flow through the constraint_label type nodes
Returns
-------
A list of nodes
"""
nodes = self.graph.query_by_node(
node=node,
step=step,
constraint_labels=constraint_labels,
constraint_node=constraint_node,
constraint_distance=constraint_distance,
block=block,
)
return nodes
def graph_query_by_intersection(
self,
nodes: list[UndirectedNode],
steps: int = 1,
constraint_labels: list[str] = None,
output_intersection_origin: bool = False,
) -> list[UndirectedNode] | list[list[list[UndirectedNode], UndirectedNode]]:
"""
search graph by node intersection, node intersected by a higher frequency has a prior order in the list
Parameters
----------
nodes : node list
step : the max steps will be searched
constraint_labels : the labels of output nodes
output_intersection_origin: output the list that contains the node which form this intersection node
Returns
-------
A list of nodes
"""
node_count = len(nodes)
assert node_count >= 2, "nodes length must >=2"
intersection_node_list = []
if output_intersection_origin:
origin_list = []
for k in range(node_count, 1, -1):
possible_combinations = combinations(nodes, k)
for possible_combination in possible_combinations:
node_list = list(possible_combination)
intersection_node_list.extend(
self.graph.get_nodes_intersection(node_list, steps=steps, constraint_labels=constraint_labels),
)
if output_intersection_origin:
for _ in range(len(intersection_node_list)):
origin_list.append(node_list)
intersection_node_list_sort_by_freq = []
for index, node in enumerate(intersection_node_list):
if node not in intersection_node_list_sort_by_freq:
if output_intersection_origin:
intersection_node_list_sort_by_freq.append([origin_list[index], node])
else:
intersection_node_list_sort_by_freq.append(node)
return intersection_node_list_sort_by_freq
rdagent/components/coder/factor_coder/CoSTEER/scheduler.py
+89
View File
@@ -0,0 +1,89 @@
import json
from pathlib import Path
from typing import Dict
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.coder.factor_coder.utils import get_data_folder_intro
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.log import RDAgentLog
from rdagent.core.prompts import Prompts
from rdagent.core.scenario import Scenario
from rdagent.oai.llm_utils import APIBackend
scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
import random
to_be_finished_task_index = random.sample(
to_be_finished_task_index,
implementation_factors_per_round,
)
RDAgentLog().info(f"The random selection is: {to_be_finished_task_index}")
return to_be_finished_task_index
def LLMSelect(
to_be_finished_task_index,
implementation_factors_per_round,
evo: FactorEvolvingItem,
former_trace: Dict,
scen: Scenario,
):
tasks = []
for i in to_be_finished_task_index:
# find corresponding former trace for each task
target_factor_task_information = evo.sub_tasks[i].get_factor_information()
if target_factor_task_information in former_trace:
tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information]))
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
scheduler_prompts["select_implementable_factor_system"],
)
.render(
scenario=scen.get_scenario_all_desc(),
)
)
while True:
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
scheduler_prompts["select_implementable_factor_user"],
)
.render(
factor_num=implementation_factors_per_round,
sub_tasks=tasks,
)
)
if (
APIBackend().build_messages_and_calculate_token(
user_prompt=user_prompt,
system_prompt=system_prompt,
)
< RD_AGENT_SETTINGS.chat_token_limit
):
break
response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=True,
)
try:
selection = json.loads(response)["selected_factor"]
if not isinstance(selection, list):
return to_be_finished_task_index
selection_index = [x for x in selection if isinstance(x, int)]
except:
return to_be_finished_task_index
return selection_index