mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 16:07:46 +00:00
New Framework for idea proposal and implementation on RD-Agent (#34)
* Commit init framework * Co-authored-by: Yuante Li (FESCO Adecco Human Resources) <v-yuanteli@microsoft.com> Co-authored-by: XianBW <XianBW@users.noreply.github.com> * add an import * refine the whole framework * benchmark related framework * fix black and isort errors * move requirements to folder * fix black again --------- Co-authored-by: Young <afe.young@gmail.com> Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
@@ -0,0 +1,734 @@
|
||||
import json
|
||||
import re
|
||||
from abc import abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
import pandas as pd
|
||||
from jinja2 import Template
|
||||
|
||||
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
|
||||
FactorEvovlingItem,
|
||||
FactorImplementTask,
|
||||
)
|
||||
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
|
||||
FACTOR_IMPLEMENT_SETTINGS,
|
||||
)
|
||||
from rdagent.core.conf import RD_AGENT_SETTINGS
|
||||
from rdagent.core.evaluation import Evaluator
|
||||
from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
|
||||
from rdagent.core.log import RDAgentLog
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.task import TaskImplementation
|
||||
from rdagent.core.utils import multiprocessing_wrapper
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
|
||||
|
||||
class FactorImplementationEvaluator(Evaluator):
|
||||
# TODO:
|
||||
# I think we should have unified interface for all evaluates, for examples.
|
||||
# So we should adjust the interface of other factors
|
||||
@abstractmethod
|
||||
def evaluate(
|
||||
self,
|
||||
gt: TaskImplementation,
|
||||
gen: TaskImplementation,
|
||||
) -> Tuple[str, object]:
|
||||
"""You can get the dataframe by
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
_, gt_df = gt.execute()
|
||||
_, gen_df = gen.execute()
|
||||
|
||||
Returns
|
||||
-------
|
||||
Tuple[str, object]
|
||||
- str: the text-based description of the evaluation result
|
||||
- object: a comparable metric (bool, integer, float ...)
|
||||
|
||||
"""
|
||||
raise NotImplementedError("Please implement the `evaluator` method")
|
||||
|
||||
def _get_df(self, gt: TaskImplementation, gen: TaskImplementation):
|
||||
_, gt_df = gt.execute()
|
||||
_, gen_df = gen.execute()
|
||||
if isinstance(gen_df, pd.Series):
|
||||
gen_df = gen_df.to_frame("source_factor")
|
||||
if isinstance(gt_df, pd.Series):
|
||||
gt_df = gt_df.to_frame("gt_factor")
|
||||
return gt_df, gen_df
|
||||
|
||||
|
||||
class FactorImplementationCodeEvaluator(Evaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorImplementTask,
|
||||
implementation: TaskImplementation,
|
||||
execution_feedback: str,
|
||||
factor_value_feedback: str = "",
|
||||
gt_implementation: TaskImplementation = None,
|
||||
**kwargs,
|
||||
):
|
||||
factor_information = target_task.get_factor_information()
|
||||
code = implementation.code
|
||||
|
||||
system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"]
|
||||
|
||||
execution_feedback_to_render = execution_feedback
|
||||
user_prompt = Template(
|
||||
evaluate_prompts["evaluator_code_feedback_v1_user"],
|
||||
).render(
|
||||
factor_information=factor_information,
|
||||
code=code,
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
factor_value_feedback=factor_value_feedback,
|
||||
gt_code=gt_implementation.code if gt_implementation else None,
|
||||
)
|
||||
while (
|
||||
APIBackend().build_messages_and_calculate_token(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
former_messages=[],
|
||||
)
|
||||
> RD_AGENT_SETTINGS.chat_token_limit
|
||||
):
|
||||
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
|
||||
user_prompt = Template(
|
||||
evaluate_prompts["evaluator_code_feedback_v1_user"],
|
||||
).render(
|
||||
factor_information=factor_information,
|
||||
code=code,
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
factor_value_feedback=factor_value_feedback,
|
||||
gt_code=gt_implementation.code if gt_implementation else None,
|
||||
)
|
||||
critic_response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=False,
|
||||
)
|
||||
|
||||
return critic_response
|
||||
|
||||
|
||||
class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
gt: TaskImplementation,
|
||||
gen: TaskImplementation,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt, gen)
|
||||
|
||||
if len(gen_df.columns) == 1 and len(gt_df.columns) == 1:
|
||||
return "Both dataframes have only one column.", True
|
||||
elif len(gen_df.columns) != 1:
|
||||
gen_df = gen_df.iloc(axis=1)[
|
||||
[
|
||||
0,
|
||||
]
|
||||
]
|
||||
return (
|
||||
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
|
||||
False,
|
||||
)
|
||||
return "", False
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
gt: TaskImplementation,
|
||||
gen: TaskImplementation,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt, gen)
|
||||
idx_name_right = gen_df.index.names == ("datetime", "instrument")
|
||||
if idx_name_right:
|
||||
return (
|
||||
'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
|
||||
True,
|
||||
)
|
||||
else:
|
||||
return (
|
||||
'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.',
|
||||
False,
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
gt: TaskImplementation,
|
||||
gen: TaskImplementation,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt, gen)
|
||||
|
||||
if gen_df.shape[0] == gt_df.shape[0]:
|
||||
return "Both dataframes have the same rows count.", True
|
||||
else:
|
||||
return (
|
||||
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorImplementationIndexEvaluator(FactorImplementationEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
gt: TaskImplementation,
|
||||
gen: TaskImplementation,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt, gen)
|
||||
|
||||
if gen_df.index.equals(gt_df.index):
|
||||
return "Both dataframes have the same index.", True
|
||||
else:
|
||||
return (
|
||||
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
gt: TaskImplementation,
|
||||
gen: TaskImplementation,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt, gen)
|
||||
|
||||
if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
|
||||
return "Both dataframes have the same missing values.", True
|
||||
else:
|
||||
return (
|
||||
f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorImplementationValuesEvaluator(FactorImplementationEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
gt: TaskImplementation,
|
||||
gen: TaskImplementation,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt, gen)
|
||||
|
||||
try:
|
||||
close_values = gen_df.sub(gt_df).abs().lt(1e-6)
|
||||
result_int = close_values.astype(int)
|
||||
pos_num = result_int.sum().sum()
|
||||
acc_rate = pos_num / close_values.size
|
||||
except:
|
||||
close_values = gen_df
|
||||
if close_values.all().iloc[0]:
|
||||
return (
|
||||
"All values in the dataframes are equal within the tolerance of 1e-6.",
|
||||
acc_rate,
|
||||
)
|
||||
else:
|
||||
return (
|
||||
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
|
||||
acc_rate,
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator):
|
||||
def __init__(self, hard_check: bool) -> None:
|
||||
self.hard_check = hard_check
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
gt: TaskImplementation,
|
||||
gen: TaskImplementation,
|
||||
) -> Tuple[str, object]:
|
||||
gt_df, gen_df = self._get_df(gt, gen)
|
||||
|
||||
concat_df = pd.concat([gen_df, gt_df], axis=1)
|
||||
concat_df.columns = ["source", "gt"]
|
||||
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
|
||||
ric = (
|
||||
concat_df.groupby("datetime")
|
||||
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
|
||||
.dropna()
|
||||
.mean()
|
||||
)
|
||||
|
||||
if self.hard_check:
|
||||
if ic > 0.99 and ric > 0.99:
|
||||
return (
|
||||
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
|
||||
True,
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
|
||||
False,
|
||||
)
|
||||
else:
|
||||
return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorImplementationValEvaluator(FactorImplementationEvaluator):
|
||||
def evaluate(self, gt: TaskImplementation, gen: TaskImplementation):
|
||||
_, gt_df = gt.execute()
|
||||
_, gen_df = gen.execute()
|
||||
# FIXME: refactor the two classes
|
||||
fiv = FactorImplementationValueEvaluator()
|
||||
return fiv.evaluate(source_df=gen_df, gt_df=gt_df)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class FactorImplementationValueEvaluator(Evaluator):
|
||||
# TODO: let's discuss the about the interface of the evaluator
|
||||
def evaluate(
|
||||
self,
|
||||
source_df: pd.DataFrame,
|
||||
gt_df: pd.DataFrame,
|
||||
**kwargs,
|
||||
) -> Tuple:
|
||||
conclusions = []
|
||||
|
||||
if isinstance(source_df, pd.Series):
|
||||
source_df = source_df.to_frame("source_factor")
|
||||
conclusions.append(
|
||||
"The source dataframe is a series, better convert it to a dataframe.",
|
||||
)
|
||||
if gt_df is not None and isinstance(gt_df, pd.Series):
|
||||
gt_df = gt_df.to_frame("gt_factor")
|
||||
conclusions.append(
|
||||
"The ground truth dataframe is a series, convert it to a dataframe.",
|
||||
)
|
||||
|
||||
# Check if both dataframe has only one columns
|
||||
if len(source_df.columns) == 1:
|
||||
conclusions.append("The source dataframe has only one column which is correct.")
|
||||
else:
|
||||
conclusions.append(
|
||||
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
|
||||
)
|
||||
source_df = source_df.iloc(axis=1)[
|
||||
[
|
||||
0,
|
||||
]
|
||||
]
|
||||
|
||||
if list(source_df.index.names) != ["datetime", "instrument"]:
|
||||
conclusions.append(
|
||||
rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.",
|
||||
)
|
||||
else:
|
||||
conclusions.append(
|
||||
'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.',
|
||||
)
|
||||
|
||||
# Check if both dataframe have the same rows count
|
||||
if gt_df is not None:
|
||||
if source_df.shape[0] == gt_df.shape[0]:
|
||||
conclusions.append("Both dataframes have the same rows count.")
|
||||
same_row_count_result = True
|
||||
else:
|
||||
conclusions.append(
|
||||
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
|
||||
)
|
||||
same_row_count_result = False
|
||||
|
||||
# Check whether both dataframe has the same index
|
||||
if source_df.index.equals(gt_df.index):
|
||||
conclusions.append("Both dataframes have the same index.")
|
||||
same_index_result = True
|
||||
else:
|
||||
conclusions.append(
|
||||
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
|
||||
)
|
||||
same_index_result = False
|
||||
|
||||
# Check for the same missing values (NaN)
|
||||
if source_df.isna().sum().sum() == gt_df.isna().sum().sum():
|
||||
conclusions.append("Both dataframes have the same missing values.")
|
||||
same_missing_values_result = True
|
||||
else:
|
||||
conclusions.append(
|
||||
f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
|
||||
)
|
||||
same_missing_values_result = False
|
||||
|
||||
# Check if the values are the same within a small tolerance
|
||||
if not same_index_result:
|
||||
conclusions.append(
|
||||
"The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless",
|
||||
)
|
||||
same_values_result = False
|
||||
high_correlation_result = False
|
||||
else:
|
||||
close_values = source_df.sub(gt_df).abs().lt(1e-6)
|
||||
if close_values.all().iloc[0]:
|
||||
conclusions.append(
|
||||
"All values in the dataframes are equal within the tolerance of 1e-6.",
|
||||
)
|
||||
same_values_result = True
|
||||
else:
|
||||
conclusions.append(
|
||||
"Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
|
||||
)
|
||||
same_values_result = False
|
||||
|
||||
# Check the ic and rankic between the two dataframes
|
||||
concat_df = pd.concat([source_df, gt_df], axis=1)
|
||||
concat_df.columns = ["source", "gt"]
|
||||
try:
|
||||
ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
|
||||
ric = (
|
||||
concat_df.groupby("datetime")
|
||||
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
|
||||
.dropna()
|
||||
.mean()
|
||||
)
|
||||
|
||||
if ic > 0.99 and ric > 0.99:
|
||||
conclusions.append(
|
||||
f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
|
||||
)
|
||||
high_correlation_result = True
|
||||
else:
|
||||
conclusions.append(
|
||||
f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
|
||||
)
|
||||
high_correlation_result = False
|
||||
|
||||
# Check for shifted alignments only in the "datetime" index
|
||||
max_shift_days = 2
|
||||
for shift in range(-max_shift_days, max_shift_days + 1):
|
||||
if shift == 0:
|
||||
continue # Skip the case where there is no shift
|
||||
|
||||
shifted_source_df = source_df.groupby(level="instrument").shift(shift)
|
||||
concat_df = pd.concat([shifted_source_df, gt_df], axis=1)
|
||||
concat_df.columns = ["source", "gt"]
|
||||
shifted_ric = (
|
||||
concat_df.groupby("datetime")
|
||||
.apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
|
||||
.dropna()
|
||||
.mean()
|
||||
)
|
||||
if shifted_ric > 0.99:
|
||||
conclusions.append(
|
||||
f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.",
|
||||
)
|
||||
break
|
||||
else:
|
||||
conclusions.append(
|
||||
f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
RDAgentLog().warning(f"Error occurred when calculating the correlation: {str(e)}")
|
||||
conclusions.append(
|
||||
f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}",
|
||||
)
|
||||
high_correlation_result = False
|
||||
|
||||
# Combine all conclusions into a single string
|
||||
conclusion_str = "\n".join(conclusions)
|
||||
|
||||
final_result = (same_values_result or high_correlation_result) if gt_df is not None else False
|
||||
return conclusion_str, final_result
|
||||
|
||||
|
||||
# TODO:
|
||||
def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
|
||||
"""When the prompt is too long. We have to shorten it.
|
||||
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
|
||||
"""
|
||||
# TODO: this should replace most of code in
|
||||
# - FactorImplementationFinalDecisionEvaluator.evaluate
|
||||
# - FactorImplementationCodeEvaluator.evaluate
|
||||
|
||||
|
||||
class FactorImplementationFinalDecisionEvaluator(Evaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorImplementTask,
|
||||
execution_feedback: str,
|
||||
value_feedback: str,
|
||||
code_feedback: str,
|
||||
**kwargs,
|
||||
) -> Tuple:
|
||||
system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
|
||||
"evaluator_final_decision_v1_system"
|
||||
]
|
||||
execution_feedback_to_render = execution_feedback
|
||||
user_prompt = Template(
|
||||
evaluate_prompts["evaluator_final_decision_v1_user"],
|
||||
).render(
|
||||
factor_information=target_task.get_factor_information(),
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
code_feedback=code_feedback,
|
||||
factor_value_feedback=(
|
||||
value_feedback
|
||||
if value_feedback is not None
|
||||
else "No Ground Truth Value provided, so no evaluation on value is performed."
|
||||
),
|
||||
)
|
||||
while (
|
||||
APIBackend().build_messages_and_calculate_token(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
former_messages=[],
|
||||
)
|
||||
> RD_AGENT_SETTINGS.chat_token_limit
|
||||
):
|
||||
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
|
||||
user_prompt = Template(
|
||||
evaluate_prompts["evaluator_final_decision_v1_user"],
|
||||
).render(
|
||||
factor_information=target_task.get_factor_information(),
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
code_feedback=code_feedback,
|
||||
factor_value_feedback=(
|
||||
value_feedback
|
||||
if value_feedback is not None
|
||||
else "No Ground Truth Value provided, so no evaluation on value is performed."
|
||||
),
|
||||
)
|
||||
|
||||
final_evaluation_dict = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
),
|
||||
)
|
||||
return (
|
||||
final_evaluation_dict["final_decision"],
|
||||
final_evaluation_dict["final_feedback"],
|
||||
)
|
||||
|
||||
|
||||
class FactorImplementationSingleFeedback:
|
||||
"""This class is a feedback to single implementation which is generated from an evaluator."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
execution_feedback: str = None,
|
||||
value_generated_flag: bool = False,
|
||||
code_feedback: str = None,
|
||||
factor_value_feedback: str = None,
|
||||
final_decision: bool = None,
|
||||
final_feedback: str = None,
|
||||
final_decision_based_on_gt: bool = None,
|
||||
) -> None:
|
||||
self.execution_feedback = execution_feedback
|
||||
self.value_generated_flag = value_generated_flag
|
||||
self.code_feedback = code_feedback
|
||||
self.factor_value_feedback = factor_value_feedback
|
||||
self.final_decision = final_decision
|
||||
self.final_feedback = final_feedback
|
||||
self.final_decision_based_on_gt = final_decision_based_on_gt
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"""------------------Factor Execution Feedback------------------
|
||||
{self.execution_feedback}
|
||||
------------------Factor Code Feedback------------------
|
||||
{self.code_feedback}
|
||||
------------------Factor Value Feedback------------------
|
||||
{self.factor_value_feedback}
|
||||
------------------Factor Final Feedback------------------
|
||||
{self.final_feedback}
|
||||
------------------Factor Final Decision------------------
|
||||
This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
|
||||
"""
|
||||
|
||||
|
||||
class FactorImplementationsMultiFeedback(
|
||||
Feedback,
|
||||
List[FactorImplementationSingleFeedback],
|
||||
):
|
||||
"""Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
|
||||
|
||||
|
||||
class FactorImplementationEvaluatorV1(FactorImplementationEvaluator):
|
||||
"""This class is the v1 version of evaluator for a single factor implementation.
|
||||
It calls several evaluators in share modules to evaluate the factor implementation.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.code_evaluator = FactorImplementationCodeEvaluator()
|
||||
self.value_evaluator = FactorImplementationValueEvaluator()
|
||||
self.final_decision_evaluator = FactorImplementationFinalDecisionEvaluator()
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: FactorImplementTask,
|
||||
implementation: TaskImplementation,
|
||||
gt_implementation: TaskImplementation = None,
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
**kwargs,
|
||||
) -> FactorImplementationSingleFeedback:
|
||||
if implementation is None:
|
||||
return None
|
||||
|
||||
target_task_information = target_task.get_factor_information()
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
):
|
||||
return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
|
||||
elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
|
||||
return FactorImplementationSingleFeedback(
|
||||
execution_feedback="This task has failed too many times, skip implementation.",
|
||||
value_generated_flag=False,
|
||||
code_feedback="This task has failed too many times, skip code evaluation.",
|
||||
factor_value_feedback="This task has failed too many times, skip value evaluation.",
|
||||
final_decision=False,
|
||||
final_feedback="This task has failed too many times, skip final decision evaluation.",
|
||||
final_decision_based_on_gt=False,
|
||||
)
|
||||
else:
|
||||
factor_feedback = FactorImplementationSingleFeedback()
|
||||
(
|
||||
factor_feedback.execution_feedback,
|
||||
source_df,
|
||||
) = implementation.execute()
|
||||
|
||||
# Remove the long list of numbers in the feedback
|
||||
pattern = r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)"
|
||||
factor_feedback.execution_feedback = re.sub(pattern, ", ", factor_feedback.execution_feedback)
|
||||
execution_feedback_lines = [
|
||||
line for line in factor_feedback.execution_feedback.split("\n") if "warning" not in line.lower()
|
||||
]
|
||||
factor_feedback.execution_feedback = "\n".join(execution_feedback_lines)
|
||||
|
||||
if source_df is None:
|
||||
factor_feedback.factor_value_feedback = "No factor value generated, skip value evaluation."
|
||||
factor_feedback.value_generated_flag = False
|
||||
value_decision = None
|
||||
else:
|
||||
factor_feedback.value_generated_flag = True
|
||||
if gt_implementation is not None:
|
||||
_, gt_df = gt_implementation.execute(store_result=True)
|
||||
else:
|
||||
gt_df = None
|
||||
try:
|
||||
source_df = source_df.sort_index()
|
||||
if gt_df is not None:
|
||||
gt_df = gt_df.sort_index()
|
||||
(
|
||||
factor_feedback.factor_value_feedback,
|
||||
value_decision,
|
||||
) = self.value_evaluator.evaluate(source_df=source_df, gt_df=gt_df)
|
||||
except Exception as e:
|
||||
RDAgentLog().warning("Value evaluation failed with exception: %s", e)
|
||||
factor_feedback.factor_value_feedback = "Value evaluation failed."
|
||||
value_decision = False
|
||||
|
||||
factor_feedback.final_decision_based_on_gt = gt_implementation is not None
|
||||
|
||||
if value_decision is not None and value_decision is True:
|
||||
# To avoid confusion, when value_decision is True, we do not need code feedback
|
||||
factor_feedback.code_feedback = "Final decision is True and there are no code critics."
|
||||
factor_feedback.final_decision = value_decision
|
||||
factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
|
||||
else:
|
||||
factor_feedback.code_feedback = self.code_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.factor_value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
(
|
||||
factor_feedback.final_decision,
|
||||
factor_feedback.final_feedback,
|
||||
) = self.final_decision_evaluator.evaluate(
|
||||
target_task=target_task,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
value_feedback=factor_feedback.factor_value_feedback,
|
||||
code_feedback=factor_feedback.code_feedback,
|
||||
)
|
||||
return factor_feedback
|
||||
|
||||
|
||||
class FactorImplementationsMultiEvaluator(Evaluator):
|
||||
def __init__(self, single_evaluator=FactorImplementationEvaluatorV1()) -> None:
|
||||
super().__init__()
|
||||
self.single_factor_implementation_evaluator = single_evaluator
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
evo: FactorEvovlingItem,
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
**kwargs,
|
||||
) -> FactorImplementationsMultiFeedback:
|
||||
multi_implementation_feedback = FactorImplementationsMultiFeedback()
|
||||
|
||||
# for index in range(len(evo.target_factor_tasks)):
|
||||
# corresponding_implementation = evo.corresponding_implementations[index]
|
||||
# corresponding_gt_implementation = (
|
||||
# evo.corresponding_gt_implementations[index]
|
||||
# if evo.corresponding_gt_implementations is not None
|
||||
# else None
|
||||
# )
|
||||
|
||||
# multi_implementation_feedback.append(
|
||||
# self.single_factor_implementation_evaluator.evaluate(
|
||||
# target_task=evo.target_factor_tasks[index],
|
||||
# implementation=corresponding_implementation,
|
||||
# gt_implementation=corresponding_gt_implementation,
|
||||
# queried_knowledge=queried_knowledge,
|
||||
# )
|
||||
# )
|
||||
|
||||
calls = []
|
||||
for index in range(len(evo.target_factor_tasks)):
|
||||
corresponding_implementation = evo.corresponding_implementations[index]
|
||||
corresponding_gt_implementation = (
|
||||
evo.corresponding_gt_implementations[index]
|
||||
if evo.corresponding_gt_implementations is not None
|
||||
else None
|
||||
)
|
||||
calls.append(
|
||||
(
|
||||
self.single_factor_implementation_evaluator.evaluate,
|
||||
(
|
||||
evo.target_factor_tasks[index],
|
||||
corresponding_implementation,
|
||||
corresponding_gt_implementation,
|
||||
queried_knowledge,
|
||||
),
|
||||
),
|
||||
)
|
||||
multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n)
|
||||
|
||||
final_decision = [
|
||||
None if single_feedback is None else single_feedback.final_decision
|
||||
for single_feedback in multi_implementation_feedback
|
||||
]
|
||||
RDAgentLog().info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
|
||||
|
||||
return multi_implementation_feedback
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from abc import abstractmethod
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from jinja2 import Template
|
||||
|
||||
from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
|
||||
FactorEvovlingItem,
|
||||
FactorImplementTask,
|
||||
FileBasedFactorImplementation,
|
||||
)
|
||||
from rdagent.components.task_implementation.factor_implementation.evolving.scheduler import (
|
||||
LLMSelect,
|
||||
RandomSelect,
|
||||
)
|
||||
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
|
||||
FACTOR_IMPLEMENT_SETTINGS,
|
||||
)
|
||||
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import (
|
||||
get_data_folder_intro,
|
||||
)
|
||||
from rdagent.core.conf import RD_AGENT_SETTINGS
|
||||
from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.task import TaskImplementation
|
||||
from rdagent.core.utils import multiprocessing_wrapper
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from rdagent.components.task_implementation.factor_implementation.evolving.knowledge_management import (
|
||||
FactorImplementationQueriedKnowledge,
|
||||
FactorImplementationQueriedKnowledgeV1,
|
||||
)
|
||||
|
||||
implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
|
||||
|
||||
class MultiProcessEvolvingStrategy(EvolvingStrategy):
|
||||
@abstractmethod
|
||||
def implement_one_factor(
|
||||
self,
|
||||
target_task: FactorImplementTask,
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
) -> TaskImplementation:
|
||||
raise NotImplementedError
|
||||
|
||||
def evolve(
|
||||
self,
|
||||
*,
|
||||
evo: FactorEvovlingItem,
|
||||
queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
|
||||
**kwargs,
|
||||
) -> FactorEvovlingItem:
|
||||
self.num_loop += 1
|
||||
new_evo = deepcopy(evo)
|
||||
|
||||
# 1.找出需要evolve的factor
|
||||
to_be_finished_task_index = []
|
||||
for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
|
||||
target_factor_task_desc = target_factor_task.get_factor_information()
|
||||
if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
|
||||
new_evo.corresponding_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
|
||||
target_factor_task_desc
|
||||
].implementation
|
||||
elif (
|
||||
target_factor_task_desc not in queried_knowledge.success_task_to_knowledge_dict
|
||||
and target_factor_task_desc not in queried_knowledge.failed_task_info_set
|
||||
):
|
||||
to_be_finished_task_index.append(index)
|
||||
|
||||
# 2. 选择selection方法
|
||||
# if the number of factors to be implemented is larger than the limit, we need to select some of them
|
||||
if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1:
|
||||
# if the number of loops is equal to the select_loop, we need to select some of them
|
||||
implementation_factors_per_round = int(
|
||||
FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index)
|
||||
)
|
||||
if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
|
||||
to_be_finished_task_index = RandomSelect(
|
||||
to_be_finished_task_index,
|
||||
implementation_factors_per_round,
|
||||
)
|
||||
|
||||
if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
|
||||
to_be_finished_task_index = LLMSelect(
|
||||
to_be_finished_task_index,
|
||||
implementation_factors_per_round,
|
||||
new_evo,
|
||||
queried_knowledge.former_traces,
|
||||
)
|
||||
|
||||
result = multiprocessing_wrapper(
|
||||
[
|
||||
(self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge))
|
||||
for target_index in to_be_finished_task_index
|
||||
],
|
||||
n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
|
||||
)
|
||||
|
||||
for index, target_index in enumerate(to_be_finished_task_index):
|
||||
new_evo.corresponding_implementations[target_index] = result[index]
|
||||
|
||||
# for target_index in to_be_finished_task_index:
|
||||
# new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
|
||||
# new_evo.target_factor_tasks[target_index], queried_knowledge
|
||||
# )
|
||||
|
||||
new_evo.corresponding_selection = to_be_finished_task_index
|
||||
|
||||
return new_evo
|
||||
|
||||
|
||||
class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
def implement_one_factor(
|
||||
self,
|
||||
target_task: FactorImplementTask,
|
||||
queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
|
||||
) -> TaskImplementation:
|
||||
factor_information_str = target_task.get_factor_information()
|
||||
|
||||
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
|
||||
return queried_knowledge.success_task_to_knowledge_dict[factor_information_str].implementation
|
||||
elif queried_knowledge is not None and factor_information_str in queried_knowledge.failed_task_info_set:
|
||||
return None
|
||||
else:
|
||||
queried_similar_successful_knowledge = (
|
||||
queried_knowledge.working_task_to_similar_successful_knowledge_dict[factor_information_str]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
)
|
||||
queried_former_failed_knowledge = (
|
||||
queried_knowledge.working_task_to_former_failed_knowledge_dict[factor_information_str]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
)
|
||||
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
|
||||
|
||||
system_prompt = Template(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
|
||||
).render(
|
||||
data_info=get_data_folder_intro(),
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
|
||||
)
|
||||
session = APIBackend(use_chat_cache=False).build_chat_session(
|
||||
session_system_prompt=system_prompt,
|
||||
)
|
||||
|
||||
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
|
||||
while True:
|
||||
user_prompt = (
|
||||
Template(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v1_user"],
|
||||
)
|
||||
.render(
|
||||
factor_information_str=factor_information_str,
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
if (
|
||||
session.build_chat_completion_message_and_calculate_token(
|
||||
user_prompt,
|
||||
)
|
||||
< RD_AGENT_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_former_failed_knowledge_to_render) > 1:
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
|
||||
elif len(queried_similar_successful_knowledge_to_render) > 1:
|
||||
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
|
||||
|
||||
code = json.loads(
|
||||
session.build_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
json_mode=True,
|
||||
),
|
||||
)["code"]
|
||||
# ast.parse(code)
|
||||
factor_implementation = FileBasedFactorImplementation(
|
||||
target_task,
|
||||
code,
|
||||
)
|
||||
|
||||
return factor_implementation
|
||||
|
||||
|
||||
class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
|
||||
def __init__(self) -> None:
|
||||
self.num_loop = 0
|
||||
self.haveSelected = False
|
||||
|
||||
def implement_one_factor(
|
||||
self,
|
||||
target_task: FactorImplementTask,
|
||||
queried_knowledge,
|
||||
) -> TaskImplementation:
|
||||
error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
|
||||
# 1. 提取因子的背景信息
|
||||
target_factor_task_information = target_task.get_factor_information()
|
||||
|
||||
# 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_factor_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
):
|
||||
return queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information].implementation
|
||||
elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
|
||||
return None
|
||||
else:
|
||||
# 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
|
||||
queried_similar_component_knowledge = (
|
||||
queried_knowledge.component_with_success_task[target_factor_task_information]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
) # A list, [success task implement knowledge]
|
||||
|
||||
queried_similar_error_knowledge = (
|
||||
queried_knowledge.error_with_success_task[target_factor_task_information]
|
||||
if queried_knowledge is not None
|
||||
else {}
|
||||
) # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
|
||||
|
||||
queried_former_failed_knowledge = (
|
||||
queried_knowledge.former_traces[target_factor_task_information] if queried_knowledge is not None else []
|
||||
)
|
||||
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
|
||||
|
||||
system_prompt = Template(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
|
||||
).render(
|
||||
data_info=get_data_folder_intro(),
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
|
||||
)
|
||||
|
||||
session = APIBackend(use_chat_cache=False).build_chat_session(
|
||||
session_system_prompt=system_prompt,
|
||||
)
|
||||
|
||||
queried_similar_component_knowledge_to_render = queried_similar_component_knowledge
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
|
||||
error_summary_critics = ""
|
||||
# 动态地防止prompt超长
|
||||
while True:
|
||||
# 总结error（可选）
|
||||
if (
|
||||
error_summary
|
||||
and len(queried_similar_error_knowledge_to_render) != 0
|
||||
and len(queried_former_failed_knowledge_to_render) != 0
|
||||
):
|
||||
error_summary_system_prompt = (
|
||||
Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
|
||||
.render(
|
||||
factor_information_str=target_factor_task_information,
|
||||
code_and_feedback=queried_former_failed_knowledge_to_render[
|
||||
-1
|
||||
].get_implementation_and_feedback_str(),
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
session_summary = APIBackend(use_chat_cache=False).build_chat_session(
|
||||
session_system_prompt=error_summary_system_prompt,
|
||||
)
|
||||
while True:
|
||||
error_summary_user_prompt = (
|
||||
Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
|
||||
.render(
|
||||
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
if (
|
||||
session_summary.build_chat_completion_message_and_calculate_token(error_summary_user_prompt)
|
||||
< RD_AGENT_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_similar_error_knowledge_to_render) > 0:
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
|
||||
error_summary_critics = session_summary.build_chat_completion(
|
||||
user_prompt=error_summary_user_prompt,
|
||||
json_mode=False,
|
||||
)
|
||||
# 构建user_prompt。开始写代码
|
||||
user_prompt = (
|
||||
Template(
|
||||
implement_prompts["evolving_strategy_factor_implementation_v2_user"],
|
||||
)
|
||||
.render(
|
||||
factor_information_str=target_factor_task_information,
|
||||
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
|
||||
queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
|
||||
error_summary=error_summary,
|
||||
error_summary_critics=error_summary_critics,
|
||||
)
|
||||
.strip("\n")
|
||||
)
|
||||
if (
|
||||
session.build_chat_completion_message_and_calculate_token(
|
||||
user_prompt,
|
||||
)
|
||||
< RD_AGENT_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
elif len(queried_former_failed_knowledge_to_render) > 1:
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
|
||||
elif len(queried_similar_component_knowledge_to_render) > len(
|
||||
queried_similar_error_knowledge_to_render,
|
||||
):
|
||||
queried_similar_component_knowledge_to_render = queried_similar_component_knowledge_to_render[:-1]
|
||||
elif len(queried_similar_error_knowledge_to_render) > 0:
|
||||
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
|
||||
|
||||
response = session.build_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
json_mode=True,
|
||||
)
|
||||
code = json.loads(response)["code"]
|
||||
factor_implementation = FileBasedFactorImplementation(target_task, code)
|
||||
return factor_implementation
|
||||
@@ -0,0 +1,251 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pickle
|
||||
import subprocess
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Tuple, Union
|
||||
|
||||
import pandas as pd
|
||||
from filelock import FileLock
|
||||
|
||||
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
|
||||
FACTOR_IMPLEMENT_SETTINGS,
|
||||
)
|
||||
from rdagent.core.evolving_framework import EvolvableSubjects
|
||||
from rdagent.core.exception import (
|
||||
CodeFormatException,
|
||||
NoOutputException,
|
||||
RuntimeErrorException,
|
||||
)
|
||||
from rdagent.core.log import RDAgentLog
|
||||
from rdagent.core.task import (
|
||||
BaseTask,
|
||||
FBTaskImplementation,
|
||||
TaskImplementation,
|
||||
TestCase,
|
||||
)
|
||||
from rdagent.oai.llm_utils import md5_hash
|
||||
|
||||
|
||||
class FactorImplementTask(BaseTask):
|
||||
# TODO: generalized the attributes into the BaseTask
|
||||
# - factor_* -> *
|
||||
def __init__(
|
||||
self,
|
||||
factor_name,
|
||||
factor_description,
|
||||
factor_formulation,
|
||||
variables: dict = {},
|
||||
resource: str = None,
|
||||
) -> None:
|
||||
self.factor_name = factor_name
|
||||
self.factor_description = factor_description
|
||||
self.factor_formulation = factor_formulation
|
||||
self.variables = variables
|
||||
self.factor_resources = resource
|
||||
|
||||
def get_factor_information(self):
|
||||
return f"""factor_name: {self.factor_name}
|
||||
factor_description: {self.factor_description}
|
||||
factor_formulation: {self.factor_formulation}
|
||||
variables: {str(self.variables)}"""
|
||||
|
||||
@staticmethod
|
||||
def from_dict(dict):
|
||||
return FactorImplementTask(**dict)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<{self.__class__.__name__}[{self.factor_name}]>"
|
||||
|
||||
|
||||
class FactorEvovlingItem(EvolvableSubjects):
|
||||
"""
|
||||
Intermediate item of factor implementation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target_factor_tasks: list[FactorImplementTask],
|
||||
corresponding_gt_implementations: list[TaskImplementation] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.target_factor_tasks = target_factor_tasks
|
||||
self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks]
|
||||
self.corresponding_selection: list = None
|
||||
if corresponding_gt_implementations is not None and len(
|
||||
corresponding_gt_implementations,
|
||||
) != len(target_factor_tasks):
|
||||
self.corresponding_gt_implementations = None
|
||||
RDAgentLog().warning(
|
||||
"The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
|
||||
)
|
||||
else:
|
||||
self.corresponding_gt_implementations = corresponding_gt_implementations
|
||||
|
||||
|
||||
class FileBasedFactorImplementation(FBTaskImplementation):
|
||||
"""
|
||||
This class is used to implement a factor by writing the code to a file.
|
||||
Input data and output factor value are also written to files.
|
||||
"""
|
||||
|
||||
# TODO: (Xiao) think raising errors may get better information for processing
|
||||
FB_FROM_CACHE = "The factor value has been executed and stored in the instance variable."
|
||||
FB_EXEC_SUCCESS = "Execution succeeded without error."
|
||||
FB_CODE_NOT_SET = "code is not set."
|
||||
FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
|
||||
FB_OUTPUT_FILE_NOT_FOUND = "\nExpected output file not found."
|
||||
FB_OUTPUT_FILE_FOUND = "\nExpected output file found."
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target_task: FactorImplementTask,
|
||||
code,
|
||||
executed_factor_value_dataframe=None,
|
||||
raise_exception=False,
|
||||
) -> None:
|
||||
super().__init__(target_task)
|
||||
self.code = code
|
||||
self.executed_factor_value_dataframe = executed_factor_value_dataframe
|
||||
self.logger = RDAgentLog()
|
||||
self.raise_exception = raise_exception
|
||||
self.workspace_path = Path(
|
||||
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_workspace,
|
||||
) / str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def link_data_to_workspace(data_path: Path, workspace_path: Path):
|
||||
data_path = Path(data_path)
|
||||
workspace_path = Path(workspace_path)
|
||||
for data_file_path in data_path.iterdir():
|
||||
workspace_data_file_path = workspace_path / data_file_path.name
|
||||
if workspace_data_file_path.exists():
|
||||
workspace_data_file_path.unlink()
|
||||
subprocess.run(
|
||||
["ln", "-s", data_file_path, workspace_data_file_path],
|
||||
check=False,
|
||||
)
|
||||
|
||||
def execute_desc(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare(self, *args, **kwargs):
|
||||
# TODO move the prepare part code in execute into here
|
||||
return super().prepare(*args, **kwargs)
|
||||
|
||||
def execute(self, store_result: bool = False) -> Tuple[str, pd.DataFrame]:
|
||||
"""
|
||||
execute the implementation and get the factor value by the following steps:
|
||||
1. make the directory in workspace path
|
||||
2. write the code to the file in the workspace path
|
||||
3. link all the source data to the workspace path folder
|
||||
4. execute the code
|
||||
5. read the factor value from the output file in the workspace path folder
|
||||
returns the execution feedback as a string and the factor value as a pandas dataframe
|
||||
|
||||
parameters:
|
||||
store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation
|
||||
"""
|
||||
if self.code is None:
|
||||
if self.raise_exception:
|
||||
raise CodeFormatException(self.FB_CODE_NOT_SET)
|
||||
else:
|
||||
# TODO: to make the interface compatible with previous code. I kept the original behavior.
|
||||
raise ValueError(self.FB_CODE_NOT_SET)
|
||||
with FileLock(self.workspace_path / "execution.lock"):
|
||||
if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
|
||||
# NOTE: cache the result for the same code
|
||||
target_file_name = md5_hash(self.code)
|
||||
cache_file_path = (
|
||||
Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl"
|
||||
)
|
||||
Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location).mkdir(
|
||||
exist_ok=True, parents=True
|
||||
)
|
||||
if cache_file_path.exists() and not self.raise_exception:
|
||||
cached_res = pickle.load(open(cache_file_path, "rb"))
|
||||
if store_result and cached_res[1] is not None:
|
||||
self.executed_factor_value_dataframe = cached_res[1]
|
||||
return cached_res
|
||||
|
||||
if self.executed_factor_value_dataframe is not None:
|
||||
return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
|
||||
|
||||
source_data_path = Path(
|
||||
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder,
|
||||
)
|
||||
self.workspace_path.mkdir(exist_ok=True, parents=True)
|
||||
source_data_path.mkdir(exist_ok=True, parents=True)
|
||||
code_path = self.workspace_path / f"{self.target_task.factor_name}.py"
|
||||
code_path.write_text(self.code)
|
||||
|
||||
self.link_data_to_workspace(source_data_path, self.workspace_path)
|
||||
|
||||
execution_feedback = self.FB_EXECUTION_SUCCEEDED
|
||||
try:
|
||||
subprocess.check_output(
|
||||
f"python {code_path}",
|
||||
shell=True,
|
||||
cwd=self.workspace_path,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
import site
|
||||
|
||||
execution_feedback = (
|
||||
e.output.decode()
|
||||
.replace(str(code_path.parent.absolute()), r"/path/to")
|
||||
.replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
|
||||
)
|
||||
if len(execution_feedback) > 2000:
|
||||
execution_feedback = (
|
||||
execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
|
||||
)
|
||||
if self.raise_exception:
|
||||
raise RuntimeErrorException(execution_feedback)
|
||||
except subprocess.TimeoutExpired:
|
||||
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
|
||||
if self.raise_exception:
|
||||
raise RuntimeErrorException(execution_feedback)
|
||||
|
||||
workspace_output_file_path = self.workspace_path / "result.h5"
|
||||
if not workspace_output_file_path.exists():
|
||||
execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND
|
||||
executed_factor_value_dataframe = None
|
||||
if self.raise_exception:
|
||||
raise NoOutputException(execution_feedback)
|
||||
else:
|
||||
try:
|
||||
executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path)
|
||||
execution_feedback += self.FB_OUTPUT_FILE_FOUND
|
||||
except Exception as e:
|
||||
execution_feedback += f"Error found when reading hdf file: {e}"[:1000]
|
||||
executed_factor_value_dataframe = None
|
||||
|
||||
if store_result and executed_factor_value_dataframe is not None:
|
||||
self.executed_factor_value_dataframe = executed_factor_value_dataframe
|
||||
|
||||
if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
|
||||
pickle.dump(
|
||||
(execution_feedback, executed_factor_value_dataframe),
|
||||
open(cache_file_path, "wb"),
|
||||
)
|
||||
return execution_feedback, executed_factor_value_dataframe
|
||||
|
||||
def __str__(self) -> str:
|
||||
# NOTE:
|
||||
# If the code cache works, the workspace will be None.
|
||||
return f"File Factor[{self.target_task.factor_name}]: {self.workspace_path}"
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
|
||||
@staticmethod
|
||||
def from_folder(task: FactorImplementTask, path: Union[str, Path], **kwargs):
|
||||
path = Path(path)
|
||||
factor_path = (path / task.factor_name).with_suffix(".py")
|
||||
with factor_path.open("r") as f:
|
||||
code = f.read()
|
||||
return FileBasedFactorImplementation(task, code=code, **kwargs)
|
||||
+914
@@ -0,0 +1,914 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from itertools import combinations
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
from jinja2 import Template
|
||||
|
||||
from rdagent.components.knowledge_management.graph import (
|
||||
UndirectedGraph,
|
||||
UndirectedNode,
|
||||
)
|
||||
from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
|
||||
FactorImplementationSingleFeedback,
|
||||
)
|
||||
from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
|
||||
FactorImplementTask,
|
||||
)
|
||||
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
|
||||
FACTOR_IMPLEMENT_SETTINGS,
|
||||
)
|
||||
from rdagent.core.evolving_framework import (
|
||||
EvolvableSubjects,
|
||||
EvoStep,
|
||||
Knowledge,
|
||||
KnowledgeBase,
|
||||
QueriedKnowledge,
|
||||
RAGStrategy,
|
||||
)
|
||||
from rdagent.core.log import RDAgentLog
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.task import TaskImplementation
|
||||
from rdagent.oai.llm_utils import (
|
||||
APIBackend,
|
||||
calculate_embedding_distance_between_str_list,
|
||||
)
|
||||
|
||||
|
||||
class FactorImplementationKnowledge(Knowledge):
|
||||
def __init__(
|
||||
self,
|
||||
target_task: FactorImplementTask,
|
||||
implementation: TaskImplementation,
|
||||
feedback: FactorImplementationSingleFeedback,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize a FactorKnowledge object. The FactorKnowledge object is used to store a factor implementation without the ground truth code and value.
|
||||
|
||||
Args:
|
||||
factor (Factor): The factor object associated with the KnowledgeManagement.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
self.target_task = target_task
|
||||
self.implementation = implementation
|
||||
self.feedback = feedback
|
||||
|
||||
def get_implementation_and_feedback_str(self) -> str:
|
||||
return f"""------------------Factor implementation code:------------------
|
||||
{self.implementation.code}
|
||||
------------------Factor implementation feedback:------------------
|
||||
{self.feedback!s}
|
||||
"""
|
||||
|
||||
|
||||
class FactorImplementationQueriedKnowledge(QueriedKnowledge):
|
||||
def __init__(self, success_task_to_knowledge_dict: dict = {}, failed_task_info_set: set = set()) -> None:
|
||||
self.success_task_to_knowledge_dict = success_task_to_knowledge_dict
|
||||
self.failed_task_info_set = failed_task_info_set
|
||||
|
||||
|
||||
class FactorImplementationKnowledgeBaseV1(KnowledgeBase):
|
||||
def __init__(self) -> None:
|
||||
self.implementation_trace: dict[str, FactorImplementationKnowledge] = dict()
|
||||
self.success_task_info_set: set[str] = set()
|
||||
|
||||
self.task_to_embedding = dict()
|
||||
|
||||
def query(self) -> QueriedKnowledge | None:
|
||||
"""
|
||||
Query the knowledge base to get the queried knowledge. So far is handled in RAG strategy.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FactorImplementationQueriedKnowledgeV1(FactorImplementationQueriedKnowledge):
|
||||
def __init__(self) -> None:
|
||||
self.working_task_to_former_failed_knowledge_dict = dict()
|
||||
self.working_task_to_similar_successful_knowledge_dict = dict()
|
||||
super().__init__()
|
||||
|
||||
|
||||
class FactorImplementationRAGStrategyV1(RAGStrategy):
|
||||
def __init__(self, knowledgebase: FactorImplementationKnowledgeBaseV1) -> None:
|
||||
super().__init__(knowledgebase)
|
||||
self.current_generated_trace_count = 0
|
||||
|
||||
def generate_knowledge(
|
||||
self,
|
||||
evolving_trace: list[EvoStep],
|
||||
*,
|
||||
return_knowledge: bool = False,
|
||||
) -> Knowledge | None:
|
||||
if len(evolving_trace) == self.current_generated_trace_count:
|
||||
return
|
||||
else:
|
||||
for trace_index in range(
|
||||
self.current_generated_trace_count,
|
||||
len(evolving_trace),
|
||||
):
|
||||
evo_step = evolving_trace[trace_index]
|
||||
implementations = evo_step.evolvable_subjects
|
||||
feedback = evo_step.feedback
|
||||
for task_index in range(len(implementations.target_factor_tasks)):
|
||||
target_task = implementations.target_factor_tasks[task_index]
|
||||
target_task_information = target_task.get_factor_information()
|
||||
implementation = implementations.corresponding_implementations[task_index]
|
||||
single_feedback = feedback[task_index]
|
||||
if single_feedback is None:
|
||||
continue
|
||||
single_knowledge = FactorImplementationKnowledge(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
feedback=single_feedback,
|
||||
)
|
||||
if target_task_information not in self.knowledgebase.success_task_info_set:
|
||||
self.knowledgebase.implementation_trace.setdefault(
|
||||
target_task_information,
|
||||
[],
|
||||
).append(single_knowledge)
|
||||
|
||||
if single_feedback.final_decision == True:
|
||||
self.knowledgebase.success_task_info_set.add(
|
||||
target_task_information,
|
||||
)
|
||||
self.current_generated_trace_count = len(evolving_trace)
|
||||
|
||||
def query(
|
||||
self,
|
||||
evo: EvolvableSubjects,
|
||||
evolving_trace: list[EvoStep],
|
||||
) -> QueriedKnowledge | None:
|
||||
v1_query_former_trace_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_former_trace_limit
|
||||
v1_query_similar_success_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_similar_success_limit
|
||||
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
|
||||
|
||||
queried_knowledge = FactorImplementationQueriedKnowledgeV1()
|
||||
for target_factor_task in evo.target_factor_tasks:
|
||||
target_factor_task_information = target_factor_task.get_factor_information()
|
||||
if target_factor_task_information in self.knowledgebase.success_task_info_set:
|
||||
queried_knowledge.success_task_to_knowledge_dict[
|
||||
target_factor_task_information
|
||||
] = self.knowledgebase.implementation_trace[target_factor_task_information][-1]
|
||||
elif (
|
||||
len(
|
||||
self.knowledgebase.implementation_trace.setdefault(
|
||||
target_factor_task_information,
|
||||
[],
|
||||
),
|
||||
)
|
||||
>= fail_task_trial_limit
|
||||
):
|
||||
queried_knowledge.failed_task_info_set.add(target_factor_task_information)
|
||||
else:
|
||||
queried_knowledge.working_task_to_former_failed_knowledge_dict[
|
||||
target_factor_task_information
|
||||
] = self.knowledgebase.implementation_trace.setdefault(
|
||||
target_factor_task_information,
|
||||
[],
|
||||
)[
|
||||
-v1_query_former_trace_limit:
|
||||
]
|
||||
|
||||
knowledge_base_success_task_list = list(
|
||||
self.knowledgebase.success_task_info_set,
|
||||
)
|
||||
similarity = calculate_embedding_distance_between_str_list(
|
||||
[target_factor_task_information],
|
||||
knowledge_base_success_task_list,
|
||||
)[0]
|
||||
similar_indexes = sorted(
|
||||
range(len(similarity)),
|
||||
key=lambda i: similarity[i],
|
||||
reverse=True,
|
||||
)[:v1_query_similar_success_limit]
|
||||
similar_successful_knowledge = [
|
||||
self.knowledgebase.implementation_trace.setdefault(
|
||||
knowledge_base_success_task_list[index],
|
||||
[],
|
||||
)[-1]
|
||||
for index in similar_indexes
|
||||
]
|
||||
queried_knowledge.working_task_to_similar_successful_knowledge_dict[
|
||||
target_factor_task_information
|
||||
] = similar_successful_knowledge
|
||||
return queried_knowledge
|
||||
|
||||
|
||||
class FactorImplementationQueriedGraphKnowledge(FactorImplementationQueriedKnowledge):
|
||||
# Aggregation of knowledge
|
||||
def __init__(
|
||||
self,
|
||||
former_traces: dict = {},
|
||||
component_with_success_task: dict = {},
|
||||
error_with_success_task: dict = {},
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.former_traces = former_traces
|
||||
self.component_with_success_task = component_with_success_task
|
||||
self.error_with_success_task = error_with_success_task
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class FactorImplementationGraphRAGStrategy(RAGStrategy):
|
||||
def __init__(self, knowledgebase: FactorImplementationGraphKnowledgeBase) -> None:
|
||||
super().__init__(knowledgebase)
|
||||
self.current_generated_trace_count = 0
|
||||
self.prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
|
||||
def generate_knowledge(
|
||||
self,
|
||||
evolving_trace: list[EvoStep],
|
||||
*,
|
||||
return_knowledge: bool = False,
|
||||
) -> Knowledge | None:
|
||||
if len(evolving_trace) == self.current_generated_trace_count:
|
||||
return None
|
||||
|
||||
else:
|
||||
for trace_index in range(self.current_generated_trace_count, len(evolving_trace)):
|
||||
evo_step = evolving_trace[trace_index]
|
||||
implementations = evo_step.evolvable_subjects
|
||||
feedback = evo_step.feedback
|
||||
for task_index in range(len(implementations.target_factor_tasks)):
|
||||
single_feedback = feedback[task_index]
|
||||
target_task = implementations.target_factor_tasks[task_index]
|
||||
target_task_information = target_task.get_factor_information()
|
||||
implementation = implementations.corresponding_implementations[task_index]
|
||||
single_feedback = feedback[task_index]
|
||||
if single_feedback is None:
|
||||
continue
|
||||
single_knowledge = FactorImplementationKnowledge(
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
feedback=single_feedback,
|
||||
)
|
||||
if (
|
||||
target_task_information not in self.knowledgebase.success_task_to_knowledge_dict
|
||||
and implementation is not None
|
||||
):
|
||||
self.knowledgebase.working_trace_knowledge.setdefault(target_task_information, []).append(
|
||||
single_knowledge,
|
||||
) # save to working trace
|
||||
if single_feedback.final_decision == True:
|
||||
self.knowledgebase.success_task_to_knowledge_dict.setdefault(
|
||||
target_task_information,
|
||||
single_knowledge,
|
||||
)
|
||||
# Do summary for the last step and update the knowledge graph
|
||||
self.knowledgebase.update_success_task(
|
||||
target_task_information,
|
||||
)
|
||||
else:
|
||||
# generate error node and store into knowledge base
|
||||
error_analysis_result = []
|
||||
if not single_feedback.value_generated_flag:
|
||||
error_analysis_result = self.analyze_error(
|
||||
single_feedback.execution_feedback,
|
||||
feedback_type="execution",
|
||||
)
|
||||
else:
|
||||
error_analysis_result = self.analyze_error(
|
||||
single_feedback.factor_value_feedback,
|
||||
feedback_type="value",
|
||||
)
|
||||
self.knowledgebase.working_trace_error_analysis.setdefault(
|
||||
target_task_information,
|
||||
[],
|
||||
).append(
|
||||
error_analysis_result,
|
||||
) # save to working trace error record, for graph update
|
||||
|
||||
self.current_generated_trace_count = len(evolving_trace)
|
||||
return None
|
||||
|
||||
def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None:
|
||||
conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler
|
||||
factor_implementation_queried_graph_knowledge = FactorImplementationQueriedGraphKnowledge(
|
||||
success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict,
|
||||
)
|
||||
|
||||
factor_implementation_queried_graph_knowledge = self.former_trace_query(
|
||||
evo,
|
||||
factor_implementation_queried_graph_knowledge,
|
||||
FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit,
|
||||
)
|
||||
factor_implementation_queried_graph_knowledge = self.component_query(
|
||||
evo,
|
||||
factor_implementation_queried_graph_knowledge,
|
||||
FACTOR_IMPLEMENT_SETTINGS.v2_query_component_limit,
|
||||
knowledge_sampler=conf_knowledge_sampler,
|
||||
)
|
||||
factor_implementation_queried_graph_knowledge = self.error_query(
|
||||
evo,
|
||||
factor_implementation_queried_graph_knowledge,
|
||||
FACTOR_IMPLEMENT_SETTINGS.v2_query_error_limit,
|
||||
knowledge_sampler=conf_knowledge_sampler,
|
||||
)
|
||||
return factor_implementation_queried_graph_knowledge
|
||||
|
||||
def analyze_component(
|
||||
self,
|
||||
target_factor_task_information,
|
||||
) -> list[UndirectedNode]: # Hardcode: certain component nodes
|
||||
all_component_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["component"])
|
||||
if not len(all_component_nodes):
|
||||
return []
|
||||
all_component_content = ""
|
||||
for _, component_node in enumerate(all_component_nodes):
|
||||
all_component_content += f"{component_node.content}, \n"
|
||||
analyze_component_system_prompt = Template(self.prompt["analyze_component_prompt_v1_system"]).render(
|
||||
all_component_content=all_component_content,
|
||||
)
|
||||
|
||||
analyze_component_user_prompt = target_factor_task_information
|
||||
try:
|
||||
component_no_list = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
system_prompt=analyze_component_system_prompt,
|
||||
user_prompt=analyze_component_user_prompt,
|
||||
json_mode=True,
|
||||
),
|
||||
)["component_no_list"]
|
||||
return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))]
|
||||
except:
|
||||
RDAgentLog().warning("Error when analyzing components.")
|
||||
analyze_component_user_prompt = "Your response is not a valid component index list."
|
||||
|
||||
return []
|
||||
|
||||
def analyze_error(
|
||||
self,
|
||||
single_feedback,
|
||||
feedback_type="execution",
|
||||
) -> list[
|
||||
UndirectedNode | str
|
||||
]: # Hardcode: Raised errors, existed error nodes + not existed error nodes(here, they are strs)
|
||||
if feedback_type == "execution":
|
||||
match = re.search(
|
||||
r'File "(?P<file>.+)", line (?P<line>\d+), in (?P<function>.+)\n\s+(?P<error_line>.+)\n(?P<error_type>\w+): (?P<error_message>.+)',
|
||||
single_feedback,
|
||||
)
|
||||
if match:
|
||||
error_details = match.groupdict()
|
||||
# last_traceback = f'File "{error_details["file"]}", line {error_details["line"]}, in {error_details["function"]}\n {error_details["error_line"]}'
|
||||
error_type = error_details["error_type"]
|
||||
error_line = error_details["error_line"]
|
||||
error_contents = [f"ErrorType: {error_type}" + "\n" + f"Error line: {error_line}"]
|
||||
else:
|
||||
error_contents = ["Undefined Error"]
|
||||
elif feedback_type == "value": # value check error
|
||||
value_check_types = r"The source dataframe and the ground truth dataframe have different rows count.|The source dataframe and the ground truth dataframe have different index.|Some values differ by more than the tolerance of 1e-6.|No sufficient correlation found when shifting up|Something wrong happens when naming the multi indices of the dataframe."
|
||||
error_contents = re.findall(value_check_types, single_feedback)
|
||||
else:
|
||||
error_contents = ["Undefined Error"]
|
||||
|
||||
all_error_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["error"])
|
||||
if not len(all_error_nodes):
|
||||
return error_contents
|
||||
else:
|
||||
error_list = []
|
||||
for error_content in error_contents:
|
||||
for error_node in all_error_nodes:
|
||||
if error_content == error_node.content:
|
||||
error_list.append(error_node)
|
||||
else:
|
||||
error_list.append(error_content)
|
||||
if error_list[-1] in error_list[:-1]:
|
||||
error_list.pop()
|
||||
|
||||
return error_list
|
||||
|
||||
def former_trace_query(
|
||||
self,
|
||||
evo: EvolvableSubjects,
|
||||
factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
|
||||
v2_query_former_trace_limit: int = 5,
|
||||
) -> Union[QueriedKnowledge, set]:
|
||||
"""
|
||||
Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
|
||||
"""
|
||||
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
|
||||
|
||||
for target_factor_task in evo.target_factor_tasks:
|
||||
target_factor_task_information = target_factor_task.get_factor_information()
|
||||
if (
|
||||
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
|
||||
and target_factor_task_information in self.knowledgebase.working_trace_knowledge
|
||||
and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information])
|
||||
>= fail_task_trial_limit
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.failed_task_info_set.add(target_factor_task_information)
|
||||
|
||||
if (
|
||||
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
|
||||
and target_factor_task_information
|
||||
not in factor_implementation_queried_graph_knowledge.failed_task_info_set
|
||||
and target_factor_task_information in self.knowledgebase.working_trace_knowledge
|
||||
):
|
||||
former_trace_knowledge = copy.copy(
|
||||
self.knowledgebase.working_trace_knowledge[target_factor_task_information],
|
||||
)
|
||||
# in former trace query we will delete the right trace in the following order:[..., value_generated_flag is True, value_generated_flag is False, ...]
|
||||
# because we think this order means a deterioration of the trial (like a wrong gradient descent)
|
||||
current_index = 1
|
||||
while current_index < len(former_trace_knowledge):
|
||||
if (
|
||||
not former_trace_knowledge[current_index].feedback.value_generated_flag
|
||||
and former_trace_knowledge[current_index - 1].feedback.value_generated_flag
|
||||
):
|
||||
former_trace_knowledge.pop(current_index)
|
||||
else:
|
||||
current_index += 1
|
||||
|
||||
factor_implementation_queried_graph_knowledge.former_traces[
|
||||
target_factor_task_information
|
||||
] = former_trace_knowledge[-v2_query_former_trace_limit:]
|
||||
else:
|
||||
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
|
||||
|
||||
return factor_implementation_queried_graph_knowledge
|
||||
|
||||
def component_query(
|
||||
self,
|
||||
evo: EvolvableSubjects,
|
||||
factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
|
||||
v2_query_component_limit: int = 5,
|
||||
knowledge_sampler: float = 1.0,
|
||||
) -> QueriedKnowledge | None:
|
||||
# queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge()
|
||||
for target_factor_task in evo.target_factor_tasks:
|
||||
target_factor_task_information = target_factor_task.get_factor_information()
|
||||
if (
|
||||
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
|
||||
or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
] = []
|
||||
else:
|
||||
if target_factor_task_information not in self.knowledgebase.task_to_component_nodes:
|
||||
self.knowledgebase.task_to_component_nodes[target_factor_task_information] = self.analyze_component(
|
||||
target_factor_task_information,
|
||||
)
|
||||
|
||||
component_analysis_result = self.knowledgebase.task_to_component_nodes[target_factor_task_information]
|
||||
|
||||
if len(component_analysis_result) > 1:
|
||||
task_des_node_list = self.knowledgebase.graph_query_by_intersection(
|
||||
component_analysis_result,
|
||||
constraint_labels=["task_description"],
|
||||
)
|
||||
single_component_constraint = (v2_query_component_limit // len(component_analysis_result)) + 1
|
||||
else:
|
||||
task_des_node_list = []
|
||||
single_component_constraint = v2_query_component_limit
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
] = []
|
||||
for component_node in component_analysis_result:
|
||||
# Reverse iterate, a trade-off with intersection search
|
||||
count = 0
|
||||
for task_des_node in self.knowledgebase.graph_query_by_node(
|
||||
node=component_node,
|
||||
step=1,
|
||||
constraint_labels=["task_description"],
|
||||
block=True,
|
||||
)[::-1]:
|
||||
if task_des_node not in task_des_node_list:
|
||||
task_des_node_list.append(task_des_node)
|
||||
count += 1
|
||||
if count >= single_component_constraint:
|
||||
break
|
||||
|
||||
for node in task_des_node_list:
|
||||
for searched_node in self.knowledgebase.graph_query_by_node(
|
||||
node=node,
|
||||
step=50,
|
||||
constraint_labels=[
|
||||
"task_success_implement",
|
||||
],
|
||||
block=True,
|
||||
):
|
||||
if searched_node.label == "task_success_implement":
|
||||
target_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[
|
||||
searched_node.id
|
||||
]
|
||||
if (
|
||||
target_knowledge
|
||||
not in factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
]
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
].append(target_knowledge)
|
||||
|
||||
# finally add embedding related knowledge
|
||||
knowledge_base_success_task_list = list(self.knowledgebase.success_task_to_knowledge_dict)
|
||||
|
||||
similarity = calculate_embedding_distance_between_str_list(
|
||||
[target_factor_task_information],
|
||||
knowledge_base_success_task_list,
|
||||
)[0]
|
||||
similar_indexes = sorted(
|
||||
range(len(similarity)),
|
||||
key=lambda i: similarity[i],
|
||||
reverse=True,
|
||||
)
|
||||
embedding_similar_successful_knowledge = [
|
||||
self.knowledgebase.success_task_to_knowledge_dict[knowledge_base_success_task_list[index]]
|
||||
for index in similar_indexes
|
||||
]
|
||||
for knowledge in embedding_similar_successful_knowledge:
|
||||
if (
|
||||
knowledge
|
||||
not in factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
]
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
].append(knowledge)
|
||||
|
||||
if knowledge_sampler > 0:
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
] = [
|
||||
knowledge
|
||||
for knowledge in factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
]
|
||||
if random.uniform(0, 1) <= knowledge_sampler
|
||||
]
|
||||
|
||||
# Make sure no less than half of the knowledge are from GT
|
||||
queried_knowledge_list = factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
]
|
||||
queried_from_gt_knowledge_list = [
|
||||
knowledge
|
||||
for knowledge in queried_knowledge_list
|
||||
if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == True
|
||||
]
|
||||
queried_without_gt_knowledge_list = [
|
||||
knowledge
|
||||
for knowledge in queried_knowledge_list
|
||||
if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False
|
||||
]
|
||||
queried_from_gt_knowledge_count = max(
|
||||
min(v2_query_component_limit // 2, len(queried_from_gt_knowledge_list)),
|
||||
v2_query_component_limit - len(queried_without_gt_knowledge_list),
|
||||
)
|
||||
factor_implementation_queried_graph_knowledge.component_with_success_task[
|
||||
target_factor_task_information
|
||||
] = (
|
||||
queried_from_gt_knowledge_list[:queried_from_gt_knowledge_count]
|
||||
+ queried_without_gt_knowledge_list[: v2_query_component_limit - queried_from_gt_knowledge_count]
|
||||
)
|
||||
|
||||
return factor_implementation_queried_graph_knowledge
|
||||
|
||||
def error_query(
|
||||
self,
|
||||
evo: EvolvableSubjects,
|
||||
factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
|
||||
v2_query_error_limit: int = 5,
|
||||
knowledge_sampler: float = 1.0,
|
||||
) -> QueriedKnowledge | None:
|
||||
# queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge()
|
||||
for task_index, target_factor_task in enumerate(evo.target_factor_tasks):
|
||||
target_factor_task_information = target_factor_task.get_factor_information()
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
|
||||
if (
|
||||
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
|
||||
or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
|
||||
):
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[
|
||||
target_factor_task_information
|
||||
] = []
|
||||
else:
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[
|
||||
target_factor_task_information
|
||||
] = []
|
||||
if (
|
||||
target_factor_task_information in self.knowledgebase.working_trace_error_analysis
|
||||
and len(self.knowledgebase.working_trace_error_analysis[target_factor_task_information]) > 0
|
||||
and len(factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information])
|
||||
> 0
|
||||
):
|
||||
queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[
|
||||
target_factor_task_information
|
||||
][-1]
|
||||
target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
|
||||
queried_last_trace,
|
||||
)
|
||||
last_knowledge_error_analysis_result = self.knowledgebase.working_trace_error_analysis[
|
||||
target_factor_task_information
|
||||
][target_index]
|
||||
else:
|
||||
last_knowledge_error_analysis_result = []
|
||||
|
||||
error_nodes = []
|
||||
for error_node in last_knowledge_error_analysis_result:
|
||||
if not isinstance(error_node, UndirectedNode):
|
||||
error_node = self.knowledgebase.graph_get_node_by_content(content=error_node)
|
||||
if error_node is None:
|
||||
continue
|
||||
error_nodes.append(error_node)
|
||||
|
||||
if len(error_nodes) > 1:
|
||||
task_trace_node_list = self.knowledgebase.graph_query_by_intersection(
|
||||
error_nodes,
|
||||
constraint_labels=["task_trace"],
|
||||
output_intersection_origin=True,
|
||||
)
|
||||
single_error_constraint = (v2_query_error_limit // len(error_nodes)) + 1
|
||||
else:
|
||||
task_trace_node_list = []
|
||||
single_error_constraint = v2_query_error_limit
|
||||
for error_node in error_nodes:
|
||||
# Reverse iterate, a trade-off with intersection search
|
||||
count = 0
|
||||
for task_trace_node in self.knowledgebase.graph_query_by_node(
|
||||
node=error_node,
|
||||
step=1,
|
||||
constraint_labels=["task_trace"],
|
||||
block=True,
|
||||
)[::-1]:
|
||||
if task_trace_node not in task_trace_node_list:
|
||||
task_trace_node_list.append([[error_node], task_trace_node])
|
||||
count += 1
|
||||
if count >= single_error_constraint:
|
||||
break
|
||||
|
||||
# for error_node in last_knowledge_error_analysis_result:
|
||||
# if not isinstance(error_node, UndirectedNode):
|
||||
# error_node = self.knowledgebase.graph_get_node_by_content(content=error_node)
|
||||
# if error_node is None:
|
||||
# continue
|
||||
# for searched_node in self.knowledgebase.graph_query_by_node(
|
||||
# node=error_node,
|
||||
# step=1,
|
||||
# constraint_labels=["task_trace"],
|
||||
# block=True,
|
||||
# ):
|
||||
# if searched_node not in [node[0] for node in task_trace_node_list]:
|
||||
# task_trace_node_list.append((searched_node, error_node.content))
|
||||
|
||||
same_error_success_knowledge_pair_list = []
|
||||
same_error_success_node_set = set()
|
||||
for error_node_list, trace_node in task_trace_node_list:
|
||||
for searched_trace_success_node in self.knowledgebase.graph_query_by_node(
|
||||
node=trace_node,
|
||||
step=50,
|
||||
constraint_labels=[
|
||||
"task_trace",
|
||||
"task_success_implement",
|
||||
"task_description",
|
||||
],
|
||||
block=True,
|
||||
):
|
||||
if (
|
||||
searched_trace_success_node not in same_error_success_node_set
|
||||
and searched_trace_success_node.label == "task_success_implement"
|
||||
):
|
||||
same_error_success_node_set.add(searched_trace_success_node)
|
||||
|
||||
trace_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[trace_node.id]
|
||||
success_knowledge = self.knowledgebase.node_to_implementation_knowledge_dict[
|
||||
searched_trace_success_node.id
|
||||
]
|
||||
error_content = ""
|
||||
for index, error_node in enumerate(error_node_list):
|
||||
error_content += f"{index+1}. {error_node.content}; "
|
||||
same_error_success_knowledge_pair_list.append(
|
||||
(
|
||||
error_content,
|
||||
(trace_knowledge, success_knowledge),
|
||||
),
|
||||
)
|
||||
|
||||
if knowledge_sampler > 0:
|
||||
same_error_success_knowledge_pair_list = [
|
||||
knowledge
|
||||
for knowledge in same_error_success_knowledge_pair_list
|
||||
if random.uniform(0, 1) <= knowledge_sampler
|
||||
]
|
||||
|
||||
same_error_success_knowledge_pair_list = same_error_success_knowledge_pair_list[:v2_query_error_limit]
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[
|
||||
target_factor_task_information
|
||||
] = same_error_success_knowledge_pair_list
|
||||
|
||||
return factor_implementation_queried_graph_knowledge
|
||||
|
||||
|
||||
class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
|
||||
def __init__(self, init_component_list=None) -> None:
|
||||
"""
|
||||
Load knowledge, offer brief information of knowledge and common handle interfaces
|
||||
"""
|
||||
self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl")
|
||||
RDAgentLog().info(f"Knowledge Graph loaded, size={self.graph.size()}")
|
||||
|
||||
if init_component_list:
|
||||
for component in init_component_list:
|
||||
exist_node = self.graph.get_node_by_content(content=component)
|
||||
node = exist_node if exist_node else UndirectedNode(content=component, label="component")
|
||||
self.graph.add_nodes(node=node, neighbors=[])
|
||||
|
||||
# A dict containing all working trace until they fail or succeed
|
||||
self.working_trace_knowledge = {}
|
||||
|
||||
# A dict containing error analysis each step aligned with working trace
|
||||
self.working_trace_error_analysis = {}
|
||||
|
||||
# Add already success task
|
||||
self.success_task_to_knowledge_dict = {}
|
||||
|
||||
# key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorImplementationKnowledge')
|
||||
self.node_to_implementation_knowledge_dict = {}
|
||||
|
||||
# store the task description to component nodes
|
||||
self.task_to_component_nodes = {}
|
||||
|
||||
def get_all_nodes_by_label(self, label: str) -> list[UndirectedNode]:
|
||||
return self.graph.get_all_nodes_by_label(label)
|
||||
|
||||
def update_success_task(
|
||||
self,
|
||||
success_task_info: str,
|
||||
): # Transfer the success tasks' working trace to knowledge storage & graph
|
||||
success_task_trace = self.working_trace_knowledge[success_task_info]
|
||||
success_task_error_analysis_record = (
|
||||
self.working_trace_error_analysis[success_task_info]
|
||||
if success_task_info in self.working_trace_error_analysis
|
||||
else []
|
||||
)
|
||||
task_des_node = UndirectedNode(content=success_task_info, label="task_description")
|
||||
self.graph.add_nodes(
|
||||
node=task_des_node,
|
||||
neighbors=self.task_to_component_nodes[success_task_info],
|
||||
) # 1st version, we assume that all component nodes are given
|
||||
for index, trace_unit in enumerate(success_task_trace): # every unit: single_knowledge
|
||||
neighbor_nodes = [task_des_node]
|
||||
if index != len(success_task_trace) - 1:
|
||||
trace_node = UndirectedNode(
|
||||
content=trace_unit.get_implementation_and_feedback_str(),
|
||||
label="task_trace",
|
||||
)
|
||||
self.node_to_implementation_knowledge_dict[trace_node.id] = trace_unit
|
||||
for node_index, error_node in enumerate(success_task_error_analysis_record[index]):
|
||||
if type(error_node).__name__ == "str":
|
||||
queried_node = self.graph.get_node_by_content(content=error_node)
|
||||
if queried_node is None:
|
||||
new_error_node = UndirectedNode(content=error_node, label="error")
|
||||
self.graph.add_node(node=new_error_node)
|
||||
success_task_error_analysis_record[index][node_index] = new_error_node
|
||||
else:
|
||||
success_task_error_analysis_record[index][node_index] = queried_node
|
||||
neighbor_nodes.extend(success_task_error_analysis_record[index])
|
||||
self.graph.add_nodes(node=trace_node, neighbors=neighbor_nodes)
|
||||
else:
|
||||
success_node = UndirectedNode(
|
||||
content=trace_unit.get_implementation_and_feedback_str(),
|
||||
label="task_success_implement",
|
||||
)
|
||||
self.graph.add_nodes(node=success_node, neighbors=neighbor_nodes)
|
||||
self.node_to_implementation_knowledge_dict[success_node.id] = trace_unit
|
||||
|
||||
def query(self):
|
||||
pass
|
||||
|
||||
def graph_get_node_by_content(self, content: str) -> UndirectedNode:
|
||||
return self.graph.get_node_by_content(content=content)
|
||||
|
||||
def graph_query_by_content(
|
||||
self,
|
||||
content: Union[str, list[str]],
|
||||
topk_k: int = 5,
|
||||
step: int = 1,
|
||||
constraint_labels: list[str] = None,
|
||||
constraint_node: UndirectedNode = None,
|
||||
similarity_threshold: float = 0.0,
|
||||
constraint_distance: float = 0,
|
||||
block: bool = False,
|
||||
) -> list[UndirectedNode]:
|
||||
"""
|
||||
search graph by content similarity and connection relationship, return empty list if nodes' chain without node
|
||||
near to constraint_node
|
||||
|
||||
Parameters
|
||||
----------
|
||||
constraint_distance
|
||||
content
|
||||
topk_k: the upper number of output for each query, if the number of fit nodes is less than topk_k, return all fit nodes's content
|
||||
step
|
||||
constraint_labels
|
||||
constraint_node
|
||||
similarity_threshold
|
||||
block: despite the start node, the search can only flow through the constraint_label type nodes
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
|
||||
return self.graph.query_by_content(
|
||||
content=content,
|
||||
topk_k=topk_k,
|
||||
step=step,
|
||||
constraint_labels=constraint_labels,
|
||||
constraint_node=constraint_node,
|
||||
similarity_threshold=similarity_threshold,
|
||||
constraint_distance=constraint_distance,
|
||||
block=block,
|
||||
)
|
||||
|
||||
def graph_query_by_node(
|
||||
self,
|
||||
node: UndirectedNode,
|
||||
step: int = 1,
|
||||
constraint_labels: list[str] = None,
|
||||
constraint_node: UndirectedNode = None,
|
||||
constraint_distance: float = 0,
|
||||
block: bool = False,
|
||||
) -> list[UndirectedNode]:
|
||||
"""
|
||||
search graph by connection, return empty list if nodes' chain without node near to constraint_node
|
||||
Parameters
|
||||
----------
|
||||
node : start node
|
||||
step : the max steps will be searched
|
||||
constraint_labels : the labels of output nodes
|
||||
constraint_node : the node that the output nodes must connect to
|
||||
constraint_distance : the max distance between output nodes and constraint_node
|
||||
block: despite the start node, the search can only flow through the constraint_label type nodes
|
||||
|
||||
Returns
|
||||
-------
|
||||
A list of nodes
|
||||
|
||||
"""
|
||||
nodes = self.graph.query_by_node(
|
||||
node=node,
|
||||
step=step,
|
||||
constraint_labels=constraint_labels,
|
||||
constraint_node=constraint_node,
|
||||
constraint_distance=constraint_distance,
|
||||
block=block,
|
||||
)
|
||||
return nodes
|
||||
|
||||
def graph_query_by_intersection(
|
||||
self,
|
||||
nodes: list[UndirectedNode],
|
||||
steps: int = 1,
|
||||
constraint_labels: list[str] = None,
|
||||
output_intersection_origin: bool = False,
|
||||
) -> list[UndirectedNode] | list[list[list[UndirectedNode], UndirectedNode]]:
|
||||
"""
|
||||
search graph by node intersection, node intersected by a higher frequency has a prior order in the list
|
||||
Parameters
|
||||
----------
|
||||
nodes : node list
|
||||
step : the max steps will be searched
|
||||
constraint_labels : the labels of output nodes
|
||||
output_intersection_origin: output the list that contains the node which form this intersection node
|
||||
|
||||
Returns
|
||||
-------
|
||||
A list of nodes
|
||||
|
||||
"""
|
||||
node_count = len(nodes)
|
||||
assert node_count >= 2, "nodes length must >=2"
|
||||
intersection_node_list = []
|
||||
if output_intersection_origin:
|
||||
origin_list = []
|
||||
for k in range(node_count, 1, -1):
|
||||
possible_combinations = combinations(nodes, k)
|
||||
for possible_combination in possible_combinations:
|
||||
node_list = list(possible_combination)
|
||||
intersection_node_list.extend(
|
||||
self.graph.get_nodes_intersection(node_list, steps=steps, constraint_labels=constraint_labels),
|
||||
)
|
||||
if output_intersection_origin:
|
||||
for _ in range(len(intersection_node_list)):
|
||||
origin_list.append(node_list)
|
||||
intersection_node_list_sort_by_freq = []
|
||||
for index, node in enumerate(intersection_node_list):
|
||||
if node not in intersection_node_list_sort_by_freq:
|
||||
if output_intersection_origin:
|
||||
intersection_node_list_sort_by_freq.append([origin_list[index], node])
|
||||
else:
|
||||
intersection_node_list_sort_by_freq.append(node)
|
||||
|
||||
return intersection_node_list_sort_by_freq
|
||||
@@ -0,0 +1,77 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from jinja2 import Template
|
||||
|
||||
from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
|
||||
FactorEvovlingItem,
|
||||
)
|
||||
from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import (
|
||||
get_data_folder_intro,
|
||||
)
|
||||
from rdagent.core.conf import RD_AGENT_SETTINGS
|
||||
from rdagent.core.log import RDAgentLog
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
|
||||
|
||||
def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
|
||||
import random
|
||||
|
||||
to_be_finished_task_index = random.sample(
|
||||
to_be_finished_task_index,
|
||||
implementation_factors_per_round,
|
||||
)
|
||||
|
||||
RDAgentLog().info(f"The random selection is: {to_be_finished_task_index}")
|
||||
return to_be_finished_task_index
|
||||
|
||||
|
||||
def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvovlingItem, former_trace):
|
||||
tasks = []
|
||||
for i in to_be_finished_task_index:
|
||||
# find corresponding former trace for each task
|
||||
target_factor_task_information = evo.target_factor_tasks[i].get_factor_information()
|
||||
if target_factor_task_information in former_trace:
|
||||
tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information]))
|
||||
|
||||
system_prompt = Template(
|
||||
scheduler_prompts["select_implementable_factor_system"],
|
||||
).render(
|
||||
data_info=get_data_folder_intro(),
|
||||
)
|
||||
|
||||
session = APIBackend(use_chat_cache=False).build_chat_session(
|
||||
session_system_prompt=system_prompt,
|
||||
)
|
||||
|
||||
while True:
|
||||
user_prompt = Template(
|
||||
scheduler_prompts["select_implementable_factor_user"],
|
||||
).render(
|
||||
factor_num=implementation_factors_per_round,
|
||||
target_factor_tasks=tasks,
|
||||
)
|
||||
if (
|
||||
session.build_chat_completion_message_and_calculate_token(
|
||||
user_prompt,
|
||||
)
|
||||
< RD_AGENT_SETTINGS.chat_token_limit
|
||||
):
|
||||
break
|
||||
|
||||
response = session.build_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
json_mode=True,
|
||||
)
|
||||
try:
|
||||
selection = json.loads(response)["selected_factor"]
|
||||
if not isinstance(selection, list):
|
||||
return to_be_finished_task_index
|
||||
selection_index = [x for x in selection if isinstance(x, int)]
|
||||
except:
|
||||
return to_be_finished_task_index
|
||||
|
||||
return selection_index
|
||||
Reference in New Issue
Block a user