import json import re from abc import abstractmethod from typing import Tuple import pandas as pd from jinja2 import Template from rdagent.oai.llm_utils import APIBackend from rdagent.core.log import RDAgentLog from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask, FactorEvovlingItem from rdagent.core.task import ( TaskImplementation, ) from typing import List, Tuple from rdagent.core.evolving_framework import QueriedKnowledge, Feedback from rdagent.core.evaluation import Evaluator from rdagent.core.prompts import Prompts from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS from rdagent.core.utils import multiprocessing_wrapper from pathlib import Path evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") class FactorImplementationEvaluator(Evaluator): # TODO: # I think we should have unified interface for all evaluates, for examples. # So we should adjust the interface of other factors @abstractmethod def evaluate( self, gt: TaskImplementation, gen: TaskImplementation, ) -> Tuple[str, object]: """You can get the dataframe by .. code-block:: python _, gt_df = gt.execute() _, gen_df = gen.execute() Returns ------- Tuple[str, object] - str: the text-based description of the evaluation result - object: a comparable metric (bool, integer, float ...) """ raise NotImplementedError("Please implement the `evaluator` method") def _get_df(self, gt: TaskImplementation, gen: TaskImplementation): _, gt_df = gt.execute() _, gen_df = gen.execute() if isinstance(gen_df, pd.Series): gen_df = gen_df.to_frame("source_factor") if isinstance(gt_df, pd.Series): gt_df = gt_df.to_frame("gt_factor") return gt_df, gen_df class FactorImplementationCodeEvaluator(Evaluator): def evaluate( self, target_task: FactorImplementTask, implementation: TaskImplementation, execution_feedback: str, factor_value_feedback: str = "", gt_implementation: TaskImplementation = None, **kwargs, ): factor_information = target_task.get_factor_information() code = implementation.code system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"] execution_feedback_to_render = execution_feedback user_prompt = Template( evaluate_prompts["evaluator_code_feedback_v1_user"], ).render( factor_information=factor_information, code=code, execution_feedback=execution_feedback_to_render, factor_value_feedback=factor_value_feedback, gt_code=gt_implementation.code if gt_implementation else None, ) while ( APIBackend().build_messages_and_calculate_token( user_prompt=user_prompt, system_prompt=system_prompt, former_messages=[], ) > RD_AGENT_SETTINGS.chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] user_prompt = Template( evaluate_prompts["evaluator_code_feedback_v1_user"], ).render( factor_information=factor_information, code=code, execution_feedback=execution_feedback_to_render, factor_value_feedback=factor_value_feedback, gt_code=gt_implementation.code if gt_implementation else None, ) critic_response = APIBackend().build_messages_and_create_chat_completion( user_prompt=user_prompt, system_prompt=system_prompt, json_mode=False, ) return critic_response class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator): def evaluate( self, gt: TaskImplementation, gen: TaskImplementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) if len(gen_df.columns) == 1 and len(gt_df.columns) == 1: return "Both dataframes have only one column.", True elif len(gen_df.columns) != 1: gen_df = gen_df.iloc(axis=1)[ [ 0, ] ] return ( "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", False, ) return "", False def __str__(self) -> str: return self.__class__.__name__ class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator): def evaluate( self, gt: TaskImplementation, gen: TaskImplementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) idx_name_right = gen_df.index.names == ("datetime", "instrument") if idx_name_right: return ( 'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.', True, ) else: return ( 'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.', False, ) def __str__(self) -> str: return self.__class__.__name__ class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator): def evaluate( self, gt: TaskImplementation, gen: TaskImplementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) if gen_df.shape[0] == gt_df.shape[0]: return "Both dataframes have the same rows count.", True else: return ( f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.", False, ) def __str__(self) -> str: return self.__class__.__name__ class FactorImplementationIndexEvaluator(FactorImplementationEvaluator): def evaluate( self, gt: TaskImplementation, gen: TaskImplementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) if gen_df.index.equals(gt_df.index): return "Both dataframes have the same index.", True else: return ( "The source dataframe and the ground truth dataframe have different index. Please check the implementation.", False, ) def __str__(self) -> str: return self.__class__.__name__ class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator): def evaluate( self, gt: TaskImplementation, gen: TaskImplementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) if gen_df.isna().sum().sum() == gt_df.isna().sum().sum(): return "Both dataframes have the same missing values.", True else: return ( f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.", False, ) def __str__(self) -> str: return self.__class__.__name__ class FactorImplementationValuesEvaluator(FactorImplementationEvaluator): def evaluate( self, gt: TaskImplementation, gen: TaskImplementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) try: close_values = gen_df.sub(gt_df).abs().lt(1e-6) result_int = close_values.astype(int) pos_num = result_int.sum().sum() acc_rate = pos_num / close_values.size except: close_values = gen_df if close_values.all().iloc[0]: return ( "All values in the dataframes are equal within the tolerance of 1e-6.", acc_rate, ) else: return ( "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.", acc_rate, ) def __str__(self) -> str: return self.__class__.__name__ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator): def __init__(self, hard_check: bool) -> None: self.hard_check = hard_check def evaluate( self, gt: TaskImplementation, gen: TaskImplementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) concat_df = pd.concat([gen_df, gt_df], axis=1) concat_df.columns = ["source", "gt"] ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean() ric = ( concat_df.groupby("datetime") .apply(lambda df: df["source"].corr(df["gt"], method="spearman")) .dropna() .mean() ) if self.hard_check: if ic > 0.99 and ric > 0.99: return ( f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.", True, ) else: return ( f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.", False, ) else: return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic def __str__(self) -> str: return self.__class__.__name__ class FactorImplementationValEvaluator(FactorImplementationEvaluator): def evaluate(self, gt: TaskImplementation, gen: TaskImplementation): _, gt_df = gt.execute() _, gen_df = gen.execute() # FIXME: refactor the two classes fiv = FactorImplementationValueEvaluator() return fiv.evaluate(source_df=gen_df, gt_df=gt_df) def __str__(self) -> str: return self.__class__.__name__ class FactorImplementationValueEvaluator(Evaluator): # TODO: let's discuss the about the interface of the evaluator def evaluate( self, source_df: pd.DataFrame, gt_df: pd.DataFrame, **kwargs, ) -> Tuple: conclusions = [] if isinstance(source_df, pd.Series): source_df = source_df.to_frame("source_factor") conclusions.append( "The source dataframe is a series, better convert it to a dataframe.", ) if gt_df is not None and isinstance(gt_df, pd.Series): gt_df = gt_df.to_frame("gt_factor") conclusions.append( "The ground truth dataframe is a series, convert it to a dataframe.", ) # Check if both dataframe has only one columns if len(source_df.columns) == 1: conclusions.append("The source dataframe has only one column which is correct.") else: conclusions.append( "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", ) source_df = source_df.iloc(axis=1)[ [ 0, ] ] if list(source_df.index.names) != ["datetime", "instrument"]: conclusions.append( rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.", ) else: conclusions.append( 'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.', ) # Check if both dataframe have the same rows count if gt_df is not None: if source_df.shape[0] == gt_df.shape[0]: conclusions.append("Both dataframes have the same rows count.") same_row_count_result = True else: conclusions.append( f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.", ) same_row_count_result = False # Check whether both dataframe has the same index if source_df.index.equals(gt_df.index): conclusions.append("Both dataframes have the same index.") same_index_result = True else: conclusions.append( "The source dataframe and the ground truth dataframe have different index. Please check the implementation.", ) same_index_result = False # Check for the same missing values (NaN) if source_df.isna().sum().sum() == gt_df.isna().sum().sum(): conclusions.append("Both dataframes have the same missing values.") same_missing_values_result = True else: conclusions.append( f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.", ) same_missing_values_result = False # Check if the values are the same within a small tolerance if not same_index_result: conclusions.append( "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless", ) same_values_result = False high_correlation_result = False else: close_values = source_df.sub(gt_df).abs().lt(1e-6) if close_values.all().iloc[0]: conclusions.append( "All values in the dataframes are equal within the tolerance of 1e-6.", ) same_values_result = True else: conclusions.append( "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.", ) same_values_result = False # Check the ic and rankic between the two dataframes concat_df = pd.concat([source_df, gt_df], axis=1) concat_df.columns = ["source", "gt"] try: ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean() ric = ( concat_df.groupby("datetime") .apply(lambda df: df["source"].corr(df["gt"], method="spearman")) .dropna() .mean() ) if ic > 0.99 and ric > 0.99: conclusions.append( f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.", ) high_correlation_result = True else: conclusions.append( f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.", ) high_correlation_result = False # Check for shifted alignments only in the "datetime" index max_shift_days = 2 for shift in range(-max_shift_days, max_shift_days + 1): if shift == 0: continue # Skip the case where there is no shift shifted_source_df = source_df.groupby(level="instrument").shift(shift) concat_df = pd.concat([shifted_source_df, gt_df], axis=1) concat_df.columns = ["source", "gt"] shifted_ric = ( concat_df.groupby("datetime") .apply(lambda df: df["source"].corr(df["gt"], method="spearman")) .dropna() .mean() ) if shifted_ric > 0.99: conclusions.append( f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.", ) break else: conclusions.append( f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.", ) except Exception as e: RDAgentLog().warning(f"Error occurred when calculating the correlation: {str(e)}") conclusions.append( f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}", ) high_correlation_result = False # Combine all conclusions into a single string conclusion_str = "\n".join(conclusions) final_result = (same_values_result or high_correlation_result) if gt_df is not None else False return conclusion_str, final_result # TODO: def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str: """When the prompt is too long. We have to shorten it. But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it. """ # TODO: this should replace most of code in # - FactorImplementationFinalDecisionEvaluator.evaluate # - FactorImplementationCodeEvaluator.evaluate class FactorImplementationFinalDecisionEvaluator(Evaluator): def evaluate( self, target_task: FactorImplementTask, execution_feedback: str, value_feedback: str, code_feedback: str, **kwargs, ) -> Tuple: system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[ "evaluator_final_decision_v1_system" ] execution_feedback_to_render = execution_feedback user_prompt = Template( evaluate_prompts["evaluator_final_decision_v1_user"], ).render( factor_information=target_task.get_factor_information(), execution_feedback=execution_feedback_to_render, code_feedback=code_feedback, factor_value_feedback=( value_feedback if value_feedback is not None else "No Ground Truth Value provided, so no evaluation on value is performed." ), ) while ( APIBackend().build_messages_and_calculate_token( user_prompt=user_prompt, system_prompt=system_prompt, former_messages=[], ) > RD_AGENT_SETTINGS.chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] user_prompt = Template( evaluate_prompts["evaluator_final_decision_v1_user"], ).render( factor_information=target_task.get_factor_information(), execution_feedback=execution_feedback_to_render, code_feedback=code_feedback, factor_value_feedback=( value_feedback if value_feedback is not None else "No Ground Truth Value provided, so no evaluation on value is performed." ), ) final_evaluation_dict = json.loads( APIBackend().build_messages_and_create_chat_completion( user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True, ), ) return ( final_evaluation_dict["final_decision"], final_evaluation_dict["final_feedback"], ) class FactorImplementationSingleFeedback: """This class is a feedback to single implementation which is generated from an evaluator.""" def __init__( self, execution_feedback: str = None, value_generated_flag: bool = False, code_feedback: str = None, factor_value_feedback: str = None, final_decision: bool = None, final_feedback: str = None, final_decision_based_on_gt: bool = None, ) -> None: self.execution_feedback = execution_feedback self.value_generated_flag = value_generated_flag self.code_feedback = code_feedback self.factor_value_feedback = factor_value_feedback self.final_decision = final_decision self.final_feedback = final_feedback self.final_decision_based_on_gt = final_decision_based_on_gt def __str__(self) -> str: return f"""------------------Factor Execution Feedback------------------ {self.execution_feedback} ------------------Factor Code Feedback------------------ {self.code_feedback} ------------------Factor Value Feedback------------------ {self.factor_value_feedback} ------------------Factor Final Feedback------------------ {self.final_feedback} ------------------Factor Final Decision------------------ This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}. """ class FactorImplementationsMultiFeedback( Feedback, List[FactorImplementationSingleFeedback], ): """Feedback contains a list, each element is the corresponding feedback for each factor implementation.""" class FactorImplementationEvaluatorV1(FactorImplementationEvaluator): """This class is the v1 version of evaluator for a single factor implementation. It calls several evaluators in share modules to evaluate the factor implementation. """ def __init__(self) -> None: self.code_evaluator = FactorImplementationCodeEvaluator() self.value_evaluator = FactorImplementationValueEvaluator() self.final_decision_evaluator = FactorImplementationFinalDecisionEvaluator() def evaluate( self, target_task: FactorImplementTask, implementation: TaskImplementation, gt_implementation: TaskImplementation = None, queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> FactorImplementationSingleFeedback: if implementation is None: return None target_task_information = target_task.get_factor_information() if ( queried_knowledge is not None and target_task_information in queried_knowledge.success_task_to_knowledge_dict ): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return FactorImplementationSingleFeedback( execution_feedback="This task has failed too many times, skip implementation.", value_generated_flag=False, code_feedback="This task has failed too many times, skip code evaluation.", factor_value_feedback="This task has failed too many times, skip value evaluation.", final_decision=False, final_feedback="This task has failed too many times, skip final decision evaluation.", final_decision_based_on_gt=False, ) else: factor_feedback = FactorImplementationSingleFeedback() ( factor_feedback.execution_feedback, source_df, ) = implementation.execute() # Remove the long list of numbers in the feedback pattern = r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)" factor_feedback.execution_feedback = re.sub(pattern, ", ", factor_feedback.execution_feedback) execution_feedback_lines = [ line for line in factor_feedback.execution_feedback.split("\n") if "warning" not in line.lower() ] factor_feedback.execution_feedback = "\n".join(execution_feedback_lines) if source_df is None: factor_feedback.factor_value_feedback = "No factor value generated, skip value evaluation." factor_feedback.value_generated_flag = False value_decision = None else: factor_feedback.value_generated_flag = True if gt_implementation is not None: _, gt_df = gt_implementation.execute(store_result=True) else: gt_df = None try: source_df = source_df.sort_index() if gt_df is not None: gt_df = gt_df.sort_index() ( factor_feedback.factor_value_feedback, value_decision, ) = self.value_evaluator.evaluate(source_df=source_df, gt_df=gt_df) except Exception as e: RDAgentLog().warning("Value evaluation failed with exception: %s", e) factor_feedback.factor_value_feedback = "Value evaluation failed." value_decision = False factor_feedback.final_decision_based_on_gt = gt_implementation is not None if value_decision is not None and value_decision is True: # To avoid confusion, when value_decision is True, we do not need code feedback factor_feedback.code_feedback = "Final decision is True and there are no code critics." factor_feedback.final_decision = value_decision factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation." else: factor_feedback.code_feedback = self.code_evaluator.evaluate( target_task=target_task, implementation=implementation, execution_feedback=factor_feedback.execution_feedback, value_feedback=factor_feedback.factor_value_feedback, gt_implementation=gt_implementation, ) ( factor_feedback.final_decision, factor_feedback.final_feedback, ) = self.final_decision_evaluator.evaluate( target_task=target_task, execution_feedback=factor_feedback.execution_feedback, value_feedback=factor_feedback.factor_value_feedback, code_feedback=factor_feedback.code_feedback, ) return factor_feedback class FactorImplementationsMultiEvaluator(Evaluator): def __init__(self, single_evaluator=FactorImplementationEvaluatorV1()) -> None: super().__init__() self.single_factor_implementation_evaluator = single_evaluator def evaluate( self, evo: FactorEvovlingItem, queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> FactorImplementationsMultiFeedback: multi_implementation_feedback = FactorImplementationsMultiFeedback() # for index in range(len(evo.target_factor_tasks)): # corresponding_implementation = evo.corresponding_implementations[index] # corresponding_gt_implementation = ( # evo.corresponding_gt_implementations[index] # if evo.corresponding_gt_implementations is not None # else None # ) # multi_implementation_feedback.append( # self.single_factor_implementation_evaluator.evaluate( # target_task=evo.target_factor_tasks[index], # implementation=corresponding_implementation, # gt_implementation=corresponding_gt_implementation, # queried_knowledge=queried_knowledge, # ) # ) calls = [] for index in range(len(evo.target_factor_tasks)): corresponding_implementation = evo.corresponding_implementations[index] corresponding_gt_implementation = ( evo.corresponding_gt_implementations[index] if evo.corresponding_gt_implementations is not None else None ) calls.append( ( self.single_factor_implementation_evaluator.evaluate, ( evo.target_factor_tasks[index], corresponding_implementation, corresponding_gt_implementation, queried_knowledge, ), ), ) multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n) final_decision = [ None if single_feedback is None else single_feedback.final_decision for single_feedback in multi_implementation_feedback ] RDAgentLog().info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}") return multi_implementation_feedback