diff --git a/rdagent/app/benchmark/factor/analysis.py b/rdagent/app/benchmark/factor/analysis.py index 385f3a42..509bf23d 100644 --- a/rdagent/app/benchmark/factor/analysis.py +++ b/rdagent/app/benchmark/factor/analysis.py @@ -6,6 +6,7 @@ import fire import matplotlib.pyplot as plt import pandas as pd import seaborn as sns +from scipy.stats import gmean from rdagent.components.benchmark.conf import BenchmarkSettings from rdagent.components.benchmark.eval_method import FactorImplementEval @@ -97,9 +98,16 @@ class BenchmarkAnalyzer: sum_df_clean["FactorRowCountEvaluator"] - format_issue = sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"] + format_issue = sum_df_clean["FactorRowCountEvaluator"].astype(bool) & sum_df_clean[ + "FactorIndexEvaluator" + ].astype(bool) + format_issue = sum_df_clean[["FactorRowCountEvaluator", "FactorIndexEvaluator"]].apply( + lambda x: gmean(x), axis=1 + ) + eval_series = format_issue.unstack() - succ_rate = eval_series.T.fillna(False).astype(bool) # false indicate failure + succ_rate = eval_series.T.fillna(False) + format_succ_rate = succ_rate.mean(axis=0).to_frame("success rate") format_succ_rate_f = self.reformat_succ_rate(format_succ_rate) diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index 3654b650..bd89e2f6 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -125,10 +125,10 @@ class BaseEval: eval_res = [] for ev in self.evaluator_l: try: + case_gen.raise_exception = True eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt))) # if the corr ev is successfully evaluated and achieve the best performance, then break - except (CoderError, AttributeError) as e: - # TODO: remove AttributeError. + except CoderError as e: return e except Exception as e: # exception when evaluation diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index 4951866b..79d7ef62 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -250,13 +250,14 @@ class FactorRowCountEvaluator(FactorEvaluator): "The source dataframe is None. Please check the implementation.", False, ) - if gen_df.shape[0] == gt_df.shape[0]: - return "Both dataframes have the same rows count.", True - else: - return ( - f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.", - False, - ) + ratio = len(gen_df) / len(gt_df) + return ( + f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. " + + "Please verify the implementation. " + if ratio <= 0.99 + else "", + ratio, + ) class FactorIndexEvaluator(FactorEvaluator): @@ -271,13 +272,15 @@ class FactorIndexEvaluator(FactorEvaluator): "The source dataframe is None. Please check the implementation.", False, ) - if gen_df.index.equals(gt_df.index): - return "Both dataframes have the same index.", True - else: - return ( - "The source dataframe and the ground truth dataframe have different index. Please check the implementation.", - False, - ) + gen_index_set, gt_index_set = set(gen_df.index), set(gt_df.index) + similarity = len(gen_index_set.intersection(gt_index_set)) / len(gen_index_set.union(gt_index_set)) + return ( + f"The source dataframe and the ground truth dataframe have different index with a similarity of {similarity:.2%}. The similarity is calculated by the number of shared indices divided by the union indices. " + + "Please check the implementation." + if similarity <= 0.99 + else "", + similarity, + ) class FactorMissingValuesEvaluator(FactorEvaluator): @@ -413,16 +416,12 @@ class FactorValueEvaluator(FactorEvaluator): else: daily_check_result = None - # Check if both dataframe have the same rows count + # Check dataframe format if gt_implementation is not None: - feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate( - implementation, gt_implementation - ) + feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) conclusions.append(feedback_str) - feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate( - implementation, gt_implementation - ) + feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation) conclusions.append(feedback_str) feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate( @@ -435,7 +434,7 @@ class FactorValueEvaluator(FactorEvaluator): ) conclusions.append(feedback_str) - if same_index_result: + if index_result > 0.99: feedback_str, high_correlation_result = FactorCorrelationEvaluator( hard_check=True, scen=self.scen ).evaluate(implementation, gt_implementation) @@ -449,7 +448,7 @@ class FactorValueEvaluator(FactorEvaluator): if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result: decision_from_value_check = True - elif single_column_result is False or output_format_result is False or daily_check_result is False: + elif row_result <= 0.99 or output_format_result is False or daily_check_result is False: decision_from_value_check = False else: decision_from_value_check = None