gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-28 16:07:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Modify FactorRowCountEvaluator and FactorIndexEvaluator to return the ratio (#328)

Browse Source 
* remove AttributeError caused by select_threshold

* develop with ground truth

* raise exception for eval_case

* Revert "develop with ground truth"

This reverts commit e68c136588685476f32a3c3696a6a33deb47acc5.

* Modify FactorRowCountEvaluator and FactorIndexEvaluator to return the ratio

* reformatted by black

* Set the threshold for FactorIndexEvaluator to 0.99

* Apply suggestions from code review

* Update rdagent/components/coder/factor_coder/CoSTEER/evaluators.py

* geometric mean for format_succ_rate

* no need to check when similarity is high enough

* black reformat

---------

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
This commit is contained in:
Tim
2024-09-27 09:34:55 +08:00
committed by GitHub
parent 0e94e74d20
commit b93f034b19
3 changed files with 34 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/benchmark/factor/analysis.py
+10 -2
View File
@@ -6,6 +6,7 @@ import fire
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from scipy.stats import gmean
from rdagent.components.benchmark.conf import BenchmarkSettings
from rdagent.components.benchmark.eval_method import FactorImplementEval
@@ -97,9 +98,16 @@ class BenchmarkAnalyzer:
sum_df_clean["FactorRowCountEvaluator"]
format_issue = sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"]
format_issue = sum_df_clean["FactorRowCountEvaluator"].astype(bool) & sum_df_clean[
"FactorIndexEvaluator"
].astype(bool)
format_issue = sum_df_clean[["FactorRowCountEvaluator", "FactorIndexEvaluator"]].apply(
lambda x: gmean(x), axis=1
)
eval_series = format_issue.unstack()
succ_rate = eval_series.T.fillna(False).astype(bool) # false indicate failure
succ_rate = eval_series.T.fillna(False)
format_succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
format_succ_rate_f = self.reformat_succ_rate(format_succ_rate)
rdagent/components/benchmark/eval_method.py
+2 -2
View File
@@ -125,10 +125,10 @@ class BaseEval:
eval_res = []
for ev in self.evaluator_l:
try:
case_gen.raise_exception = True
eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
# if the corr ev is successfully evaluated and achieve the best performance, then break
except (CoderError, AttributeError) as e:
# TODO: remove AttributeError.
except CoderError as e:
return e
except Exception as e:
# exception when evaluation
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+22 -23
View File
@@ -250,13 +250,14 @@ class FactorRowCountEvaluator(FactorEvaluator):
"The source dataframe is None. Please check the implementation.",
False,
)
if gen_df.shape[0] == gt_df.shape[0]:
return "Both dataframes have the same rows count.", True
else:
return (
f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
False,
)
ratio = len(gen_df) / len(gt_df)
return (
f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. "
+ "Please verify the implementation. "
if ratio <= 0.99
else "",
ratio,
)
class FactorIndexEvaluator(FactorEvaluator):
@@ -271,13 +272,15 @@ class FactorIndexEvaluator(FactorEvaluator):
"The source dataframe is None. Please check the implementation.",
False,
)
if gen_df.index.equals(gt_df.index):
return "Both dataframes have the same index.", True
else:
return (
"The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
False,
)
gen_index_set, gt_index_set = set(gen_df.index), set(gt_df.index)
similarity = len(gen_index_set.intersection(gt_index_set)) / len(gen_index_set.union(gt_index_set))
return (
f"The source dataframe and the ground truth dataframe have different index with a similarity of {similarity:.2%}. The similarity is calculated by the number of shared indices divided by the union indices. "
+ "Please check the implementation."
if similarity <= 0.99
else "",
similarity,
)
class FactorMissingValuesEvaluator(FactorEvaluator):
@@ -413,16 +416,12 @@ class FactorValueEvaluator(FactorEvaluator):
else:
daily_check_result = None
# Check if both dataframe have the same rows count
# Check dataframe format
if gt_implementation is not None:
feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(
@@ -435,7 +434,7 @@ class FactorValueEvaluator(FactorEvaluator):
)
conclusions.append(feedback_str)
if same_index_result:
if index_result > 0.99:
feedback_str, high_correlation_result = FactorCorrelationEvaluator(
hard_check=True, scen=self.scen
).evaluate(implementation, gt_implementation)
@@ -449,7 +448,7 @@ class FactorValueEvaluator(FactorEvaluator):
if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
decision_from_value_check = True
elif single_column_result is False or output_format_result is False or daily_check_result is False:
elif row_result <= 0.99 or output_format_result is False or daily_check_result is False:
decision_from_value_check = False
else:
decision_from_value_check = None