gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-04 10:47:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: partial bug in bench (#368)

Browse Source 
* Add more log

* Fix eval

* update

* keep ratio <= 1

* feat: cache exception (#369)

* 0 instead of NaN

* remove unused evaluators

* save gen_factor_l_all_rounds

* black reformat

* cache exception

---------

Co-authored-by: Tim <illking@foxmail.com>
This commit is contained in:
you-n-g
2024-09-29 09:59:54 +08:00
committed by GitHub
parent 3ea9d6744e
commit 346261c1ac
6 changed files with 82 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+13 -13
View File
@@ -250,7 +250,7 @@ class FactorRowCountEvaluator(FactorEvaluator):
"The source dataframe is None. Please check the implementation.",
False,
)
ratio = len(gen_df) / len(gt_df)
ratio = min(len(gen_df), len(gt_df)) / max(len(gen_df), len(gt_df))
return (
f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. "
+ "Please verify the implementation. "
@@ -304,7 +304,7 @@ class FactorMissingValuesEvaluator(FactorEvaluator):
)
class FactorEqualValueCountEvaluator(FactorEvaluator):
class FactorEqualValueRatioEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Workspace,
@@ -392,6 +392,7 @@ class FactorValueEvaluator(FactorEvaluator):
output_format_result = None
equal_value_ratio_result = 0
high_correlation_result = False
row_result = None
# Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
if version == 1:
@@ -429,7 +430,7 @@ class FactorValueEvaluator(FactorEvaluator):
)
conclusions.append(feedback_str)
feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate(
feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
conclusions.append(feedback_str)
@@ -448,16 +449,15 @@ class FactorValueEvaluator(FactorEvaluator):
if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
decision_from_value_check = True
if version == 1:
if row_result <= 0.99 or output_format_result is False or daily_check_result is False:
decision_from_value_check = False
else:
decision_from_value_check = None
elif version == 2:
if output_format_result is False or daily_check_result is False:
decision_from_value_check = False
else:
decision_from_value_check = None
elif (
row_result is not None
and row_result <= 0.99
or output_format_result is False
or daily_check_result is False
):
decision_from_value_check = False
else:
decision_from_value_check = None
return conclusion_str, decision_from_value_check