gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-01 09:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reject the factor code if factor is not daily in COSTEER coder (#73)

Browse Source 
* when not build from docker file we allow user not provide the dockerfile path

* Reject the factor code if factor is not daily in COSTEER coder
This commit is contained in:
Xu Yang
2024-07-16 11:37:38 +08:00
committed by GitHub
parent d70292959e
commit d36a316509
1 changed files with 21 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+21 -9
View File
@@ -344,7 +344,9 @@ class FactorValueEvaluator(FactorEvaluator):
feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
feedback_str, _ = FactorDatetimeDailyEvaluator(self.scen).evaluate(implementation, gt_implementation)
feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
conclusions.append(feedback_str)
# Check if both dataframe have the same rows count
@@ -377,10 +379,13 @@ class FactorValueEvaluator(FactorEvaluator):
# Combine all conclusions into a single string
conclusion_str = "\n".join(conclusions)
same_value_or_high_correlation = (
((equal_value_ratio_result > 0.99) or high_correlation_result) if gt_implementation is not None else False
)
return conclusion_str, same_value_or_high_correlation
if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
decision_from_value_check = True
elif daily_check_result is False:
decision_from_value_check = False
else:
decision_from_value_check = None
return conclusion_str, decision_from_value_check
class FactorFinalDecisionEvaluator(Evaluator):
@@ -542,21 +547,27 @@ class FactorEvaluatorForCoder(FactorEvaluator):
if gen_df is None:
factor_feedback.factor_value_feedback = "No factor value generated, skip value evaluation."
factor_feedback.value_generated_flag = False
same_value_or_high_correlation = None
decision_from_value_check = None
else:
factor_feedback.value_generated_flag = True
(
factor_feedback.factor_value_feedback,
same_value_or_high_correlation,
decision_from_value_check,
) = self.value_evaluator.evaluate(implementation=implementation, gt_implementation=gt_implementation)
factor_feedback.final_decision_based_on_gt = gt_implementation is not None
if same_value_or_high_correlation is not None and same_value_or_high_correlation is True:
if decision_from_value_check is not None and decision_from_value_check is True:
# To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback
factor_feedback.code_feedback = "Final decision is True and there are no code critics."
factor_feedback.final_decision = same_value_or_high_correlation
factor_feedback.final_decision = decision_from_value_check
factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
elif decision_from_value_check is not None and decision_from_value_check is False:
factor_feedback.code_feedback = (
"Final decision is False because value evaluation gets a confident rejection to the result."
)
factor_feedback.final_decision = decision_from_value_check
factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
else:
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
target_task=target_task,
@@ -574,6 +585,7 @@ class FactorEvaluatorForCoder(FactorEvaluator):
value_feedback=factor_feedback.factor_value_feedback,
code_feedback=factor_feedback.code_feedback,
)
RDAgentLog().info(factor_feedback.final_decision)
return factor_feedback