gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: partial bug in bench (#368)

Browse Source 
* Add more log

* Fix eval

* update

* keep ratio <= 1

* feat: cache exception (#369)

* 0 instead of NaN

* remove unused evaluators

* save gen_factor_l_all_rounds

* black reformat

* cache exception

---------

Co-authored-by: Tim <illking@foxmail.com>
This commit is contained in:
you-n-g
2024-09-29 09:59:54 +08:00
committed by GitHub
parent 3ea9d6744e
commit 346261c1ac
6 changed files with 82 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+13 -13
View File
@@ -250,7 +250,7 @@ class FactorRowCountEvaluator(FactorEvaluator):
"The source dataframe is None. Please check the implementation.",
False,
)
ratio = len(gen_df) / len(gt_df)
ratio = min(len(gen_df), len(gt_df)) / max(len(gen_df), len(gt_df))
return (
f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. "
+ "Please verify the implementation. "
@@ -304,7 +304,7 @@ class FactorMissingValuesEvaluator(FactorEvaluator):
)
class FactorEqualValueCountEvaluator(FactorEvaluator):
class FactorEqualValueRatioEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Workspace,
@@ -392,6 +392,7 @@ class FactorValueEvaluator(FactorEvaluator):
output_format_result = None
equal_value_ratio_result = 0
high_correlation_result = False
row_result = None
# Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
if version == 1:
@@ -429,7 +430,7 @@ class FactorValueEvaluator(FactorEvaluator):
)
conclusions.append(feedback_str)
feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate(
feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
conclusions.append(feedback_str)
@@ -448,16 +449,15 @@ class FactorValueEvaluator(FactorEvaluator):
if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
decision_from_value_check = True
if version == 1:
if row_result <= 0.99 or output_format_result is False or daily_check_result is False:
decision_from_value_check = False
else:
decision_from_value_check = None
elif version == 2:
if output_format_result is False or daily_check_result is False:
decision_from_value_check = False
else:
decision_from_value_check = None
elif (
row_result is not None
and row_result <= 0.99
or output_format_result is False
or daily_check_result is False
):
decision_from_value_check = False
else:
decision_from_value_check = None
return conclusion_str, decision_from_value_check
rdagent/components/coder/factor_coder/factor.py
+15 -5
View File
@@ -117,11 +117,14 @@ class FactorFBWorkspace(FBWorkspace):
target_file_name = md5_hash(data_type + self.code_dict["factor.py"])
cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.cache_location) / f"{target_file_name}.pkl"
Path(FACTOR_IMPLEMENT_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
if cache_file_path.exists() and not self.raise_exception:
if cache_file_path.exists():
cached_res = pickle.load(open(cache_file_path, "rb"))
if store_result and cached_res[1] is not None:
self.executed_factor_value_dataframe = cached_res[1]
return cached_res
if not self.raise_exception or len(cached_res) == 3:
if cached_res[2]:
raise cached_res[2]
if store_result and cached_res[1] is not None:
self.executed_factor_value_dataframe = cached_res[1]
return cached_res[:1]
if self.executed_factor_value_dataframe is not None:
return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
@@ -147,6 +150,7 @@ class FactorFBWorkspace(FBWorkspace):
execution_feedback = self.FB_EXECUTION_SUCCEEDED
execution_success = False
execution_error = None
if self.target_task.version == 1:
execution_code_path = code_path
@@ -177,10 +181,14 @@ class FactorFBWorkspace(FBWorkspace):
)
if self.raise_exception:
raise CustomRuntimeError(execution_feedback)
else:
execution_error = CustomRuntimeError(execution_feedback)
except subprocess.TimeoutExpired:
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
if self.raise_exception:
raise CustomRuntimeError(execution_feedback)
else:
execution_error = CustomRuntimeError(execution_feedback)
workspace_output_file_path = self.workspace_path / "result.h5"
if workspace_output_file_path.exists() and execution_success:
@@ -195,13 +203,15 @@ class FactorFBWorkspace(FBWorkspace):
executed_factor_value_dataframe = None
if self.raise_exception:
raise NoOutputError(execution_feedback)
else:
execution_error = NoOutputError(execution_feedback)
if store_result and executed_factor_value_dataframe is not None:
self.executed_factor_value_dataframe = executed_factor_value_dataframe
if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
pickle.dump(
(execution_feedback, executed_factor_value_dataframe),
(execution_feedback, executed_factor_value_dataframe, execution_error),
open(cache_file_path, "wb"),
)
return execution_feedback, executed_factor_value_dataframe