fix: partial bug in bench (#368)

* Add more log

* Fix eval

* update

* keep ratio <= 1

* feat: cache exception (#369)

* 0 instead of NaN

* remove unused evaluators

* save gen_factor_l_all_rounds

* black reformat

* cache exception

---------

Co-authored-by: Tim <illking@foxmail.com>

rdagent/components/benchmark/eval_method.py

@@ -8,11 +8,9 @@ from tqdm import tqdm
 from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
     FactorCorrelationEvaluator,
-    FactorEqualValueCountEvaluator,
+    FactorEqualValueRatioEvaluator,
     FactorEvaluator,
     FactorIndexEvaluator,
-    FactorMissingValuesEvaluator,
-    FactorOutputFormatEvaluator,
     FactorRowCountEvaluator,
     FactorSingleColumnEvaluator,
 )
@@ -151,20 +149,16 @@ class FactorImplementEval(BaseEval):
     ):
         online_evaluator_l = [
             FactorSingleColumnEvaluator(scen),
-            FactorOutputFormatEvaluator(scen),
             FactorRowCountEvaluator(scen),
             FactorIndexEvaluator(scen),
-            FactorMissingValuesEvaluator(scen),
-            FactorEqualValueCountEvaluator(scen),
+            FactorEqualValueRatioEvaluator(scen),
             FactorCorrelationEvaluator(hard_check=False, scen=scen),
         ]
         super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)
         self.test_round = test_round
 
-    def eval(self):
+    def develop(self):
         gen_factor_l_all_rounds = []
-        test_cases_all_rounds = []
-        res = defaultdict(list)
         for _ in tqdm(range(self.test_round), desc="Rounds of Eval"):
             print("\n========================================================")
             print(f"Eval {_}-th times...")
@@ -181,8 +175,14 @@ class FactorImplementEval(BaseEval):
                     "The number of cases to eval should be equal to the number of test cases.",
                 )
             gen_factor_l_all_rounds.extend(gen_factor_l.sub_workspace_list)
-            test_cases_all_rounds.extend(self.test_cases.ground_truth)
 
+        return gen_factor_l_all_rounds
+
+    def eval(self, gen_factor_l_all_rounds):
+        test_cases_all_rounds = []
+        res = defaultdict(list)
+        for _ in range(self.test_round):
+            test_cases_all_rounds.extend(self.test_cases.ground_truth)
         eval_res_list = multiprocessing_wrapper(
             [
                 (self.eval_case, (gt_case, gen_factor))