feat: run benchmark on gpt-4o & llama 3.1 (#497)

* Run benchmark on gpt-4o & llama 3.1

* update link

rdagent/app/benchmark/factor/analysis.py

@@ -13,9 +13,10 @@ from rdagent.components.benchmark.eval_method import FactorImplementEval
 
 
 class BenchmarkAnalyzer:
-    def __init__(self, settings):
+    def __init__(self, settings, only_correct_format=False):
         self.settings = settings
         self.index_map = self.load_index_map()
+        self.only_correct_format = only_correct_format
 
     def load_index_map(self):
         index_map = {}
@@ -119,11 +120,13 @@ class BenchmarkAnalyzer:
         format_succ_rate_f = self.reformat_index(format_succ_rate)
 
         corr = sum_df_clean["FactorCorrelationEvaluator"].fillna(0.0)
-        corr = corr.unstack().T.mean(axis=0).to_frame("corr(only success)")
-        corr_res = self.reformat_index(corr)
-        corr_max = sum_df_clean["FactorCorrelationEvaluator"]
+        if self.only_correct_format:
+            corr = corr.loc[format_issue == 1.0]
 
-        corr_max = corr_max.unstack().T.max(axis=0).to_frame("corr(only success)")
+        corr_res = corr.unstack().T.mean(axis=0).to_frame("corr(only success)")
+        corr_res = self.reformat_index(corr_res)
+
+        corr_max = corr.unstack().T.max(axis=0).to_frame("corr(only success)")
         corr_max_res = self.reformat_index(corr_max)
 
         value_max = sum_df_clean["FactorEqualValueRatioEvaluator"]
@@ -150,9 +153,15 @@ class BenchmarkAnalyzer:
             axis=1,
         )
 
-        df = result_all.sort_index(axis=1, key=self.result_all_key_order)
+        df = result_all.sort_index(axis=1, key=self.result_all_key_order).sort_index(axis=0)
         print(df)
 
+        print()
+        print(df.groupby("Category").mean())
+
+        print()
+        print(df.mean())
+
         # Calculate the mean of each column
         mean_values = df.fillna(0.0).mean()
         mean_df = pd.DataFrame(mean_values).T
@@ -196,9 +205,10 @@ def main(
     path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
     round=1,
     title="Comparison of Different Methods",
+    only_correct_format=False,
 ):
     settings = BenchmarkSettings()
-    benchmark = BenchmarkAnalyzer(settings)
+    benchmark = BenchmarkAnalyzer(settings, only_correct_format=only_correct_format)
     results = {
         f"{round} round experiment": path,
     }

rdagent/app/benchmark/factor/eval.py

@@ -1,16 +1,9 @@
-import os
-import pickle
-import time
-from pathlib import Path
-from pprint import pprint
-
 from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
 from rdagent.components.benchmark.conf import BenchmarkSettings
 from rdagent.components.benchmark.eval_method import FactorImplementEval
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
-from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
 from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
     FactorTestCaseLoaderFromJsonFile,
 )
@@ -25,7 +18,7 @@ if __name__ == "__main__":
     # 3.declare the method to be tested and pass the arguments.
 
     scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
-    generate_method = import_class(bs.bench_method_cls)(scen=scen)
+    generate_method = import_class(bs.bench_method_cls)(scen=scen, **bs.bench_method_extra_kwargs)
     # 4.declare the eval method and pass the arguments.
     eval_method = FactorImplementEval(
         method=generate_method,
@@ -36,7 +29,7 @@ if __name__ == "__main__":
     )
 
     # 5.run the eval
-    res = eval_method.eval()
+    res = eval_method.eval(eval_method.develop())
 
     # 6.save the result
     logger.log_object(res)

rdagent/components/benchmark/conf.py

@@ -12,9 +12,6 @@ class BenchmarkSettings(ExtendedBaseSettings):
         env_prefix = "BENCHMARK_"
         """Use `BENCHMARK_` as prefix for environment variables"""
 
-    ground_truth_dir: Path = DIRNAME / "ground_truth"
-    """ground truth dir"""
-
     bench_data_path: Path = DIRNAME / "example.json"
     """data for benchmark"""
 
@@ -24,7 +21,7 @@ class BenchmarkSettings(ExtendedBaseSettings):
     bench_test_case_n: Optional[int] = None
     """how many test cases to run; If not given, all test cases will be run"""
 
-    bench_method_cls: str = "rdagent.components.coder.CoSTEER.FactorCoSTEER"
+    bench_method_cls: str = "rdagent.components.coder.factor_coder.FactorCoSTEER"
     """method to be used for test cases"""
 
     bench_method_extra_kwargs: dict = field(

rdagent/components/coder/factor_coder/eva_utils.py

@@ -221,15 +221,11 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
                     str(resp_dict["output_format_feedback"]),
                     resp_dict["output_format_decision"],
                 )
-
-            except json.JSONDecodeError as e:
-                raise ValueError("Failed to decode JSON response from API.") from e
-
-            except KeyError as e:
+            except (KeyError, json.JSONDecodeError) as e:
                 attempts += 1
                 if attempts >= max_attempts:
                     raise KeyError(
-                        "Response from API is missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
+                        "Wrong JSON Response or missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
                     ) from e
 
         return "Failed to evaluate output format after multiple attempts.", False

rdagent/components/coder/factor_coder/evolving_strategy.py

@@ -158,14 +158,20 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
                 queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[:-1]
             elif len(queried_similar_error_knowledge_to_render) > 0:
                 queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
-        code = json.loads(
-            APIBackend(
-                use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
-            ).build_messages_and_create_chat_completion(
-                user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True
-            )
-        )["code"]
-        return code
+        for _ in range(10):
+            try:
+                code = json.loads(
+                    APIBackend(
+                        use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
+                    ).build_messages_and_create_chat_completion(
+                        user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True
+                    )
+                )["code"]
+                return code
+            except json.decoder.JSONDecodeError:
+                pass
+        else:
+            return ""  # return empty code if failed to get code after 10 attempts
 
     def assign_code_list_to_evo(self, code_list, evo):
         for index in range(len(evo.sub_tasks)):