fix: refactor Bench (#302)

* refactor for better bench

* autolint

* add cmd

* lint

rdagent/app/benchmark/factor/analysis.py

@@ -2,6 +2,7 @@ import json
 import pickle
 from pathlib import Path
 
+import fire
 import matplotlib.pyplot as plt
 import pandas as pd
 import seaborn as sns
@@ -168,11 +169,11 @@ class Plotter:
         plt.savefig(file_name)
 
 
-if __name__ == "__main__":
+def main(path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl"):
     settings = BenchmarkSettings()
     benchmark = BenchmarkAnalyzer(settings)
     results = {
-        "1 round experiment": "git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
+        "1 round experiment": path,
     }
     final_results = benchmark.process_results(results)
     final_results_df = pd.DataFrame(final_results)
@@ -180,4 +181,8 @@ if __name__ == "__main__":
     Plotter.change_fs(20)
     plot_data = final_results_df.drop(["max. accuracy", "avg. accuracy"], axis=0).T
     plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b")
-    Plotter.plot_data(plot_data, "rdagent/app/quant_factor_benchmark/comparison_plot.png")
+    Plotter.plot_data(plot_data, "./comparison_plot.png")
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/components/benchmark/eval_method.py

@@ -20,7 +20,7 @@ from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.developer import Developer
 from rdagent.core.exception import CoderError
-from rdagent.core.experiment import Task, Workspace
+from rdagent.core.experiment import Experiment, Task, Workspace
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import multiprocessing_wrapper
 
@@ -33,11 +33,34 @@ EVAL_RES = Dict[
 class TestCase:
     def __init__(
         self,
-        target_task: list[Task] = [],
-        ground_truth: list[Workspace] = [],
+        target_task: Task,
+        ground_truth: Workspace,
     ):
-        self.ground_truth = ground_truth
         self.target_task = target_task
+        self.ground_truth = ground_truth
+
+
+class TestCases:
+    def __init__(self, test_case_l: list[TestCase] = []):
+        # self.test_case_l = [TestCase(task, gt) for task, gt in zip(target_task, ground_truth)]
+        self.test_case_l = test_case_l
+
+    def __getitem__(self, item):
+        return self.test_case_l[item]
+
+    def __len__(self):
+        return len(self.test_case_l)
+
+    def get_exp(self):
+        return Experiment([case.target_task for case in self.test_case_l])
+
+    @property
+    def target_task(self):
+        return [case.target_task for case in self.test_case_l]
+
+    @property
+    def ground_truth(self):
+        return [case.ground_truth for case in self.test_case_l]
 
 
 class BaseEval:
@@ -48,13 +71,13 @@ class BaseEval:
     def __init__(
         self,
         evaluator_l: List[FactorEvaluator],
-        test_cases: List[TestCase],
+        test_cases: TestCases,
         generate_method: Developer,
         catch_eval_except: bool = True,
     ):
         """Parameters
         ----------
-        test_cases : List[TestCase]
+        test_cases : TestCases
             cases to be evaluated, ground truth are included in the test cases.
         evaluator_l : List[FactorEvaluator]
             A list of evaluators to evaluate the generated code.
@@ -105,6 +128,7 @@ class BaseEval:
                 eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
                 # if the corr ev is successfully evaluated and achieve the best performance, then break
             except (CoderError, AttributeError) as e:
+                # TODO: remove AttributeError.
                 return e
             except Exception as e:
                 # exception when evaluation
@@ -118,7 +142,7 @@ class BaseEval:
 class FactorImplementEval(BaseEval):
     def __init__(
         self,
-        test_cases: TestCase,
+        test_cases: TestCases,
         method: Developer,
         *args,
         scen: Scenario,
@@ -146,7 +170,7 @@ class FactorImplementEval(BaseEval):
             print(f"Eval {_}-th times...")
             print("========================================================\n")
             try:
-                gen_factor_l = self.generate_method.develop(self.test_cases.target_task)
+                gen_factor_l = self.generate_method.develop(self.test_cases.get_exp())
             except KeyboardInterrupt:
                 # TODO: Why still need to save result after KeyboardInterrupt?
                 print("Manually interrupted the evaluation. Saving existing results")

rdagent/components/coder/factor_coder/factor.py

@@ -31,13 +31,15 @@ class FactorTask(Task):
         factor_implementation: bool = False,
         **kwargs,
     ) -> None:
-        self.factor_name = factor_name
+        self.factor_name = (
+            factor_name  # TODO: remove it in the later version. Keep it only for pickle version compatibility
+        )
         self.factor_description = factor_description
         self.factor_formulation = factor_formulation
         self.variables = variables
         self.factor_resources = resource
         self.factor_implementation = factor_implementation
-        super().__init__(*args, **kwargs)
+        super().__init__(name=factor_name, *args, **kwargs)
 
     def get_task_information(self):
         return f"""factor_name: {self.factor_name}

rdagent/components/coder/model_coder/model.py

@@ -23,14 +23,13 @@ class ModelTask(Task):
         model_type: Optional[str] = None,
         **kwargs,
     ) -> None:
-        self.name: str = name
         self.description: str = description
         self.formulation: str = formulation
         self.architecture: str = architecture
         self.variables: str = variables
         self.hyperparameters: str = hyperparameters
         self.model_type: str = model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
-        super().__init__(*args, **kwargs)
+        super().__init__(name=name, *args, **kwargs)
 
     def get_task_information(self):
         task_desc = f"""name: {self.name}

rdagent/core/experiment.py

@@ -17,13 +17,14 @@ This file contains the all the class about organizing the task in RD-Agent.
 
 
 class Task(ABC):
-    def __init__(self, version: int = 1) -> None:
+    def __init__(self, name: str, version: int = 1) -> None:
         """
         The version of the task, default is 1
         Because qlib tasks execution and kaggle tasks execution are different, we need to distinguish them.
         TODO: We may align them in the future.
         """
         self.version = version
+        self.name = name
 
     @abstractmethod
     def get_task_information(self) -> str:

rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py

@@ -1,7 +1,7 @@
 import json
 from pathlib import Path
 
-from rdagent.components.benchmark.eval_method import TestCase
+from rdagent.components.benchmark.eval_method import TestCase, TestCases
 from rdagent.components.coder.factor_coder.factor import (
     FactorExperiment,
     FactorFBWorkspace,
@@ -42,12 +42,12 @@ class FactorExperimentLoaderFromJsonString(FactorExperimentLoader):
 
 
 # TODO loader only supports generic of task or experiment, testcase might cause CI error here
-# class FactorTestCaseLoaderFromJsonFile(Loader[TestCase]):
+# class FactorTestCaseLoaderFromJsonFile(Loader[TestCases]):
 class FactorTestCaseLoaderFromJsonFile:
-    def load(self, json_file_path: Path) -> list:
+    def load(self, json_file_path: Path) -> TestCases:
         with open(json_file_path, "r") as file:
             factor_dict = json.load(file)
-        TestData = TestCase(target_task=Experiment(sub_tasks=[]))
+        test_cases = TestCases()
         for factor_name, factor_data in factor_dict.items():
             task = FactorTask(
                 factor_name=factor_name,
@@ -58,7 +58,6 @@ class FactorTestCaseLoaderFromJsonFile:
             gt = FactorFBWorkspace(task, raise_exception=True)
             code = {"factor.py": factor_data["gt_code"]}
             gt.inject_code(**code)
-            TestData.target_task.sub_tasks.append(task)
-            TestData.ground_truth.append(gt)
+            test_cases.test_case_l.append(TestCase(task, gt))
 
-        return TestData
+        return test_cases