From 444ded9a5dd488304f4d21eb39693ead0ff80ae6 Mon Sep 17 00:00:00 2001 From: you-n-g Date: Mon, 23 Sep 2024 19:53:05 +0800 Subject: [PATCH] fix: refactor Bench (#302) * refactor for better bench * autolint * add cmd * lint --- rdagent/app/benchmark/factor/analysis.py | 11 +++-- rdagent/components/benchmark/eval_method.py | 40 +++++++++++++++---- .../components/coder/factor_coder/factor.py | 6 ++- rdagent/components/coder/model_coder/model.py | 3 +- rdagent/core/experiment.py | 3 +- .../factor_experiment_loader/json_loader.py | 13 +++--- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/rdagent/app/benchmark/factor/analysis.py b/rdagent/app/benchmark/factor/analysis.py index 6167ab67..385f3a42 100644 --- a/rdagent/app/benchmark/factor/analysis.py +++ b/rdagent/app/benchmark/factor/analysis.py @@ -2,6 +2,7 @@ import json import pickle from pathlib import Path +import fire import matplotlib.pyplot as plt import pandas as pd import seaborn as sns @@ -168,11 +169,11 @@ class Plotter: plt.savefig(file_name) -if __name__ == "__main__": +def main(path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl"): settings = BenchmarkSettings() benchmark = BenchmarkAnalyzer(settings) results = { - "1 round experiment": "git_ignore_folder/eval_results/res_promptV220240724-060037.pkl", + "1 round experiment": path, } final_results = benchmark.process_results(results) final_results_df = pd.DataFrame(final_results) @@ -180,4 +181,8 @@ if __name__ == "__main__": Plotter.change_fs(20) plot_data = final_results_df.drop(["max. accuracy", "avg. accuracy"], axis=0).T plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b") - Plotter.plot_data(plot_data, "rdagent/app/quant_factor_benchmark/comparison_plot.png") + Plotter.plot_data(plot_data, "./comparison_plot.png") + + +if __name__ == "__main__": + fire.Fire(main) diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index 3bf998b6..3654b650 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -20,7 +20,7 @@ from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.developer import Developer from rdagent.core.exception import CoderError -from rdagent.core.experiment import Task, Workspace +from rdagent.core.experiment import Experiment, Task, Workspace from rdagent.core.scenario import Scenario from rdagent.core.utils import multiprocessing_wrapper @@ -33,11 +33,34 @@ EVAL_RES = Dict[ class TestCase: def __init__( self, - target_task: list[Task] = [], - ground_truth: list[Workspace] = [], + target_task: Task, + ground_truth: Workspace, ): - self.ground_truth = ground_truth self.target_task = target_task + self.ground_truth = ground_truth + + +class TestCases: + def __init__(self, test_case_l: list[TestCase] = []): + # self.test_case_l = [TestCase(task, gt) for task, gt in zip(target_task, ground_truth)] + self.test_case_l = test_case_l + + def __getitem__(self, item): + return self.test_case_l[item] + + def __len__(self): + return len(self.test_case_l) + + def get_exp(self): + return Experiment([case.target_task for case in self.test_case_l]) + + @property + def target_task(self): + return [case.target_task for case in self.test_case_l] + + @property + def ground_truth(self): + return [case.ground_truth for case in self.test_case_l] class BaseEval: @@ -48,13 +71,13 @@ class BaseEval: def __init__( self, evaluator_l: List[FactorEvaluator], - test_cases: List[TestCase], + test_cases: TestCases, generate_method: Developer, catch_eval_except: bool = True, ): """Parameters ---------- - test_cases : List[TestCase] + test_cases : TestCases cases to be evaluated, ground truth are included in the test cases. evaluator_l : List[FactorEvaluator] A list of evaluators to evaluate the generated code. @@ -105,6 +128,7 @@ class BaseEval: eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt))) # if the corr ev is successfully evaluated and achieve the best performance, then break except (CoderError, AttributeError) as e: + # TODO: remove AttributeError. return e except Exception as e: # exception when evaluation @@ -118,7 +142,7 @@ class BaseEval: class FactorImplementEval(BaseEval): def __init__( self, - test_cases: TestCase, + test_cases: TestCases, method: Developer, *args, scen: Scenario, @@ -146,7 +170,7 @@ class FactorImplementEval(BaseEval): print(f"Eval {_}-th times...") print("========================================================\n") try: - gen_factor_l = self.generate_method.develop(self.test_cases.target_task) + gen_factor_l = self.generate_method.develop(self.test_cases.get_exp()) except KeyboardInterrupt: # TODO: Why still need to save result after KeyboardInterrupt? print("Manually interrupted the evaluation. Saving existing results") diff --git a/rdagent/components/coder/factor_coder/factor.py b/rdagent/components/coder/factor_coder/factor.py index 1d3b8cef..10c354e9 100644 --- a/rdagent/components/coder/factor_coder/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -31,13 +31,15 @@ class FactorTask(Task): factor_implementation: bool = False, **kwargs, ) -> None: - self.factor_name = factor_name + self.factor_name = ( + factor_name # TODO: remove it in the later version. Keep it only for pickle version compatibility + ) self.factor_description = factor_description self.factor_formulation = factor_formulation self.variables = variables self.factor_resources = resource self.factor_implementation = factor_implementation - super().__init__(*args, **kwargs) + super().__init__(name=factor_name, *args, **kwargs) def get_task_information(self): return f"""factor_name: {self.factor_name} diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index 86f0531c..7e66ba95 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -23,14 +23,13 @@ class ModelTask(Task): model_type: Optional[str] = None, **kwargs, ) -> None: - self.name: str = name self.description: str = description self.formulation: str = formulation self.architecture: str = architecture self.variables: str = variables self.hyperparameters: str = hyperparameters self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model - super().__init__(*args, **kwargs) + super().__init__(name=name, *args, **kwargs) def get_task_information(self): task_desc = f"""name: {self.name} diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index 6cac715f..7e86198c 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -17,13 +17,14 @@ This file contains the all the class about organizing the task in RD-Agent. class Task(ABC): - def __init__(self, version: int = 1) -> None: + def __init__(self, name: str, version: int = 1) -> None: """ The version of the task, default is 1 Because qlib tasks execution and kaggle tasks execution are different, we need to distinguish them. TODO: We may align them in the future. """ self.version = version + self.name = name @abstractmethod def get_task_information(self) -> str: diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py index 5e04a0d4..ecbeeb7c 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py @@ -1,7 +1,7 @@ import json from pathlib import Path -from rdagent.components.benchmark.eval_method import TestCase +from rdagent.components.benchmark.eval_method import TestCase, TestCases from rdagent.components.coder.factor_coder.factor import ( FactorExperiment, FactorFBWorkspace, @@ -42,12 +42,12 @@ class FactorExperimentLoaderFromJsonString(FactorExperimentLoader): # TODO loader only supports generic of task or experiment, testcase might cause CI error here -# class FactorTestCaseLoaderFromJsonFile(Loader[TestCase]): +# class FactorTestCaseLoaderFromJsonFile(Loader[TestCases]): class FactorTestCaseLoaderFromJsonFile: - def load(self, json_file_path: Path) -> list: + def load(self, json_file_path: Path) -> TestCases: with open(json_file_path, "r") as file: factor_dict = json.load(file) - TestData = TestCase(target_task=Experiment(sub_tasks=[])) + test_cases = TestCases() for factor_name, factor_data in factor_dict.items(): task = FactorTask( factor_name=factor_name, @@ -58,7 +58,6 @@ class FactorTestCaseLoaderFromJsonFile: gt = FactorFBWorkspace(task, raise_exception=True) code = {"factor.py": factor_data["gt_code"]} gt.inject_code(**code) - TestData.target_task.sub_tasks.append(task) - TestData.ground_truth.append(gt) + test_cases.test_case_l.append(TestCase(task, gt)) - return TestData + return test_cases