From 07a77dd802dfe2d87496c3f379d32f3872e00566 Mon Sep 17 00:00:00 2001 From: Haoxue Date: Thu, 27 Jun 2024 09:39:17 +0100 Subject: [PATCH] Eval process (#31) * test data load process and fix bug * fix bug when evaluating * refine json content --------- Co-authored-by: USTCKevinF Co-authored-by: xuyang1 --- .../factor_implementation/eval_implement.py | 16 +++---- rdagent/benchmark/conf.py | 10 ++--- rdagent/benchmark/eval_method.py | 43 ++++++++----------- rdagent/core/task.py | 4 +- .../factor_implementation/evolving/factor.py | 2 +- .../factor_implementation_config.py | 6 +-- .../task_loader/json_loader.py | 25 ++++++++++- 7 files changed, 57 insertions(+), 49 deletions(-) diff --git a/rdagent/app/factor_implementation/eval_implement.py b/rdagent/app/factor_implementation/eval_implement.py index 06b653c5..0ae577b8 100644 --- a/rdagent/app/factor_implementation/eval_implement.py +++ b/rdagent/app/factor_implementation/eval_implement.py @@ -1,18 +1,18 @@ -from rdagent.core.conf import BenchmarkSettings +from rdagent.benchmark.conf import BenchmarkSettings from rdagent.core.utils import import_class from rdagent.benchmark.eval_method import FactorImplementEval -from rdagent.benchmark.data_process import load_eval_data +from rdagent.factor_implementation.task_loader.json_loader import FactorTestCaseLoaderFromJsonFile # 1.read the settings bs = BenchmarkSettings() # 2.read and prepare the eval_data -test_cases = load_eval_data(bs.bench_version) +test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path) # 3.declare the method to be tested and pass the arguments. -# TODO: Whether it is necessary to define two Eval method classes for two data type? + method_cls = import_class(bs.bench_method_cls) -generate_method = method_cls(bs.bench_method_extra_kwargs) +generate_method = method_cls() # 4.declare the eval method and pass the arguments. eval_method = FactorImplementEval( @@ -23,8 +23,4 @@ eval_method = FactorImplementEval( ) # 5.run the eval -eval_method.eval() - -# 6.save the result -eval_method.save(output_path = bs.bench_result_path) - +res = eval_method.eval() diff --git a/rdagent/benchmark/conf.py b/rdagent/benchmark/conf.py index 10241b90..bcbbc030 100644 --- a/rdagent/benchmark/conf.py +++ b/rdagent/benchmark/conf.py @@ -2,24 +2,22 @@ from dotenv import load_dotenv load_dotenv(verbose=True, override=True) from dataclasses import field from pathlib import Path -from typing import Literal, Optional, Union +from typing import Optional from pydantic_settings import BaseSettings DIRNAME = Path(__file__).absolute().resolve().parent -BENCHMARK_VERSION = Literal["paper", "amcV01", "amcV02train", "amcV02test"] - class BenchmarkSettings(BaseSettings): ground_truth_dir: Path = DIRNAME / "ground_truth" - bench_version: Union[BENCHMARK_VERSION, str] = "paper" + bench_data_path: Path = DIRNAME / "example.json" - bench_test_round: int = 20 + bench_test_round: int = 10 bench_test_case_n: Optional[int] = None # how many test cases to run; If not given, all test cases will be run - bench_method_cls: str = "scripts.factor_implementation.baselines.naive.one_shot.OneshotFactorGen" + bench_method_cls: str = "rdagent.factor_implementation.CoSTEER.CoSTEERFG" bench_method_extra_kwargs: dict = field( default_factory=dict, ) # extra kwargs for the method to be tested except the task list diff --git a/rdagent/benchmark/eval_method.py b/rdagent/benchmark/eval_method.py index 2735e70b..262d248b 100644 --- a/rdagent/benchmark/eval_method.py +++ b/rdagent/benchmark/eval_method.py @@ -3,7 +3,7 @@ from typing import List, Tuple, Union from tqdm import tqdm from collections import defaultdict -from rdagent.core.conf import RD_AGENT_SETTINGS +from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS from rdagent.core.exception import ImplementRunException from rdagent.core.task import ( TaskImplementation, @@ -102,29 +102,25 @@ class BaseEval: class FactorImplementEval(BaseEval): def __init__( self, - test_case: TestCase, + test_cases: TestCase, method: TaskGenerator, test_round: int = 10, *args, **kwargs, ): - # evaluator collection for online evaluation - online_evaluator_l = ( - [ - FactorImplementationCorrelationEvaluator, - FactorImplementationIndexEvaluator, - FactorImplementationIndexFormatEvaluator, - FactorImplementationMissingValuesEvaluator, - FactorImplementationRowCountEvaluator, - FactorImplementationSingleColumnEvaluator, - FactorImplementationValuesEvaluator, - ], - ) - super().__init__(online_evaluator_l, test_case, method, *args, **kwargs) + online_evaluator_l = [ + FactorImplementationSingleColumnEvaluator(), + FactorImplementationIndexFormatEvaluator(), + FactorImplementationRowCountEvaluator(), + FactorImplementationIndexEvaluator(), + FactorImplementationMissingValuesEvaluator(), + FactorImplementationValuesEvaluator(), + FactorImplementationCorrelationEvaluator(hard_check=False), + ] + super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs) self.test_round = test_round def eval(self): - gen_factor_l_all_rounds = [] test_cases_all_rounds = [] res = defaultdict(list) @@ -139,25 +135,22 @@ class FactorImplementEval(BaseEval): print("Manually interrupted the evaluation. Saving existing results") break - if len(gen_factor_l) != len(self.test_cases): + if len(gen_factor_l.corresponding_implementations) != len(self.test_cases.ground_truth): raise ValueError( "The number of cases to eval should be equal to the number of test cases.", ) - gen_factor_l_all_rounds.extend(gen_factor_l) - test_cases_all_rounds.extend(self.test_cases) - - eval_res_l = [] + gen_factor_l_all_rounds.extend(gen_factor_l.corresponding_implementations) + test_cases_all_rounds.extend(self.test_cases.ground_truth) eval_res_list = multiprocessing_wrapper( [ - (self.eval_case, (gt_case.ground_truth, gen_factor)) + (self.eval_case, (gt_case, gen_factor)) for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds) ], - n=RD_AGENT_SETTINGS.evo_multi_proc_n, + n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n, ) for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)): - res[gt_case.task.factor_name].append((gen_factor, eval_res)) - eval_res_l.append(eval_res) + res[gt_case.target_task.factor_name].append((gen_factor, eval_res)) return res diff --git a/rdagent/core/task.py b/rdagent/core/task.py index 8e544948..bb2d758d 100644 --- a/rdagent/core/task.py +++ b/rdagent/core/task.py @@ -118,8 +118,8 @@ class TestCase: def __init__( self, - target_task: BaseTask, - ground_truth: TaskImplementation, + target_task: list[BaseTask] = [], + ground_truth: list[TaskImplementation] = [], ): self.ground_truth = ground_truth self.target_task = target_task diff --git a/rdagent/factor_implementation/evolving/factor.py b/rdagent/factor_implementation/evolving/factor.py index 6fd0cf7d..f2cee55d 100644 --- a/rdagent/factor_implementation/evolving/factor.py +++ b/rdagent/factor_implementation/evolving/factor.py @@ -173,7 +173,7 @@ class FileBasedFactorImplementation(TaskImplementation): FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder, ) self.workspace_path.mkdir(exist_ok=True, parents=True) - + source_data_path.mkdir(exist_ok=True, parents=True) code_path = self.workspace_path / f"{self.target_task.factor_name}.py" code_path.write_text(self.code) diff --git a/rdagent/factor_implementation/share_modules/factor_implementation_config.py b/rdagent/factor_implementation/share_modules/factor_implementation_config.py index 515e1193..16607143 100644 --- a/rdagent/factor_implementation/share_modules/factor_implementation_config.py +++ b/rdagent/factor_implementation/share_modules/factor_implementation_config.py @@ -9,13 +9,13 @@ SELECT_METHOD = Literal["random", "scheduler"] class FactorImplementSettings(BaseSettings): file_based_execution_data_folder: str = str( - (Path().cwd() / "factor_implementation_source_data").absolute(), + (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(), ) file_based_execution_workspace: str = str( - (Path().cwd() / "factor_implementation_workspace").absolute(), + (Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(), ) implementation_execution_cache_location: str = str( - (Path().cwd() / "factor_implementation_execution_cache").absolute(), + (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(), ) enable_execution_cache: bool = True # whether to enable the execution cache diff --git a/rdagent/factor_implementation/task_loader/json_loader.py b/rdagent/factor_implementation/task_loader/json_loader.py index 722b2ecc..d626d98a 100644 --- a/rdagent/factor_implementation/task_loader/json_loader.py +++ b/rdagent/factor_implementation/task_loader/json_loader.py @@ -1,7 +1,8 @@ import json from pathlib import Path from rdagent.core.task import TaskLoader -from rdagent.factor_implementation.evolving.factor import FactorImplementTask +from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FileBasedFactorImplementation +from rdagent.core.task import TestCase class FactorImplementationTaskLoaderFromDict(TaskLoader): @@ -21,7 +22,8 @@ class FactorImplementationTaskLoaderFromDict(TaskLoader): class FactorImplementationTaskLoaderFromJsonFile(TaskLoader): def load(self, json_file_path: Path) -> list: - factor_dict = json.load(json_file_path) + with open(json_file_path, 'r') as file: + factor_dict = json.load(file) return FactorImplementationTaskLoaderFromDict().load(factor_dict) @@ -29,3 +31,22 @@ class FactorImplementationTaskLoaderFromJsonString(TaskLoader): def load(self, json_string: str) -> list: factor_dict = json.loads(json_string) return FactorImplementationTaskLoaderFromDict().load(factor_dict) + +class FactorTestCaseLoaderFromJsonFile(TaskLoader): + def load(self, json_file_path: Path) -> list: + with open(json_file_path, 'r') as file: + factor_dict = json.load(file) + TestData = TestCase() + for factor_name, factor_data in factor_dict.items(): + task = FactorImplementTask( + factor_name=factor_name, + factor_description=factor_data["description"], + factor_formulation=factor_data["formulation"], + variables=factor_data["variables"], + ) + gt = FileBasedFactorImplementation(task, code=factor_data["gt_code"]) + gt.execute() + TestData.target_task.append(task) + TestData.ground_truth.append(gt) + + return TestData \ No newline at end of file