Eval process (#31)

* test data load process and fix bug

* fix bug when evaluating

* refine json content

---------

Co-authored-by: USTCKevinF <fengwenjun@mail.ustc.edu.cn>
Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/app/factor_implementation/eval_implement.py

@@ -1,18 +1,18 @@
-from rdagent.core.conf import BenchmarkSettings
+from rdagent.benchmark.conf import BenchmarkSettings
 from rdagent.core.utils import import_class
 from rdagent.benchmark.eval_method import FactorImplementEval
-from rdagent.benchmark.data_process import load_eval_data
+from rdagent.factor_implementation.task_loader.json_loader import FactorTestCaseLoaderFromJsonFile
 
 # 1.read the settings
 bs = BenchmarkSettings()
 
 # 2.read and prepare the eval_data
-test_cases = load_eval_data(bs.bench_version)
+test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
 
 # 3.declare the method to be tested and pass the arguments.
-# TODO: Whether it is necessary to define two Eval method classes for two data type?
+
 method_cls = import_class(bs.bench_method_cls)
-generate_method = method_cls(bs.bench_method_extra_kwargs)
+generate_method = method_cls()
 
 # 4.declare the eval method and pass the arguments.
 eval_method = FactorImplementEval(
@@ -23,8 +23,4 @@ eval_method = FactorImplementEval(
 )
 
 # 5.run the eval
-eval_method.eval()
-
-# 6.save the result
-eval_method.save(output_path = bs.bench_result_path)
-
+res = eval_method.eval()

rdagent/benchmark/conf.py

@@ -2,24 +2,22 @@ from dotenv import load_dotenv
 load_dotenv(verbose=True, override=True)
 from dataclasses import field
 from pathlib import Path
-from typing import Literal, Optional, Union
+from typing import Optional 
 
 from pydantic_settings import BaseSettings
 
 DIRNAME = Path(__file__).absolute().resolve().parent
 
-BENCHMARK_VERSION = Literal["paper", "amcV01", "amcV02train", "amcV02test"]
-
 class BenchmarkSettings(BaseSettings):
 
     ground_truth_dir: Path = DIRNAME / "ground_truth"
 
-    bench_version: Union[BENCHMARK_VERSION, str] = "paper"
+    bench_data_path: Path = DIRNAME / "example.json"
 
-    bench_test_round: int = 20 
+    bench_test_round: int = 10
     bench_test_case_n: Optional[int] = None  # how many test cases to run; If not given, all test cases will be run
 
-    bench_method_cls: str = "scripts.factor_implementation.baselines.naive.one_shot.OneshotFactorGen"
+    bench_method_cls: str = "rdagent.factor_implementation.CoSTEER.CoSTEERFG"
     bench_method_extra_kwargs: dict = field(
         default_factory=dict,
     )  # extra kwargs for the method to be tested except the task list

rdagent/benchmark/eval_method.py

@@ -3,7 +3,7 @@ from typing import List, Tuple, Union
 
 from tqdm import tqdm
 from collections import defaultdict
-from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.core.exception import ImplementRunException
 from rdagent.core.task import (
     TaskImplementation,
@@ -102,29 +102,25 @@ class BaseEval:
 class FactorImplementEval(BaseEval):
     def __init__(
         self,
-        test_case: TestCase,
+        test_cases: TestCase,
         method: TaskGenerator,
         test_round: int = 10,
         *args,
         **kwargs,
     ):
-        # evaluator collection for online evaluation
-        online_evaluator_l = (
-            [
-                FactorImplementationCorrelationEvaluator,
-                FactorImplementationIndexEvaluator,
-                FactorImplementationIndexFormatEvaluator,
-                FactorImplementationMissingValuesEvaluator,
-                FactorImplementationRowCountEvaluator,
-                FactorImplementationSingleColumnEvaluator,
-                FactorImplementationValuesEvaluator,
-            ],
-        )
-        super().__init__(online_evaluator_l, test_case, method, *args, **kwargs)
+        online_evaluator_l = [
+                FactorImplementationSingleColumnEvaluator(),
+                FactorImplementationIndexFormatEvaluator(),
+                FactorImplementationRowCountEvaluator(),
+                FactorImplementationIndexEvaluator(),
+                FactorImplementationMissingValuesEvaluator(),
+                FactorImplementationValuesEvaluator(),
+                FactorImplementationCorrelationEvaluator(hard_check=False),
+            ]
+        super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)
         self.test_round = test_round
 
     def eval(self):
-
         gen_factor_l_all_rounds = []
         test_cases_all_rounds = []
         res = defaultdict(list)
@@ -139,25 +135,22 @@ class FactorImplementEval(BaseEval):
                 print("Manually interrupted the evaluation. Saving existing results")
                 break
 
-            if len(gen_factor_l) != len(self.test_cases):
+            if len(gen_factor_l.corresponding_implementations) != len(self.test_cases.ground_truth):
                 raise ValueError(
                     "The number of cases to eval should be equal to the number of test cases.",
                 )
-            gen_factor_l_all_rounds.extend(gen_factor_l)
-            test_cases_all_rounds.extend(self.test_cases)
-
-        eval_res_l = []
+            gen_factor_l_all_rounds.extend(gen_factor_l.corresponding_implementations)
+            test_cases_all_rounds.extend(self.test_cases.ground_truth)
 
         eval_res_list = multiprocessing_wrapper(
             [
-                (self.eval_case, (gt_case.ground_truth, gen_factor))
+                (self.eval_case, (gt_case, gen_factor))
                 for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds)
             ],
-            n=RD_AGENT_SETTINGS.evo_multi_proc_n,
+            n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
         )
 
         for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)):
-            res[gt_case.task.factor_name].append((gen_factor, eval_res))
-            eval_res_l.append(eval_res)
+            res[gt_case.target_task.factor_name].append((gen_factor, eval_res))
 
         return res

rdagent/core/task.py

@@ -118,8 +118,8 @@ class TestCase:
 
     def __init__(
         self,
-        target_task: BaseTask,
-        ground_truth: TaskImplementation,
+        target_task: list[BaseTask] = [],
+        ground_truth: list[TaskImplementation] = [],
     ):
         self.ground_truth = ground_truth
         self.target_task = target_task

rdagent/factor_implementation/evolving/factor.py

@@ -173,7 +173,7 @@ class FileBasedFactorImplementation(TaskImplementation):
                 FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder,
             )
             self.workspace_path.mkdir(exist_ok=True, parents=True)
-
+            source_data_path.mkdir(exist_ok=True, parents=True)
             code_path = self.workspace_path / f"{self.target_task.factor_name}.py"
             code_path.write_text(self.code)
 

rdagent/factor_implementation/share_modules/factor_implementation_config.py

@@ -9,13 +9,13 @@ SELECT_METHOD = Literal["random", "scheduler"]
 
 class FactorImplementSettings(BaseSettings):
     file_based_execution_data_folder: str = str(
-        (Path().cwd() / "factor_implementation_source_data").absolute(),
+        (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
     )
     file_based_execution_workspace: str = str(
-        (Path().cwd() / "factor_implementation_workspace").absolute(),
+        (Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(),
     )
     implementation_execution_cache_location: str = str(
-        (Path().cwd() / "factor_implementation_execution_cache").absolute(),
+        (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(),
     )
     enable_execution_cache: bool = True  # whether to enable the execution cache
 

rdagent/factor_implementation/task_loader/json_loader.py

@@ -1,7 +1,8 @@
 import json
 from pathlib import Path
 from rdagent.core.task import TaskLoader
-from rdagent.factor_implementation.evolving.factor import FactorImplementTask
+from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FileBasedFactorImplementation
+from rdagent.core.task import TestCase
 
 
 class FactorImplementationTaskLoaderFromDict(TaskLoader):
@@ -21,7 +22,8 @@ class FactorImplementationTaskLoaderFromDict(TaskLoader):
 
 class FactorImplementationTaskLoaderFromJsonFile(TaskLoader):
     def load(self, json_file_path: Path) -> list:
-        factor_dict = json.load(json_file_path)
+        with open(json_file_path, 'r') as file:
+            factor_dict = json.load(file)
         return FactorImplementationTaskLoaderFromDict().load(factor_dict)
 
 
@@ -29,3 +31,22 @@ class FactorImplementationTaskLoaderFromJsonString(TaskLoader):
     def load(self, json_string: str) -> list:
         factor_dict = json.loads(json_string)
         return FactorImplementationTaskLoaderFromDict().load(factor_dict)
+
+class FactorTestCaseLoaderFromJsonFile(TaskLoader):
+    def load(self, json_file_path: Path) -> list:
+        with open(json_file_path, 'r') as file:
+            factor_dict = json.load(file)
+        TestData = TestCase()
+        for factor_name, factor_data in factor_dict.items():
+            task = FactorImplementTask(
+                factor_name=factor_name,
+                factor_description=factor_data["description"],
+                factor_formulation=factor_data["formulation"],
+                variables=factor_data["variables"],
+            )
+            gt = FileBasedFactorImplementation(task, code=factor_data["gt_code"])
+            gt.execute()
+            TestData.target_task.append(task)
+            TestData.ground_truth.append(gt)
+
+        return TestData