Eval process (#31)

* test data load process and fix bug

* fix bug when evaluating

* refine json content

---------

Co-authored-by: USTCKevinF <fengwenjun@mail.ustc.edu.cn>
Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/factor_implementation/evolving/factor.py

@@ -173,7 +173,7 @@ class FileBasedFactorImplementation(TaskImplementation):
                 FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder,
             )
             self.workspace_path.mkdir(exist_ok=True, parents=True)
-
+            source_data_path.mkdir(exist_ok=True, parents=True)
             code_path = self.workspace_path / f"{self.target_task.factor_name}.py"
             code_path.write_text(self.code)
 

rdagent/factor_implementation/share_modules/factor_implementation_config.py

@@ -9,13 +9,13 @@ SELECT_METHOD = Literal["random", "scheduler"]
 
 class FactorImplementSettings(BaseSettings):
     file_based_execution_data_folder: str = str(
-        (Path().cwd() / "factor_implementation_source_data").absolute(),
+        (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
     )
     file_based_execution_workspace: str = str(
-        (Path().cwd() / "factor_implementation_workspace").absolute(),
+        (Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(),
     )
     implementation_execution_cache_location: str = str(
-        (Path().cwd() / "factor_implementation_execution_cache").absolute(),
+        (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(),
     )
     enable_execution_cache: bool = True  # whether to enable the execution cache
 

rdagent/factor_implementation/task_loader/json_loader.py

@@ -1,7 +1,8 @@
 import json
 from pathlib import Path
 from rdagent.core.task import TaskLoader
-from rdagent.factor_implementation.evolving.factor import FactorImplementTask
+from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FileBasedFactorImplementation
+from rdagent.core.task import TestCase
 
 
 class FactorImplementationTaskLoaderFromDict(TaskLoader):
@@ -21,7 +22,8 @@ class FactorImplementationTaskLoaderFromDict(TaskLoader):
 
 class FactorImplementationTaskLoaderFromJsonFile(TaskLoader):
     def load(self, json_file_path: Path) -> list:
-        factor_dict = json.load(json_file_path)
+        with open(json_file_path, 'r') as file:
+            factor_dict = json.load(file)
         return FactorImplementationTaskLoaderFromDict().load(factor_dict)
 
 
@@ -29,3 +31,22 @@ class FactorImplementationTaskLoaderFromJsonString(TaskLoader):
     def load(self, json_string: str) -> list:
         factor_dict = json.loads(json_string)
         return FactorImplementationTaskLoaderFromDict().load(factor_dict)
+
+class FactorTestCaseLoaderFromJsonFile(TaskLoader):
+    def load(self, json_file_path: Path) -> list:
+        with open(json_file_path, 'r') as file:
+            factor_dict = json.load(file)
+        TestData = TestCase()
+        for factor_name, factor_data in factor_dict.items():
+            task = FactorImplementTask(
+                factor_name=factor_name,
+                factor_description=factor_data["description"],
+                factor_formulation=factor_data["formulation"],
+                variables=factor_data["variables"],
+            )
+            gt = FileBasedFactorImplementation(task, code=factor_data["gt_code"])
+            gt.execute()
+            TestData.target_task.append(task)
+            TestData.ground_truth.append(gt)
+
+        return TestData