gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-03 02:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Eval process (#31)

Browse Source 
* test data load process and fix bug

* fix bug when evaluating

* refine json content

---------

Co-authored-by: USTCKevinF <fengwenjun@mail.ustc.edu.cn>
Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
Haoxue
2024-06-27 09:39:17 +01:00
committed by GitHub
parent f20dc4482b
commit 28e282d6c4
8 changed files with 87 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/factor_implementation/eval_implement.py
+6 -10
View File
@@ -1,18 +1,18 @@
from rdagent.core.conf import BenchmarkSettings
from rdagent.benchmark.conf import BenchmarkSettings
from rdagent.core.utils import import_class
from rdagent.benchmark.eval_method import FactorImplementEval
from rdagent.benchmark.data_process import load_eval_data
from rdagent.factor_implementation.task_loader.json_loader import FactorTestCaseLoaderFromJsonFile
# 1.read the settings
bs = BenchmarkSettings()
# 2.read and prepare the eval_data
test_cases = load_eval_data(bs.bench_version)
test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
# 3.declare the method to be tested and pass the arguments.
# TODO: Whether it is necessary to define two Eval method classes for two data type?
method_cls = import_class(bs.bench_method_cls)
generate_method = method_cls(bs.bench_method_extra_kwargs)
generate_method = method_cls()
# 4.declare the eval method and pass the arguments.
eval_method = FactorImplementEval(
@@ -23,8 +23,4 @@ eval_method = FactorImplementEval(
)
# 5.run the eval
eval_method.eval()
# 6.save the result
eval_method.save(output_path = bs.bench_result_path)
res = eval_method.eval()
rdagent/benchmark/conf.py
+4 -6
View File
@@ -2,24 +2,22 @@ from dotenv import load_dotenv
load_dotenv(verbose=True, override=True)
from dataclasses import field
from pathlib import Path
from typing import Literal, Optional, Union
from typing import Optional
from pydantic_settings import BaseSettings
DIRNAME = Path(__file__).absolute().resolve().parent
BENCHMARK_VERSION = Literal["paper", "amcV01", "amcV02train", "amcV02test"]
class BenchmarkSettings(BaseSettings):
ground_truth_dir: Path = DIRNAME / "ground_truth"
bench_version: Union[BENCHMARK_VERSION, str] = "paper"
bench_data_path: Path = DIRNAME / "example.json"
bench_test_round: int = 20
bench_test_round: int = 10
bench_test_case_n: Optional[int] = None # how many test cases to run; If not given, all test cases will be run
bench_method_cls: str = "scripts.factor_implementation.baselines.naive.one_shot.OneshotFactorGen"
bench_method_cls: str = "rdagent.factor_implementation.CoSTEER.CoSTEERFG"
bench_method_extra_kwargs: dict = field(
default_factory=dict,
) # extra kwargs for the method to be tested except the task list
rdagent/benchmark/eval_method.py
+18 -25
View File
@@ -3,7 +3,7 @@ from typing import List, Tuple, Union
from tqdm import tqdm
from collections import defaultdict
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.core.exception import ImplementRunException
from rdagent.core.task import (
TaskImplementation,
@@ -102,29 +102,25 @@ class BaseEval:
class FactorImplementEval(BaseEval):
def __init__(
self,
test_case: TestCase,
test_cases: TestCase,
method: TaskGenerator,
test_round: int = 10,
*args,
**kwargs,
):
# evaluator collection for online evaluation
online_evaluator_l = (
[
FactorImplementationCorrelationEvaluator,
FactorImplementationIndexEvaluator,
FactorImplementationIndexFormatEvaluator,
FactorImplementationMissingValuesEvaluator,
FactorImplementationRowCountEvaluator,
FactorImplementationSingleColumnEvaluator,
FactorImplementationValuesEvaluator,
],
)
super().__init__(online_evaluator_l, test_case, method, *args, **kwargs)
online_evaluator_l = [
FactorImplementationSingleColumnEvaluator(),
FactorImplementationIndexFormatEvaluator(),
FactorImplementationRowCountEvaluator(),
FactorImplementationIndexEvaluator(),
FactorImplementationMissingValuesEvaluator(),
FactorImplementationValuesEvaluator(),
FactorImplementationCorrelationEvaluator(hard_check=False),
]
super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)
self.test_round = test_round
def eval(self):
gen_factor_l_all_rounds = []
test_cases_all_rounds = []
res = defaultdict(list)
@@ -139,25 +135,22 @@ class FactorImplementEval(BaseEval):
print("Manually interrupted the evaluation. Saving existing results")
break
if len(gen_factor_l) != len(self.test_cases):
if len(gen_factor_l.corresponding_implementations) != len(self.test_cases.ground_truth):
raise ValueError(
"The number of cases to eval should be equal to the number of test cases.",
)
gen_factor_l_all_rounds.extend(gen_factor_l)
test_cases_all_rounds.extend(self.test_cases)
eval_res_l = []
gen_factor_l_all_rounds.extend(gen_factor_l.corresponding_implementations)
test_cases_all_rounds.extend(self.test_cases.ground_truth)
eval_res_list = multiprocessing_wrapper(
[
(self.eval_case, (gt_case.ground_truth, gen_factor))
(self.eval_case, (gt_case, gen_factor))
for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds)
],
n=RD_AGENT_SETTINGS.evo_multi_proc_n,
n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
)
for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)):
res[gt_case.task.factor_name].append((gen_factor, eval_res))
eval_res_l.append(eval_res)
res[gt_case.target_task.factor_name].append((gen_factor, eval_res))
return res
rdagent/benchmark/example.json
+30
View File
@@ -0,0 +1,30 @@
{
"Turnover_Rate_Factor": {
"description": "A traditional factor based on 20-day average turnover rate, adjusted for market capitalization, which is further improved by applying the information distribution theory.",
"formulation": "\\text{Adjusted Turnover Rate} = \\frac{\\text{mean}(20\\text{-day turnover rate})}{\\text{Market Capitalization}}",
"variables": {
"20-day turnover rate": "Average turnover rate over the past 20 days.",
"Market Capitalization": "Total market value of a company's outstanding shares."
},
"gt_code": "import pandas as pd\n\ndata_f = pd.read_hdf('daily_f.h5')\n\ndata = data_f.reset_index()\nwindow_size = 20\n\nnominator=data.groupby('instrument')[['30\u65e5\u6362\u624b\u7387']].rolling(window=window_size).mean().reset_index(0, drop=True)\n# transfer to series\nnew=nominator['30\u65e5\u6362\u624b\u7387']\ndata['Turnover_Rate_Factor']=new/data['\u6d41\u901aA\u80a1']\n\n# # set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['Turnover_Rate_Factor']).set_index(data_f.index)\n\n# transfer the result to series\nresult=result['Turnover_Rate_Factor']\nresult.to_hdf(\"result.h5\", key=\"data\")\n"
},
"PctTurn20": {
"description": "A factor representing the percentage change in turnover rate over the past 20 trading days, market-value neutralized.",
"formulation": "\\text{PctTurn20} = \\frac{1}{N} \\sum_{i=1}^{N} \\left( \\frac{\\text{Turnover}_{i, t} - \\text{Turnover}_{i, t-20}}{\\text{Turnover}_{i, t-20}} \\right)",
"variables": {
"N": "Number of stocks in the market.",
"Turnover_{i, t}": "Turnover of stock i at day t.",
"Turnover_{i, t-20}": "Turnover of stock i at day t-20."
},
"gt_code": "import pandas as pd\nfrom statsmodels import api as sm\n\n\ndef fill_mean(s: pd.Series) -> pd.Series:\n return s.fillna(s.mean()).fillna(0.0)\n\n\ndef market_value_neutralize(s: pd.Series, mv: pd.Series) -> pd.Series:\n s = s.groupby(\"datetime\", group_keys=False).apply(fill_mean)\n mv = mv.groupby(\"datetime\", group_keys=False).apply(fill_mean)\n\n df_f = mv.to_frame(\"\u5e02\u503c\")\n df_f[\"const\"] = 1\n X = df_f[[\"\u5e02\u503c\", \"const\"]]\n\n # Perform the Ordinary Least Squares (OLS) regression\n model = sm.OLS(s, X)\n results = model.fit()\n\n # Calculate the residuals\n df_f[\"residual\"] = results.resid\n df_f[\"norm_resi\"] = df_f.groupby(level=\"datetime\", group_keys=False)[\"residual\"].apply(\n lambda x: (x - x.mean()) / x.std(),\n )\n return df_f[\"norm_resi\"]\n\n\n# get_turnover\ndf_pv = pd.read_hdf(\"daily_pv.h5\", key=\"data\")\ndf_f = pd.read_hdf(\"daily_f.h5\", key=\"data\")\nturnover = df_pv[\"$money\"] / df_f[\"\u6d41\u901a\u5e02\u503c\"]\n\nf = turnover.groupby(\"instrument\").pct_change(periods=20)\n\nf_neutralized = market_value_neutralize(f, df_f[\"\u6d41\u901a\u5e02\u503c\"])\n\nf_neutralized.to_hdf(\"result.h5\", key=\"data\")\n"
},
"PB_ROE": {
"description": "Constructed using the ranking difference between PB and ROE, with PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
"formulation": "\\text{rank}(PB\\_t) - rank(ROE_t)",
"variables": {
"\\text{rank}(PB_t)": "Ranking PB on cross-section at time t.",
"\\text{rank}(ROE_t)": "Ranking single-quarter ROE on cross-section at time t."
},
"gt_code": "#!/usr/bin/env python\n\nimport pandas as pd\n\ndata_f = pd.read_hdf('daily_f.h5')\n\ndata = data_f.reset_index()\n\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank'] - data['ROE_rank']\n\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf(\"result.h5\", key=\"data\")\n"
}
}
rdagent/core/task.py
+2 -2
View File
@@ -118,8 +118,8 @@ class TestCase:
def __init__(
self,
target_task: BaseTask,
ground_truth: TaskImplementation,
target_task: list[BaseTask] = [],
ground_truth: list[TaskImplementation] = [],
):
self.ground_truth = ground_truth
self.target_task = target_task
rdagent/factor_implementation/evolving/factor.py
+1 -1
View File
@@ -173,7 +173,7 @@ class FileBasedFactorImplementation(TaskImplementation):
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder,
)
self.workspace_path.mkdir(exist_ok=True, parents=True)
source_data_path.mkdir(exist_ok=True, parents=True)
code_path = self.workspace_path / f"{self.target_task.factor_name}.py"
code_path.write_text(self.code)
rdagent/factor_implementation/share_modules/factor_implementation_config.py
+3 -3
View File
@@ -9,13 +9,13 @@ SELECT_METHOD = Literal["random", "scheduler"]
class FactorImplementSettings(BaseSettings):
file_based_execution_data_folder: str = str(
(Path().cwd() / "factor_implementation_source_data").absolute(),
(Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
)
file_based_execution_workspace: str = str(
(Path().cwd() / "factor_implementation_workspace").absolute(),
(Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(),
)
implementation_execution_cache_location: str = str(
(Path().cwd() / "factor_implementation_execution_cache").absolute(),
(Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(),
)
enable_execution_cache: bool = True # whether to enable the execution cache
rdagent/factor_implementation/task_loader/json_loader.py
+23 -2
View File
@@ -1,7 +1,8 @@
import json
from pathlib import Path
from rdagent.core.task import TaskLoader
from rdagent.factor_implementation.evolving.factor import FactorImplementTask
from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FileBasedFactorImplementation
from rdagent.core.task import TestCase
class FactorImplementationTaskLoaderFromDict(TaskLoader):
@@ -21,7 +22,8 @@ class FactorImplementationTaskLoaderFromDict(TaskLoader):
class FactorImplementationTaskLoaderFromJsonFile(TaskLoader):
def load(self, json_file_path: Path) -> list:
factor_dict = json.load(json_file_path)
with open(json_file_path, 'r') as file:
factor_dict = json.load(file)
return FactorImplementationTaskLoaderFromDict().load(factor_dict)
@@ -29,3 +31,22 @@ class FactorImplementationTaskLoaderFromJsonString(TaskLoader):
def load(self, json_string: str) -> list:
factor_dict = json.loads(json_string)
return FactorImplementationTaskLoaderFromDict().load(factor_dict)
class FactorTestCaseLoaderFromJsonFile(TaskLoader):
def load(self, json_file_path: Path) -> list:
with open(json_file_path, 'r') as file:
factor_dict = json.load(file)
TestData = TestCase()
for factor_name, factor_data in factor_dict.items():
task = FactorImplementTask(
factor_name=factor_name,
factor_description=factor_data["description"],
factor_formulation=factor_data["formulation"],
variables=factor_data["variables"],
)
gt = FileBasedFactorImplementation(task, code=factor_data["gt_code"])
gt.execute()
TestData.target_task.append(task)
TestData.ground_truth.append(gt)
return TestData