From b6cd6b14fe6aac07e15d409ad74bc0751821db87 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Thu, 23 Dec 2021 10:35:20 +0100 Subject: [PATCH] feat(Config): feature extractors are enabled one-by-one with a bool, added previous model to model.fit() (#77) * feat(Config): feature extractors are enabled one-by-one with a bool, added previous model to model.fit() * fix(Sweep): removed unused `other_features` parameter that fails sweep * feat(Config): using preset names for defining feature extractors again * fix(Tests): fixed model stub classes --- config.py | 7 ++- .../feature_extractor_presets.py | 59 +++++++++++++------ models/average.py | 2 +- models/base.py | 4 +- models/momentum.py | 2 +- models/naive.py | 2 +- reporting/wandb.py | 13 ++-- run_pipeline.py | 6 +- sweep.yaml | 14 ++--- tests/test_evaluation.py | 2 +- tests/test_walk_forward.py | 2 +- training/walk_forward.py | 2 +- utils/helpers.py | 3 + utils/typing.py | 4 +- 14 files changed, 78 insertions(+), 44 deletions(-) diff --git a/config.py b/config.py index de7b902..928fdb1 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,6 @@ +from collections import defaultdict from utils.load_data import get_crypto_assets -import feature_extractors.feature_extractor_presets as feature_extractor_presets +from feature_extractors.feature_extractor_presets import presets from models.model_map import model_names_classification, model_names_regression def get_default_config() -> tuple[dict, dict, dict]: @@ -18,8 +19,8 @@ def get_default_config() -> tuple[dict, dict, dict]: load_other_assets= False, log_returns= True, forecasting_horizon = 1, - own_features= feature_extractor_presets.date + feature_extractor_presets.level1, - other_features= [], + own_features = ['level_1', 'date_days'], + other_features = [], index_column= 'int', method= 'classification', ) diff --git a/feature_extractors/feature_extractor_presets.py b/feature_extractors/feature_extractor_presets.py index 536ba76..cfd0d7b 100644 --- a/feature_extractors/feature_extractor_presets.py +++ b/feature_extractors/feature_extractor_presets.py @@ -1,24 +1,49 @@ from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead +from utils.typing import FeatureExtractorConfig +from utils.helpers import flatten -debug_future_lookahead = [('debug_future', feature_debug_future_lookahead, [1])] +__presets = dict( + debug_future_lookahead = [('debug_future', feature_debug_future_lookahead, [1])], + single_mom = [('mom', feature_mom, [30])], + single_vol = [('vol', feature_vol, [30])], + mom = [('mom', feature_mom, [10, 20, 30, 60, 90])], + vol = [('vol', feature_vol, [10, 20, 30, 60])], + lags_up_to_5 = [('lag', feature_lag, [1,2,3,4,5])], + lags_up_to_10 = [('lag', feature_lag, [1,2,3,4,5,6,7,8,9,10])], + date_all = [ + ('day_of_week', feature_day_of_week, [0]), + ('day_of_month', feature_day_of_month, [0]), + ('month', feature_month, [0])], + date_days = [ + ('day_of_week', feature_day_of_week, [0]), + ('day_of_month', feature_day_of_month, [0]), + ], + roc = [('roc', feature_ROC, [10, 30])], + rsi = [('rsi', feature_ROC, [10, 30, 100])], + stod = [('stod', feature_STOD, [10, 30, 200])], + stok = [('stok', feature_STOK, [10, 30, 200])], +) -lags = [('lag', feature_lag, [1,2,3,4,5,6,7,8,9])] +presets = __presets | dict( + level_1 = __presets["mom"] + __presets["vol"], + level_2 = __presets["mom"] + __presets["vol"] + __presets["roc"] + __presets["rsi"] + __presets["stod"] + __presets["stok"], +) -only_mom = [('mom', feature_mom, [30])] +def preprocess_feature_extractors_config(data_dict: dict) -> dict: + keys = ['own_features', 'other_features'] + for key in keys: + preset_names = data_dict[key] + data_dict[key] = flatten([presets[preset_name] for preset_name in preset_names]) + return data_dict -date = [ - ('day_of_week', feature_day_of_week, [0]), - ('day_of_month', feature_day_of_month, [0]), - ('month', feature_month, [0])] -level1 = [ - ('mom', feature_mom, [10, 20, 30, 60, 90]), - ('vol', feature_vol, [10, 20, 30, 60]), -] +# Use this if ever we want to create an independent boolean for each featureextractor +# def preprocess_feature_extractors_config(data_dict: dict) -> dict: +# prefixes = ['own_features', 'other_features'] +# features_dict = dict() +# for prefix in prefixes: +# features_to_include = [key.replace(prefix + "_", "") for key, value in data_dict.items() if key.startswith(prefix) and value == True] +# features_dict[prefix] = flatten([presets[feature_name] for feature_name in features_to_include]) -level2 = level1 + [ - ('roc', feature_ROC, [10, 30]), - ('rsi', feature_RSI, [10, 30, 100]), - ('stod', feature_STOD, [10, 30, 200]), - ('stok', feature_STOK, [10, 30, 200]), -] \ No newline at end of file +# data_dict = {k: v for k, v in data_dict.items() if not (k.startswith(prefixes[0]) or k.startswith(prefixes[1]))} +# return (data_dict | features_dict) \ No newline at end of file diff --git a/models/average.py b/models/average.py index 8d7ddf1..0b490d5 100644 --- a/models/average.py +++ b/models/average.py @@ -10,7 +10,7 @@ class StaticAverageModel(Model): data_scaling = 'unscaled' only_column = 'model_' - def fit(self, X, y): + def fit(self, X, y, prev_model): # This is a static model, it can' learn anything pass diff --git a/models/base.py b/models/base.py index c44398e..1be5bda 100644 --- a/models/base.py +++ b/models/base.py @@ -11,7 +11,7 @@ class Model(ABC): only_column: Optional[str] @abstractmethod - def fit(self, X, y): + def fit(self, X, y, prev_model): pass @abstractmethod @@ -32,7 +32,7 @@ class SKLearnModel(Model): def __init__(self, model): self.model = model - def fit(self, X, y): + def fit(self, X, y, prev_model): self.model.fit(X, y) def predict(self, X): diff --git a/models/momentum.py b/models/momentum.py index e3a6ee9..e838fb6 100644 --- a/models/momentum.py +++ b/models/momentum.py @@ -14,7 +14,7 @@ class StaticMomentumModel(Model): super().__init__() self.allow_short = allow_short - def fit(self, X, y): + def fit(self, X, y, prev_model): # This is a static model, it can' learn anything pass diff --git a/models/naive.py b/models/naive.py index e578c2d..7822a7f 100644 --- a/models/naive.py +++ b/models/naive.py @@ -10,7 +10,7 @@ class StaticNaiveModel(Model): data_scaling = 'unscaled' only_column = None - def fit(self, X, y): + def fit(self, X, y, prev_model): # This is a static model, it can' learn anything pass diff --git a/reporting/wandb.py b/reporting/wandb.py index 4a0abf8..e28e775 100644 --- a/reporting/wandb.py +++ b/reporting/wandb.py @@ -15,15 +15,16 @@ def launch_wandb(project_name:str, default_config:dict, sweep:bool=False): return wandb -def seperate_configs(wandb: Optional[object], model_config:dict, training_config:dict, data_config:dict) -> tuple[dict,dict,dict]: +def register_config_with_wandb(wandb: Optional[object], model_config:dict, training_config:dict, data_config:dict): config: dict = wandb.config if type(wandb) is not type(None): - for k in training_config: training_config[k] = config[k] - for k in model_config: model_config[k] = config[k] - # for k in data_config: data_config[k] = config[k] - - return model_config, training_config, data_config + for k in training_config: + training_config[k] = config[k] + for k in model_config: + model_config[k] = config[k] + for k in data_config: + data_config[k] = config[k] def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_name: str, model_name: str): diff --git a/run_pipeline.py b/run_pipeline.py index 4811ae3..baa578c 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -1,8 +1,9 @@ from utils.load_data import load_data import pandas as pd from training.training import run_single_asset_trainig -from reporting.wandb import launch_wandb, send_report_to_wandb, seperate_configs +from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_with_wandb from models.model_map import map_model_name_to_function +from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config from config import get_default_config, validate_config, get_model_name def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): @@ -11,9 +12,10 @@ def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): wandb = None if with_wandb: wandb = launch_wandb(project_name=project_name, default_config=dict(**model_config, **training_config, **data_config), sweep=sweep) - model_config, training_config, data_config = seperate_configs(wandb, model_config, training_config, data_config) + register_config_with_wandb(wandb, model_config, training_config, data_config) model_config = map_model_name_to_function(model_config, data_config['method']) + data_config = preprocess_feature_extractors_config(data_config) pipeline(project_name, wandb, sweep, model_config, training_config, data_config) diff --git a/sweep.yaml b/sweep.yaml index 387a81c..1f2a8ae 100644 --- a/sweep.yaml +++ b/sweep.yaml @@ -15,10 +15,10 @@ parameters: values: [True, False] distribution: categorical sliding_window_size: - values: [90, 130, 160, 180, 280, 380] + values: [180, 280, 380] distribution: categorical retrain_every: - values: [14, 30, 60, 100] + values: [10, 20, 30] distribution: categorical scaler: values: ['minmax', 'normalize', 'minmax', 'standardize', 'none'] @@ -37,10 +37,6 @@ parameters: log_returns: values: [True, False] distribution: categorical - own_features: - value: [] - other_features: - value: [] index_column: value: 'int' level_1_models: @@ -49,4 +45,8 @@ parameters: level_2_models: value: [] distribution: constant - + own_features: + values: [['only_mom', 'date_days'], [], ['level_1', 'date_days'], ['level_1', 'date_days', 'level_2']] + distribution: categorical + other_features: + value: [] \ No newline at end of file diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 8b8fd5b..58be78f 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -42,7 +42,7 @@ class EvenOddStubModel(Model): super().__init__() self.window_length = window_length - def fit(self, X, y): + def fit(self, X, y, prev_model): assert len(X) == self.window_length for i in range(len(X)): assert y[i] == -1 if X[i][0] == 1 else 1 diff --git a/tests/test_walk_forward.py b/tests/test_walk_forward.py index ce763f4..d1a1506 100644 --- a/tests/test_walk_forward.py +++ b/tests/test_walk_forward.py @@ -40,7 +40,7 @@ class IncrementingStubModel(Model): super().__init__() self.window_length = window_length - def fit(self, X, y): + def fit(self, X, y, prev_model): assert len(X) == self.window_length for i in range(len(X)): assert X[i][0] + 1 == y[i] diff --git a/training/walk_forward.py b/training/walk_forward.py index 37eadcd..171e571 100644 --- a/training/walk_forward.py +++ b/training/walk_forward.py @@ -58,7 +58,7 @@ def walk_forward_train_test( X_slice = X_slice.to_numpy() current_model = model.clone() - current_model.fit(X_slice, y_slice.to_numpy()) + current_model.fit(X_slice, y_slice.to_numpy(), models[index-1]) iterations_before_retrain = retrain_every else: current_model = models[index-1] diff --git a/utils/helpers.py b/utils/helpers.py index dd23d7a..b7e4868 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -4,3 +4,6 @@ import numpy as np def get_first_valid_return_index(series: pd.Series) -> int: return np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series))))[0][0] + +def flatten(list_of_lists: list) -> list: + return [item for sublist in list_of_lists for item in sublist] \ No newline at end of file diff --git a/utils/typing.py b/utils/typing.py index d09ccd2..002d8f0 100644 --- a/utils/typing.py +++ b/utils/typing.py @@ -4,4 +4,6 @@ import pandas as pd Period = int IsLogReturn = bool -FeatureExtractor = Callable[[pd.DataFrame, Period, IsLogReturn], Union[pd.DataFrame, pd.Series]] \ No newline at end of file +FeatureExtractor = Callable[[pd.DataFrame, Period, IsLogReturn], Union[pd.DataFrame, pd.Series]] +Name = str +FeatureExtractorConfig = tuple[Name, FeatureExtractor, list[Period]] \ No newline at end of file