From fc5eba4e2d8837c082168ad6c317eeadd01d9efc Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Sat, 8 Jan 2022 12:02:42 +0100 Subject: [PATCH] feat(Models): added lightGBM, moved other models to separate files (#128) * feat(Models): added lightGBM, moved other models to separate files * feat(Models): added non-working statsmodel wrapper * fix(Models): added work-in-progress comment to StatsModels --- config/config.py | 2 +- environment.yml | 1 + feature_selection/feature_selection.py | 3 +- models/base.py | 67 ------------------- models/lightgbm.py | 2 + models/model_map.py | 13 +++- models/neural.py | 42 ++++++++++++ .../pytorch}/pytorch_dataset.py | 0 models/sklearn.py | 34 ++++++++++ models/statsmodels.py | 35 ++++++++++ models/xgboost.py | 36 ++++++++++ run_pipeline.py | 3 +- 12 files changed, 165 insertions(+), 73 deletions(-) create mode 100644 models/lightgbm.py create mode 100644 models/neural.py rename {data_loader => models/pytorch}/pytorch_dataset.py (100%) create mode 100644 models/sklearn.py create mode 100644 models/statsmodels.py create mode 100644 models/xgboost.py diff --git a/config/config.py b/config/config.py index c6f0f56..2a1c473 100644 --- a/config/config.py +++ b/config/config.py @@ -118,7 +118,7 @@ def get_default_level_2_daily_config() -> tuple[dict, dict, dict]: regression_models = ["Lasso", "KNN", "RF"] regression_ensemble_model = 'KNN' - classification_models = ['SVC', 'LDA', 'KNN', 'CART', 'NB', 'AB', 'RF', 'StaticMom'] + classification_models = ['SVC', 'LDA', 'KNN', 'CART', 'NB', 'AB', 'RF', 'XGB_two_class', 'LGBM', 'StaticMom'] classification_ensemble_model = 'LDA' model_config = dict( diff --git a/environment.yml b/environment.yml index 37c203e..f72f883 100644 --- a/environment.yml +++ b/environment.yml @@ -26,6 +26,7 @@ dependencies: - pip - pandas-ta - xgboost + - lightgbm - alphalens-reloaded - pyfolio-reloaded - pip: diff --git a/feature_selection/feature_selection.py b/feature_selection/feature_selection.py index d02512e..fe3ee36 100644 --- a/feature_selection/feature_selection.py +++ b/feature_selection/feature_selection.py @@ -1,7 +1,8 @@ from sklearn.feature_selection import RFE from sklearn.model_selection import TimeSeriesSplit import pandas as pd -from models.base import Model, SKLearnModel +from models.base import Model +from models.sklearn import SKLearnModel from utils.scaler import get_scaler from utils.types import ScalerTypes from utils.hashing import hash_df, hash_series diff --git a/models/base.py b/models/base.py index 1914183..c042cc8 100644 --- a/models/base.py +++ b/models/base.py @@ -1,14 +1,9 @@ from __future__ import annotations from typing import Literal, Optional, Union -from sklearn.base import clone from abc import ABC, abstractmethod import numpy as np -import copy -import pytorch_lightning as pl - import numpy as np -from data_loader.pytorch_dataset import get_dataloader class Model(ABC): @@ -39,68 +34,6 @@ class Model(ABC): def initialize_network(self, input_dim:int, output_dim:int): pass -class SKLearnModel(Model): - data_scaling = 'scaled' - only_column = None - feature_selection = 'on' - model_type = 'ml' - predict_window_size = 'single_timestamp' - - def __init__(self, model): - self.model = model - - def fit(self, X: np.ndarray, y: np.ndarray) -> None: - self.model.fit(X, y) - - def predict(self, X) -> tuple[float, np.ndarray]: - pred = self.model.predict(X).item() - probability = self.model.predict_proba(X).squeeze() - return (pred, probability) - - def clone(self) -> SKLearnModel: - return SKLearnModel(clone(self.model)) - - def get_name(self) -> str: - return self.model.__class__.__name__ - - def initialize_network(self, input_dim:int, output_dim:int): - pass - -class LightningNeuralNetModel(Model): - - data_scaling = 'scaled' - only_column = None - feature_selection = 'off' - model_type = 'ml' - - ''' Standard lightning methods ''' - - def __init__(self, model, max_epochs=5): - self.model = model - self.trainer = pl.Trainer(max_epochs=max_epochs) - - def fit(self, X: np.ndarray, y: np.ndarray) -> None: - train_dataloader = self.__prepare_data(X.astype(float), y.astype(float)) - self.trainer.fit(self.model, train_dataloader) - - def predict(self, X: np.ndarray) -> tuple[float, np.ndarray]: - return self.model(X) - - def clone(self): - model_copy = copy.deepcopy(self.model) - return LightningNeuralNetModel(model_copy) - - ''' Non-standard lightning methods ''' - def __prepare_data(self, X:np.ndarray, y:np.ndarray): - dataloader = get_dataloader(X, y) - return dataloader - - def initialize_network(self, input_dim:int, output_dim:int): - self.model.initialize_network(input_dim, output_dim) - - def get_name(self) -> str: - return self.model.__class__.__name__ - \ No newline at end of file diff --git a/models/lightgbm.py b/models/lightgbm.py new file mode 100644 index 0000000..5ab6330 --- /dev/null +++ b/models/lightgbm.py @@ -0,0 +1,2 @@ +import lightgbm as lgb + diff --git a/models/model_map.py b/models/model_map.py index b4ea05f..59cbfd4 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -8,13 +8,19 @@ from sklearn.naive_bayes import GaussianNB from sklearn.neural_network import MLPRegressor, MLPClassifier from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier from sklearnex.ensemble import RandomForestClassifier -from models.base import SKLearnModel, LightningNeuralNetModel +from models.sklearn import SKLearnModel +from models.neural import LightningNeuralNetModel from models.momentum import StaticMomentumModel from models.average import StaticAverageModel from models.naive import StaticNaiveModel from models.pytorch.neural_nets import MultiLayerPerceptron +from models.xgboost import XGBoostModel +from models.statsmodels import StatsModel from xgboost import XGBClassifier import torch.nn.functional as F +from lightgbm import LGBMClassifier +from statsmodels.tsa.api import ExponentialSmoothing + model_map = { @@ -46,10 +52,11 @@ model_map = { AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)), RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)), SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True)), - XGB_three_class= SKLearnModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, use_label_encoder=True, objective='multi:softprob', eval_metric='mlogloss')), - XGB_two_class= SKLearnModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', eval_metric='mlogloss')), + XGB_two_class= XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')), + LGBM = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)), StaticMom= StaticMomentumModel(allow_short=True), Ensemble_Average= StaticAverageModel(), + # ExpSmoothing = SKLearnModel(ExponentialSmoothing(trend='add', seasonal='add', seasonal_periods=30)), ), } diff --git a/models/neural.py b/models/neural.py new file mode 100644 index 0000000..ad392e8 --- /dev/null +++ b/models/neural.py @@ -0,0 +1,42 @@ +from __future__ import annotations +from models.base import Model +import numpy as np +from models.pytorch.pytorch_dataset import get_dataloader +import copy +import pytorch_lightning as pl + +class LightningNeuralNetModel(Model): + + data_scaling = 'scaled' + only_column = None + feature_selection = 'off' + model_type = 'ml' + + ''' Standard lightning methods ''' + + def __init__(self, model, max_epochs=5): + self.model = model + self.trainer = pl.Trainer(max_epochs=max_epochs) + + def fit(self, X: np.ndarray, y: np.ndarray) -> None: + train_dataloader = self.__prepare_data(X.astype(float), y.astype(float)) + self.trainer.fit(self.model, train_dataloader) + + def predict(self, X: np.ndarray) -> tuple[float, np.ndarray]: + return self.model(X) + + def clone(self): + model_copy = copy.deepcopy(self.model) + return LightningNeuralNetModel(model_copy) + + ''' Non-standard lightning methods ''' + def __prepare_data(self, X:np.ndarray, y:np.ndarray): + dataloader = get_dataloader(X, y) + return dataloader + + def initialize_network(self, input_dim:int, output_dim:int): + self.model.initialize_network(input_dim, output_dim) + + def get_name(self) -> str: + return self.model.__class__.__name__ + \ No newline at end of file diff --git a/data_loader/pytorch_dataset.py b/models/pytorch/pytorch_dataset.py similarity index 100% rename from data_loader/pytorch_dataset.py rename to models/pytorch/pytorch_dataset.py diff --git a/models/sklearn.py b/models/sklearn.py new file mode 100644 index 0000000..e2e33f0 --- /dev/null +++ b/models/sklearn.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from models.base import Model +import numpy as np +from sklearn.base import clone + + +class SKLearnModel(Model): + + data_scaling = 'scaled' + only_column = None + feature_selection = 'on' + model_type = 'ml' + predict_window_size = 'single_timestamp' + + def __init__(self, model): + self.model = model + + def fit(self, X: np.ndarray, y: np.ndarray) -> None: + self.model.fit(X, y) + + def predict(self, X) -> tuple[float, np.ndarray]: + pred = self.model.predict(X).item() + probability = self.model.predict_proba(X).squeeze() + return (pred, probability) + + def clone(self) -> SKLearnModel: + return SKLearnModel(clone(self.model)) + + def get_name(self) -> str: + return self.model.__class__.__name__ + + def initialize_network(self, input_dim:int, output_dim:int): + pass + \ No newline at end of file diff --git a/models/statsmodels.py b/models/statsmodels.py new file mode 100644 index 0000000..7c1b30a --- /dev/null +++ b/models/statsmodels.py @@ -0,0 +1,35 @@ +from __future__ import annotations +from statsmodels.tsa.base.tsa_model import TimeSeriesModel +from models.base import Model +import numpy as np +from copy import deepcopy + + +class StatsModel(Model): + + # This is work in progress + data_scaling = 'scaled' + only_column = None + feature_selection = 'on' + model_type = 'ml' + predict_window_size = 'single_timestamp' + + def __init__(self, model: TimeSeriesModel): + self.model = model + + def fit(self, X: np.ndarray, y: np.ndarray) -> None: + self.model.fit(X, y) + + def predict(self, X) -> tuple[float, np.ndarray]: + pred = self.model.predict(X).item() + return (pred, np.array([0])) + + def clone(self) -> StatsModel: + return StatsModel(deepcopy(self.model)) + + def get_name(self) -> str: + return self.model.__class__.__name__ + + def initialize_network(self, input_dim:int, output_dim:int): + pass + \ No newline at end of file diff --git a/models/xgboost.py b/models/xgboost.py new file mode 100644 index 0000000..9d1d09c --- /dev/null +++ b/models/xgboost.py @@ -0,0 +1,36 @@ +from __future__ import annotations +from models.base import Model +import numpy as np +from xgboost import XGBClassifier +from sklearn.base import clone + +class XGBoostModel(Model): + + data_scaling = 'scaled' + only_column = None + feature_selection = 'on' + model_type = 'ml' + predict_window_size = 'single_timestamp' + + def __init__(self, model: XGBClassifier): + self.model = model + + def fit(self, X: np.ndarray, y: np.ndarray) -> None: + def map_to_xgb(y): return np.array([1 if i == 1 else 0 for i in y]) + self.model.fit(X, map_to_xgb(y)) + + def predict(self, X) -> tuple[float, np.ndarray]: + pred = self.model.predict(X).item() + probability = self.model.predict_proba(X).squeeze() + def map_from_xgb(y): return 1 if y == 1 else -1 + return (map_from_xgb(pred), probability) + + def clone(self) -> XGBoostModel: + return XGBoostModel(clone(self.model)) + + def get_name(self) -> str: + return self.model.__class__.__name__ + + def initialize_network(self, input_dim:int, output_dim:int): + pass + \ No newline at end of file diff --git a/run_pipeline.py b/run_pipeline.py index a16bb73..adfefc0 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -88,7 +88,8 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): all_models_for_all_assets[asset[1]] = dict( name=asset[1], - models=all_models_for_single_asset) + models=all_models_for_single_asset + ) # 4. Train a Meta-Labeling model for each Level-1 model and replace its predictions with the meta-labeling predictions if training_config['meta_labeling_lvl_1'] == True: