feat(Models): added lightGBM, moved other models to separate files (#128)

* feat(Models): added lightGBM, moved other models to separate files

* feat(Models): added non-working statsmodel wrapper

* fix(Models): added work-in-progress comment to StatsModels

config/config.py

@@ -118,7 +118,7 @@ def get_default_level_2_daily_config() -> tuple[dict, dict, dict]:
 
     regression_models = ["Lasso", "KNN", "RF"]
     regression_ensemble_model = 'KNN'
-    classification_models = ['SVC', 'LDA', 'KNN', 'CART', 'NB', 'AB', 'RF', 'StaticMom']
+    classification_models = ['SVC', 'LDA', 'KNN', 'CART', 'NB', 'AB', 'RF', 'XGB_two_class', 'LGBM', 'StaticMom']
     classification_ensemble_model = 'LDA'
 
     model_config = dict(

environment.yml

@@ -26,6 +26,7 @@ dependencies:
   - pip
   - pandas-ta
   - xgboost
+  - lightgbm
   - alphalens-reloaded
   - pyfolio-reloaded
   - pip:

feature_selection/feature_selection.py

@@ -1,7 +1,8 @@
 from sklearn.feature_selection import RFE
 from sklearn.model_selection import TimeSeriesSplit
 import pandas as pd
-from models.base import Model, SKLearnModel
+from models.base import Model
+from models.sklearn import SKLearnModel
 from utils.scaler import get_scaler
 from utils.types import ScalerTypes
 from utils.hashing import hash_df, hash_series

models/base.py

@@ -1,14 +1,9 @@
 from __future__ import annotations
 from typing import Literal, Optional, Union
-from sklearn.base import clone
 from abc import ABC, abstractmethod
 import numpy as np
 
-import copy
-import pytorch_lightning as pl
-
 import numpy as np
-from data_loader.pytorch_dataset import get_dataloader
 
 class Model(ABC):
 
@@ -39,68 +34,6 @@ class Model(ABC):
     def initialize_network(self, input_dim:int, output_dim:int):
         pass
 
-class SKLearnModel(Model):
 
-    data_scaling = 'scaled'
-    only_column = None
-    feature_selection = 'on'
-    model_type = 'ml'
-    predict_window_size = 'single_timestamp'
-
-    def __init__(self, model):
-        self.model = model
-    
-    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
-        self.model.fit(X, y)
-
-    def predict(self, X) -> tuple[float, np.ndarray]:
-        pred = self.model.predict(X).item()
-        probability = self.model.predict_proba(X).squeeze()
-        return (pred, probability)
-    
-    def clone(self) -> SKLearnModel:
-        return SKLearnModel(clone(self.model))
-
-    def get_name(self) -> str:
-        return self.model.__class__.__name__
-    
-    def initialize_network(self, input_dim:int, output_dim:int):
-        pass
     
     
-    
-class LightningNeuralNetModel(Model):
-
-    data_scaling = 'scaled'
-    only_column = None
-    feature_selection = 'off'
-    model_type = 'ml'
-
-    ''' Standard lightning methods '''
-    
-    def __init__(self, model, max_epochs=5):
-        self.model = model
-        self.trainer = pl.Trainer(max_epochs=max_epochs)
-        
-    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
-        train_dataloader = self.__prepare_data(X.astype(float), y.astype(float))
-        self.trainer.fit(self.model, train_dataloader)
-        
-    def predict(self, X: np.ndarray) -> tuple[float, np.ndarray]:
-        return self.model(X)
-
-    def clone(self):
-        model_copy = copy.deepcopy(self.model)
-        return LightningNeuralNetModel(model_copy)
-    
-    ''' Non-standard lightning methods '''
-    def __prepare_data(self, X:np.ndarray, y:np.ndarray):
-        dataloader = get_dataloader(X, y)
-        return dataloader
-
-    def initialize_network(self, input_dim:int, output_dim:int):
-        self.model.initialize_network(input_dim, output_dim)
-
-    def get_name(self) -> str:
-        return self.model.__class__.__name__
-    

models/lightgbm.py

@@ -0,0 +1,2 @@
+import lightgbm as lgb
+

models/model_map.py

@@ -8,13 +8,19 @@ from sklearn.naive_bayes import GaussianNB
 from sklearn.neural_network import MLPRegressor, MLPClassifier
 from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier
 from sklearnex.ensemble import RandomForestClassifier
-from models.base import SKLearnModel, LightningNeuralNetModel
+from models.sklearn import SKLearnModel
+from models.neural import LightningNeuralNetModel
 from models.momentum import StaticMomentumModel
 from models.average import StaticAverageModel
 from models.naive import StaticNaiveModel
 from models.pytorch.neural_nets import MultiLayerPerceptron
+from models.xgboost import XGBoostModel
+from models.statsmodels import StatsModel
 from xgboost import XGBClassifier
 import torch.nn.functional as F
+from lightgbm import LGBMClassifier
+from statsmodels.tsa.api import ExponentialSmoothing
+
 
 
 model_map = {
@@ -46,10 +52,11 @@ model_map = {
         AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)),
         RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)),
         SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True)),
-        XGB_three_class= SKLearnModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, use_label_encoder=True, objective='multi:softprob', eval_metric='mlogloss')),
-        XGB_two_class= SKLearnModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', eval_metric='mlogloss')),
+        XGB_two_class= XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')),
+        LGBM = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)),
         StaticMom= StaticMomentumModel(allow_short=True),
         Ensemble_Average= StaticAverageModel(),
+        # ExpSmoothing = SKLearnModel(ExponentialSmoothing(trend='add', seasonal='add', seasonal_periods=30)),
     ),
 }
 

models/neural.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+from models.base import Model
+import numpy as np
+from models.pytorch.pytorch_dataset import get_dataloader
+import copy
+import pytorch_lightning as pl
+
+class LightningNeuralNetModel(Model):
+
+    data_scaling = 'scaled'
+    only_column = None
+    feature_selection = 'off'
+    model_type = 'ml'
+
+    ''' Standard lightning methods '''
+    
+    def __init__(self, model, max_epochs=5):
+        self.model = model
+        self.trainer = pl.Trainer(max_epochs=max_epochs)
+        
+    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
+        train_dataloader = self.__prepare_data(X.astype(float), y.astype(float))
+        self.trainer.fit(self.model, train_dataloader)
+        
+    def predict(self, X: np.ndarray) -> tuple[float, np.ndarray]:
+        return self.model(X)
+
+    def clone(self):
+        model_copy = copy.deepcopy(self.model)
+        return LightningNeuralNetModel(model_copy)
+    
+    ''' Non-standard lightning methods '''
+    def __prepare_data(self, X:np.ndarray, y:np.ndarray):
+        dataloader = get_dataloader(X, y)
+        return dataloader
+
+    def initialize_network(self, input_dim:int, output_dim:int):
+        self.model.initialize_network(input_dim, output_dim)
+
+    def get_name(self) -> str:
+        return self.model.__class__.__name__
+    

models/sklearn.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+from models.base import Model
+import numpy as np
+from sklearn.base import clone
+
+
+class SKLearnModel(Model):
+
+    data_scaling = 'scaled'
+    only_column = None
+    feature_selection = 'on'
+    model_type = 'ml'
+    predict_window_size = 'single_timestamp'
+
+    def __init__(self, model):
+        self.model = model
+    
+    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
+        self.model.fit(X, y)
+
+    def predict(self, X) -> tuple[float, np.ndarray]:
+        pred = self.model.predict(X).item()
+        probability = self.model.predict_proba(X).squeeze()
+        return (pred, probability)
+    
+    def clone(self) -> SKLearnModel:
+        return SKLearnModel(clone(self.model))
+
+    def get_name(self) -> str:
+        return self.model.__class__.__name__
+    
+    def initialize_network(self, input_dim:int, output_dim:int):
+        pass
+    

models/statsmodels.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+from statsmodels.tsa.base.tsa_model import TimeSeriesModel
+from models.base import Model
+import numpy as np
+from copy import deepcopy
+
+
+class StatsModel(Model):
+
+    # This is work in progress
+    data_scaling = 'scaled'
+    only_column = None
+    feature_selection = 'on'
+    model_type = 'ml'
+    predict_window_size = 'single_timestamp'
+
+    def __init__(self, model: TimeSeriesModel):
+        self.model = model
+    
+    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
+        self.model.fit(X, y)
+
+    def predict(self, X) -> tuple[float, np.ndarray]:
+        pred = self.model.predict(X).item()
+        return (pred, np.array([0]))
+    
+    def clone(self) -> StatsModel:
+        return StatsModel(deepcopy(self.model))
+
+    def get_name(self) -> str:
+        return self.model.__class__.__name__
+    
+    def initialize_network(self, input_dim:int, output_dim:int):
+        pass
+    

models/xgboost.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+from models.base import Model
+import numpy as np
+from xgboost import XGBClassifier
+from sklearn.base import clone
+
+class XGBoostModel(Model):
+
+    data_scaling = 'scaled'
+    only_column = None
+    feature_selection = 'on'
+    model_type = 'ml'
+    predict_window_size = 'single_timestamp'
+
+    def __init__(self, model: XGBClassifier):
+        self.model = model
+    
+    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
+        def map_to_xgb(y): return np.array([1 if i == 1 else 0 for i in y])
+        self.model.fit(X, map_to_xgb(y))
+
+    def predict(self, X) -> tuple[float, np.ndarray]:
+        pred = self.model.predict(X).item()
+        probability = self.model.predict_proba(X).squeeze()
+        def map_from_xgb(y): return 1 if y == 1 else -1
+        return (map_from_xgb(pred), probability)
+    
+    def clone(self) -> XGBoostModel:
+        return XGBoostModel(clone(self.model))
+
+    def get_name(self) -> str:
+        return self.model.__class__.__name__
+    
+    def initialize_network(self, input_dim:int, output_dim:int):
+        pass
+    

run_pipeline.py

@@ -88,7 +88,8 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
         
         all_models_for_all_assets[asset[1]] = dict(
             name=asset[1], 
-            models=all_models_for_single_asset)
+            models=all_models_for_single_asset
+        )
         
         # 4. Train a Meta-Labeling model for each Level-1 model and replace its predictions with the meta-labeling predictions
         if training_config['meta_labeling_lvl_1'] == True: