Feature(Speed): Python launches faster by conditionally importing models. (#169)

* feat: Added optional import of models.

* fix: Models weren't wrapped into abstract class, fixed it.

* chore: Deleted leftover comments.

* fix: Same merge commit as on remote.

* fix: System wasn't putting in RF because there was no differentiation between RF as regressor and RF as classificator.

* fix(Models): use the XGBoostModel wrapper

Co-authored-by: Daniel Szemerey <szemereydaniel@gmail.com>
Co-authored-by: Mark Aron Szulyovszky <mark.szulyovszky@gmail.com>

config/config.py

@@ -72,8 +72,8 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]:
         narrow_format = False,
     )
 
-    regression_models = ["Lasso", "KNN", "RF"]
+    regression_models = ["Lasso", "KNN", "RFR"]
-    classification_models = ["LR_two_class", "LDA", "NB", "RF", "XGB_two_class", "LGBM", "StaticMom"]
+    classification_models = ["LR_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
     meta_labeling_models = ['LR_two_class', 'LGBM']
     ensemble_model = 'Average'
 

config/preprocess.py

@@ -1,7 +1,7 @@
 
 from utils.helpers import flatten
 from feature_extractors.feature_extractor_presets import presets as feature_extractor_presets
-from models.model_map import model_map
+from models.model_map import get_model_map
 from data_loader.collections import data_collections
 
 def preprocess_config(model_config:dict, training_config:dict, data_config:dict) -> tuple[dict, dict, dict]:
@@ -21,6 +21,7 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
     return data_dict
 
 def __preprocess_model_config(model_config:dict, method:str) -> dict:
+    model_map, _, _, _, _ = get_model_map(model_config)
     model_config['primary_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in  model_config['primary_models']]
     if len(model_config['meta_labeling_models']) > 0:
         model_config['meta_labeling_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in  model_config['meta_labeling_models']]

data_loader/process_data.py

@@ -3,16 +3,18 @@ from operator import itemgetter
 
 from utils.helpers import has_enough_samples_to_train
 from feature_selection.dim_reduction import reduce_dimensionality
-from models.model_map import default_feature_selector_regression, default_feature_selector_classification
+from models.model_map import get_model_map
 from feature_selection.feature_selection import select_features
 import warnings
 
 
 
 
+
 def process_data(X:pd.DataFrame, y:pd.Series, configs: dict) -> tuple[pd.DataFrame,pd.DataFrame]:
     
     model_config, training_config, data_config = itemgetter('model_config', 'training_config', 'data_config')(configs)
+    _, _, _, default_feature_selector_regression, default_feature_selector_classification = get_model_map(model_config)
     
     original_X = X.copy()
     

models/model_map.py

@@ -1,72 +1,111 @@
-from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, Ridge
-from sklearn.linear_model import LogisticRegression
-from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX
-from sklearn.tree import DecisionTreeClassifier
-from sklearnex.neighbors import KNeighborsRegressor, KNeighborsClassifier
-from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
-from sklearnex.svm import SVR, SVC
-from sklearn.naive_bayes import GaussianNB
-from sklearn.neural_network import MLPRegressor, MLPClassifier
-from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier
-from sklearnex.ensemble import RandomForestClassifier
 from models.sklearn import SKLearnModel
-from models.neural import LightningNeuralNetModel
+from sklearn.ensemble import RandomForestClassifier
-from models.momentum import StaticMomentumModel
+from sklearn.ensemble import RandomForestRegressor
-from models.average import StaticAverageModel
-from models.naive import StaticNaiveModel
-from models.pytorch.neural_nets import MultiLayerPerceptron
-from models.xgboost import XGBoostModel
-from models.statsmodels import StatsModel
-from xgboost import XGBClassifier
-import torch.nn.functional as F
-from lightgbm import LGBMClassifier
-from statsmodels.tsa.api import ExponentialSmoothing
-
-
 
 model_map = {
-    "regression_models": dict(
+    "regression_models": dict(),
-        LR= SKLearnModel(LinearRegression(n_jobs=-1)),
+    "classification_models": dict(),
-        Lasso= SKLearnModel(Lasso(alpha=100, random_state=1)),
+    "ensemble_models": dict()
-        Ridge= SKLearnModel(Ridge(alpha=0.1)),
-        BayesianRidge= SKLearnModel(BayesianRidge()),
-        KNN= SKLearnModel(KNeighborsRegressor(n_neighbors=25)),
-        AB= SKLearnModel(AdaBoostRegressor(random_state=1)),
-        MLP= SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
-        RF= SKLearnModel(RandomForestRegressor(n_jobs=-1, max_depth=20, random_state=1)),
-        SVR= SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1)),
-        StaticNaive= StaticNaiveModel(),
-        DNN = LightningNeuralNetModel(
-            MultiLayerPerceptron(
-                hidden_layers_ratio = [1.0], 
-                probabilities = False, 
-                loss_function = F.mse_loss), 
-            max_epochs=15
-        ) 
-    ),
-    "classification_models": dict(
-        LR_two_class= SKLearnModel(LogisticRegression(C=10, random_state=1, solver='liblinear', max_iter=1000)),
-        LR_three_class= SKLearnModel(LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1)),
-        LDA= SKLearnModel(LinearDiscriminantAnalysis()),
-        KNN= SKLearnModel(KNeighborsClassifier()),
-        CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)),
-        NB= SKLearnModel(GaussianNB()),
-        AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)),
-        RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)),
-        SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True, random_state=1)),
-        XGB_two_class= XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')),
-        LGBM = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)),
-        StaticMom= StaticMomentumModel(allow_short=True),
-        # ExpSmoothing = SKLearnModel(ExponentialSmoothing(trend='add', seasonal='add', seasonal_periods=30)),
-    ),
-    "ensemble_models": dict(
-        Average= StaticAverageModel(),
-    )
 }
 
-model_names_classification = list(model_map["classification_models"].keys())
+def get_model_map(config:dict):
-model_names_regression = list(model_map["regression_models"].keys())
+    
+    if len(config['primary_models']) > 0 and isinstance(config['primary_models'][0], str):
+        print("Going to Load models")
+        combined_list = config['primary_models'] + config['meta_labeling_models'] + [config['ensemble_model']]
+        for model_name in combined_list:
+            if model_name == 'LR':
+                from sklearn.linear_model import LinearRegression
+                model_map['regression_models']['LR'] = SKLearnModel(LinearRegression(n_jobs=-1))
+            elif model_name == 'Lasso':
+                from sklearn.linear_model import Lasso
+                model_map['regression_models']['Lasso'] = SKLearnModel(Lasso(alpha=100, random_state=1))
+            elif model_name == 'Ridge':
+                from sklearn.linear_model import Ridge
+                model_map['regression_models']['Ridge'] = SKLearnModel(Ridge(alpha=0.1))
+            elif model_name == 'BayesianRidge':
+                from sklearn.linear_model import BayesianRidge
+                model_map['regression_models']['BayesianRidge'] = SKLearnModel(BayesianRidge())
+            elif model_name == 'KNN':
+                from sklearnex.neighbors import KNeighborsRegressor
+                model_map['regression_models']['KNN'] = SKLearnModel(KNeighborsRegressor(n_neighbors=25))
+            elif model_name == 'AB':
+                from sklearn.ensemble import AdaBoostRegressor
+                model_map['regression_models']['AB'] = SKLearnModel(AdaBoostRegressor(random_state=1))
+            elif model_name == 'MLP':
+                from sklearn.neural_network import MLPRegressor
+                model_map['regression_models']['MLP'] = SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000))
+            elif model_name == 'RFR':
+                # from sklearn.ensemble import RandomForestRegressor
+                model_map['regression_models']['RFR'] = SKLearnModel(RandomForestRegressor(n_jobs=-1, max_depth=20, random_state=1))
+            elif model_name == 'SVR':
+                from sklearnex.svm import SVR
+                model_map['regression_models']['SVR'] = SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1))
+            elif model_name == 'StaticNaive':
+                from models.naive import StaticNaiveModel
+                model_map['regression_models']['StaticNaive'] = StaticNaiveModel()
+            elif model_name == 'DNN':
+                from models.neural import LightningNeuralNetModel
+                from models.pytorch.neural_nets import MultiLayerPerceptron
+                import torch.nn.functional as F
+                model_map['regression_models']['DNN'] = LightningNeuralNetModel(
+                    MultiLayerPerceptron(
+                        hidden_layers_ratio = [1.0], 
+                        probabilities = False, 
+                        loss_function = F.mse_loss), 
+                    max_epochs=15
+                )
+            
+            
+            elif model_name == 'LR_two_class':
+                from sklearn.linear_model import LogisticRegression
+                model_map['classification_models']['LR_two_class'] = SKLearnModel(LogisticRegression(C=10, random_state=1, solver='liblinear', max_iter=1000))
+            elif model_name == 'LR_three_class':
+                from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX
+                model_map['classification_models']['LR_three_class'] = SKLearnModel(LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1))
+            elif model_name == 'LDA':
+                from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
+                model_map['classification_models']['LDA'] = SKLearnModel(LinearDiscriminantAnalysis())
+            elif model_name == 'KNN':
+                from sklearn.neighbors import KNeighborsClassifier
+                model_map['classification_models']['KNN'] = SKLearnModel(KNeighborsClassifier())
+            elif model_name == 'CART':
+                from sklearn.tree import DecisionTreeClassifier
+                model_map['classification_models']['CART'] = SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1))
+            elif model_name == 'NB':
+                from sklearn.naive_bayes import GaussianNB
+                model_map['classification_models']['NB'] = SKLearnModel(GaussianNB())
+            elif model_name == 'AB':
+                from sklearn.ensemble import AdaBoostClassifier
+                model_map['classification_models']['AB'] = SKLearnModel(AdaBoostClassifier(n_estimators=15))
+            elif model_name == 'RFC':
+                # from sklearn.ensemble import RandomForestClassifier
+                model_map['classification_models']['RFC'] = SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1))
+            elif model_name == 'SVC':
+                from sklearn.svm import SVC
+                model_map['classification_models']['SVC'] = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True, random_state=1))
+            elif model_name == 'XGB_two_class':
+                from xgboost import XGBClassifier
+                from models.xgboost import XGBoostModel
+                model_map['classification_models']['XGB_two_class'] = XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss'))
+            elif model_name == 'LGBM':
+                from lightgbm import LGBMClassifier
+                model_map['classification_models']['LGBM'] = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1))
+            elif model_name == 'StaticMom':
+                from models.momentum import StaticMomentumModel
+                model_map['classification_models']['StaticMom'] = StaticMomentumModel(allow_short=True)
+            elif model_name == 'Average':
+                from models.average import StaticAverageModel
+                model_map['ensemble_models']['Average'] = StaticAverageModel()
+           
+    
+    model_names_classification = list(model_map["classification_models"].keys())
+    model_names_regression = list(model_map["regression_models"].keys())
 
-default_feature_selector_regression = model_map['regression_models']['RF']
+    
-default_feature_selector_classification = model_map['classification_models']['RF']
+    default_feature_selector_classification = SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1))
+    default_feature_selector_regression = SKLearnModel(RandomForestRegressor(n_jobs=-1, max_depth=20, random_state=1))
+
+    
+    return model_map, model_names_classification, model_names_regression, default_feature_selector_regression, default_feature_selector_classification
 

sweep_ensemble.yaml

@@ -47,9 +47,9 @@ parameters:
     primary_models: 
       distribution: categorical
       values:
-      - ["LDA", "LR_two_class", "KNN", "SVC", "CART", "NB", "AB", "RF", "XGB_two_class", "LGBM", "StaticMom"]
+      - ["LDA", "LR_two_class", "KNN", "SVC", "CART", "NB", "AB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
-      - ["LR_two_class", "LDA", "NB", "RF", "XGB_two_class", "LGBM", "StaticMom"]
+      - ["LR_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
-      - ["LR_two_class", "LDA", "LGBM", "RF", "XGB_two_class"]
+      - ["LR_two_class", "LDA", "LGBM", "RFC", "XGB_two_class"]
     meta_labeling_models: 
       value: ["LGBM", "LR_two_class"]
     own_features:

sweep_meta_labeling.yaml

@@ -51,9 +51,9 @@ parameters:
     index_column: 
       value: 'int'
     primary_models: 
-      value: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"]
+      value: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RFC", "StaticMom"]
     meta_labeling_models: 
-      values: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RF"]
+      values: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RFC"]
       distribution: categorical
     own_features:
       values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']]

sweep_primary_models.yaml

@@ -45,7 +45,7 @@ parameters:
     index_column: 
       value: 'int'
     primary_models: 
-      values: [['LR_two_class'], ['SVC'], ['LDA'], ['KNN'], ['CART'], ['MNB'], ['NB'], ['AB'], ['RF'], ['XGB_two_class'], ['LGBM']]
+      values: [['LR_two_class'], ['SVC'], ['LDA'], ['KNN'], ['CART'], ['MNB'], ['NB'], ['AB'], ['RFC'], ['XGB_two_class'], ['LGBM']]
       distribution: categorical
     meta_labeling_models: 
       value: ["LGBM", "LR_two_class"]

training/meta_labeling.py

@@ -3,7 +3,7 @@ from utils.helpers import random_string, equal_except_nan, drop_until_first_vali
 from training.primary_model import train_primary_model
 from feature_selection.feature_selection import select_features
 import pandas as pd
-from models.model_map import default_feature_selector_regression, default_feature_selector_classification
+from models.model_map import get_model_map
 from models.base import Model
 from reporting.types import Reporting
 from typing import Union
@@ -23,7 +23,7 @@ def train_meta_labeling_model(
                             preloaded_models: Union[list[Reporting.Single_Model], None] = None
                         ) -> tuple[pd.Series, pd.Series, pd.DataFrame, list[Reporting.Single_Model]]:
 
-    
+    _, _, _, default_feature_selector_regression, default_feature_selector_classification = get_model_map(model_config)
     discretize = discretize_threeway_threshold(0.33)
     discretized_predictions = input_predictions.apply(discretize)
     meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply(equal_except_nan, axis = 1)