From 31dc847be1b37b66aadbafacd526d8c94384c9a5 Mon Sep 17 00:00:00 2001 From: Daniel Szemerey Date: Fri, 14 Jan 2022 14:29:24 +0100 Subject: [PATCH] Feature(Speed): Python launches faster by conditionally importing models. (#169) * feat: Added optional import of models. * fix: Models weren't wrapped into abstract class, fixed it. * chore: Deleted leftover comments. * fix: Same merge commit as on remote. * fix: System wasn't putting in RF because there was no differentiation between RF as regressor and RF as classificator. * fix(Models): use the XGBoostModel wrapper Co-authored-by: Daniel Szemerey Co-authored-by: Mark Aron Szulyovszky --- config/config.py | 4 +- config/preprocess.py | 3 +- data_loader/process_data.py | 4 +- models/model_map.py | 169 ++++++++++++++++++++++-------------- sweep_ensemble.yaml | 6 +- sweep_meta_labeling.yaml | 4 +- sweep_primary_models.yaml | 2 +- training/meta_labeling.py | 4 +- 8 files changed, 119 insertions(+), 77 deletions(-) diff --git a/config/config.py b/config/config.py index d808792..145c132 100644 --- a/config/config.py +++ b/config/config.py @@ -72,8 +72,8 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]: narrow_format = False, ) - regression_models = ["Lasso", "KNN", "RF"] - classification_models = ["LR_two_class", "LDA", "NB", "RF", "XGB_two_class", "LGBM", "StaticMom"] + regression_models = ["Lasso", "KNN", "RFR"] + classification_models = ["LR_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"] meta_labeling_models = ['LR_two_class', 'LGBM'] ensemble_model = 'Average' diff --git a/config/preprocess.py b/config/preprocess.py index c97fcb2..38d1abb 100644 --- a/config/preprocess.py +++ b/config/preprocess.py @@ -1,7 +1,7 @@ from utils.helpers import flatten from feature_extractors.feature_extractor_presets import presets as feature_extractor_presets -from models.model_map import model_map +from models.model_map import get_model_map from data_loader.collections import data_collections def preprocess_config(model_config:dict, training_config:dict, data_config:dict) -> tuple[dict, dict, dict]: @@ -21,6 +21,7 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict: return data_dict def __preprocess_model_config(model_config:dict, method:str) -> dict: + model_map, _, _, _, _ = get_model_map(model_config) model_config['primary_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in model_config['primary_models']] if len(model_config['meta_labeling_models']) > 0: model_config['meta_labeling_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in model_config['meta_labeling_models']] diff --git a/data_loader/process_data.py b/data_loader/process_data.py index ae1f764..d2bbb5d 100644 --- a/data_loader/process_data.py +++ b/data_loader/process_data.py @@ -3,16 +3,18 @@ from operator import itemgetter from utils.helpers import has_enough_samples_to_train from feature_selection.dim_reduction import reduce_dimensionality -from models.model_map import default_feature_selector_regression, default_feature_selector_classification +from models.model_map import get_model_map from feature_selection.feature_selection import select_features import warnings + def process_data(X:pd.DataFrame, y:pd.Series, configs: dict) -> tuple[pd.DataFrame,pd.DataFrame]: model_config, training_config, data_config = itemgetter('model_config', 'training_config', 'data_config')(configs) + _, _, _, default_feature_selector_regression, default_feature_selector_classification = get_model_map(model_config) original_X = X.copy() diff --git a/models/model_map.py b/models/model_map.py index 98b3aff..46246cd 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -1,72 +1,111 @@ -from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, Ridge -from sklearn.linear_model import LogisticRegression -from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX -from sklearn.tree import DecisionTreeClassifier -from sklearnex.neighbors import KNeighborsRegressor, KNeighborsClassifier -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis -from sklearnex.svm import SVR, SVC -from sklearn.naive_bayes import GaussianNB -from sklearn.neural_network import MLPRegressor, MLPClassifier -from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier -from sklearnex.ensemble import RandomForestClassifier from models.sklearn import SKLearnModel -from models.neural import LightningNeuralNetModel -from models.momentum import StaticMomentumModel -from models.average import StaticAverageModel -from models.naive import StaticNaiveModel -from models.pytorch.neural_nets import MultiLayerPerceptron -from models.xgboost import XGBoostModel -from models.statsmodels import StatsModel -from xgboost import XGBClassifier -import torch.nn.functional as F -from lightgbm import LGBMClassifier -from statsmodels.tsa.api import ExponentialSmoothing - - +from sklearn.ensemble import RandomForestClassifier +from sklearn.ensemble import RandomForestRegressor model_map = { - "regression_models": dict( - LR= SKLearnModel(LinearRegression(n_jobs=-1)), - Lasso= SKLearnModel(Lasso(alpha=100, random_state=1)), - Ridge= SKLearnModel(Ridge(alpha=0.1)), - BayesianRidge= SKLearnModel(BayesianRidge()), - KNN= SKLearnModel(KNeighborsRegressor(n_neighbors=25)), - AB= SKLearnModel(AdaBoostRegressor(random_state=1)), - MLP= SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)), - RF= SKLearnModel(RandomForestRegressor(n_jobs=-1, max_depth=20, random_state=1)), - SVR= SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1)), - StaticNaive= StaticNaiveModel(), - DNN = LightningNeuralNetModel( - MultiLayerPerceptron( - hidden_layers_ratio = [1.0], - probabilities = False, - loss_function = F.mse_loss), - max_epochs=15 - ) - ), - "classification_models": dict( - LR_two_class= SKLearnModel(LogisticRegression(C=10, random_state=1, solver='liblinear', max_iter=1000)), - LR_three_class= SKLearnModel(LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1)), - LDA= SKLearnModel(LinearDiscriminantAnalysis()), - KNN= SKLearnModel(KNeighborsClassifier()), - CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)), - NB= SKLearnModel(GaussianNB()), - AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)), - RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)), - SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True, random_state=1)), - XGB_two_class= XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')), - LGBM = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)), - StaticMom= StaticMomentumModel(allow_short=True), - # ExpSmoothing = SKLearnModel(ExponentialSmoothing(trend='add', seasonal='add', seasonal_periods=30)), - ), - "ensemble_models": dict( - Average= StaticAverageModel(), - ) + "regression_models": dict(), + "classification_models": dict(), + "ensemble_models": dict() } -model_names_classification = list(model_map["classification_models"].keys()) -model_names_regression = list(model_map["regression_models"].keys()) +def get_model_map(config:dict): + + if len(config['primary_models']) > 0 and isinstance(config['primary_models'][0], str): + print("Going to Load models") + combined_list = config['primary_models'] + config['meta_labeling_models'] + [config['ensemble_model']] + for model_name in combined_list: + if model_name == 'LR': + from sklearn.linear_model import LinearRegression + model_map['regression_models']['LR'] = SKLearnModel(LinearRegression(n_jobs=-1)) + elif model_name == 'Lasso': + from sklearn.linear_model import Lasso + model_map['regression_models']['Lasso'] = SKLearnModel(Lasso(alpha=100, random_state=1)) + elif model_name == 'Ridge': + from sklearn.linear_model import Ridge + model_map['regression_models']['Ridge'] = SKLearnModel(Ridge(alpha=0.1)) + elif model_name == 'BayesianRidge': + from sklearn.linear_model import BayesianRidge + model_map['regression_models']['BayesianRidge'] = SKLearnModel(BayesianRidge()) + elif model_name == 'KNN': + from sklearnex.neighbors import KNeighborsRegressor + model_map['regression_models']['KNN'] = SKLearnModel(KNeighborsRegressor(n_neighbors=25)) + elif model_name == 'AB': + from sklearn.ensemble import AdaBoostRegressor + model_map['regression_models']['AB'] = SKLearnModel(AdaBoostRegressor(random_state=1)) + elif model_name == 'MLP': + from sklearn.neural_network import MLPRegressor + model_map['regression_models']['MLP'] = SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)) + elif model_name == 'RFR': + # from sklearn.ensemble import RandomForestRegressor + model_map['regression_models']['RFR'] = SKLearnModel(RandomForestRegressor(n_jobs=-1, max_depth=20, random_state=1)) + elif model_name == 'SVR': + from sklearnex.svm import SVR + model_map['regression_models']['SVR'] = SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1)) + elif model_name == 'StaticNaive': + from models.naive import StaticNaiveModel + model_map['regression_models']['StaticNaive'] = StaticNaiveModel() + elif model_name == 'DNN': + from models.neural import LightningNeuralNetModel + from models.pytorch.neural_nets import MultiLayerPerceptron + import torch.nn.functional as F + model_map['regression_models']['DNN'] = LightningNeuralNetModel( + MultiLayerPerceptron( + hidden_layers_ratio = [1.0], + probabilities = False, + loss_function = F.mse_loss), + max_epochs=15 + ) + + + elif model_name == 'LR_two_class': + from sklearn.linear_model import LogisticRegression + model_map['classification_models']['LR_two_class'] = SKLearnModel(LogisticRegression(C=10, random_state=1, solver='liblinear', max_iter=1000)) + elif model_name == 'LR_three_class': + from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX + model_map['classification_models']['LR_three_class'] = SKLearnModel(LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1)) + elif model_name == 'LDA': + from sklearn.discriminant_analysis import LinearDiscriminantAnalysis + model_map['classification_models']['LDA'] = SKLearnModel(LinearDiscriminantAnalysis()) + elif model_name == 'KNN': + from sklearn.neighbors import KNeighborsClassifier + model_map['classification_models']['KNN'] = SKLearnModel(KNeighborsClassifier()) + elif model_name == 'CART': + from sklearn.tree import DecisionTreeClassifier + model_map['classification_models']['CART'] = SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)) + elif model_name == 'NB': + from sklearn.naive_bayes import GaussianNB + model_map['classification_models']['NB'] = SKLearnModel(GaussianNB()) + elif model_name == 'AB': + from sklearn.ensemble import AdaBoostClassifier + model_map['classification_models']['AB'] = SKLearnModel(AdaBoostClassifier(n_estimators=15)) + elif model_name == 'RFC': + # from sklearn.ensemble import RandomForestClassifier + model_map['classification_models']['RFC'] = SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)) + elif model_name == 'SVC': + from sklearn.svm import SVC + model_map['classification_models']['SVC'] = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True, random_state=1)) + elif model_name == 'XGB_two_class': + from xgboost import XGBClassifier + from models.xgboost import XGBoostModel + model_map['classification_models']['XGB_two_class'] = XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')) + elif model_name == 'LGBM': + from lightgbm import LGBMClassifier + model_map['classification_models']['LGBM'] = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)) + elif model_name == 'StaticMom': + from models.momentum import StaticMomentumModel + model_map['classification_models']['StaticMom'] = StaticMomentumModel(allow_short=True) + elif model_name == 'Average': + from models.average import StaticAverageModel + model_map['ensemble_models']['Average'] = StaticAverageModel() + + + model_names_classification = list(model_map["classification_models"].keys()) + model_names_regression = list(model_map["regression_models"].keys()) -default_feature_selector_regression = model_map['regression_models']['RF'] -default_feature_selector_classification = model_map['classification_models']['RF'] + + default_feature_selector_classification = SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)) + default_feature_selector_regression = SKLearnModel(RandomForestRegressor(n_jobs=-1, max_depth=20, random_state=1)) + + + return model_map, model_names_classification, model_names_regression, default_feature_selector_regression, default_feature_selector_classification diff --git a/sweep_ensemble.yaml b/sweep_ensemble.yaml index 5bb9147..ccdaaf6 100644 --- a/sweep_ensemble.yaml +++ b/sweep_ensemble.yaml @@ -47,9 +47,9 @@ parameters: primary_models: distribution: categorical values: - - ["LDA", "LR_two_class", "KNN", "SVC", "CART", "NB", "AB", "RF", "XGB_two_class", "LGBM", "StaticMom"] - - ["LR_two_class", "LDA", "NB", "RF", "XGB_two_class", "LGBM", "StaticMom"] - - ["LR_two_class", "LDA", "LGBM", "RF", "XGB_two_class"] + - ["LDA", "LR_two_class", "KNN", "SVC", "CART", "NB", "AB", "RFC", "XGB_two_class", "LGBM", "StaticMom"] + - ["LR_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"] + - ["LR_two_class", "LDA", "LGBM", "RFC", "XGB_two_class"] meta_labeling_models: value: ["LGBM", "LR_two_class"] own_features: diff --git a/sweep_meta_labeling.yaml b/sweep_meta_labeling.yaml index 42d03f5..8f1c35f 100644 --- a/sweep_meta_labeling.yaml +++ b/sweep_meta_labeling.yaml @@ -51,9 +51,9 @@ parameters: index_column: value: 'int' primary_models: - value: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"] + value: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RFC", "StaticMom"] meta_labeling_models: - values: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RF"] + values: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RFC"] distribution: categorical own_features: values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']] diff --git a/sweep_primary_models.yaml b/sweep_primary_models.yaml index 4b2048c..37fa414 100644 --- a/sweep_primary_models.yaml +++ b/sweep_primary_models.yaml @@ -45,7 +45,7 @@ parameters: index_column: value: 'int' primary_models: - values: [['LR_two_class'], ['SVC'], ['LDA'], ['KNN'], ['CART'], ['MNB'], ['NB'], ['AB'], ['RF'], ['XGB_two_class'], ['LGBM']] + values: [['LR_two_class'], ['SVC'], ['LDA'], ['KNN'], ['CART'], ['MNB'], ['NB'], ['AB'], ['RFC'], ['XGB_two_class'], ['LGBM']] distribution: categorical meta_labeling_models: value: ["LGBM", "LR_two_class"] diff --git a/training/meta_labeling.py b/training/meta_labeling.py index 88fbd91..ffeed84 100644 --- a/training/meta_labeling.py +++ b/training/meta_labeling.py @@ -3,7 +3,7 @@ from utils.helpers import random_string, equal_except_nan, drop_until_first_vali from training.primary_model import train_primary_model from feature_selection.feature_selection import select_features import pandas as pd -from models.model_map import default_feature_selector_regression, default_feature_selector_classification +from models.model_map import get_model_map from models.base import Model from reporting.types import Reporting from typing import Union @@ -23,7 +23,7 @@ def train_meta_labeling_model( preloaded_models: Union[list[Reporting.Single_Model], None] = None ) -> tuple[pd.Series, pd.Series, pd.DataFrame, list[Reporting.Single_Model]]: - + _, _, _, default_feature_selector_regression, default_feature_selector_classification = get_model_map(model_config) discretize = discretize_threeway_threshold(0.33) discretized_predictions = input_predictions.apply(discretize) meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply(equal_except_nan, axis = 1)