refactor(Naming): use new convention, added Ensemble model parameter back, support multiple Meta-Labeling models (#132)

* refactor(Naming): use `primary_models` & `meta_labeling_models`

* refactor(Naming): using primary * meta_labeling across config and in pipeline

* feat(Pipeline): added back Ensemble models

* fix(Pipeline): compiler error

* fix(Config): typo

* chore(Pipeline): removed unused averaging step

* revert the changes in discretizing

* chore(Pipeline): remove sharpe improvement logging

* fix(Pipeline): ensemble predictions should be a pd.Series instead of a DataFrame

* fix(Pipeline): discard unnecessary ensemble_probabilities

* fix(Pipeline): fixes regarding various meta-labeling ensemble bugs

* fix(Reporting): use the new naming convention

* fix(Reporting): use the right variable

* feat(Sweep): new sweep for ensemble models

* fix(Sweep): config reference

* fix(Config): simplified dev config

* fix(Models): use the faster LR model

* fix(Models): use LGBM in the meta-labeling model for speed

* fix(Selection): always use the first model for feature selection, commented out caching from select_features() as it's close to redundant in terms of speed

config/config.py

@@ -1,15 +1,14 @@
 
-def get_default_level_1_daily_config() -> tuple[dict, dict, dict]:
+def get_dev_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
-        meta_labeling_lvl_1 = False,
+        primary_models_meta_labeling = False,
         dimensionality_reduction = True,
         n_features_to_select = 30,
-        dynamic_feature_selection = True,
-        expanding_window_level1 = False,
-        expanding_window_level2 = False,
-        sliding_window_size_level1 = 380,
-        sliding_window_size_level2 = 1,
+        expanding_window_primary = False,
+        expanding_window_meta_labeling = False,
+        sliding_window_size_primary = 380,
+        sliding_window_size_meta_labeling = 1,
         retrain_every = 20,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
     )
@@ -26,76 +25,33 @@ def get_default_level_1_daily_config() -> tuple[dict, dict, dict]:
         exogenous_features = ['standard_scaling'],
         index_column= 'int',
         method= 'classification',
-        no_of_classes= 'three-balanced',
+        no_of_classes= 'two',
         narrow_format = False,
     )
 
     regression_models = ["Lasso"]
-    classification_models = ["KNN"]
+    classification_models = ["LR_two_class"]
 
     model_config = dict(
-        level_1_models = regression_models if data_config['method'] == 'regression' else classification_models,
-        level_2_model = None
+        primary_models = regression_models if data_config['method'] == 'regression' else classification_models,
+        meta_labeling_models = [],
+        ensemble_model = None
     )
     
     return model_config, training_config, data_config
 
 
-def get_default_level_2_hourly_config() -> tuple[dict, dict, dict]:
+
+def get_default_ensemble_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
-        meta_labeling_lvl_1 = True,
+        primary_models_meta_labeling = True,
         dimensionality_reduction = True,
         n_features_to_select = 30,
-        dynamic_feature_selection = True,
-        expanding_window_level1 = True,
-        expanding_window_level2 = False,
-        sliding_window_size_level1 = 2480,
-        sliding_window_size_level2 = 1,
-        retrain_every = 100,
-        scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
-    )
-
-    data_config = dict(
-        assets = ['hourly_crypto'],
-        other_assets = [],
-        exogenous_data = [],
-        load_non_target_asset= True,
-        log_returns= True,
-        forecasting_horizon = 1,
-        own_features = ['level_2', 'date_days', 'lags_up_to_5'],
-        other_features = ['level_2'],
-        exogenous_features = ['standard_scaling'],
-        index_column= 'int',
-        method= 'classification',
-        no_of_classes= 'three-balanced',
-        narrow_format = False,
-    )
-
-    regression_models = ["Lasso", "KNN", "RF"]
-    regression_ensemble_model = 'KNN'
-    classification_models = ["LDA", "KNN", "CART", "RF", "StaticMom"]
-    classification_ensemble_model = 'Ensemble_Average'
-
-    model_config = dict(
-        level_1_models = regression_models if data_config['method'] == 'regression' else classification_models,
-        level_2_model = regression_ensemble_model if data_config['method'] == 'regression' else classification_ensemble_model
-    )
-    
-    return model_config, training_config, data_config
-
-
-def get_default_level_2_daily_config() -> tuple[dict, dict, dict]:
-  
-    training_config = dict(
-        meta_labeling_lvl_1 = True,
-        dimensionality_reduction = True,
-        n_features_to_select = 30,
-        dynamic_feature_selection = True,
-        expanding_window_level1 = False,
-        expanding_window_level2 = True,
-        sliding_window_size_level1 = 380,
-        sliding_window_size_level2 = 240,
+        expanding_window_primary = False,
+        expanding_window_meta_labeling = True,
+        sliding_window_size_primary = 380,
+        sliding_window_size_meta_labeling = 240,
         retrain_every = 20,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
     )
@@ -117,13 +73,14 @@ def get_default_level_2_daily_config() -> tuple[dict, dict, dict]:
     )
 
     regression_models = ["Lasso", "KNN", "RF"]
-    regression_ensemble_model = 'KNN'
-    classification_models = ['SVC', 'LDA', 'KNN', 'CART', 'NB', 'AB', 'RF', 'XGB_two_class', 'LGBM', 'StaticMom']
-    classification_ensemble_model = 'LDA'
+    classification_models = ['LR_two_class', 'SVC', 'KNN', 'CART', 'NB', 'AB', 'RF', 'XGB_two_class', 'LGBM', 'StaticMom']
+    meta_labeling_models = ['LR_two_class', 'LGBM']
+    ensemble_model = 'Average'
 
     model_config = dict(
-        level_1_models = regression_models if data_config['method'] == 'regression' else classification_models,
-        level_2_model = regression_ensemble_model if data_config['method'] == 'regression' else classification_ensemble_model
+        primary_models = regression_models if data_config['method'] == 'regression' else classification_models,
+        meta_labeling_models = meta_labeling_models,
+        ensemble_model = ensemble_model
     )
     
     return model_config, training_config, data_config

config/preprocess.py

@@ -21,9 +21,11 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
     return data_dict
 
 def __preprocess_model_config(model_config:dict, method:str) -> dict:
-    model_config['level_1_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in  model_config['level_1_models']]
-    if model_config['level_2_model'] is not None:
-        model_config['level_2_model'] = (model_config['level_2_model'], model_map[method + '_models'][model_config['level_2_model']])
+    model_config['primary_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in  model_config['primary_models']]
+    if len(model_config['meta_labeling_models']) > 0:
+        model_config['meta_labeling_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in  model_config['meta_labeling_models']]
+    if model_config['ensemble_model'] is not None:
+        model_config['ensemble_model'] = (model_config['ensemble_model'], model_map['ensemble_models'][model_config['ensemble_model']])
 
     return model_config
 
@@ -39,15 +41,15 @@ def __preprocess_data_collections_config(data_dict: dict) -> dict:
 def validate_config(model_config:dict, training_config:dict, data_config:dict):
     # We need to make sure there's only one output from the pipeline
     # If level-2 model is there, we need more than one level-1 models to train
-    if model_config["level_2_model"] is not None: assert len(model_config["level_1_models"]) > 0
+    if len(model_config["meta_labeling_models"]) > 1: assert len(model_config["primary_models"]) > 0
     # If there's no level-2 model, we need to have only one level-1 model
-    if model_config["level_2_model"] is None: assert len(model_config["level_1_models"]) == 1
+    if len(model_config["meta_labeling_models"]) == 0: assert len(model_config["primary_models"]) == 1
 
 def get_model_name(model_config:dict) -> str:
-    if model_config["level_2_model"] is not None:
-        return model_config["level_2_model"][0]
-    elif len(model_config["level_1_models"]) == 1:
-        return model_config["level_1_models"][0][0]
+    if len(model_config["meta_labeling_models"]) > 0:
+        return model_config["meta_labeling_models"][0][0]
+    elif len(model_config["primary_models"]) == 1:
+        return model_config["primary_models"][0][0]
     else:
         raise Exception("No model name found")
 

exploration.ipynb

@@ -25,13 +25,13 @@
    "source": [
     "import pandas as pd\n",
     "import pandas_ta as ta\n",
-    "from config.config import get_default_level_2_daily_config\n",
+    "from config.config import get_default_ensemble_config\n",
     "from config.preprocess import preprocess_config\n",
     "from data_loader.load_data import load_data\n",
     "import seaborn as sns\n",
     "import numpy as np\n",
     "\n",
-    "model_config, training_config, data_config = get_default_level_2_daily_config()\n",
+    "model_config, training_config, data_config = get_default_ensemble_config()\n",
     "model_config, training_config, data_config = preprocess_config(model_config, training_config, data_config)\n",
     "\n",
     "data_config['target_asset'] = data_config['assets'][0]\n",

feature_selection/feature_selection.py

@@ -5,20 +5,20 @@ from models.base import Model
 from models.sklearn import SKLearnModel
 from utils.scaler import get_scaler
 from utils.types import ScalerTypes
-from utils.hashing import hash_df, hash_series
-from diskcache import Cache
-cache = Cache(".cachedir/feature_selection")
+# from utils.hashing import hash_df, hash_series
+# from diskcache import Cache
+# cache = Cache(".cachedir/feature_selection")
 
-def select_features(**kwargs) -> pd.DataFrame:
-    hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling'] + str(kwargs['dynamic_feature_selection'])
-    if hashed in cache:
-        return cache.get(hashed)
-    else:
-        return_value = __select_features(**kwargs)
-        cache[hashed] = return_value
-        return return_value
+# def select_features(**kwargs) -> pd.DataFrame:
+#     hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling']
+#     if hashed in cache:
+#         return cache.get(hashed)
+#     else:
+#         return_value = __select_features(**kwargs)
+#         cache[hashed] = return_value
+#         return return_value
 
-def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel, scaling: ScalerTypes, dynamic_feature_selection: bool, data_config_hash: str) -> pd.DataFrame:
+def select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel, scaling: ScalerTypes) -> pd.DataFrame:
     ''' Select features using RFECV, returns a pd.DataFrame (X) with only the selected features.'''
     if model.model_type != 'ml': return X
 
@@ -34,7 +34,7 @@ def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to
         feat_selector_model = backup_model.model
 
 #     selector = RFECV(feat_selector_model, cv = cv, step=5, min_features_to_select=min_features_to_select)
-    step = 0.05 if dynamic_feature_selection else 5
+    step = 0.05
     selector = RFE(feat_selector_model, n_features_to_select= n_features_to_select, step=step)
     selector = selector.fit(X_scaled, y)
     print("Kept %d features out of %d" % (selector.n_features_, X_scaled.shape[1]))

models/lightgbm.py

@@ -1,2 +0,0 @@
-import lightgbm as lgb
-

models/model_map.py

@@ -1,5 +1,6 @@
 from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, Ridge
-from sklearnex.linear_model import LogisticRegression
+from sklearn.linear_model import LogisticRegression
+from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX
 from sklearn.tree import DecisionTreeClassifier
 from sklearnex.neighbors import KNeighborsRegressor, KNeighborsClassifier
 from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
@@ -44,7 +45,8 @@ model_map = {
         ) 
     ),
     "classification_models": dict(
-        LR= SKLearnModel(LogisticRegression(C=10, random_state=1, max_iter=1000, n_jobs=-1)),
+        LR_two_class= SKLearnModel(LogisticRegression(C=10, random_state=1, solver='liblinear', max_iter=1000)),
+        LR_three_class= SKLearnModel(LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1)),
         LDA= SKLearnModel(LinearDiscriminantAnalysis()),
         KNN= SKLearnModel(KNeighborsClassifier()),
         CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)),
@@ -55,9 +57,11 @@ model_map = {
         XGB_two_class= XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')),
         LGBM = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)),
         StaticMom= StaticMomentumModel(allow_short=True),
-        Ensemble_Average= StaticAverageModel(),
         # ExpSmoothing = SKLearnModel(ExponentialSmoothing(trend='add', seasonal='add', seasonal_periods=30)),
     ),
+    "ensemble_models": dict(
+        Average= StaticAverageModel(),
+    )
 }
 
 model_names_classification = list(model_map["classification_models"].keys())

reporting/reporting.py

@@ -5,36 +5,36 @@ from utils.helpers import weighted_average
 
 def report_results(results:pd.DataFrame, all_predictions:pd.DataFrame, model_config:dict, wandb, sweep: bool, project_name:str):
 
-    level1_columns = results[[column for column in results.columns if 'lvl1' in column]]
-    level2_columns = results[[column for column in results.columns if 'lvl2' in column]]
+    primary_results = results[[column for column in results.columns if 'primary' in column]]
+    ensemble_results = results[[column for column in results.columns if 'ensemble' in column]]
 
     # Only send the results of the final model to wandb
-    results_to_send = level2_columns if level2_columns.shape[1] > 0 else level1_columns
+    results_to_send = ensemble_results if ensemble_results.shape[1] > 0 else primary_results
     send_report_to_wandb(results_to_send, wandb, project_name, get_model_name(model_config))
     results.to_csv('output/results.csv')
 
-    level1_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl1' in column]]
-    level2_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl2' in column]]
-    predictions_to_save = level2_predictions if level2_predictions.shape[1] > 0 else level1_predictions
+    primary_weights = all_predictions[[column for column in all_predictions.columns if 'primary' in column]]
+    ensemble_weights = all_predictions[[column for column in all_predictions.columns if 'ensemble' in column]]
+    predictions_to_save = ensemble_weights if ensemble_weights.shape[1] > 0 else primary_weights
     predictions_to_save.to_csv('output/predictions.csv')
 
     print("\n--------\n")
     all_avg_results = weighted_average(results, 'no_of_samples')
-    lvl1_avg_results = weighted_average(level1_columns, 'no_of_samples')
-    lvl2_avg_results = weighted_average(level2_columns, 'no_of_samples')
+    primary_avg_results = weighted_average(primary_results, 'no_of_samples')
+    ensemble_avg_results = weighted_average(ensemble_results, 'no_of_samples')
 
     print("Benchmark buy-and-hold sharpe: ", round(all_avg_results.loc['benchmark_sharpe'], 3))
 
-    print("Level-1: Number of samples evaluated: ", level1_columns.loc['no_of_samples'].sum())
-    print("Mean Sharpe ratio for Level-1 models: ", round(lvl1_avg_results.loc['sharpe'], 3))
-    print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(lvl1_avg_results.loc['prob_sharpe'].mean(), 3))
+    print("Level-1: Number of samples evaluated: ", primary_results.loc['no_of_samples'].sum())
+    print("Mean Sharpe ratio for Level-1 models: ", round(primary_avg_results.loc['sharpe'], 3))
+    print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(primary_avg_results.loc['prob_sharpe'].mean(), 3))
 
-    if model_config['level_2_model'] is not None: 
-        print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum())
-        print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(lvl2_avg_results.loc['sharpe'].mean(), 3))
-        print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(lvl2_avg_results.loc['prob_sharpe'].mean(), 3))
+    if len(model_config['meta_labeling_models']) > 0: 
+        print("Level-2 (Ensemble): Number of samples evaluated: ", ensemble_results.loc['no_of_samples'].sum())
+        print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_avg_results.loc['sharpe'].mean(), 3))
+        print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_avg_results.loc['prob_sharpe'].mean(), 3))
 
-    lvl2_avg_results.to_csv('output/results_level2.csv')
+    ensemble_avg_results.to_csv('output/results_level2.csv')
 
     if sweep:
         if wandb.run is not None:

run_feature_selection.py

@@ -1,12 +1,12 @@
 # #%%
 # import pandas as pd
 # import pandas_ta as ta
-# from config.config import get_default_level_2_daily_config
+# from config.config import get_default_ensemble_config
 # from config.preprocess import preprocess_config
 # from data_loader.load_data import load_data
 
 # # %%
-# model_config, training_config, data_config = get_default_level_2_daily_config()
+# model_config, training_config, data_config = get_default_ensemble_config()
 # model_config, training_config, data_config = preprocess_config(model_config, training_config, data_config)
 
 # data_config['target_asset'] = data_config['assets'][0]

run_model_dev.py

@@ -1,5 +1,5 @@
 from run_pipeline import run_pipeline
-from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config
+from config.config import get_dev_config
 
 
-run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_level_1_daily_config)
+run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_dev_config)

run_pipeline.py

@@ -1,16 +1,15 @@
 from config.hashing import hash_data_config
 from data_loader.load_data import load_data
 import pandas as pd
-from training.training import run_single_asset_trainig
-from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_with_wandb
+from training.primary_model import train_primary_model
+from reporting.wandb import launch_wandb, register_config_with_wandb
 from models.model_map import default_feature_selector_regression, default_feature_selector_classification
 from utils.helpers import get_first_valid_return_index
-from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config
+from config.config import get_default_ensemble_config
 from config.preprocess import validate_config, preprocess_config
 from feature_selection.feature_selection import select_features
 from feature_selection.dim_reduction import reduce_dimensionality
-from training.meta_labeling import run_meta_labeling_training
-from training.averaged import average_and_evaluate_predictions
+from training.meta_labeling import train_meta_labeling_model
 from reporting.reporting import report_results
 from typing import Callable, Optional
 import ray
@@ -52,7 +51,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
         original_X = X.copy()
         first_valid_index = get_first_valid_return_index(X.iloc[:,0])
         samples_to_train = len(y) - first_valid_index
-        if samples_to_train < training_config['sliding_window_size_level1'] * 3:
+        if samples_to_train < training_config['sliding_window_size_primary'] * 3:
             print("Not enough samples to train")
             continue
 
@@ -67,23 +66,24 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
         print("Feature Selection started")
         # TODO: this needs to be done per model!
         backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
-        X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], dynamic_feature_selection = training_config['dynamic_feature_selection'], data_config_hash = hash_data_config(data_params))
+        X = select_features(X = X, y = y, model = model_config['primary_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'])
 
-        # 3. Train Level-1 models
-        current_result, current_predictions, current_probabilities, all_models_for_single_asset = run_single_asset_trainig(
+        # 3. Train Primary models
+        current_result, current_predictions, current_probabilities, all_models_for_single_asset = train_primary_model(
             ticker_to_predict = asset[1],
             original_X = original_X,
             X = X,
             y = y,
             target_returns = target_returns,
-            models = model_config['level_1_models'],
+            models = model_config['primary_models'],
             method = data_config['method'],
-            expanding_window = training_config['expanding_window_level1'],
-            sliding_window_size = training_config['sliding_window_size_level1'],
+            expanding_window = training_config['expanding_window_primary'],
+            sliding_window_size = training_config['sliding_window_size_primary'],
             retrain_every =  training_config['retrain_every'],
             scaler =  training_config['scaler'],
             no_of_classes = data_config['no_of_classes'],
-            level = 1
+            level = 'primary',
+            print_results= True
         )
         
         all_models_for_all_assets[asset[1]] = dict(
@@ -91,25 +91,24 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
             models=all_models_for_single_asset
         )
         
-        # 4. Train a Meta-Labeling model for each Level-1 model and replace its predictions with the meta-labeling predictions
-        if training_config['meta_labeling_lvl_1'] == True:
+        # 4. Train a Meta-Labeling model for each Primary model and replace their predictions with the meta-labeling predictions
+        if training_config['primary_models_meta_labeling'] == True:
             for model_name in current_result.columns:
-                lvl1_model_predictions = current_predictions[model_name]
-                prev_sharpe = current_result[model_name]['sharpe']
-                lvl1_meta_result, lvl1_meta_preds, lvl1_meta_probabilities, meta_labeling_models = run_meta_labeling_training(
+                primary_model_predictions = current_predictions[model_name]
+                primary_meta_result, primary_meta_preds, primary_meta_probabilities, meta_labeling_models = train_meta_labeling_model(
                     target_asset=asset[1],
                     X_pca = X_pca,
-                    input_predictions= lvl1_model_predictions,
+                    input_predictions= primary_model_predictions,
                     y = y,
                     target_returns = target_returns,
+                    models = model_config['meta_labeling_models'],
                     data_config= data_config,
                     model_config= model_config,
-                    training_config= training_config
+                    training_config= training_config,
+                    model_suffix = 'meta'
                 )
-                new_sharpe = lvl1_meta_result['sharpe']
-                print("Improvement in sharpe for the meta model: ", ((new_sharpe / prev_sharpe) - 1) * 100, "%")
-                current_result[model_name] = lvl1_meta_result
-                current_predictions[model_name] = lvl1_meta_preds
+                current_result[model_name] = primary_meta_result
+                current_predictions[model_name] = primary_meta_preds
 
                 all_models_for_all_assets[asset[1]][model_name] = meta_labeling_models
         
@@ -118,26 +117,46 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
         all_predictions = pd.concat([all_predictions, current_predictions], axis=1).fillna(0.)
         all_probabilities = pd.concat([all_probabilities, current_probabilities], axis=1).fillna(0.)
 
-        if model_config['level_2_model'] is not None: 
+        # 5. Ensemble primary model predictions (If Ensemble model is present)
+        if model_config['ensemble_model'] is not None:
 
-            # 3. Average the Level-1 model predictions
-            averaged_predictions, averaged_results = average_and_evaluate_predictions(current_predictions, y, target_returns, data_config)
-
-            # 3. Train a Meta-labeling model on the averaged level-1 model predictions
-            meta_result, avg_predictions_with_sizing, meta_probabilities, meta_labeling_models = run_meta_labeling_training(
-                target_asset=asset[1],
-                X_pca = X_pca,
-                input_predictions= averaged_predictions,
+            ensemble_result, ensemble_predictions, _, _ = train_primary_model(
+                ticker_to_predict = asset[1],
+                original_X = current_predictions,
+                X = current_predictions,
                 y = y,
                 target_returns = target_returns,
-                data_config= data_config,
-                model_config= model_config,
-                training_config= training_config
+                models = [model_config['ensemble_model']],
+                method = data_config['method'],
+                expanding_window = False,
+                sliding_window_size = 1,
+                retrain_every = training_config['retrain_every'],
+                scaler = training_config['scaler'],
+                no_of_classes = data_config['no_of_classes'],
+                level = 'ensemble',
+                print_results= True,
             )
+            ensemble_result, ensemble_predictions = ensemble_result.iloc[:,0], ensemble_predictions.iloc[:,0]
 
-            results = pd.concat([results, meta_result], axis=1)
-            all_predictions = pd.concat([all_predictions, avg_predictions_with_sizing], axis=1)
-            all_probabilities = pd.concat([all_probabilities, meta_probabilities], axis=1).fillna(0.)
+            if len(model_config['meta_labeling_models']) > 0: 
+
+                # 3. Train a Meta-labeling model on the averaged level-1 model predictions
+                ensemble_meta_result, ensemble_meta_predictions, ensemble_meta_probabilities, ensemble_meta_labeling_models = train_meta_labeling_model(
+                    target_asset=asset[1],
+                    X_pca = X_pca,
+                    input_predictions= ensemble_predictions,
+                    y = y,
+                    target_returns = target_returns,
+                    models = model_config['meta_labeling_models'],
+                    data_config= data_config,
+                    model_config= model_config,
+                    training_config= training_config,
+                    model_suffix = 'ensemble'
+                )
+
+                results = pd.concat([results, ensemble_meta_result], axis=1)
+                all_predictions = pd.concat([all_predictions, ensemble_meta_predictions], axis=1)
+                all_probabilities = pd.concat([all_probabilities, ensemble_meta_probabilities], axis=1).fillna(0.)
         
     return results, all_predictions, all_probabilities
     
@@ -145,4 +164,4 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
 
     
 if __name__ == '__main__':
-    run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_level_2_daily_config)
+    run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_ensemble_config)

run_sweep.py

@@ -1,4 +1,4 @@
 from run_pipeline import run_pipeline
-from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config
+from config.config import get_default_ensemble_config
 
-run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True, get_config= get_default_level_2_daily_config)
+run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True, get_config= get_default_ensemble_config)

sweep_ensemble.yaml

@@ -6,10 +6,7 @@ metric:
   goal: maximize
   name: sharpe
 parameters:
-    dynamic_feature_selection:
-      values: [True, False]
-      distribution: 'categorical'
-    meta_labeling_lvl_1:
+    primary_models_meta_labeling:
       value: True
     assets:
       value: ['daily_crypto']
@@ -17,18 +14,16 @@ parameters:
       value: ['daily_etf']
     exogenous_data:
       value: ['daily_glassnode']
-    expanding_window_level1:
-      values: [False, True]
-      distribution: 'categorical'
-    expanding_window_level2:
+    expanding_window_primary:
       value: True
-    sliding_window_size_level1: 
+    expanding_window_meta_labeling:
+      value: True
+    sliding_window_size_primary: 
       value: 380
-    sliding_window_size_level2: 
+    sliding_window_size_meta_labeling: 
       value: 380
     n_features_to_select:
-      values: [30, 50, 70, 80]
-      distribution: 'categorical'
+      value: 50
     dimensionality_reduction:
       value: True
     retrain_every: 
@@ -47,10 +42,10 @@ parameters:
       value: True
     index_column: 
       value: 'int'
-    level_1_models: 
-      value: ["LDA", "KNN", "SVC", "CART", "NB", "AB", "RF", "XGB_two_class", "StaticMom"]
-    level_2_model: 
-      values: ["LDA", "XGB_two_class"]
+    primary_models: 
+      value: ["LDA", "KNN", "SVC", "CART", "NB", "AB", "RF", "XGB_two_class", "LGBM", "StaticMom"]
+    meta_labeling_models: 
+      values: [["LDA"], ["XGB_two_class"], ["LR_two_class"], ["LGBM"], ["LGBM", "LR_two_class"], ["XGB_two_class", "LDA"], ["XGB_two_class", "LR_two_class"]]
       distribution: categorical
     own_features:
       value: ['date_days', 'level_2', 'lags_up_to_5']

sweep_meta_labeling.yaml

@@ -6,7 +6,7 @@ metric:
   goal: maximize
   name: sharpe
 parameters:
-    meta_labeling_lvl_1:
+    primary_models_meta_labeling:
       value: True
     assets:
       value: ['daily_crypto']
@@ -14,10 +14,10 @@ parameters:
       value: ['daily_etf']
     exogenous_data:
       value: ['daily_glassnode']
-    expanding_window_level1:
+    expanding_window_primary:
       values: [True, False]
       distribution: categorical
-    expanding_window_level2:
+    expanding_window_meta_labeling:
       values: [True, False]
       distribution: categorical
     n_features_to_select:
@@ -25,10 +25,10 @@ parameters:
       distribution: categorical
     dimensionality_reduction:
       value: True
-    sliding_window_size_level1: 
+    sliding_window_size_primary: 
       values: [180, 280, 380]
       distribution: categorical
-    sliding_window_size_level2: 
+    sliding_window_size_meta_labeling: 
       values: [180, 280, 380]
       distribution: categorical
     retrain_every: 
@@ -50,10 +50,10 @@ parameters:
       value: True
     index_column: 
       value: 'int'
-    level_1_models: 
-      value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"]
-    level_2_model: 
-      values: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "Ensemble_Average"]
+    primary_models: 
+      value: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"]
+    meta_labeling_models: 
+      values: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RF"]
       distribution: categorical
     own_features:
       values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']]

sweep_primary_models.yaml

@@ -6,7 +6,7 @@ metric:
   goal: maximize
   name: sharpe
 parameters:
-    meta_labeling_lvl_1:
+    primary_models_meta_labeling:
       value: True
     assets:
       value: ['daily_crypto']
@@ -14,20 +14,20 @@ parameters:
       value: ['daily_etf']
     exogenous_data:
       value: ['daily_glassnode']
-    expanding_window_level1:
+    expanding_window_primary:
       values: [True, False]
       distribution: categorical
-    expanding_window_level2:
+    expanding_window_meta_labeling:
       value: False
     n_features_to_select:
       values: [10, 20, 30]
       distribution: categorical
     dimensionality_reduction:
       value: True
-    sliding_window_size_level1: 
+    sliding_window_size_primary: 
       values: [180, 280, 380, 480, 580]
       distribution: categorical
-    sliding_window_size_level2: 
+    sliding_window_size_meta_labeling: 
       value: 1
     retrain_every: 
       values: [10, 20, 30]
@@ -48,11 +48,11 @@ parameters:
       value: True
     index_column: 
       value: 'int'
-    level_1_models: 
-      values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"]]
+    primary_models: 
+      values: [["LR_two_class"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"]]
       distribution: categorical
-    level_2_model: 
-      value: None
+    meta_labeling_models: 
+      value: []
     own_features:
       values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']]
       distribution: categorical

tests/test_evaluation.py

@@ -95,6 +95,7 @@ def test_evaluation():
         y_true=y,
         method='classification',
         no_of_classes='two',
+        print_results = False,
         discretize=True
     )
 

training/averaged.py

@@ -1,25 +0,0 @@
-
-import pandas as pd
-from utils.evaluate import evaluate_predictions
-
-def average_and_evaluate_predictions(predictions: pd.DataFrame, y: pd.Series, target_returns: pd.Series, data_config: dict) -> tuple[pd.Series, pd.DataFrame]:
-    averaged_predictions = predictions.mean(axis = 1)
-    non_discretized_result = evaluate_predictions(
-        model_name = 'Averaged - Non-discrete',
-        target_returns = target_returns,
-        y_pred = averaged_predictions,
-        y_true = y,
-        method = 'classification',
-        no_of_classes = data_config['no_of_classes'],
-        discretize=False
-    )
-    discretized_result = evaluate_predictions(
-        model_name = 'Averaged - Discrete',
-        target_returns = target_returns,
-        y_pred = averaged_predictions,
-        y_true = y,
-        method = 'classification',
-        no_of_classes = data_config['no_of_classes'],
-        discretize=True
-    )
-    return averaged_predictions, pd.concat([non_discretized_result, discretized_result], axis = 1)

training/meta_labeling.py

@@ -1,20 +1,23 @@
 from utils.evaluate import discretize_threeway_threshold, evaluate_predictions
 from utils.helpers import random_string, equal_except_nan, drop_until_first_valid_index
-from training.training import run_single_asset_trainig
+from training.primary_model import train_primary_model
 from feature_selection.feature_selection import select_features
 import pandas as pd
 from models.model_map import default_feature_selector_regression, default_feature_selector_classification
+from models.base import Model
 
 
-def run_meta_labeling_training(
+def train_meta_labeling_model(
                             target_asset: str,
                             X_pca: pd.DataFrame,
                             input_predictions: pd.Series,
                             y: pd.Series,
                             target_returns: pd.Series,
+                            models: list[tuple[str, Model]],
                             data_config: dict,
                             model_config: dict,
-                            training_config: dict
+                            training_config: dict,
+                            model_suffix: str
                         ) -> tuple[pd.Series, pd.Series, pd.DataFrame, dict]:
 
     discretize = discretize_threeway_threshold(0.33)
@@ -24,29 +27,34 @@ def run_meta_labeling_training(
     print("Feature Selection started")
     backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
     meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X_pca, meta_y)
-    feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], dynamic_feature_selection = training_config['dynamic_feature_selection'], data_config_hash = random_string(10))
+    feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = models[0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'])
     meta_selected_features_X = X_pca[feature_selection_output.columns]
 
     meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1)
 
-    _, meta_preds, meta_probabilities, all_models_single_asset = run_single_asset_trainig(
+    _, meta_preds, meta_probabilities, all_models_single_asset = train_primary_model(
         ticker_to_predict = "prediction_correct",
         original_X = meta_X,
         X = meta_X,
         y = meta_y,
         target_returns = target_returns,
-        models = [model_config['level_2_model']],
-        method = data_config['method'],
-        expanding_window = training_config['expanding_window_level2'],
-        sliding_window_size = training_config['sliding_window_size_level2'],
+        models = models,
+        method = 'classification',
+        expanding_window = training_config['expanding_window_meta_labeling'],
+        sliding_window_size = training_config['sliding_window_size_meta_labeling'],
         retrain_every = training_config['retrain_every'],
         scaler = training_config['scaler'],
         no_of_classes = 'two',
-        level = 2
+        level = 'meta_labeling',
+        print_results = False
     )
-    bet_size = meta_probabilities.iloc[:,1]
+    if len(models) > 1:
+        meta_preds = meta_preds.mean(axis = 1)
+        bet_size = meta_probabilities[meta_probabilities.columns[1::2]].mean(axis = 1)
+    else:
+        bet_size = meta_probabilities.iloc[:,1]
     avg_predictions_with_sizing = input_predictions * bet_size
-    avg_predictions_with_sizing.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True)
+    avg_predictions_with_sizing.rename("model_" + target_asset + "_" + model_suffix, inplace=True)
 
     meta_result = evaluate_predictions(
         model_name = "Meta",
@@ -54,9 +62,10 @@ def run_meta_labeling_training(
         y_pred = avg_predictions_with_sizing,
         y_true = y,
         method = 'classification',
-        no_of_classes = data_config['no_of_classes'],
+        no_of_classes = 'two',
+        print_results = True,
         discretize=False
     )
-    meta_result.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True)
+    meta_result.rename("model_" + target_asset + "_" + model_suffix, inplace=True)
 
     return meta_result, avg_predictions_with_sizing, meta_probabilities, all_models_single_asset

training/primary_model.py

@@ -6,7 +6,7 @@ from models.base import Model
 from utils.scaler import get_scaler
 from utils.types import ScalerTypes
 
-def run_single_asset_trainig(
+def train_primary_model(
                     ticker_to_predict: str,
                     original_X: pd.DataFrame,
                     X: pd.DataFrame,
@@ -19,10 +19,10 @@ def run_single_asset_trainig(
                     retrain_every: int,
                     scaler: ScalerTypes,
                     no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
-                    level: int
+                    level: str,
+                    print_results: bool
     ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]:
 
-
     scaler = get_scaler(scaler)
 
     results = pd.DataFrame()
@@ -51,14 +51,15 @@ def run_single_asset_trainig(
             y_true = y,
             method = method,
             no_of_classes=no_of_classes,
+            print_results = print_results,
             discretize=True
         )
-        column_name = "model_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level)
+        column_name = "model_" + ticker_to_predict + "_" + model_name + "_" + level
         results[column_name] = result
         all_models_single_asset[model_name] = model_over_time
         # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary
         predictions[column_name] = preds
-        probs_column_name = "probs_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level)
+        probs_column_name = "probs_" + ticker_to_predict + "_" + model_name + "_" + level
         probs.columns = [probs_column_name + "_" + c for c in probs.columns]
         probabilities = pd.concat([probabilities, probs], axis=1)
         

utils/evaluate.py

@@ -37,6 +37,7 @@ def evaluate_predictions(
                         y_true: pd.Series,
                         method: Literal['classification', 'regression'],
                         no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
+                        print_results: bool,
                         discretize: bool = False,
                         ) -> pd.Series:
     # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
@@ -100,8 +101,9 @@ def evaluate_predictions(
     #     scorecard.loc['edge_to_mae'] = 0.
 
     scorecard = scorecard.round(3)
-    print("Model name: ", model_name)
-    print(scorecard)
+    if print_results:        
+        print("Model name: ", model_name)
+        print(scorecard)
     return scorecard  
 
 

utils/helpers.py

@@ -21,7 +21,7 @@ def flatten(list_of_lists: list) -> list:
     return [item for sublist in list_of_lists for item in sublist]
 
 def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.Series:
-    if df.shape[0] == 0:
+    if df.shape[1] == 0:
         return df
     mean_df = df.iloc[:,0]
     weights = df.loc[weights_source]