feat(Sweep): separated level-1 and level-2 sweep configs, skip assets with too few samples to train on, simplified model mapping (#84)

* feat: Added ensemble models to sweep and configured naming convention.

* fix: Default value was misconfigured.

* feat(Sweep): separated level-1 and level-2 sweep configs, skip assets with too few samples to train on, simplified model mapping

* fix(Sweep): syntax error

* chore(Sweep): set sweep names accordingly

* fix(Sweep): set sliding window

* fix(Sweep): adjusted sweep config

* fix(Sweep): removed invalid feature extractor preset

Co-authored-by: Mark Aron Szulyovszky <mark.szulyovszky@gmail.com>

config.py

@@ -7,8 +7,8 @@ def get_default_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
         expanding_window = False,
-        sliding_window_size = 200,
-        retrain_every = 100,
+        sliding_window_size = 220,
+        retrain_every = 20,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
         include_original_data_in_ensemble = True,
     )
@@ -26,12 +26,10 @@ def get_default_config() -> tuple[dict, dict, dict]:
         no_of_classes= 'two'
     )
 
-    # regression_models = ["Lasso", "Ridge", "BayesianRidge", "KNN", "AB", "LR", "MLP", "RF", "SVR"]
     regression_models = ["Lasso", "KNN", "RF"]
-    regression_ensemble_models = ['Ensemble_Average']
-    classification_models = ["LR", "LDA", "KNN", "CART", "RF"]
-    # classification_models = model_names_classification        
-    classification_ensemble_models = ['Ensemble_Average']
+    regression_ensemble_models = ['KNN']
+    classification_models = ["LR", "LDA", "KNN", "CART", "RF", "StaticMom"]
+    classification_ensemble_models = ['LR']
 
     model_config = dict(
         level_1_models = regression_models if data_config['method'] == 'regression' else classification_models,

models/model_map.py

@@ -35,15 +35,8 @@ model_map = {
         AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)),
         RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)),
         StaticMom= StaticMomentumModel(allow_short=True),
-    ),     
-    "classification_ensemble_models": dict(
-        Ensemble_CART = SKLearnModel(DecisionTreeClassifier()),
         Ensemble_Average = StaticAverageModel(),
     ),
-    "regression_ensemble_models": dict(
-        Ensemble_Ridge = SKLearnModel(Ridge(alpha=0.1)),
-        Ensemble_Average = StaticAverageModel(),
-    )
 }
 
 model_names_classification = list(model_map["classification_models"].keys())
@@ -52,7 +45,7 @@ model_names_regression = list(model_map["regression_models"].keys())
 
 def map_model_name_to_function(model_config:dict, method:str) -> dict:
     for level in ['level_1_models', 'level_2_models']:
-        model_category = method + '_models' if level=='level_1_models' else method + '_ensemble_models'
+        model_category = method + '_models'
         model_config[level] = [(model_name, model_map[model_category][model_name]) for model_name in  model_config[level]]
 
     return model_config

run_pipeline.py

@@ -5,6 +5,7 @@ from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_
 from models.model_map import map_model_name_to_function
 from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config
 from config import get_default_config, validate_config, get_model_name
+from utils.helpers import get_first_valid_return_index
 
 def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
     model_config, training_config, data_config = get_default_config()
@@ -33,6 +34,11 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
         data_params['target_asset'] = asset
 
         X, y, target_returns = load_data(**data_params)
+        first_valid_index = get_first_valid_return_index(X.iloc[:,0])
+        samples_to_train = len(y) - first_valid_index
+        if samples_to_train < training_config['sliding_window_size'] * 2.6:
+            print("Not enough samples to train")
+            continue
 
         # 2. Train Level-1 models
         current_result, current_predictions = run_single_asset_trainig(
@@ -46,14 +52,14 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
             sliding_window_size = training_config['sliding_window_size'],
             retrain_every =  training_config['retrain_every'],
             scaler =  training_config['scaler'],
-            no_of_classes = data_config['no_of_classes']
+            no_of_classes = data_config['no_of_classes'],
+            level = 1
         )
         results = pd.concat([results, current_result], axis=1)
         all_predictions = pd.concat([all_predictions, current_predictions], axis=1)
 
         if len(model_config['level_2_models']) > 0: 
             # 3. Train Level-2 (Ensemble) model
-            
             ensemble_X = all_predictions
             if training_config['include_original_data_in_ensemble']:
                 ensemble_X = pd.concat([ensemble_X, X], axis=1)
@@ -69,7 +75,8 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
                 sliding_window_size = training_config['sliding_window_size'],
                 retrain_every = training_config['retrain_every'],
                 scaler = training_config['scaler'],
-                no_of_classes = data_config['no_of_classes']
+                no_of_classes = data_config['no_of_classes'],
+                level = 2
             )
 
             results = pd.concat([results, ensemble_result], axis=1)

sweep_level_1.yaml

@@ -1,10 +1,7 @@
 program: run_sweep.py
 method: bayes
 project: price-forecasting
-name: Finding best hyperparameters for price prediction
-# early_terminate:
-#   type: hyperband
-#   min_iter: 2000
+name: Level-1 models
 metric:
   goal: maximize
   name: sharpe
@@ -15,14 +12,13 @@ parameters:
       values: [True, False]
       distribution: categorical
     sliding_window_size: 
-      values: [180, 280, 380]
+      values: [180, 280, 380, 480, 580]
       distribution: categorical
     retrain_every: 
       values: [10, 20, 30]
       distribution: categorical
     scaler:
-      values: ['minmax', 'none']
-      distribution: categorical
+      value: 'minmax'
     include_original_data_in_ensemble:
       value: False
     method: 
@@ -45,7 +41,8 @@ parameters:
     level_2_models: 
       value: []
     own_features:
-      values: [['only_mom', 'date_days'], [], ['level_1', 'date_days'], ['level_1', 'date_days', 'level_2']]
+      values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']]
       distribution: categorical
     other_features:
-      value: []
+      values: [[], ['level_1'], ['level_2']]
+      distribution: categorical

sweep_level_2.yaml

@@ -0,0 +1,49 @@
+program: run_sweep.py
+method: bayes
+project: price-forecasting
+name: Level-2 models
+metric:
+  goal: maximize
+  name: sharpe
+parameters:
+    path : 
+      value: 'data/'
+    expanding_window:
+      values: [True, False]
+      distribution: categorical
+    sliding_window_size: 
+      values: [180, 280, 380]
+      distribution: categorical
+    retrain_every: 
+      values: [10, 20, 30]
+      distribution: categorical
+    scaler:
+      value: 'minmax'
+    include_original_data_in_ensemble:
+      values: [True, False]
+      distribution: categorical
+    method: 
+      value: 'classification'
+    no_of_classes:
+      values: ['two', 'three-balanced', 'three-imbalanced']
+      distribution: categorical
+    forecasting_horizon: 
+      value: 1
+    load_other_assets: 
+      values: [True, False]
+      distribution: categorical
+    log_returns: 
+      value: True
+    index_column: 
+      value: 'int'
+    level_1_models: 
+      value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"]
+    level_2_models: 
+      values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"], ["Ensemble_Average"]]
+      distribution: categorical
+    own_features:
+      values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']]
+      distribution: categorical
+    other_features:
+      values: [[], ['level_1'], ['level_2']]
+      distribution: categorical

training/training.py

@@ -26,7 +26,8 @@ def run_single_asset_trainig(
                     sliding_window_size: int,
                     retrain_every: int,
                     scaler: Literal['normalize', 'minmax', 'standardize', 'none'],
-                    no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
+                    no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
+                    level: int
     ) -> tuple[pd.DataFrame, pd.DataFrame]:
 
 
@@ -56,7 +57,7 @@ def run_single_asset_trainig(
             method = method,
             no_of_classes=no_of_classes
         )
-        column_name = ticker_to_predict + "_" + model_name
+        column_name = ticker_to_predict + "_" + model_name + "_" + str(level)
         results[column_name] = result
         # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary
         predictions["model_" + column_name] = preds

utils/evaluate.py

@@ -1,10 +1,9 @@
 from typing import Literal
-from sklearn.metrics import mean_absolute_error, accuracy_score, r2_score, f1_score, precision_score, recall_score
+from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
 from quantstats.stats import skew, sortino
-from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio, average_holding_period
+from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio
 from utils.helpers import get_first_valid_return_index
 import pandas as pd
-import numpy as np
 
 def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.00) -> pd.Series:
     delta_pos = signal.diff(1).abs().fillna(0.)