feat(Selection): added toggleable feature selection step into the pipeline (#83)

* feat(Selection): added prototype feature selection python script

* feat(Utils): added some helpers for the future from Advances in Financial ML book

* feat(Selection): added RFECV

* feat(Selection): added configurable feature selection step into pipeline

* feat(Config): added level_1 & level_2 default config, PCA before feature selection process starts

* feat(Selection): added backup feature selector models if current one can't output feature importance, removed unnecessary array for level-2 models

* fix(Training): deal with zero first value coming out of static models

* feat(Sweep): added feature selection sweep

* fix(Sweep): config problem

* fix(Sweep): config

* chore(Utils): removed unnecessary purged k-fold crossval class

* feat(Config): added dimensionality_reduction as a separate flag

* fix(Sweep): config updated

* fix(Sweep): sweep name

* chore(Config): updated level_2 config to the best performing configuation

config.py

@@ -3,37 +3,77 @@ from utils.load_data import get_crypto_assets
 from feature_extractors.feature_extractor_presets import presets
 from models.model_map import model_names_classification, model_names_regression
 
-def get_default_config() -> tuple[dict, dict, dict]:
+
+def get_default_level_1_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
+        dimensionality_reduction = False,
+        feature_selection = True,
         expanding_window = False,
-        sliding_window_size = 220,
+        sliding_window_size = 380,
         retrain_every = 20,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
-        include_original_data_in_ensemble = True,
+        include_original_data_in_ensemble = False,
     )
 
     data_config = dict(
         path='data/',
         all_assets = get_crypto_assets('data/'),
-        load_other_assets= False,
+        load_other_assets= True,
         log_returns= True,
         forecasting_horizon = 1,
-        own_features = ['level_1', 'date_days'],
-        other_features = [],
+        own_features = ['level_2', 'date_days'],
+        other_features = ['single_mom'],
         index_column= 'int',
         method= 'classification',
-        no_of_classes= 'two'
+        no_of_classes= 'three-balanced'
     )
 
-    regression_models = ["Lasso", "KNN", "RF"]
-    regression_ensemble_models = ['KNN']
-    classification_models = ["LR", "LDA", "KNN", "CART", "RF", "StaticMom"]
-    classification_ensemble_models = ['LR']
+    regression_models = ["Lasso"]
+    classification_models = ["KNN"]
 
     model_config = dict(
         level_1_models = regression_models if data_config['method'] == 'regression' else classification_models,
-        level_2_models = regression_ensemble_models if data_config['method'] == 'regression' else classification_ensemble_models
+        level_2_model = None
+    )
+    
+    return model_config, training_config, data_config
+
+
+
+def get_default_level_2_config() -> tuple[dict, dict, dict]:
+  
+    training_config = dict(
+        dimensionality_reduction = True,
+        feature_selection = True,
+        expanding_window = True,
+        sliding_window_size = 380,
+        retrain_every = 20,
+        scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
+        include_original_data_in_ensemble = False,
+    )
+
+    data_config = dict(
+        path='data/',
+        all_assets = get_crypto_assets('data/'),
+        load_other_assets= True,
+        log_returns= True,
+        forecasting_horizon = 1,
+        own_features = ['level_2', 'date_days', 'lags_up_to_5'],
+        other_features = ['level_2', 'lags_up_to_5'],
+        index_column= 'int',
+        method= 'classification',
+        no_of_classes= 'three-balanced'
+    )
+
+    regression_models = ["Lasso", "KNN", "RF"]
+    regression_ensemble_model = 'KNN'
+    classification_models = ["LR", "LDA", "KNN", "CART", "RF", "StaticMom"]
+    classification_ensemble_model = 'Ensemble_Average'
+
+    model_config = dict(
+        level_1_models = regression_models if data_config['method'] == 'regression' else classification_models,
+        level_2_model = regression_ensemble_model if data_config['method'] == 'regression' else classification_ensemble_model
     )
     
     return model_config, training_config, data_config
@@ -41,16 +81,14 @@ def get_default_config() -> tuple[dict, dict, dict]:
 
 def validate_config(model_config:dict, training_config:dict, data_config:dict):
     # We need to make sure there's only one output from the pipeline
-    # We're not prepared for more than 1 level-2 models at the moment
-    assert len(model_config["level_2_models"]) <= 1
     # If level-2 model is there, we need more than one level-1 models to train
-    if len(model_config["level_2_models"]) == 1: assert len(model_config["level_1_models"]) > 0
+    if model_config["level_2_model"] is not None: assert len(model_config["level_1_models"]) > 0
     # If there's no level-2 model, we need to have only one level-1 model
-    if len(model_config["level_2_models"]) == 0: assert len(model_config["level_1_models"]) == 1
+    if model_config["level_2_model"] is None: assert len(model_config["level_1_models"]) == 1
 
 def get_model_name(model_config:dict) -> str:
-    if len(model_config["level_2_models"]) == 1:
-        return model_config["level_2_models"][0][0]
+    if model_config["level_2_model"] is not None:
+        return model_config["level_2_model"][0]
     elif len(model_config["level_1_models"]) == 1:
         return model_config["level_1_models"][0][0]
     else:

environment.yml

@@ -20,4 +20,5 @@ dependencies:
   - pytest
   - wandb
   - python-dotenv
+  - tscv
 prefix: /usr/local/anaconda3/envs/quant

feature_extractors/feature_extractor_presets.py

@@ -30,6 +30,7 @@ presets = __presets | dict(
 )
 
 def preprocess_feature_extractors_config(data_dict: dict) -> dict:
+    data_dict = data_dict.copy()
     keys = ['own_features', 'other_features']
     for key in keys:
         preset_names = data_dict[key]

feature_extractors/utils/fracdiff.py

@@ -0,0 +1,240 @@
+"""
+Fractional differentiation is a technique to make a time series stationary but also
+retain as much memory as possible.  This is done by differencing by a positive real
+number. Fractionally differenced series can be used as a feature in machine learning
+process.
+"""
+
+import numpy as np
+import pandas as pd
+
+
+class FractionalDifferentiation:
+    """
+    FractionalDifferentiation class encapsulates the functions that can
+    be used to compute fractionally differentiated series.
+    """
+
+    @staticmethod
+    def get_weights(diff_amt, size):
+        """
+        Advances in Financial Machine Learning, Chapter 5, section 5.4.2, page 79.
+
+        The helper function generates weights that are used to compute fractionally
+        differentiated series. It computes the weights that get used in the computation
+        of  fractionally differentiated series. This generates a non-terminating series
+        that approaches zero asymptotically. The side effect of this function is that
+        it leads to negative drift "caused by an expanding window's added weights"
+        (see page 83 AFML)
+
+        When diff_amt is real (non-integer) positive number then it preserves memory.
+
+        The book does not discuss what should be expected if d is a negative real
+        number. Conceptually (from set theory) negative d leads to set of negative
+        number of elements. And that translates into a set whose elements can be
+        selected more than once or as many times as one chooses (multisets with
+        unbounded multiplicity) - see http://faculty.uml.edu/jpropp/msri-up12.pdf.
+
+        :param diff_amt: (float) Differencing amount
+        :param size: (int) Length of the series
+        :return: (np.ndarray) Weight vector
+        """
+
+        # The algorithm below executes the iterative estimation (section 5.4.2, page 78)
+        weights = [1.]  # create an empty list and initialize the first element with 1.
+        for k in range(1, size):
+            weights_ = -weights[-1] * (diff_amt - k + 1) / k  # compute the next weight
+            weights.append(weights_)
+
+        # Now, reverse the list, convert into a numpy column vector
+        weights = np.array(weights[::-1]).reshape(-1, 1)
+        return weights
+
+    @staticmethod
+    def frac_diff(series, diff_amt, thresh=0.01):
+        """
+        Advances in Financial Machine Learning, Chapter 5, section 5.5, page 82.
+
+        References:
+        https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086
+        https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf
+        https://en.wikipedia.org/wiki/Fractional_calculus
+
+        The steps are as follows:
+        - Compute weights (this is a one-time exercise)
+        - Iteratively apply the weights to the price series and generate output points
+
+        This is the expanding window variant of the fracDiff algorithm
+        Note 1: For thresh-1, nothing is skipped
+        Note 2: diff_amt can be any positive fractional, not necessarility bounded [0, 1]
+
+        :param series: (pd.Series) A time series that needs to be differenced
+        :param diff_amt: (float) Differencing amount
+        :param thresh: (float) Threshold or epsilon
+        :return: (pd.DataFrame) Differenced series
+        """
+
+        # 1. Compute weights for the longest series
+        weights = get_weights(diff_amt, series.shape[0])
+
+        # 2. Determine initial calculations to be skipped based on weight-loss threshold
+        weights_ = np.cumsum(abs(weights))
+        weights_ /= weights_[-1]
+        skip = weights_[weights_ > thresh].shape[0]
+
+        # 3. Apply weights to values
+        output_df = {}
+        for name in series.columns:
+            series_f = series[[name]].fillna(method='ffill').dropna()
+            output_df_ = pd.Series(index=series.index, dtype='float64')
+
+            for iloc in range(skip, series_f.shape[0]):
+                loc = series_f.index[iloc]
+
+                # At this point all entries are non-NAs so no need for the following check
+                # if np.isfinite(series.loc[loc, name]):
+                output_df_[loc] = np.dot(weights[-(iloc + 1):, :].T, series_f.loc[:loc])[0, 0]
+
+            output_df[name] = output_df_.copy(deep=True)
+        output_df = pd.concat(output_df, axis=1)
+        return output_df
+
+    @staticmethod
+    def get_weights_ffd(diff_amt, thresh, lim):
+        """
+        Advances in Financial Machine Learning, Chapter 5, section 5.4.2, page 83.
+
+        The helper function generates weights that are used to compute fractionally
+        differentiate dseries. It computes the weights that get used in the computation
+        of fractionally differentiated series. The series is of fixed width and same
+        weights (generated by this function) can be used when creating fractional
+        differentiated series.
+        This makes the process more efficient. But the side-effect is that the
+        fractionally differentiated series is skewed and has excess kurtosis. In
+        other words, it is not Gaussian any more.
+
+        The discussion of positive and negative d is similar to that in get_weights
+        (see the function get_weights)
+
+        :param diff_amt: (float) Differencing amount
+        :param thresh: (float) Threshold for minimum weight
+        :param lim: (int) Maximum length of the weight vector
+        :return: (np.ndarray) Weight vector
+        """
+
+        weights = [1.]
+        k = 1
+
+        # The algorithm below executes the iterativetive estimation (section 5.4.2, page 78)
+        # The output weights array is of the indicated length (specified by lim)
+        ctr = 0
+        while True:
+            # compute the next weight
+            weights_ = -weights[-1] * (diff_amt - k + 1) / k
+
+            if abs(weights_) < thresh:
+                break
+
+            weights.append(weights_)
+            k += 1
+            ctr += 1
+            if ctr == lim - 1:  # if we have reached the size limit, exit the loop
+                break
+
+        # Now, reverse the list, convert into a numpy column vector
+        weights = np.array(weights[::-1]).reshape(-1, 1)
+        return weights
+
+    @staticmethod
+    def frac_diff_ffd(series, diff_amt, thresh=1e-5):
+        """
+        Advances in Financial Machine Learning, Chapter 5, section 5.5, page 83.
+
+        References:
+
+        * https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086
+        * https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf
+        * https://en.wikipedia.org/wiki/Fractional_calculus
+
+        The steps are as follows:
+
+        - Compute weights (this is a one-time exercise)
+        - Iteratively apply the weights to the price series and generate output points
+
+        Constant width window (new solution)
+        Note 1: thresh determines the cut-off weight for the window
+        Note 2: diff_amt can be any positive fractional, not necessarity bounded [0, 1].
+
+        :param series: (pd.Series) A time series that needs to be differenced
+        :param diff_amt: (float) Differencing amount
+        :param thresh: (float) Threshold for minimum weight
+        :return: (pd.DataFrame) A data frame of differenced series
+        """
+
+        # 1) Compute weights for the longest series
+        weights = get_weights_ffd(diff_amt, thresh, series.shape[0])
+        width = len(weights) - 1
+
+        # 2) Apply weights to values
+        # 2.1) Start by creating a dictionary to hold all the fractionally differenced series
+        output_df = {}
+
+        # 2.2) compute fractionally differenced series for each stock
+        for name in series.columns:
+            series_f = series[[name]].fillna(method='ffill').dropna()
+            temp_df_ = pd.Series(index=series.index, dtype='float64')
+            for iloc1 in range(width, series_f.shape[0]):
+                loc0 = series_f.index[iloc1 - width]
+                loc1 = series.index[iloc1]
+
+                # At this point all entries are non-NAs, hence no need for the following check
+                # if np.isfinite(series.loc[loc1, name]):
+                temp_df_[loc1] = np.dot(weights.T, series_f.loc[loc0:loc1])[0, 0]
+
+            output_df[name] = temp_df_.copy(deep=True)
+
+        # transform the dictionary into a data frame
+        output_df = pd.concat(output_df, axis=1)
+        return output_df
+
+
+def get_weights(diff_amt, size):
+    """ This is a pass-through function """
+    return FractionalDifferentiation.get_weights(diff_amt, size)
+
+
+def frac_diff(series, diff_amt, thresh=0.01):
+    """ This is a pass-through function """
+    return FractionalDifferentiation.frac_diff(series, diff_amt, thresh)
+
+
+def get_weights_ffd(diff_amt, thresh, lim):
+    """ This is a pass-through function """
+    return FractionalDifferentiation.get_weights_ffd(diff_amt, thresh, lim)
+
+
+def frac_diff_ffd(series, diff_amt, thresh=1e-5):
+    """
+    Advances in Financial Machine Learning, Chapter 5, section 5.5, page 83.
+
+    References:
+
+    * https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086
+    * https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf
+    * https://en.wikipedia.org/wiki/Fractional_calculus
+
+    The steps are as follows:
+
+    - Compute weights (this is a one-time exercise)
+    - Iteratively apply the weights to the price series and generate output points
+
+    Constant width window (new solution)
+    Note 1: thresh determines the cut-off weight for the window
+    Note 2: diff_amt can be any positive fractional, not necessarity bounded [0, 1].
+
+    :param series: (pd.Series) A time series that needs to be differenced
+    :param diff_amt: (float) Differencing amount
+    :param thresh: (float) Threshold for minimum weight
+    :return: (pd.DataFrame) A data frame of differenced series
+    """
+    return FractionalDifferentiation.frac_diff_ffd(series, diff_amt, thresh)

feature_selection/dim_reduction.py

@@ -0,0 +1,8 @@
+import pandas as pd
+from sklearn.decomposition import PCA
+
+def reduce_dimensionality(X: pd.DataFrame, no_of_compoments: int) -> pd.DataFrame:
+    pca = PCA(n_components= no_of_compoments)
+    result = pd.DataFrame(pca.fit_transform(X), index= X.index)
+    result.columns = ['PCA_' + str(i) for i in range(1, no_of_compoments+1)]
+    return result

feature_selection/feature_selection.py

@@ -0,0 +1,23 @@
+from sklearn.feature_selection import RFE
+from sklearn.model_selection import TimeSeriesSplit
+import pandas as pd
+from models.base import Model, SKLearnModel
+from sklearn.decomposition import PCA
+
+def select_features(X: pd.DataFrame, y: pd.Series, model: Model, min_features_to_select: int, backup_model: SKLearnModel) -> pd.DataFrame:
+    ''' Select features using RFECV, returns a pd.DataFrame (X) with only the selected features.'''
+    if model.model_type != 'ml': return X
+
+    # 2. Recursive feature selection
+    cv = TimeSeriesSplit(n_splits=5)
+
+    feat_selector_model = model.model
+    if hasattr(feat_selector_model, 'feature_importances_') == False and hasattr(feat_selector_model, 'coef_') == False:
+        feat_selector_model = backup_model.model
+
+#     selector = RFECV(feat_selector_model, cv = cv, step=5, min_features_to_select=min_features_to_select)
+    selector = RFE(feat_selector_model, n_features_to_select=10)
+    selector = selector.fit(X, y)
+    print("Kept %d features out of %d" % (selector.n_features_, X.shape[1]))
+
+    return pd.DataFrame(X[X.columns[selector.support_]], index= X.index)

models/average.py

@@ -6,9 +6,10 @@ class StaticAverageModel(Model):
     Model that averages .
     '''
 
-    # data_format = 'dataframe'
     data_scaling = 'unscaled'
     only_column = 'model_'
+    feature_selection = 'off'
+    model_type = 'static'
 
     def fit(self, X, y, prev_model):
         # This is a static model, it can' learn anything

models/base.py

@@ -5,10 +5,11 @@ from abc import ABC, abstractmethod, abstractproperty
 
 class Model(ABC):
 
-    # data_format: Literal['dataframe', 'numpy'] 
     data_scaling: Literal["scaled", "unscaled"]
+    feature_selection: Literal["on", "off"]
     # data_format: Literal["wide", "narrow"]
     only_column: Optional[str]
+    model_type: Literal['ml', 'static']
 
     @abstractmethod
     def fit(self, X, y, prev_model):
@@ -25,9 +26,10 @@ class Model(ABC):
 
 class SKLearnModel(Model):
 
-    # data_format = 'numpy'
     data_scaling = 'scaled'
     only_column = None
+    feature_selection = 'on'
+    model_type = 'ml'
 
     def __init__(self, model):
         self.model = model

models/model_map.py

@@ -22,12 +22,12 @@ model_map = {
         KNN = SKLearnModel(KNeighborsRegressor(n_neighbors=25)),
         AB = SKLearnModel(AdaBoostRegressor(random_state=1)),
         MLP = SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
-        RF = SKLearnModel(RandomForestRegressor(n_jobs=-1, random_state=1)),
+        RF = SKLearnModel(RandomForestRegressor(n_jobs=-1, max_depth=20, random_state=1)),
         SVR = SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1)),
         StaticNaive = StaticNaiveModel(),
     ),
     "classification_models": dict(
-        LR= SKLearnModel(LogisticRegression(C=10, random_state=1, max_iter=1000)),
+        LR= SKLearnModel(LogisticRegression(C=10, random_state=1, max_iter=1000, n_jobs=-1)),
         LDA= SKLearnModel(LinearDiscriminantAnalysis()),
         KNN= SKLearnModel(KNeighborsClassifier()),
         CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)),
@@ -42,10 +42,12 @@ model_map = {
 model_names_classification = list(model_map["classification_models"].keys())
 model_names_regression = list(model_map["regression_models"].keys())
 
+default_feature_selector_regression = model_map['regression_models']['RF']
+default_feature_selector_classification = model_map['classification_models']['RF']
 
 def map_model_name_to_function(model_config:dict, method:str) -> dict:
-    for level in ['level_1_models', 'level_2_models']:
-        model_category = method + '_models'
-        model_config[level] = [(model_name, model_map[model_category][model_name]) for model_name in  model_config[level]]
+    model_config['level_1_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in  model_config['level_1_models']]
+    if model_config['level_2_model'] is not None:
+        model_config['level_2_model'] = (model_config['level_2_model'], model_map[method + '_models'][model_config['level_2_model']])
 
     return model_config

models/momentum.py

@@ -6,9 +6,10 @@ class StaticMomentumModel(Model):
     Model that uses only one feature: momentum. It's positive if momentum is greater than 0, otherwise it's negative.
     '''
 
-    # data_format = 'dataframe'
     data_scaling = 'unscaled'
     only_column = 'mom'
+    feature_selection = 'off'
+    model_type = 'static'
 
     def __init__(self, allow_short: bool) -> None:
         super().__init__()

models/naive.py

@@ -6,9 +6,10 @@ class StaticNaiveModel(Model):
     Model that carries the last observation (from returns) to the next one, naively.
     '''
 
-    # data_format = 'dataframe'
     data_scaling = 'unscaled'
     only_column = None
+    feature_selection = 'off'
+    model_type = 'static'
 
     def fit(self, X, y, prev_model):
         # This is a static model, it can' learn anything

reporting/wandb.py

@@ -17,15 +17,16 @@ def launch_wandb(project_name:str, default_config:dict, sweep:bool=False):
     
     
 def register_config_with_wandb(wandb: Optional[object], model_config:dict, training_config:dict, data_config:dict):
-    config: dict = wandb.config
+    if wandb is None: return
 
-    if type(wandb) is not type(None):
-        for k in training_config:
-            training_config[k] = config[k]
-        for k in model_config:
-            model_config[k] = config[k]
-        for k in data_config:
-            data_config[k] = config[k]
+    config: dict = wandb.config
+    
+    for k in training_config:
+        training_config[k] = config[k]
+    for k in model_config:
+        model_config[k] = config[k]
+    for k in data_config:
+        data_config[k] = config[k]
 
 
 def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_name: str, model_name: str):

run_pipeline.py

@@ -2,13 +2,15 @@ from utils.load_data import load_data
 import pandas as pd
 from training.training import run_single_asset_trainig
 from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_with_wandb
-from models.model_map import map_model_name_to_function
+from models.model_map import map_model_name_to_function, default_feature_selector_regression, default_feature_selector_classification
 from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config
-from config import get_default_config, validate_config, get_model_name
 from utils.helpers import get_first_valid_return_index, weighted_average
+from config import get_default_level_1_config, get_default_level_2_config, validate_config, get_model_name
+from feature_selection.feature_selection import select_features
+from feature_selection.dim_reduction import reduce_dimensionality
 
 def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
-    model_config, training_config, data_config = get_default_config()
+    model_config, training_config, data_config = get_default_level_2_config()
     
     wandb = None
     if with_wandb: 
@@ -34,15 +36,30 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
         data_params['target_asset'] = asset
 
         X, y, target_returns = load_data(**data_params)
+        original_X = X.copy()
         first_valid_index = get_first_valid_return_index(X.iloc[:,0])
         samples_to_train = len(y) - first_valid_index
         if samples_to_train < training_config['sliding_window_size'] * 3:
             print("Not enough samples to train")
             continue
 
-        # 2. Train Level-1 models
+        # 2a. Dimensionality Reduction (optional)
+        if training_config['dimensionality_reduction']:
+            X = reduce_dimensionality(X, int(len(X.columns) / 2))
+
+        # 2b. Feature Selection (optional)
+        if training_config['feature_selection']:
+            print("Feature Selection started")
+            # TODO: this needs to be done per model!
+            backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
+            X = select_features(X, y, model_config['level_1_models'][0][1], min_features_to_select = 10, backup_model = backup_model)
+            print("Feature Selection ended")
+
+
+        # 3. Train Level-1 models
         current_result, current_predictions = run_single_asset_trainig(
             ticker_to_predict = asset,
+            original_X = original_X,
             X = X,
             y = y,
             target_returns = target_returns,
@@ -56,20 +73,22 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
             level = 1
         )
         results = pd.concat([results, current_result], axis=1)
-        all_predictions = pd.concat([all_predictions, current_predictions], axis=1)
+        # With static models, because of the lag in the indicator, the first prediction is NA, so we fill it with zero.
+        all_predictions = pd.concat([all_predictions, current_predictions], axis=1).fillna(0.)
 
-        if len(model_config['level_2_models']) > 0: 
-            # 3. Train Level-2 (Ensemble) model
+        # 3. Train Level-2 (Ensemble) model (Optional)
+        if model_config['level_2_model'] is not None: 
             ensemble_X = all_predictions
             if training_config['include_original_data_in_ensemble']:
                 ensemble_X = pd.concat([ensemble_X, X], axis=1)
 
             ensemble_result, ensemble_preds = run_single_asset_trainig(
                 ticker_to_predict = asset,
+                original_X = ensemble_X,
                 X = ensemble_X,
                 y = y,
                 target_returns = target_returns,
-                models = model_config['level_2_models'],
+                models = [model_config['level_2_model']],
                 method = data_config['method'],
                 expanding_window = training_config['expanding_window'],
                 sliding_window_size = training_config['sliding_window_size'],
@@ -82,7 +101,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
             results = pd.concat([results, ensemble_result], axis=1)
             all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
         
-    # 4. Save & report results
+
     results.to_csv('results.csv')
 
     level1_columns = results[[column for column in results.columns if 'lvl1' in column]]
@@ -99,9 +118,10 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
     print("Mean Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['sharpe'], 3))
     print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
 
-    print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum())
-    print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['sharpe'].mean(), 3))
-    print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
+    if model_config['level_2_model'] is not None: 
+        print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum())
+        print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['sharpe'].mean(), 3))
+        print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
 
     if sweep:
         if wandb.run is not None:

sweep_level_1.yaml

@@ -38,8 +38,8 @@ parameters:
     level_1_models: 
       values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"]]
       distribution: categorical
-    level_2_models: 
-      value: []
+    level_2_model: 
+      value: None
     own_features:
       values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']]
       distribution: categorical

sweep_level_2.yaml

@@ -38,8 +38,8 @@ parameters:
       value: 'int'
     level_1_models: 
       value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"]
-    level_2_models: 
-      values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"], ["Ensemble_Average"]]
+    level_2_model: 
+      values: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "Ensemble_Average"]
       distribution: categorical
     own_features:
       values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']]

sweep_selection.yaml

@@ -0,0 +1,47 @@
+program: run_sweep.py
+method: bayes
+project: price-forecasting
+name: Feature selection
+metric:
+  goal: maximize
+  name: sharpe
+parameters:
+    path : 
+      value: 'data/'
+    expanding_window:
+      value: True
+    sliding_window_size: 
+      value: 380
+    feature_selection:
+      values: [True, False]
+      distribution: categorical
+    dimensionality_reduction:
+      values: [True, False]
+      distribution: categorical
+    retrain_every: 
+      value: 20
+    scaler:
+      value: 'minmax'
+    include_original_data_in_ensemble:
+      value: False
+    method: 
+      value: 'classification'
+    no_of_classes:
+      values: ['two', 'three-balanced', 'three-imbalanced']
+      distribution: categorical
+    forecasting_horizon: 
+      value: 1
+    load_other_assets: 
+      value: True
+    log_returns: 
+      value: True
+    index_column: 
+      value: 'int'
+    level_1_models: 
+      value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"]
+    level_2_model: 
+      value: "Ensemble_Average"
+    own_features:
+      value: ['date_days', 'level_2', 'lags_up_to_5']
+    other_features:
+      value: ['level_2', 'lags_up_to_5']

training/training.py

@@ -17,6 +17,7 @@ def __get_scaler(type: Literal['normalize', 'minmax', 'standardize', 'none']):
 
 def run_single_asset_trainig(
                     ticker_to_predict: str,
+                    original_X: pd.DataFrame,
                     X: pd.DataFrame,
                     y: pd.Series,
                     target_returns: pd.Series,
@@ -40,7 +41,7 @@ def run_single_asset_trainig(
         model_over_time, preds = walk_forward_train_test(
             model_name=model_name,
             model = model,
-            X = X,
+            X = X if model.feature_selection == 'on' else original_X,
             y = y,
             target_returns = target_returns,
             expanding_window = expanding_window,

utils/helpers.py

@@ -15,6 +15,8 @@ def flatten(list_of_lists: list) -> list:
     return [item for sublist in list_of_lists for item in sublist]
 
 def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.DataFrame:
+    if df.shape[0] == 0:
+        return df
     mean_df = df.iloc[:,0]
     weights = df.loc[weights_source]
 

utils/rolling.py

@@ -1,6 +0,0 @@
-import numpy as np
-
-def rolling_window(a, window):
-    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
-    strides = a.strides + (a.strides[-1],)
-    return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)