feat(Transformations): replaced feature selection pre-processing step with online version (with cache) (#170)

* feat(Transformations): removed feature-selection pre-processing step completely

* fix(Core): removed unnecessary `original_X`

* fix(Transformations): use the X_expanding_window to transform subsequent data

* fix(RFE): should check for model correctly

* fix(Config): only re-train the model every 40 timestamp

* fix(MetaLabeling): pass in the correct X to meta-labeling step

* fix(Transformation): PCA should at least keep as many features as sliding_window_size

* feat(Transformations): cache transformations across the same asset

* fix(Tests): missing preloaded_transformations arg

* chore(Config): got rid of unnecessary 'classification_models' and 'regression_models' dictionary keys

training/meta_labeling.py

@@ -1,9 +1,8 @@
 from utils.evaluate import discretize_threeway_threshold, evaluate_predictions
-from utils.helpers import random_string, equal_except_nan, drop_until_first_valid_index
+from utils.helpers import equal_except_nan
 from training.primary_model import train_primary_model
-from feature_selection.feature_selection import select_features
 import pandas as pd
-from models.model_map import get_model_map
+from models.model_map import default_feature_selector_classification, default_feature_selector_regression
 from models.base import Model
 from reporting.types import Reporting
 from typing import Union
@@ -23,22 +22,14 @@ def train_meta_labeling_model(
                             preloaded_models: Union[list[Reporting.Single_Model], None] = None
                         ) -> tuple[pd.Series, pd.Series, pd.DataFrame, list[Reporting.Single_Model]]:
 
-    _, _, _, default_feature_selector_regression, default_feature_selector_classification = get_model_map(model_config)
     discretize = discretize_threeway_threshold(0.33)
     discretized_predictions = input_predictions.apply(discretize)
     meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply(equal_except_nan, axis = 1)
 
-    print("Feature Selection started")
-    backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
-    meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X, meta_y)
-    feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = models[0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'])
-    meta_selected_features_X = X[feature_selection_output.columns]
-
-    meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1)
+    meta_X = pd.concat([X, input_predictions, discretized_predictions], axis = 1)
 
     _, meta_preds, meta_probabilities, all_models_single_asset = train_primary_model(
         ticker_to_predict = "prediction_correct",
-        original_X = meta_X,
         X = meta_X,
         y = meta_y,
         target_returns = target_returns,

training/primary_model.py

@@ -6,10 +6,11 @@ from models.base import Model
 from utils.scaler import get_scaler
 from utils.types import ScalerTypes
 from reporting.types import Reporting
+from transformations.rfe import RFETransformation
+from transformations.pca import PCATransformation
 
 def train_primary_model(
                     ticker_to_predict: str,
-                    original_X: pd.DataFrame,
                     X: pd.DataFrame,
                     y: pd.Series,
                     target_returns: pd.Series,
@@ -29,28 +30,35 @@ def train_primary_model(
     predictions = pd.DataFrame(index=y.index)
     probabilities = pd.DataFrame(index=y.index)
     all_models_single_asset: list[Reporting.Single_Model] = []
-    
+
     if preloaded_models is not None:
         models = preloaded_models
     
+    transformations_over_time = None
+    
     for model_name, model in models:
         if preloaded_models is None:
             model_over_time, transformations_over_time = walk_forward_train(
                 model_name=model_name,
                 model = model,
-                X = X if model.feature_selection == 'on' else original_X,
+                X = X,
                 y = y,
                 target_returns = target_returns,
                 expanding_window = expanding_window,
                 window_size = sliding_window_size,
                 retrain_every = retrain_every,
-                transformations= [get_scaler(scaler)],
+                transformations= [
+                    get_scaler(scaler),
+                    PCATransformation(ratio_components_to_keep=0.5, sliding_window_size=sliding_window_size),
+                    RFETransformation(n_feature_to_select=40, model=model)
+                ],
+                preloaded_transformations=transformations_over_time,
             )
         preds, probs = walk_forward_inference(
             model_name = model_name,
             model_over_time= model_over_time if preloaded_models is None else pd.Series(model),
             transformations_over_time = transformations_over_time,
-            X = X if model.feature_selection == 'on' else original_X,
+            X = X,
             expanding_window = expanding_window,
             window_size = sliding_window_size
         )

training/training_steps.py

@@ -11,7 +11,6 @@ from typing import Union
 def primary_step(
                 X: pd.DataFrame,
                 y:pd.Series,
-                original_X:pd.DataFrame,
                 asset:list,
                 target_returns:pd.Series,
                 configs: dict,
@@ -24,7 +23,6 @@ def primary_step(
     # 3. Train Primary models
     current_result, current_predictions, current_probabilities, all_models_for_single_asset = train_primary_model(
         ticker_to_predict = asset[1],
-        original_X = original_X,
         X = X,
         y = y,
         target_returns = target_returns,
@@ -48,7 +46,7 @@ def primary_step(
             primary_model_predictions = current_predictions[model_name]
             primary_meta_result, primary_meta_preds, primary_meta_probabilities, meta_labeling_models = train_meta_labeling_model(
                 target_asset=asset[1],
-                X = original_X,
+                X = X,
                 input_predictions= primary_model_predictions,
                 y = y,
                 target_returns = target_returns,
@@ -75,7 +73,6 @@ def primary_step(
 def secondary_step(
                 X:pd.DataFrame,
                 y:pd.Series,
-                original_X:pd.DataFrame,
                 current_predictions:pd.DataFrame,
                 asset:list,
                 target_returns:pd.Series,
@@ -89,7 +86,6 @@ def secondary_step(
     if model_config['ensemble_model'] is not None:
         ensemble_result, ensemble_predictions, _, ensemble_models_one_asset = train_primary_model(
             ticker_to_predict = asset[1],
-            original_X = current_predictions,
             X = current_predictions,
             y = y,
             target_returns = target_returns,
@@ -116,7 +112,7 @@ def secondary_step(
             # 3. Train a Meta-labeling model on the averaged level-1 model predictions
             ensemble_meta_result, ensemble_meta_predictions, ensemble_meta_probabilities, ensemble_meta_labeling_models = train_meta_labeling_model(
                 target_asset=asset[1],
-                X = original_X,
+                X = X,
                 input_predictions= ensemble_predictions,
                 y = y,
                 target_returns = target_returns,

training/walk_forward.py

@@ -3,10 +3,8 @@ from models.base import Model
 import numpy as np
 from utils.helpers import get_first_valid_return_index
 from tqdm import tqdm
-from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler
-from typing import Union
-from sklearn.base import clone
 from transformations.base import Transformation
+from typing import Optional
 
 def walk_forward_train(
                         model_name: str,
@@ -18,6 +16,7 @@ def walk_forward_train(
                         window_size: int,
                         retrain_every: int,
                         transformations: list[Transformation],
+                        preloaded_transformations: Optional[list[pd.Series]],
                     ) -> tuple[pd.Series, list[pd.Series]]:
     assert len(X) == len(y)
     models_over_time = pd.Series(index=y.index).rename(model_name)
@@ -44,9 +43,12 @@ def walk_forward_train(
             X_expanding_window = X[first_nonzero_return:train_window_end]
             y_expanding_window = y[first_nonzero_return:train_window_end]
 
-            current_transformations = [t.clone() for t in transformations]
-            for transformation_index, transformation in enumerate(current_transformations):
-                transformation.fit_transform(X_expanding_window, y_expanding_window)
+            if preloaded_transformations is not None and len(transformations) > 0:
+                current_transformations = [transformation_over_time[index] for transformation_over_time in preloaded_transformations]
+            else:
+                current_transformations = [t.clone() for t in transformations]
+                for transformation_index, transformation in enumerate(current_transformations):
+                    X_expanding_window = transformation.fit_transform(X_expanding_window, y_expanding_window)
 
             X_slice = X[train_window_start:train_window_end]