feat(DataLoader): caching MVP, added ability to use standard scaling for exogenous data, scaling is now also done before feature selection (#105)

* fix(FeatureExtractor): apply log to transform some series to normality

* feat(DataLoader): add ability of not returning returns when they're not needed (exogenous data), applied log to certain features

* feat(FeatureExtractors): added standard scaling for exogenous data

* feat(FeatureSelection): scale data with the passed in scaler before doing feature-selection

* fix(Config): sweep config

* feat(Models): output probability, store it

* feat(Core): added caching to select_features() and load_data()

* fix(Dependencies): added diskcache

* fix(Training): error when creating results DF

* feat(Models): added xgboost, fixed tests

* refactor(Cache): moved hashing to a separate function, created wrapper functions to separate business logic and caching

* fix(Tests): new syntax

* fix(Model): XGboost can't handle -1 class, so we'll use the deprecated label_encoder fornow

* fix(Model): XGBoost config

* feat(Cache): add run_clear_cache script

* fix(Pipeline) accidentally re-instatiating all_predictions for each asset

training/training.py

@@ -1,19 +1,10 @@
 import pandas as pd
 from typing import Literal
 from training.walk_forward import walk_forward_train_test
-from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler
 from utils.evaluate import evaluate_predictions
 from models.base import Model
-
-def __get_scaler(type: Literal['normalize', 'minmax', 'standardize', 'none']):
-    if type == 'normalize':
-        return Normalizer()
-    elif type == 'minmax':
-        return MinMaxScaler(feature_range= (-1, 1))
-    elif type == 'standardize':
-        return StandardScaler()
-    else:
-        return None
+from utils.scaler import get_scaler
+from utils.types import ScalerTypes
 
 def run_single_asset_trainig(
                     ticker_to_predict: str,
@@ -26,19 +17,20 @@ def run_single_asset_trainig(
                     expanding_window: bool,
                     sliding_window_size: int,
                     retrain_every: int,
-                    scaler: Literal['normalize', 'minmax', 'standardize', 'none'],
+                    scaler: ScalerTypes,
                     no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                     level: int
-    ) -> tuple[pd.DataFrame, pd.DataFrame]:
+    ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
 
 
-    scaler = __get_scaler(scaler)
+    scaler = get_scaler(scaler)
 
     results = pd.DataFrame()
-    predictions = pd.DataFrame()
+    predictions = pd.DataFrame(index=y.index)
+    probabilities = pd.DataFrame(index=y.index)
     
     for model_name, model in models:
-        model_over_time, preds = walk_forward_train_test(
+        model_over_time, preds, probs = walk_forward_train_test(
             model_name=model_name,
             model = model,
             X = X if model.feature_selection == 'on' else original_X,
@@ -58,10 +50,13 @@ def run_single_asset_trainig(
             method = method,
             no_of_classes=no_of_classes
         )
-        column_name = ticker_to_predict + "_" + model_name + "_lvl" + str(level)
+        column_name = "model_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level)
         results[column_name] = result
         # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary
-        predictions["model_" + column_name] = preds
+        predictions[column_name] = preds
+        probs_column_name = "probs_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level)
+        probs.columns = [probs_column_name + "_" + c for c in probs.columns]
+        probabilities = pd.concat([probabilities, probs], axis=1)
         
 
-    return results, predictions
+    return results, predictions, probabilities

training/walk_forward.py

@@ -16,9 +16,10 @@ def walk_forward_train_test(
                             window_size: int,
                             retrain_every: int,
                             scaler,
-                        ) -> tuple[pd.Series, pd.Series]:
+                        ) -> tuple[pd.Series, pd.Series, pd.DataFrame]:
     assert len(X) == len(y)
     predictions = pd.Series(index=y.index).rename(model_name)
+    probabilities = pd.DataFrame(index=y.index)
     models = pd.Series(index=y.index).rename(model_name)
 
     first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(X.iloc[:,0]))
@@ -45,8 +46,8 @@ def walk_forward_train_test(
             train_window_end = index - 1
             
             if is_scaling_on:
-                # First we need to fit on the expanding window data slice
-                # This is our only way to avoid lookahead bia
+                # We need to fit on the expanding window data slice
+                # This is our only way to avoid lookahead bias
                 X_expanding_window = X[first_nonzero_return:train_window_end]
                 scaler.fit(X_expanding_window.values)
 
@@ -59,7 +60,7 @@ def walk_forward_train_test(
                 X_slice = X_slice.to_numpy()
 
             current_model = model.clone()
-            current_model.fit(X_slice, y_slice.to_numpy(), models[index-1])
+            current_model.fit(X_slice, y_slice.to_numpy())
             iterations_before_retrain = retrain_every
         else:
             current_model = models[index-1]
@@ -70,8 +71,12 @@ def walk_forward_train_test(
         if is_scaling_on:
             next_timestep = scaler.transform(next_timestep)
 
-        prediction = current_model.predict(next_timestep).item()
+        prediction, probs = current_model.predict(next_timestep)
         predictions[index] = prediction
+        if len(probabilities.columns) != len(probs):
+            probabilities = probabilities.reindex(columns = ["prob_" + str(num) for num in range(0, len(probs.T))])
+        probabilities.iloc[index] = probs
+
         iterations_before_retrain -= 1
 
-    return models, predictions
+    return models, predictions, probabilities