refactor(WalkForward): separate train / test functions to help with inference later (#158)

* refactor(WalkForward): separate train / test functions (draft) to potentially help with inference later

* fix(Training): use the new separate train / test functions

* feat(Training): return and pass in scalers that are necessary for inference

* fix(Project): runtime errors

* fix(WalkForward): use the correct `train_from` value

* fix(Tests): for new walk_forward functions()

* refactor(WalkForward): rename `walk_forward_test()` to `walk_forward_inference()`

config/config.py

@@ -10,7 +10,7 @@ def get_dev_config() -> tuple[dict, dict, dict]:
         sliding_window_size_primary = 380,
         sliding_window_size_meta_labeling = 1,
         retrain_every = 20,
-        scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
+        scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
     )
 
     data_config = dict(
@@ -53,7 +53,7 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]:
         sliding_window_size_primary = 380,
         sliding_window_size_meta_labeling = 240,
         retrain_every = 20,
-        scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
+        scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
     )
 
     data_config = dict(
@@ -98,7 +98,7 @@ def get_lightweight_ensemble_config() -> tuple[dict, dict, dict]:
         sliding_window_size_primary = 380,
         sliding_window_size_meta_labeling = 240,
         retrain_every = 20,
-        scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
+        scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
     )
 
     data_config = dict(

feature_selection/feature_selection.py

@@ -25,9 +25,7 @@ def select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_s
     # 2. Recursive feature selection
     cv = TimeSeriesSplit(n_splits=5)
     scaler = get_scaler(scaling)
-    X_scaled = X.copy()
+    X_scaled = scaler.fit_transform(X)
-    if scaler is not None:
-        X_scaled = scaler.fit_transform(X_scaled)
 
     feat_selector_model = model.model
     if hasattr(feat_selector_model, 'feature_importances_') == False and hasattr(feat_selector_model, 'coef_') == False:

tests/test_evaluation.py

@@ -1,9 +1,10 @@
 
 import numpy as np
 import pandas as pd
-from training.walk_forward import walk_forward_train_test
+from training.walk_forward import walk_forward_train, walk_forward_inference
 from models.base import Model
 from utils.evaluate import evaluate_predictions
+from sklearn.preprocessing import MinMaxScaler
 
 no_of_rows = 100
 
@@ -67,9 +68,9 @@ def test_evaluation():
     window_length = 10
 
     model = EvenOddStubModel(window_length = window_length)
-    scaler = None
+    scaler = MinMaxScaler()
     
-    models, predictions, probs = walk_forward_train_test(
+    models, scalers = walk_forward_train(
         model_name='test',
         model=model,
         X=X,
@@ -78,7 +79,14 @@ def test_evaluation():
         expanding_window=False,
         window_size=window_length,
         retrain_every=10,
-        scaler=scaler
+        scaler=scaler)
+    predictions, probs = walk_forward_inference(
+        model_name='test',
+        models=models,
+        scalers=scalers,
+        X=X,
+        expanding_window=False,
+        window_size=window_length
     )
     
     # verify if predictions are the same as y

tests/test_walk_forward.py

@@ -1,7 +1,8 @@
 import numpy as np
 import pandas as pd
-from training.walk_forward import walk_forward_train_test
+from training.walk_forward import walk_forward_train, walk_forward_inference
 from models.base import Model
+from sklearn.preprocessing import MinMaxScaler
 
 no_of_rows = 100
 
@@ -65,9 +66,9 @@ def test_walk_forward_train_test():
     window_length = 10
 
     model = IncrementingStubModel(window_length = window_length)
-    scaler = None
+    scaler = MinMaxScaler()
 
-    models, predictions, probs = walk_forward_train_test(
+    models, scalers = walk_forward_train(
         model_name='test',
         model=model,
         X=X,
@@ -77,6 +78,14 @@ def test_walk_forward_train_test():
         window_size=window_length,
         retrain_every=10,
         scaler=scaler)
+    predictions, probs = walk_forward_inference(
+        model_name='test',
+        models=models,
+        scalers=scalers,
+        X=X,
+        expanding_window=False,
+        window_size=window_length
+    )
     
     # verify if predictions are the same as y
     for i in range(window_length+2, no_of_rows):

training/primary_model.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from typing import Literal
-from training.walk_forward import walk_forward_train_test
+from training.walk_forward import walk_forward_train, walk_forward_inference
 from utils.evaluate import evaluate_predictions
 from models.base import Model
 from utils.scaler import get_scaler
@@ -31,7 +31,7 @@ def train_primary_model(
     probabilities = pd.DataFrame(index=y.index)
     
     for model_name, model in models:
-        model_over_time, preds, probs = walk_forward_train_test(
+        model_over_time, scaler_over_time = walk_forward_train(
             model_name=model_name,
             model = model,
             X = X if model.feature_selection == 'on' else original_X,
@@ -42,6 +42,14 @@ def train_primary_model(
             retrain_every = retrain_every,
             scaler = scaler
         )
+        preds, probs = walk_forward_inference(
+            model_name = model_name,
+            models = model_over_time,
+            X = X if model.feature_selection == 'on' else original_X,
+            expanding_window = expanding_window,
+            window_size = sliding_window_size,
+            scalers = scaler_over_time
+        )
         
         assert len(preds) == len(y)
         result = evaluate_predictions(

training/walk_forward.py

@@ -3,24 +3,24 @@ from models.base import Model
 import numpy as np
 from utils.helpers import get_first_valid_return_index
 from tqdm import tqdm
-
+from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler
+from typing import Union
 from sklearn.base import clone
 
-def walk_forward_train_test(
+def walk_forward_train(
-                            model_name: str,
+                        model_name: str,
-                            model: Model,
+                        model: Model,
-                            X: pd.DataFrame,
+                        X: pd.DataFrame,
-                            y: pd.Series,
+                        y: pd.Series,
-                            target_returns: pd.Series,
+                        target_returns: pd.Series,
-                            expanding_window: bool,
+                        expanding_window: bool,
-                            window_size: int,
+                        window_size: int,
-                            retrain_every: int,
+                        retrain_every: int,
-                            scaler,
+                        scaler: Union[MinMaxScaler, Normalizer, StandardScaler],
-                        ) -> tuple[pd.Series, pd.Series, pd.DataFrame]:
+                    ) -> tuple[pd.Series, pd.Series]:
     assert len(X) == len(y)
-    predictions = pd.Series(index=y.index).rename(model_name)
-    probabilities = pd.DataFrame(index=y.index)
     models = pd.Series(index=y.index).rename(model_name)
+    scalers = pd.Series(index=y.index).rename("scaler_" + model_name)
 
     first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y))
     train_from = first_nonzero_return + window_size + 1
@@ -29,10 +29,8 @@ def walk_forward_train_test(
 
     if model.only_column is not None:
         X = X[[column for column in X.columns if model.only_column in column]]
-
         
-        
+    is_scaling_on = model.data_scaling == 'scaled'
-    is_scaling_on = scaler is not None and model.data_scaling == 'scaled'
 
     if is_scaling_on:
         scaler = clone(scaler)
@@ -47,39 +45,71 @@ def walk_forward_train_test(
 
             train_window_end = index - 1
             
+            current_scaler = None
             if is_scaling_on:
                 # We need to fit on the expanding window data slice
                 # This is our only way to avoid lookahead bias
+                current_scaler = clone(scaler)
                 X_expanding_window = X[first_nonzero_return:train_window_end]
-                scaler.fit(X_expanding_window.values)
+                current_scaler.fit(X_expanding_window.values)
 
-            X_slice = X[train_window_start:train_window_end]
+            X_slice = X[train_window_start:train_window_end].to_numpy()
-            y_slice = y[train_window_start:train_window_end]
+            y_slice = y[train_window_start:train_window_end].to_numpy()
 
             if is_scaling_on:
-                X_slice = scaler.transform(X_slice.values)
+                X_slice = current_scaler.transform(X_slice)
-            else:
-                X_slice = X_slice.to_numpy()
-            
             
             current_model = model.clone()
 
             current_model.initialize_network(input_dim = len(X_slice[0]), output_dim=1) 
-            current_model.fit(X_slice, y_slice.to_numpy())
+            current_model.fit(X_slice, y_slice)
 
             iterations_before_retrain = retrain_every
-        else:
-            current_model = models[index-1]
 
         models[index] = current_model
+        scalers[index] = current_scaler
 
-        if model.predict_window_size == 'window_size': 
+        iterations_before_retrain -= 1
+
+    return models, scalers
+
+def walk_forward_inference(
+                    model_name: str,
+                    models: pd.Series,
+                    scalers: pd.Series,
+                    X: pd.DataFrame,
+                    expanding_window: bool,
+                    window_size: int,
+                ) -> tuple[pd.Series, pd.DataFrame]:
+    predictions = pd.Series(index=X.index).rename(model_name)
+    probabilities = pd.DataFrame(index=X.index)
+
+    first_nonzero_return = get_first_valid_return_index(models)
+    train_from = first_nonzero_return
+    train_till = X.shape[0]
+    first_model = models[first_nonzero_return]
+
+    if first_model.only_column is not None:
+        X = X[[column for column in X.columns if first_model.only_column in column]]
+
+    is_scaling_on = first_model.data_scaling == 'scaled'
+
+    for index in tqdm(range(train_from, train_till)):
+        if expanding_window:
+            train_window_start = first_nonzero_return
+        else:
+            train_window_start = index - window_size - 1
+
+        current_model = models[index]
+        curren_scaler = scalers[index]
+
+        if current_model.predict_window_size == 'window_size': 
             next_timestep = X.iloc[train_window_start:index].to_numpy()#.reshape(1, -1)
         else: 
             next_timestep = X.iloc[index].to_numpy().reshape(1, -1)
         
         if is_scaling_on:
-            next_timestep = scaler.transform(next_timestep)
+            next_timestep = curren_scaler.transform(next_timestep)
 
         prediction, probs = current_model.predict(next_timestep)
         predictions[index] = prediction
@@ -87,6 +117,4 @@ def walk_forward_train_test(
             probabilities = probabilities.reindex(columns = ["prob_" + str(num) for num in range(0, len(probs.T))])
         probabilities.iloc[index] = probs
 
-        iterations_before_retrain -= 1
+    return predictions, probabilities
-
-    return models, predictions, probabilities

utils/scaler.py

@@ -1,8 +1,8 @@
 from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler
-from typing import Optional, Union
+from typing import Union
 from utils.types import ScalerTypes
 
-def get_scaler(type: ScalerTypes) -> Optional[Union[MinMaxScaler, Normalizer, StandardScaler]]:
+def get_scaler(type: ScalerTypes) -> Union[MinMaxScaler, Normalizer, StandardScaler]:
     if type == 'normalize':
         return Normalizer()
     elif type == 'minmax':
@@ -10,4 +10,4 @@ def get_scaler(type: ScalerTypes) -> Optional[Union[MinMaxScaler, Normalizer, St
     elif type == 'standardize':
         return StandardScaler()
     else:
-        return None
+        raise Exception("Scaler type not supported")