diff --git a/config/config.py b/config/config.py index f5a2e87..d808792 100644 --- a/config/config.py +++ b/config/config.py @@ -10,7 +10,7 @@ def get_dev_config() -> tuple[dict, dict, dict]: sliding_window_size_primary = 380, sliding_window_size_meta_labeling = 1, retrain_every = 20, - scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' + scaler = 'minmax', # 'normalize' 'minmax' 'standardize' ) data_config = dict( @@ -53,7 +53,7 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]: sliding_window_size_primary = 380, sliding_window_size_meta_labeling = 240, retrain_every = 20, - scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' + scaler = 'minmax', # 'normalize' 'minmax' 'standardize' ) data_config = dict( @@ -98,7 +98,7 @@ def get_lightweight_ensemble_config() -> tuple[dict, dict, dict]: sliding_window_size_primary = 380, sliding_window_size_meta_labeling = 240, retrain_every = 20, - scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' + scaler = 'minmax', # 'normalize' 'minmax' 'standardize' ) data_config = dict( diff --git a/feature_selection/feature_selection.py b/feature_selection/feature_selection.py index 623045c..1f85cf8 100644 --- a/feature_selection/feature_selection.py +++ b/feature_selection/feature_selection.py @@ -25,9 +25,7 @@ def select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_s # 2. Recursive feature selection cv = TimeSeriesSplit(n_splits=5) scaler = get_scaler(scaling) - X_scaled = X.copy() - if scaler is not None: - X_scaled = scaler.fit_transform(X_scaled) + X_scaled = scaler.fit_transform(X) feat_selector_model = model.model if hasattr(feat_selector_model, 'feature_importances_') == False and hasattr(feat_selector_model, 'coef_') == False: diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 83df433..471d876 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -1,9 +1,10 @@ import numpy as np import pandas as pd -from training.walk_forward import walk_forward_train_test +from training.walk_forward import walk_forward_train, walk_forward_inference from models.base import Model from utils.evaluate import evaluate_predictions +from sklearn.preprocessing import MinMaxScaler no_of_rows = 100 @@ -67,9 +68,9 @@ def test_evaluation(): window_length = 10 model = EvenOddStubModel(window_length = window_length) - scaler = None + scaler = MinMaxScaler() - models, predictions, probs = walk_forward_train_test( + models, scalers = walk_forward_train( model_name='test', model=model, X=X, @@ -78,7 +79,14 @@ def test_evaluation(): expanding_window=False, window_size=window_length, retrain_every=10, - scaler=scaler + scaler=scaler) + predictions, probs = walk_forward_inference( + model_name='test', + models=models, + scalers=scalers, + X=X, + expanding_window=False, + window_size=window_length ) # verify if predictions are the same as y diff --git a/tests/test_walk_forward.py b/tests/test_walk_forward.py index b0cb7b4..4bc071d 100644 --- a/tests/test_walk_forward.py +++ b/tests/test_walk_forward.py @@ -1,7 +1,8 @@ import numpy as np import pandas as pd -from training.walk_forward import walk_forward_train_test +from training.walk_forward import walk_forward_train, walk_forward_inference from models.base import Model +from sklearn.preprocessing import MinMaxScaler no_of_rows = 100 @@ -65,9 +66,9 @@ def test_walk_forward_train_test(): window_length = 10 model = IncrementingStubModel(window_length = window_length) - scaler = None + scaler = MinMaxScaler() - models, predictions, probs = walk_forward_train_test( + models, scalers = walk_forward_train( model_name='test', model=model, X=X, @@ -77,6 +78,14 @@ def test_walk_forward_train_test(): window_size=window_length, retrain_every=10, scaler=scaler) + predictions, probs = walk_forward_inference( + model_name='test', + models=models, + scalers=scalers, + X=X, + expanding_window=False, + window_size=window_length + ) # verify if predictions are the same as y for i in range(window_length+2, no_of_rows): diff --git a/training/primary_model.py b/training/primary_model.py index 7918f9c..f04a778 100644 --- a/training/primary_model.py +++ b/training/primary_model.py @@ -1,6 +1,6 @@ import pandas as pd from typing import Literal -from training.walk_forward import walk_forward_train_test +from training.walk_forward import walk_forward_train, walk_forward_inference from utils.evaluate import evaluate_predictions from models.base import Model from utils.scaler import get_scaler @@ -31,7 +31,7 @@ def train_primary_model( probabilities = pd.DataFrame(index=y.index) for model_name, model in models: - model_over_time, preds, probs = walk_forward_train_test( + model_over_time, scaler_over_time = walk_forward_train( model_name=model_name, model = model, X = X if model.feature_selection == 'on' else original_X, @@ -42,6 +42,14 @@ def train_primary_model( retrain_every = retrain_every, scaler = scaler ) + preds, probs = walk_forward_inference( + model_name = model_name, + models = model_over_time, + X = X if model.feature_selection == 'on' else original_X, + expanding_window = expanding_window, + window_size = sliding_window_size, + scalers = scaler_over_time + ) assert len(preds) == len(y) result = evaluate_predictions( diff --git a/training/walk_forward.py b/training/walk_forward.py index 68af042..4f28fae 100644 --- a/training/walk_forward.py +++ b/training/walk_forward.py @@ -3,24 +3,24 @@ from models.base import Model import numpy as np from utils.helpers import get_first_valid_return_index from tqdm import tqdm - +from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler +from typing import Union from sklearn.base import clone -def walk_forward_train_test( - model_name: str, - model: Model, - X: pd.DataFrame, - y: pd.Series, - target_returns: pd.Series, - expanding_window: bool, - window_size: int, - retrain_every: int, - scaler, - ) -> tuple[pd.Series, pd.Series, pd.DataFrame]: +def walk_forward_train( + model_name: str, + model: Model, + X: pd.DataFrame, + y: pd.Series, + target_returns: pd.Series, + expanding_window: bool, + window_size: int, + retrain_every: int, + scaler: Union[MinMaxScaler, Normalizer, StandardScaler], + ) -> tuple[pd.Series, pd.Series]: assert len(X) == len(y) - predictions = pd.Series(index=y.index).rename(model_name) - probabilities = pd.DataFrame(index=y.index) models = pd.Series(index=y.index).rename(model_name) + scalers = pd.Series(index=y.index).rename("scaler_" + model_name) first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y)) train_from = first_nonzero_return + window_size + 1 @@ -29,10 +29,8 @@ def walk_forward_train_test( if model.only_column is not None: X = X[[column for column in X.columns if model.only_column in column]] - - - is_scaling_on = scaler is not None and model.data_scaling == 'scaled' + is_scaling_on = model.data_scaling == 'scaled' if is_scaling_on: scaler = clone(scaler) @@ -47,39 +45,71 @@ def walk_forward_train_test( train_window_end = index - 1 + current_scaler = None if is_scaling_on: # We need to fit on the expanding window data slice # This is our only way to avoid lookahead bias + current_scaler = clone(scaler) X_expanding_window = X[first_nonzero_return:train_window_end] - scaler.fit(X_expanding_window.values) + current_scaler.fit(X_expanding_window.values) - X_slice = X[train_window_start:train_window_end] - y_slice = y[train_window_start:train_window_end] + X_slice = X[train_window_start:train_window_end].to_numpy() + y_slice = y[train_window_start:train_window_end].to_numpy() if is_scaling_on: - X_slice = scaler.transform(X_slice.values) - else: - X_slice = X_slice.to_numpy() - + X_slice = current_scaler.transform(X_slice) current_model = model.clone() current_model.initialize_network(input_dim = len(X_slice[0]), output_dim=1) - current_model.fit(X_slice, y_slice.to_numpy()) + current_model.fit(X_slice, y_slice) iterations_before_retrain = retrain_every - else: - current_model = models[index-1] models[index] = current_model + scalers[index] = current_scaler - if model.predict_window_size == 'window_size': + iterations_before_retrain -= 1 + + return models, scalers + +def walk_forward_inference( + model_name: str, + models: pd.Series, + scalers: pd.Series, + X: pd.DataFrame, + expanding_window: bool, + window_size: int, + ) -> tuple[pd.Series, pd.DataFrame]: + predictions = pd.Series(index=X.index).rename(model_name) + probabilities = pd.DataFrame(index=X.index) + + first_nonzero_return = get_first_valid_return_index(models) + train_from = first_nonzero_return + train_till = X.shape[0] + first_model = models[first_nonzero_return] + + if first_model.only_column is not None: + X = X[[column for column in X.columns if first_model.only_column in column]] + + is_scaling_on = first_model.data_scaling == 'scaled' + + for index in tqdm(range(train_from, train_till)): + if expanding_window: + train_window_start = first_nonzero_return + else: + train_window_start = index - window_size - 1 + + current_model = models[index] + curren_scaler = scalers[index] + + if current_model.predict_window_size == 'window_size': next_timestep = X.iloc[train_window_start:index].to_numpy()#.reshape(1, -1) else: next_timestep = X.iloc[index].to_numpy().reshape(1, -1) if is_scaling_on: - next_timestep = scaler.transform(next_timestep) + next_timestep = curren_scaler.transform(next_timestep) prediction, probs = current_model.predict(next_timestep) predictions[index] = prediction @@ -87,6 +117,4 @@ def walk_forward_train_test( probabilities = probabilities.reindex(columns = ["prob_" + str(num) for num in range(0, len(probs.T))]) probabilities.iloc[index] = probs - iterations_before_retrain -= 1 - - return models, predictions, probabilities + return predictions, probabilities diff --git a/utils/scaler.py b/utils/scaler.py index d36e5d0..d021d4b 100644 --- a/utils/scaler.py +++ b/utils/scaler.py @@ -1,8 +1,8 @@ from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler -from typing import Optional, Union +from typing import Union from utils.types import ScalerTypes -def get_scaler(type: ScalerTypes) -> Optional[Union[MinMaxScaler, Normalizer, StandardScaler]]: +def get_scaler(type: ScalerTypes) -> Union[MinMaxScaler, Normalizer, StandardScaler]: if type == 'normalize': return Normalizer() elif type == 'minmax': @@ -10,4 +10,4 @@ def get_scaler(type: ScalerTypes) -> Optional[Union[MinMaxScaler, Normalizer, St elif type == 'standardize': return StandardScaler() else: - return None \ No newline at end of file + raise Exception("Scaler type not supported") \ No newline at end of file