diff --git a/feature_selection/feature_selection.py b/feature_selection/feature_selection.py index 1f85cf8..55f5751 100644 --- a/feature_selection/feature_selection.py +++ b/feature_selection/feature_selection.py @@ -25,7 +25,7 @@ def select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_s # 2. Recursive feature selection cv = TimeSeriesSplit(n_splits=5) scaler = get_scaler(scaling) - X_scaled = scaler.fit_transform(X) + X_scaled = scaler.fit_transform(X, y) feat_selector_model = model.model if hasattr(feat_selector_model, 'feature_importances_') == False and hasattr(feat_selector_model, 'coef_') == False: diff --git a/models/average.py b/models/average.py index ddb4670..ac77642 100644 --- a/models/average.py +++ b/models/average.py @@ -7,7 +7,7 @@ class StaticAverageModel(Model): Model that averages . ''' - data_scaling = 'unscaled' + data_transformation = 'original' only_column = 'model_' feature_selection = 'off' model_type = 'static' diff --git a/models/base.py b/models/base.py index 3ea6343..5ed44ce 100644 --- a/models/base.py +++ b/models/base.py @@ -7,7 +7,7 @@ import numpy as np class Model(ABC): - data_scaling: Literal["scaled", "unscaled"] + data_transformation: Literal["transformed", "original"] feature_selection: Literal["on", "off"] # data_format: Literal["wide", "narrow"] only_column: Optional[str] diff --git a/models/momentum.py b/models/momentum.py index 65929ec..a4f17f6 100644 --- a/models/momentum.py +++ b/models/momentum.py @@ -7,7 +7,7 @@ class StaticMomentumModel(Model): Model that uses only one feature: momentum. It's positive if momentum is greater than 0, otherwise it's negative. ''' - data_scaling = 'unscaled' + data_transformation = 'original' only_column = 'mom' feature_selection = 'off' model_type = 'static' diff --git a/models/naive.py b/models/naive.py index 4f3eba5..bbf6688 100644 --- a/models/naive.py +++ b/models/naive.py @@ -7,7 +7,7 @@ class StaticNaiveModel(Model): Model that carries the last observation (from returns) to the next one, naively. ''' - data_scaling = 'unscaled' + data_transformation = 'original' only_column = None feature_selection = 'off' model_type = 'static' diff --git a/models/neural.py b/models/neural.py index ad392e8..0114ebc 100644 --- a/models/neural.py +++ b/models/neural.py @@ -7,7 +7,7 @@ import pytorch_lightning as pl class LightningNeuralNetModel(Model): - data_scaling = 'scaled' + data_transformation = 'transformed' only_column = None feature_selection = 'off' model_type = 'ml' diff --git a/models/sklearn.py b/models/sklearn.py index e2e33f0..269fcf9 100644 --- a/models/sklearn.py +++ b/models/sklearn.py @@ -6,7 +6,7 @@ from sklearn.base import clone class SKLearnModel(Model): - data_scaling = 'scaled' + data_transformation = 'transformed' only_column = None feature_selection = 'on' model_type = 'ml' diff --git a/models/statsmodels.py b/models/statsmodels.py index 7c1b30a..4ac8e20 100644 --- a/models/statsmodels.py +++ b/models/statsmodels.py @@ -8,7 +8,7 @@ from copy import deepcopy class StatsModel(Model): # This is work in progress - data_scaling = 'scaled' + data_transformation = 'transformed' only_column = None feature_selection = 'on' model_type = 'ml' diff --git a/models/xgboost.py b/models/xgboost.py index 9d1d09c..d6e9189 100644 --- a/models/xgboost.py +++ b/models/xgboost.py @@ -6,7 +6,7 @@ from sklearn.base import clone class XGBoostModel(Model): - data_scaling = 'scaled' + data_transformation = 'transformed' only_column = None feature_selection = 'on' model_type = 'ml' diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 471d876..419df86 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -4,7 +4,6 @@ import pandas as pd from training.walk_forward import walk_forward_train, walk_forward_inference from models.base import Model from utils.evaluate import evaluate_predictions -from sklearn.preprocessing import MinMaxScaler no_of_rows = 100 @@ -36,7 +35,7 @@ class EvenOddStubModel(Model): It verifies that the X[n][any_column] == 1 if n is even, ''' - data_scaling = "unscaled" + data_transformation = "original" only_column = None predict_window_size = 'single_timestamp' @@ -68,9 +67,8 @@ def test_evaluation(): window_length = 10 model = EvenOddStubModel(window_length = window_length) - scaler = MinMaxScaler() - models, scalers = walk_forward_train( + model_over_time, transformations_over_time = walk_forward_train( model_name='test', model=model, X=X, @@ -79,11 +77,11 @@ def test_evaluation(): expanding_window=False, window_size=window_length, retrain_every=10, - scaler=scaler) - predictions, probs = walk_forward_inference( + transformations=[]) + predictions, _ = walk_forward_inference( model_name='test', - models=models, - scalers=scalers, + model_over_time=model_over_time, + transformations_over_time=transformations_over_time, X=X, expanding_window=False, window_size=window_length diff --git a/tests/test_walk_forward.py b/tests/test_walk_forward.py index 4bc071d..e5c1e44 100644 --- a/tests/test_walk_forward.py +++ b/tests/test_walk_forward.py @@ -2,7 +2,6 @@ import numpy as np import pandas as pd from training.walk_forward import walk_forward_train, walk_forward_inference from models.base import Model -from sklearn.preprocessing import MinMaxScaler no_of_rows = 100 @@ -34,7 +33,7 @@ class IncrementingStubModel(Model): It verifies that the X[n][any_column]+1 == y[n] ''' - data_scaling = "unscaled" + data_transformation = "original" only_column = None predict_window_size = 'single_timestamp' @@ -66,9 +65,8 @@ def test_walk_forward_train_test(): window_length = 10 model = IncrementingStubModel(window_length = window_length) - scaler = MinMaxScaler() - models, scalers = walk_forward_train( + model_over_time, transformations_over_time = walk_forward_train( model_name='test', model=model, X=X, @@ -77,11 +75,11 @@ def test_walk_forward_train_test(): expanding_window=False, window_size=window_length, retrain_every=10, - scaler=scaler) - predictions, probs = walk_forward_inference( + transformations=[]) + predictions, _ = walk_forward_inference( model_name='test', - models=models, - scalers=scalers, + model_over_time=model_over_time, + transformations_over_time=transformations_over_time, X=X, expanding_window=False, window_size=window_length diff --git a/training/primary_model.py b/training/primary_model.py index 993aecb..39f76a4 100644 --- a/training/primary_model.py +++ b/training/primary_model.py @@ -6,6 +6,7 @@ from models.base import Model from utils.scaler import get_scaler from utils.types import ScalerTypes from utils.encapsulation import Training_Step, Single_Model, Asset +from transformations.sklearn import SKLearnTransformation def train_primary_model( ticker_to_predict: str, @@ -24,8 +25,6 @@ def train_primary_model( print_results: bool, ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, list[Single_Model]]: - scaler = get_scaler(scaler) - results = pd.DataFrame() predictions = pd.DataFrame(index=y.index) probabilities = pd.DataFrame(index=y.index) @@ -34,7 +33,7 @@ def train_primary_model( for model_name, model in models: - model_over_time, scaler_over_time = walk_forward_train( + model_over_time, transformations_over_time = walk_forward_train( model_name=model_name, model = model, X = X if model.feature_selection == 'on' else original_X, @@ -43,15 +42,15 @@ def train_primary_model( expanding_window = expanding_window, window_size = sliding_window_size, retrain_every = retrain_every, - scaler = scaler + transformations= [get_scaler(scaler)], ) preds, probs = walk_forward_inference( model_name = model_name, - models = model_over_time, + model_over_time= model_over_time, + transformations_over_time = transformations_over_time, X = X if model.feature_selection == 'on' else original_X, expanding_window = expanding_window, - window_size = sliding_window_size, - scalers = scaler_over_time + window_size = sliding_window_size ) assert len(preds) == len(y) diff --git a/training/walk_forward.py b/training/walk_forward.py index 4f28fae..ac4dfc5 100644 --- a/training/walk_forward.py +++ b/training/walk_forward.py @@ -6,6 +6,7 @@ from tqdm import tqdm from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler from typing import Union from sklearn.base import clone +from transformations.base import Transformation def walk_forward_train( model_name: str, @@ -16,11 +17,11 @@ def walk_forward_train( expanding_window: bool, window_size: int, retrain_every: int, - scaler: Union[MinMaxScaler, Normalizer, StandardScaler], - ) -> tuple[pd.Series, pd.Series]: + transformations: list[Transformation], + ) -> tuple[pd.Series, list[pd.Series]]: assert len(X) == len(y) - models = pd.Series(index=y.index).rename(model_name) - scalers = pd.Series(index=y.index).rename("scaler_" + model_name) + models_over_time = pd.Series(index=y.index).rename(model_name) + transformations_over_time = [pd.Series(index=y.index).rename(t.get_name()) for t in transformations] first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y)) train_from = first_nonzero_return + window_size + 1 @@ -29,36 +30,32 @@ def walk_forward_train( if model.only_column is not None: X = X[[column for column in X.columns if model.only_column in column]] - - is_scaling_on = model.data_scaling == 'scaled' - - if is_scaling_on: - scaler = clone(scaler) - + + if model.data_transformation == 'original': + transformations = [] + for index in tqdm(range(train_from, train_till)): - if expanding_window: - train_window_start = first_nonzero_return - else: - train_window_start = index - window_size - 1 + train_window_start = first_nonzero_return if expanding_window else index - window_size - 1 - if iterations_before_retrain <= 0 or pd.isna(models[index-1]): + if iterations_before_retrain <= 0 or pd.isna(models_over_time[index-1]): train_window_end = index - 1 - current_scaler = None - if is_scaling_on: - # We need to fit on the expanding window data slice - # This is our only way to avoid lookahead bias - current_scaler = clone(scaler) - X_expanding_window = X[first_nonzero_return:train_window_end] - current_scaler.fit(X_expanding_window.values) + X_expanding_window = X[first_nonzero_return:train_window_end] + y_expanding_window = y[first_nonzero_return:train_window_end] - X_slice = X[train_window_start:train_window_end].to_numpy() + current_transformations = [t.clone() for t in transformations] + for transformation_index, transformation in enumerate(current_transformations): + transformation.fit_transform(X_expanding_window, y_expanding_window) + + X_slice = X[train_window_start:train_window_end] + + for transformation in current_transformations: + X_slice = transformation.transform(X_slice) + + X_slice = X_slice.to_numpy() y_slice = y[train_window_start:train_window_end].to_numpy() - if is_scaling_on: - X_slice = current_scaler.transform(X_slice) - current_model = model.clone() current_model.initialize_network(input_dim = len(X_slice[0]), output_dim=1) @@ -66,17 +63,18 @@ def walk_forward_train( iterations_before_retrain = retrain_every - models[index] = current_model - scalers[index] = current_scaler + models_over_time[index] = current_model + for transformation_index, transformation in enumerate(current_transformations): + transformations_over_time[transformation_index][index] = transformation iterations_before_retrain -= 1 - return models, scalers + return models_over_time, transformations_over_time def walk_forward_inference( model_name: str, - models: pd.Series, - scalers: pd.Series, + model_over_time: pd.Series, + transformations_over_time: list[pd.Series], X: pd.DataFrame, expanding_window: bool, window_size: int, @@ -84,32 +82,33 @@ def walk_forward_inference( predictions = pd.Series(index=X.index).rename(model_name) probabilities = pd.DataFrame(index=X.index) - first_nonzero_return = get_first_valid_return_index(models) - train_from = first_nonzero_return - train_till = X.shape[0] - first_model = models[first_nonzero_return] + inference_from = get_first_valid_return_index(model_over_time) + inference_till = X.shape[0] + first_model = model_over_time[inference_from] if first_model.only_column is not None: X = X[[column for column in X.columns if first_model.only_column in column]] + + if first_model.data_transformation == 'original': + transformations_over_time = [] - is_scaling_on = first_model.data_scaling == 'scaled' + for index in tqdm(range(inference_from, inference_till)): + + train_window_start = inference_from if expanding_window else index - window_size - 1 - for index in tqdm(range(train_from, train_till)): - if expanding_window: - train_window_start = first_nonzero_return - else: - train_window_start = index - window_size - 1 - - current_model = models[index] - curren_scaler = scalers[index] + current_model = model_over_time[index] + current_transformations = [transformation_over_time[index] for transformation_over_time in transformations_over_time] if current_model.predict_window_size == 'window_size': - next_timestep = X.iloc[train_window_start:index].to_numpy()#.reshape(1, -1) + next_timestep = X.iloc[train_window_start:index] else: - next_timestep = X.iloc[index].to_numpy().reshape(1, -1) + # we need to get a Dataframe out of it, since the transformation step always expects a 2D array, but it's equivalent to X.iloc[index] + next_timestep = X.iloc[index:index+1] - if is_scaling_on: - next_timestep = curren_scaler.transform(next_timestep) + for transformation in current_transformations: + next_timestep = transformation.transform(next_timestep) + + next_timestep = next_timestep.to_numpy() prediction, probs = current_model.predict(next_timestep) predictions[index] = prediction diff --git a/transformations/base.py b/transformations/base.py new file mode 100644 index 0000000..bafa10c --- /dev/null +++ b/transformations/base.py @@ -0,0 +1,31 @@ +from __future__ import annotations +from typing import Literal, Optional, Union +from abc import ABC, abstractmethod +import pandas as pd + +class Transformation(ABC): + + @abstractmethod + def fit(self, X: pd.DataFrame, y: Optional[pd.Series]) -> None: + raise NotImplementedError + + @abstractmethod + def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> pd.DataFrame: + raise NotImplementedError + + @abstractmethod + def transform(self, X: pd.DataFrame) -> pd.DataFrame: + raise NotImplementedError + + @abstractmethod + def clone(self) -> Transformation: + raise NotImplementedError + + @abstractmethod + def get_name(self) -> str: + raise NotImplementedError + + + + + diff --git a/transformations/sklearn.py b/transformations/sklearn.py new file mode 100644 index 0000000..7940218 --- /dev/null +++ b/transformations/sklearn.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from transformations.base import Transformation +from typing import Literal, Optional, Union +from sklearn.base import clone, BaseEstimator +import pandas as pd + +class SKLearnTransformation(Transformation): + + transformer: BaseEstimator + + def __init__(self, transformer: BaseEstimator): + self.transformer = transformer + + def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> None: + self.transformer.fit(X, y) + + def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series]) -> pd.DataFrame: + self.fit(X, y) + return self.transform(X) + + def transform(self, X: pd.DataFrame) -> pd.DataFrame: + return pd.DataFrame(self.transformer.transform(X), index = X.index, columns = X.columns) + + def clone(self) -> SKLearnTransformation: + return SKLearnTransformation(clone(self.transformer)) + + def get_name(self) -> str: + return self.transformer.__class__.__name__ + + + + + + diff --git a/utils/scaler.py b/utils/scaler.py index d021d4b..17f3630 100644 --- a/utils/scaler.py +++ b/utils/scaler.py @@ -1,13 +1,13 @@ from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler -from typing import Union +from transformations.sklearn import SKLearnTransformation from utils.types import ScalerTypes -def get_scaler(type: ScalerTypes) -> Union[MinMaxScaler, Normalizer, StandardScaler]: +def get_scaler(type: ScalerTypes) -> SKLearnTransformation: if type == 'normalize': - return Normalizer() + return SKLearnTransformation(Normalizer()) elif type == 'minmax': - return MinMaxScaler(feature_range= (-1, 1)) + return SKLearnTransformation(MinMaxScaler(feature_range= (-1, 1))) elif type == 'standardize': - return StandardScaler() + return SKLearnTransformation(StandardScaler()) else: raise Exception("Scaler type not supported") \ No newline at end of file