fix(FeatureExtractor): use a rolling z-score instead of StandardScaler with unavoidable lookahead bias (#146)

* fix(FeatureExtractor): use a rolling z-score instead of StandardScaler with unavoidable lookahead bias

* chore(Archive): removed archived models

* fix(FeatureExtractors): syntax

* fix(FeatureExtractors): mistake with expanding window

archive/keras_models/classification.py

@@ -1,102 +0,0 @@
-import keras
-
-def create_basic_lstm_model(input_shape, num_classes):
-    model = keras.Sequential()
-    model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=input_shape))
-    model.add(keras.layers.Flatten())
-    model.add(keras.layers.Dropout(0.3))
-    model.add(keras.layers.Dense(units = 64, activation = 'sigmoid'))
-    model.add(keras.layers.Dropout(0.3))
-    model.add(keras.layers.Dense(units = 32, activation = 'sigmoid'))
-    model.add(keras.layers.Dropout(0.3))
-    model.add(keras.layers.Dense(units = num_classes, activation = 'linear'))
-    return model
-
-def create_basic_cnn_model(input_shape, num_classes):
-    model = keras.Sequential()
-    model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same", input_shape=input_shape))
-    model.add(keras.layers.BatchNormalization())
-    model.add(keras.layers.ReLU())
-    model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same"))
-    model.add(keras.layers.BatchNormalization())
-    model.add(keras.layers.ReLU())
-    model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same"))
-    model.add(keras.layers.BatchNormalization())
-    model.add(keras.layers.ReLU())
-    model.add(keras.layers.GlobalAveragePooling1D())
-    model.add(keras.layers.Dense(num_classes, activation="linear"))
-    return model
-
-
-def create_resnet_cnn_model(input_shape, num_classes):
-    n_feature_maps = 24
-
-    input_layer = keras.layers.Input(input_shape)
-
-
-    conv_x = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=8, padding='same')(input_layer)
-    conv_x = keras.layers.BatchNormalization()(conv_x)
-    conv_x = keras.layers.Activation('relu')(conv_x)
-
-    conv_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=5, padding='same')(conv_x)
-    conv_y = keras.layers.BatchNormalization()(conv_y)
-    conv_y = keras.layers.Activation('relu')(conv_y)
-
-    conv_z = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=3, padding='same')(conv_y)
-    conv_z = keras.layers.BatchNormalization()(conv_z)
-
-    # expand channels for the sum
-    shortcut_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=1, padding='same')(input_layer)
-    shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
-
-    output_block_1 = keras.layers.add([shortcut_y, conv_z])
-    output_block_1 = keras.layers.Activation('relu')(output_block_1)
-
-    # BLOCK 2
-
-    conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_1)
-    conv_x = keras.layers.BatchNormalization()(conv_x)
-    conv_x = keras.layers.Activation('relu')(conv_x)
-
-    conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)
-    conv_y = keras.layers.BatchNormalization()(conv_y)
-    conv_y = keras.layers.Activation('relu')(conv_y)
-
-    conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)
-    conv_z = keras.layers.BatchNormalization()(conv_z)
-
-    # expand channels for the sum
-    shortcut_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=1, padding='same')(output_block_1)
-    shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
-
-    output_block_2 = keras.layers.add([shortcut_y, conv_z])
-    output_block_2 = keras.layers.Activation('relu')(output_block_2)
-
-    # BLOCK 3
-
-    conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_2)
-    conv_x = keras.layers.BatchNormalization()(conv_x)
-    conv_x = keras.layers.Activation('relu')(conv_x)
-
-    conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)
-    conv_y = keras.layers.BatchNormalization()(conv_y)
-    conv_y = keras.layers.Activation('relu')(conv_y)
-
-    conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)
-    conv_z = keras.layers.BatchNormalization()(conv_z)
-
-    # no need to expand channels because they are equal
-    shortcut_y = keras.layers.BatchNormalization()(output_block_2)
-
-    output_block_3 = keras.layers.add([shortcut_y, conv_z])
-    output_block_3 = keras.layers.Activation('relu')(output_block_3)
-
-    # FINAL
-
-    gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3)
-
-    output_layer = keras.layers.Dense(num_classes, activation='linear')(gap_layer)
-    model = keras.models.Model(inputs=input_layer, outputs=output_layer)
-
-
-    return model

archive/keras_models/classification_transformer.py

@@ -1,41 +0,0 @@
-from keras import layers
-import keras
-
-def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):
-    # Normalization and Attention
-    x = layers.LayerNormalization(epsilon=1e-6)(inputs)
-    x = layers.MultiHeadAttention(
-        key_dim=head_size, num_heads=num_heads, dropout=dropout
-    )(x, x)
-    x = layers.Dropout(dropout)(x)
-    res = x + inputs
-
-    # Feed Forward Part
-    x = layers.LayerNormalization(epsilon=1e-6)(res)
-    x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation="relu")(x)
-    x = layers.Dropout(dropout)(x)
-    x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)
-    return x + res
-
-def create_basic_transformer_model(
-    input_shape,
-    n_classes,
-    head_size,
-    num_heads,
-    ff_dim,
-    num_transformer_blocks,
-    mlp_units,
-    dropout=0,
-    mlp_dropout=0,
-):
-    inputs = keras.Input(shape=input_shape)
-    x = inputs
-    for _ in range(num_transformer_blocks):
-        x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)
-
-    x = layers.GlobalAveragePooling1D(data_format="channels_first")(x)
-    for dim in mlp_units:
-        x = layers.Dense(dim, activation="relu")(x)
-        x = layers.Dropout(mlp_dropout)(x)
-    outputs = layers.Dense(n_classes, activation="softmax")(x)
-    return keras.Model(inputs, outputs)

archive/model_classification_keras.py

@@ -1,134 +0,0 @@
-#%% Import all the stuff, load data, define constants
-from sklearn.utils import shuffle
-from data_loader.load_data import load_files, create_target_classes
-import pandas as pd
-from tensorflow import keras
-from utils.normalize import normalize
-import tensorflow as tf
-from utils.visualize import visualize_loss
-from sklearn.preprocessing import MinMaxScaler
-from utils.evaluate import print_classification_metrics
-import numpy as np
-from utils.rolling import rolling_window
-from keras_models.classification import create_basic_cnn_model, create_basic_lstm_model, create_resnet_cnn_model
-from keras_models.classification_transformer import create_basic_transformer_model
-
-data = load_files(path='data/',
-    own_asset='BTC_ETH',
-    own_asset_lags=[1,2,3,4,5,6,8,10,15],
-    load_non_target_asset=True,
-    other_asset_lags=[1,2,3,4],
-    log_returns=False,
-    add_date_features=True,
-    own_technical_features='level2',
-    other_technical_features='level2',
-    exogenous_features='none',
-    index_column='int'
-)
-
-
-target_col = 'target'
-data = create_target_classes(data, 'BTC_ETH_returns', 1, 'two')
-
-num_classes = 2
-learning_rate = 0.002
-batch_size = 64
-epochs = 100
-
-split_fraction = 0.8
-train_split = int(split_fraction * int(data.shape[0]))
-
-past = 60
-future = 10
-
-start = past + future
-end = start + train_split
-
-#%% split data into training - validation sets
-train_data = data.loc[0 : train_split - 1]
-val_data = data.loc[train_split:]
-
-#%% create features and target for training set & keras dataset
-feature_scaler = MinMaxScaler(feature_range= (-1, 1))
-
-x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std
-y_train = keras.utils.to_categorical(data.iloc[start:end][target_col].values)
-
-dataset_train = keras.preprocessing.timeseries_dataset_from_array(
-    x_train,
-    y_train,
-    sequence_length=past,
-    batch_size=batch_size,
-)
-
-
-#%% create features and target for validation set & keras dataset
-x_end = len(val_data) - past - future
-label_start = train_split + past + future
-
-x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std
-y_val = keras.utils.to_categorical(data.iloc[label_start:][target_col].values)
-
-dataset_val = keras.utils.timeseries_dataset_from_array(
-    x_val,
-    y_val,
-    sequence_length=past,
-    batch_size=batch_size,
-    shuffle=False,
-)
-
-
-#%%
-
-for batch in dataset_train.take(1):
-    batch_inputs, batch_targets = batch
-
-print("Input shape:", batch_inputs.shape)
-print("Target shape:", batch_targets.shape)
-
-n_timestamps = batch_inputs.shape[1]
-n_features = batch_inputs.shape[2]
-# print(batch_inputs)
-# print(batch_targets)
-
-# %%
-# model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
-# model = create_basic_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
-model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
-# model = create_basic_transformer_model(
-#     input_shape=(n_timestamps, n_features),
-#     n_classes=num_classes,
-#     head_size=64,
-#     num_heads=4,
-#     ff_dim=4,
-#     num_transformer_blocks=4,
-#     mlp_units=[64],
-#     mlp_dropout=0.4,
-#     dropout=0.25,
-# )
-
-optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
-loss = keras.losses.CategoricalCrossentropy(from_logits=True)
-model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
-model.summary()
-
-# %%
-
-path_checkpoint = "model_checkpoint.h5"
-
-history = model.fit(
-    dataset_train,
-    epochs=epochs,
-    validation_data=dataset_val,
-)
-
-#%%
-
-# pred = model.predict(rolling_window(x_val, 11))
-# pred = pred.reshape(pred.shape[0], 1)
-
-
-#%%
-# print_classification_metrics(y_val, pred)
-
-visualize_loss(history, "Training and Validation Loss")

archive/model_classification_sktime.py

@@ -1,97 +0,0 @@
-#%% Import all the stuff, load data, define constants
-from data_loader.load_data import load_files, create_target_classes
-import pandas as pd
-import numpy as np
-from utils.sktime import from_df_to_sktime_data
-from sktime.utils.plotting import plot_series
-from sktime.forecasting.model_selection import temporal_train_test_split
-from sklearn.metrics import accuracy_score
-from sklearn.pipeline import Pipeline
-from sktime.classification.interval_based import (
-    TimeSeriesForestClassifier,
-    SupervisedTimeSeriesForest,
-
-)
-from sktime.forecasting.model_selection import SlidingWindowSplitter
-from sktime.forecasting.model_selection import ForecastingRandomizedSearchCV
-
-from sktime.forecasting.compose import make_reduction
-
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.metrics import confusion_matrix
-from sklearn.preprocessing import StandardScaler
-from sktime.forecasting.model_selection import (
-    SlidingWindowSplitter,
-    ForecastingGridSearchCV,
-)
-from utils.evaluate import print_classification_metrics, format_data_for_backtest
-from sklearn.ensemble import RandomForestRegressor
-from sklearnex import patch_sklearn
-patch_sklearn()
-
-
-ticket_to_predict = 'BTC_USD'
-print('Predicting: ', ticket_to_predict)
-
-data = load_files(path='data/',
-    own_asset=ticket_to_predict,
-    own_asset_lags=[1,2,3,4,5,6,8,10,15],
-    load_non_target_asset=False,
-    other_asset_lags=[1,2,3,4],
-    log_returns=True,
-    add_date_features=True,
-    own_technical_features='level2',
-    other_technical_features='none',
-    exogenous_features='none',
-    index_column='int'
-)
-
-target_col = 'target'
-returns_col = ticket_to_predict + '_returns'
-data = create_target_classes(data, returns_col, 1, 'two')
-
-X = data.drop(columns=[target_col])
-y = data[target_col]
-
-X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2)
-X_train = from_df_to_sktime_data(X_train)
-X_test = from_df_to_sktime_data(X_test)
-
-#%%
-
-# pipe = RecursiveTabularRegressionForecaster(steps=[
-#     # ("deseasonalizer", OptionalPassthrough(Deseasonalizer())),
-#     ("scaler", StandardScaler()),
-#     ("classifier", TimeSeriesForestClassifier(n_estimators=200, random_state=1)),
-# ])
-
-# pipe.fit(X_train, y_train)
-# regressor = RandomForestRegressor(n_estimators=20)
-# model = DecisionTreeClassifier(random_state=1)
-model = TimeSeriesForestClassifier(n_estimators=50, random_state=1)
-model.fit(y = y_train, X = X_train)
-# forecaster = make_reduction(model, scitype="tabular-regressor")
-# nested_params = {"window_length": list(range(2,30)), 
-#                  "estimator__max_depth": list(range(5,16))}
-#                 # "estimator__n_estimators": list(range(10,200))}
-#%%
-
-# cv = SlidingWindowSplitter(initial_window=40, window_length=30)
-# nrcv = ForecastingRandomizedSearchCV(forecaster, strategy="refit", cv=cv, 
-#                                      param_distributions=nested_params, 
-#                                      n_iter=5, random_state=42)
-# nrcv.fit(y = y_train, X = X_train, fh=np.array([1]))
-# print(nrcv.best_params_)
-# print(nrcv.best_score_)
-
-# model = DecisionTreeClassifier(random_state=1)
-# model.fit(X_train, y_train)
-
-# preds = nrcv.best_forecaster_.predict( X=X_test)
-# # print(preds)
-preds = model.predict(X_test)
-print(print_classification_metrics(y_test, preds))
-
-# backtest_data = format_data_for_backtest(data, returns_col, X_test, preds)
-# print(backtest_data)
-# %%

config/config.py

@@ -22,7 +22,7 @@ def get_dev_config() -> tuple[dict, dict, dict]:
         forecasting_horizon = 1,
         own_features = ['level_2', 'date_days'],
         other_features = ['single_mom'],
-        exogenous_features = ['standard_scaling'],
+        exogenous_features = ['z_score'],
         index_column= 'int',
         method= 'classification',
         no_of_classes= 'two',
@@ -65,7 +65,7 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]:
         forecasting_horizon = 1,
         own_features = ['level_2', 'date_days', 'lags_up_to_5'],
         other_features = ['level_2', 'lags_up_to_5'],
-        exogenous_features = ['standard_scaling'],
+        exogenous_features = ['z_score'],
         index_column= 'int',
         method= 'classification',
         no_of_classes= 'two',

feature_extractors/feature_extractor_presets.py

@@ -1,4 +1,4 @@
-from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_standard_scaling, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
+from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_expanding_zscore, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
 from utils.types import FeatureExtractorConfig
 from utils.helpers import flatten
 from feature_extractors.fractional_differentiation import feature_fractional_differentiation, feature_fractional_differentiation_log
@@ -25,7 +25,7 @@ __presets = dict(
     stok = [('stok', feature_STOK, [10, 30, 200])],
     fracdiff = [('fracdiff', feature_fractional_differentiation, [10, 30])],
     fracdiff_log = [('fracdiff_log', feature_fractional_differentiation_log, [10, 30])],
-    standard_scaling = [('standard_scaling', feature_standard_scaling, [0])],
+    z_score = [('z_score', feature_expanding_zscore, [10])],
 )
 
 presets = __presets | dict(

feature_extractors/feature_extractors.py

@@ -2,7 +2,6 @@ import pandas as pd
 import numpy as np
 from feature_extractors.utils import get_close_low_high
 from feature_extractors.utils import apply_log_if_necessary_series
-from sklearn.preprocessing import StandardScaler
 
 def feature_debug_future_lookahead(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
     return df['returns'].shift(-period)
@@ -11,9 +10,9 @@ def feature_lag(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series
     assert period > 0
     return df['returns'].shift(period)
 
-def feature_standard_scaling(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
+def feature_expanding_zscore(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
-    scaler = StandardScaler()
+    close = df['close']
-    return pd.Series(scaler.fit_transform(df['close'].to_numpy().reshape(-1, 1)).squeeze(), index = df.index)
+    return (close - close.expanding(period).mean()) / close.expanding(period).std()
 
 def feature_day_of_week(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.DataFrame:
     return pd.get_dummies(pd.DatetimeIndex(df.index).dayofweek, drop_first=True, prefix="date_day_week").set_index(df.index)

sweep_ensemble.yaml

@@ -52,4 +52,4 @@ parameters:
     other_features:
       value: ['level_2', 'lags_up_to_5']
     exogenous_features:
-      value: ['standard_scaling']
+      value: ['z_score']

sweep_primary_models.yaml

@@ -54,4 +54,4 @@ parameters:
     other_features:
       value: ['level_2', 'lags_up_to_5']
     exogenous_features:
-      value: ['standard_scaling']
+      value: ['z_score']