From 18768c3925d278b527f1fadccea2a5f16404c61b Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Mon, 10 Jan 2022 14:24:51 +0100 Subject: [PATCH] fix(FeatureExtractor): use a rolling z-score instead of StandardScaler with unavoidable lookahead bias (#146) * fix(FeatureExtractor): use a rolling z-score instead of StandardScaler with unavoidable lookahead bias * chore(Archive): removed archived models * fix(FeatureExtractors): syntax * fix(FeatureExtractors): mistake with expanding window --- archive/keras_models/classification.py | 102 ------------- .../classification_transformer.py | 41 ------ archive/model_classification_keras.py | 134 ------------------ archive/model_classification_sktime.py | 97 ------------- config/config.py | 4 +- .../feature_extractor_presets.py | 4 +- feature_extractors/feature_extractors.py | 7 +- sweep_ensemble.yaml | 2 +- sweep_primary_models.yaml | 2 +- 9 files changed, 9 insertions(+), 384 deletions(-) delete mode 100644 archive/keras_models/classification.py delete mode 100644 archive/keras_models/classification_transformer.py delete mode 100644 archive/model_classification_keras.py delete mode 100644 archive/model_classification_sktime.py diff --git a/archive/keras_models/classification.py b/archive/keras_models/classification.py deleted file mode 100644 index b840b3e..0000000 --- a/archive/keras_models/classification.py +++ /dev/null @@ -1,102 +0,0 @@ -import keras - -def create_basic_lstm_model(input_shape, num_classes): - model = keras.Sequential() - model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=input_shape)) - model.add(keras.layers.Flatten()) - model.add(keras.layers.Dropout(0.3)) - model.add(keras.layers.Dense(units = 64, activation = 'sigmoid')) - model.add(keras.layers.Dropout(0.3)) - model.add(keras.layers.Dense(units = 32, activation = 'sigmoid')) - model.add(keras.layers.Dropout(0.3)) - model.add(keras.layers.Dense(units = num_classes, activation = 'linear')) - return model - -def create_basic_cnn_model(input_shape, num_classes): - model = keras.Sequential() - model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same", input_shape=input_shape)) - model.add(keras.layers.BatchNormalization()) - model.add(keras.layers.ReLU()) - model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same")) - model.add(keras.layers.BatchNormalization()) - model.add(keras.layers.ReLU()) - model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same")) - model.add(keras.layers.BatchNormalization()) - model.add(keras.layers.ReLU()) - model.add(keras.layers.GlobalAveragePooling1D()) - model.add(keras.layers.Dense(num_classes, activation="linear")) - return model - - -def create_resnet_cnn_model(input_shape, num_classes): - n_feature_maps = 24 - - input_layer = keras.layers.Input(input_shape) - - - conv_x = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=8, padding='same')(input_layer) - conv_x = keras.layers.BatchNormalization()(conv_x) - conv_x = keras.layers.Activation('relu')(conv_x) - - conv_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=5, padding='same')(conv_x) - conv_y = keras.layers.BatchNormalization()(conv_y) - conv_y = keras.layers.Activation('relu')(conv_y) - - conv_z = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=3, padding='same')(conv_y) - conv_z = keras.layers.BatchNormalization()(conv_z) - - # expand channels for the sum - shortcut_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=1, padding='same')(input_layer) - shortcut_y = keras.layers.BatchNormalization()(shortcut_y) - - output_block_1 = keras.layers.add([shortcut_y, conv_z]) - output_block_1 = keras.layers.Activation('relu')(output_block_1) - - # BLOCK 2 - - conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_1) - conv_x = keras.layers.BatchNormalization()(conv_x) - conv_x = keras.layers.Activation('relu')(conv_x) - - conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x) - conv_y = keras.layers.BatchNormalization()(conv_y) - conv_y = keras.layers.Activation('relu')(conv_y) - - conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y) - conv_z = keras.layers.BatchNormalization()(conv_z) - - # expand channels for the sum - shortcut_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=1, padding='same')(output_block_1) - shortcut_y = keras.layers.BatchNormalization()(shortcut_y) - - output_block_2 = keras.layers.add([shortcut_y, conv_z]) - output_block_2 = keras.layers.Activation('relu')(output_block_2) - - # BLOCK 3 - - conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_2) - conv_x = keras.layers.BatchNormalization()(conv_x) - conv_x = keras.layers.Activation('relu')(conv_x) - - conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x) - conv_y = keras.layers.BatchNormalization()(conv_y) - conv_y = keras.layers.Activation('relu')(conv_y) - - conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y) - conv_z = keras.layers.BatchNormalization()(conv_z) - - # no need to expand channels because they are equal - shortcut_y = keras.layers.BatchNormalization()(output_block_2) - - output_block_3 = keras.layers.add([shortcut_y, conv_z]) - output_block_3 = keras.layers.Activation('relu')(output_block_3) - - # FINAL - - gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3) - - output_layer = keras.layers.Dense(num_classes, activation='linear')(gap_layer) - model = keras.models.Model(inputs=input_layer, outputs=output_layer) - - - return model \ No newline at end of file diff --git a/archive/keras_models/classification_transformer.py b/archive/keras_models/classification_transformer.py deleted file mode 100644 index 49d855e..0000000 --- a/archive/keras_models/classification_transformer.py +++ /dev/null @@ -1,41 +0,0 @@ -from keras import layers -import keras - -def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0): - # Normalization and Attention - x = layers.LayerNormalization(epsilon=1e-6)(inputs) - x = layers.MultiHeadAttention( - key_dim=head_size, num_heads=num_heads, dropout=dropout - )(x, x) - x = layers.Dropout(dropout)(x) - res = x + inputs - - # Feed Forward Part - x = layers.LayerNormalization(epsilon=1e-6)(res) - x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation="relu")(x) - x = layers.Dropout(dropout)(x) - x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x) - return x + res - -def create_basic_transformer_model( - input_shape, - n_classes, - head_size, - num_heads, - ff_dim, - num_transformer_blocks, - mlp_units, - dropout=0, - mlp_dropout=0, -): - inputs = keras.Input(shape=input_shape) - x = inputs - for _ in range(num_transformer_blocks): - x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout) - - x = layers.GlobalAveragePooling1D(data_format="channels_first")(x) - for dim in mlp_units: - x = layers.Dense(dim, activation="relu")(x) - x = layers.Dropout(mlp_dropout)(x) - outputs = layers.Dense(n_classes, activation="softmax")(x) - return keras.Model(inputs, outputs) \ No newline at end of file diff --git a/archive/model_classification_keras.py b/archive/model_classification_keras.py deleted file mode 100644 index 92b0b0b..0000000 --- a/archive/model_classification_keras.py +++ /dev/null @@ -1,134 +0,0 @@ -#%% Import all the stuff, load data, define constants -from sklearn.utils import shuffle -from data_loader.load_data import load_files, create_target_classes -import pandas as pd -from tensorflow import keras -from utils.normalize import normalize -import tensorflow as tf -from utils.visualize import visualize_loss -from sklearn.preprocessing import MinMaxScaler -from utils.evaluate import print_classification_metrics -import numpy as np -from utils.rolling import rolling_window -from keras_models.classification import create_basic_cnn_model, create_basic_lstm_model, create_resnet_cnn_model -from keras_models.classification_transformer import create_basic_transformer_model - -data = load_files(path='data/', - own_asset='BTC_ETH', - own_asset_lags=[1,2,3,4,5,6,8,10,15], - load_non_target_asset=True, - other_asset_lags=[1,2,3,4], - log_returns=False, - add_date_features=True, - own_technical_features='level2', - other_technical_features='level2', - exogenous_features='none', - index_column='int' -) - - -target_col = 'target' -data = create_target_classes(data, 'BTC_ETH_returns', 1, 'two') - -num_classes = 2 -learning_rate = 0.002 -batch_size = 64 -epochs = 100 - -split_fraction = 0.8 -train_split = int(split_fraction * int(data.shape[0])) - -past = 60 -future = 10 - -start = past + future -end = start + train_split - -#%% split data into training - validation sets -train_data = data.loc[0 : train_split - 1] -val_data = data.loc[train_split:] - -#%% create features and target for training set & keras dataset -feature_scaler = MinMaxScaler(feature_range= (-1, 1)) - -x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std -y_train = keras.utils.to_categorical(data.iloc[start:end][target_col].values) - -dataset_train = keras.preprocessing.timeseries_dataset_from_array( - x_train, - y_train, - sequence_length=past, - batch_size=batch_size, -) - - -#%% create features and target for validation set & keras dataset -x_end = len(val_data) - past - future -label_start = train_split + past + future - -x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std -y_val = keras.utils.to_categorical(data.iloc[label_start:][target_col].values) - -dataset_val = keras.utils.timeseries_dataset_from_array( - x_val, - y_val, - sequence_length=past, - batch_size=batch_size, - shuffle=False, -) - - -#%% - -for batch in dataset_train.take(1): - batch_inputs, batch_targets = batch - -print("Input shape:", batch_inputs.shape) -print("Target shape:", batch_targets.shape) - -n_timestamps = batch_inputs.shape[1] -n_features = batch_inputs.shape[2] -# print(batch_inputs) -# print(batch_targets) - -# %% -# model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) -# model = create_basic_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) -model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) -# model = create_basic_transformer_model( -# input_shape=(n_timestamps, n_features), -# n_classes=num_classes, -# head_size=64, -# num_heads=4, -# ff_dim=4, -# num_transformer_blocks=4, -# mlp_units=[64], -# mlp_dropout=0.4, -# dropout=0.25, -# ) - -optimizer = keras.optimizers.Adam(learning_rate=learning_rate) -loss = keras.losses.CategoricalCrossentropy(from_logits=True) -model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy']) -model.summary() - -# %% - -path_checkpoint = "model_checkpoint.h5" - -history = model.fit( - dataset_train, - epochs=epochs, - validation_data=dataset_val, -) - -#%% - -# pred = model.predict(rolling_window(x_val, 11)) -# pred = pred.reshape(pred.shape[0], 1) - - -#%% -# print_classification_metrics(y_val, pred) - -visualize_loss(history, "Training and Validation Loss") \ No newline at end of file diff --git a/archive/model_classification_sktime.py b/archive/model_classification_sktime.py deleted file mode 100644 index dae70b8..0000000 --- a/archive/model_classification_sktime.py +++ /dev/null @@ -1,97 +0,0 @@ -#%% Import all the stuff, load data, define constants -from data_loader.load_data import load_files, create_target_classes -import pandas as pd -import numpy as np -from utils.sktime import from_df_to_sktime_data -from sktime.utils.plotting import plot_series -from sktime.forecasting.model_selection import temporal_train_test_split -from sklearn.metrics import accuracy_score -from sklearn.pipeline import Pipeline -from sktime.classification.interval_based import ( - TimeSeriesForestClassifier, - SupervisedTimeSeriesForest, - -) -from sktime.forecasting.model_selection import SlidingWindowSplitter -from sktime.forecasting.model_selection import ForecastingRandomizedSearchCV - -from sktime.forecasting.compose import make_reduction - -from sklearn.tree import DecisionTreeClassifier -from sklearn.metrics import confusion_matrix -from sklearn.preprocessing import StandardScaler -from sktime.forecasting.model_selection import ( - SlidingWindowSplitter, - ForecastingGridSearchCV, -) -from utils.evaluate import print_classification_metrics, format_data_for_backtest -from sklearn.ensemble import RandomForestRegressor -from sklearnex import patch_sklearn -patch_sklearn() - - -ticket_to_predict = 'BTC_USD' -print('Predicting: ', ticket_to_predict) - -data = load_files(path='data/', - own_asset=ticket_to_predict, - own_asset_lags=[1,2,3,4,5,6,8,10,15], - load_non_target_asset=False, - other_asset_lags=[1,2,3,4], - log_returns=True, - add_date_features=True, - own_technical_features='level2', - other_technical_features='none', - exogenous_features='none', - index_column='int' -) - -target_col = 'target' -returns_col = ticket_to_predict + '_returns' -data = create_target_classes(data, returns_col, 1, 'two') - -X = data.drop(columns=[target_col]) -y = data[target_col] - -X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2) -X_train = from_df_to_sktime_data(X_train) -X_test = from_df_to_sktime_data(X_test) - -#%% - -# pipe = RecursiveTabularRegressionForecaster(steps=[ -# # ("deseasonalizer", OptionalPassthrough(Deseasonalizer())), -# ("scaler", StandardScaler()), -# ("classifier", TimeSeriesForestClassifier(n_estimators=200, random_state=1)), -# ]) - -# pipe.fit(X_train, y_train) -# regressor = RandomForestRegressor(n_estimators=20) -# model = DecisionTreeClassifier(random_state=1) -model = TimeSeriesForestClassifier(n_estimators=50, random_state=1) -model.fit(y = y_train, X = X_train) -# forecaster = make_reduction(model, scitype="tabular-regressor") -# nested_params = {"window_length": list(range(2,30)), -# "estimator__max_depth": list(range(5,16))} -# # "estimator__n_estimators": list(range(10,200))} -#%% - -# cv = SlidingWindowSplitter(initial_window=40, window_length=30) -# nrcv = ForecastingRandomizedSearchCV(forecaster, strategy="refit", cv=cv, -# param_distributions=nested_params, -# n_iter=5, random_state=42) -# nrcv.fit(y = y_train, X = X_train, fh=np.array([1])) -# print(nrcv.best_params_) -# print(nrcv.best_score_) - -# model = DecisionTreeClassifier(random_state=1) -# model.fit(X_train, y_train) - -# preds = nrcv.best_forecaster_.predict( X=X_test) -# # print(preds) -preds = model.predict(X_test) -print(print_classification_metrics(y_test, preds)) - -# backtest_data = format_data_for_backtest(data, returns_col, X_test, preds) -# print(backtest_data) -# %% diff --git a/config/config.py b/config/config.py index dfe66ff..8108cd3 100644 --- a/config/config.py +++ b/config/config.py @@ -22,7 +22,7 @@ def get_dev_config() -> tuple[dict, dict, dict]: forecasting_horizon = 1, own_features = ['level_2', 'date_days'], other_features = ['single_mom'], - exogenous_features = ['standard_scaling'], + exogenous_features = ['z_score'], index_column= 'int', method= 'classification', no_of_classes= 'two', @@ -65,7 +65,7 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]: forecasting_horizon = 1, own_features = ['level_2', 'date_days', 'lags_up_to_5'], other_features = ['level_2', 'lags_up_to_5'], - exogenous_features = ['standard_scaling'], + exogenous_features = ['z_score'], index_column= 'int', method= 'classification', no_of_classes= 'two', diff --git a/feature_extractors/feature_extractor_presets.py b/feature_extractors/feature_extractor_presets.py index 6dba829..0f012c7 100644 --- a/feature_extractors/feature_extractor_presets.py +++ b/feature_extractors/feature_extractor_presets.py @@ -1,4 +1,4 @@ -from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_standard_scaling, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead +from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_expanding_zscore, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead from utils.types import FeatureExtractorConfig from utils.helpers import flatten from feature_extractors.fractional_differentiation import feature_fractional_differentiation, feature_fractional_differentiation_log @@ -25,7 +25,7 @@ __presets = dict( stok = [('stok', feature_STOK, [10, 30, 200])], fracdiff = [('fracdiff', feature_fractional_differentiation, [10, 30])], fracdiff_log = [('fracdiff_log', feature_fractional_differentiation_log, [10, 30])], - standard_scaling = [('standard_scaling', feature_standard_scaling, [0])], + z_score = [('z_score', feature_expanding_zscore, [10])], ) presets = __presets | dict( diff --git a/feature_extractors/feature_extractors.py b/feature_extractors/feature_extractors.py index 154efc5..16ba345 100644 --- a/feature_extractors/feature_extractors.py +++ b/feature_extractors/feature_extractors.py @@ -2,7 +2,6 @@ import pandas as pd import numpy as np from feature_extractors.utils import get_close_low_high from feature_extractors.utils import apply_log_if_necessary_series -from sklearn.preprocessing import StandardScaler def feature_debug_future_lookahead(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series: return df['returns'].shift(-period) @@ -11,9 +10,9 @@ def feature_lag(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series assert period > 0 return df['returns'].shift(period) -def feature_standard_scaling(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series: - scaler = StandardScaler() - return pd.Series(scaler.fit_transform(df['close'].to_numpy().reshape(-1, 1)).squeeze(), index = df.index) +def feature_expanding_zscore(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series: + close = df['close'] + return (close - close.expanding(period).mean()) / close.expanding(period).std() def feature_day_of_week(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.DataFrame: return pd.get_dummies(pd.DatetimeIndex(df.index).dayofweek, drop_first=True, prefix="date_day_week").set_index(df.index) diff --git a/sweep_ensemble.yaml b/sweep_ensemble.yaml index 7b50ef0..3ea323e 100644 --- a/sweep_ensemble.yaml +++ b/sweep_ensemble.yaml @@ -52,4 +52,4 @@ parameters: other_features: value: ['level_2', 'lags_up_to_5'] exogenous_features: - value: ['standard_scaling'] + value: ['z_score'] diff --git a/sweep_primary_models.yaml b/sweep_primary_models.yaml index b66c279..4b2048c 100644 --- a/sweep_primary_models.yaml +++ b/sweep_primary_models.yaml @@ -54,4 +54,4 @@ parameters: other_features: value: ['level_2', 'lags_up_to_5'] exogenous_features: - value: ['standard_scaling'] \ No newline at end of file + value: ['z_score'] \ No newline at end of file