gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-07-27 18:57:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(FeatureExtractor): use a rolling z-score instead of StandardScaler with unavoidable lookahead bias (#146)

Browse Source 
* fix(FeatureExtractor): use a rolling z-score instead of StandardScaler with unavoidable lookahead bias

* chore(Archive): removed archived models

* fix(FeatureExtractors): syntax

* fix(FeatureExtractors): mistake with expanding window
This commit is contained in:
Mark Aron Szulyovszky
2022-01-10 14:24:51 +01:00
committed by GitHub
parent 73cfc67336
commit 18768c3925
9 changed files with 9 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files
archive/keras_models/classification.py
-102
View File
@@ -1,102 +0,0 @@
import keras
def create_basic_lstm_model(input_shape, num_classes):
model = keras.Sequential()
model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=input_shape))
model.add(keras.layers.Flatten())
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(units = 64, activation = 'sigmoid'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(units = 32, activation = 'sigmoid'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(units = num_classes, activation = 'linear'))
return model
def create_basic_cnn_model(input_shape, num_classes):
model = keras.Sequential()
model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same", input_shape=input_shape))
model.add(keras.layers.BatchNormalization())
model.add(keras.layers.ReLU())
model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same"))
model.add(keras.layers.BatchNormalization())
model.add(keras.layers.ReLU())
model.add(keras.layers.Conv1D(filters=32, kernel_size=3, padding="same"))
model.add(keras.layers.BatchNormalization())
model.add(keras.layers.ReLU())
model.add(keras.layers.GlobalAveragePooling1D())
model.add(keras.layers.Dense(num_classes, activation="linear"))
return model
def create_resnet_cnn_model(input_shape, num_classes):
n_feature_maps = 24
input_layer = keras.layers.Input(input_shape)
conv_x = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=8, padding='same')(input_layer)
conv_x = keras.layers.BatchNormalization()(conv_x)
conv_x = keras.layers.Activation('relu')(conv_x)
conv_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=5, padding='same')(conv_x)
conv_y = keras.layers.BatchNormalization()(conv_y)
conv_y = keras.layers.Activation('relu')(conv_y)
conv_z = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=3, padding='same')(conv_y)
conv_z = keras.layers.BatchNormalization()(conv_z)
# expand channels for the sum
shortcut_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=1, padding='same')(input_layer)
shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
output_block_1 = keras.layers.add([shortcut_y, conv_z])
output_block_1 = keras.layers.Activation('relu')(output_block_1)
# BLOCK 2
conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_1)
conv_x = keras.layers.BatchNormalization()(conv_x)
conv_x = keras.layers.Activation('relu')(conv_x)
conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)
conv_y = keras.layers.BatchNormalization()(conv_y)
conv_y = keras.layers.Activation('relu')(conv_y)
conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)
conv_z = keras.layers.BatchNormalization()(conv_z)
# expand channels for the sum
shortcut_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=1, padding='same')(output_block_1)
shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
output_block_2 = keras.layers.add([shortcut_y, conv_z])
output_block_2 = keras.layers.Activation('relu')(output_block_2)
# BLOCK 3
conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_2)
conv_x = keras.layers.BatchNormalization()(conv_x)
conv_x = keras.layers.Activation('relu')(conv_x)
conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)
conv_y = keras.layers.BatchNormalization()(conv_y)
conv_y = keras.layers.Activation('relu')(conv_y)
conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)
conv_z = keras.layers.BatchNormalization()(conv_z)
# no need to expand channels because they are equal
shortcut_y = keras.layers.BatchNormalization()(output_block_2)
output_block_3 = keras.layers.add([shortcut_y, conv_z])
output_block_3 = keras.layers.Activation('relu')(output_block_3)
# FINAL
gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3)
output_layer = keras.layers.Dense(num_classes, activation='linear')(gap_layer)
model = keras.models.Model(inputs=input_layer, outputs=output_layer)
return model
archive/keras_models/classification_transformer.py
-41
View File
@@ -1,41 +0,0 @@
from keras import layers
import keras
def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):
# Normalization and Attention
x = layers.LayerNormalization(epsilon=1e-6)(inputs)
x = layers.MultiHeadAttention(
key_dim=head_size, num_heads=num_heads, dropout=dropout
)(x, x)
x = layers.Dropout(dropout)(x)
res = x + inputs
# Feed Forward Part
x = layers.LayerNormalization(epsilon=1e-6)(res)
x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation="relu")(x)
x = layers.Dropout(dropout)(x)
x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)
return x + res
def create_basic_transformer_model(
input_shape,
n_classes,
head_size,
num_heads,
ff_dim,
num_transformer_blocks,
mlp_units,
dropout=0,
mlp_dropout=0,
):
inputs = keras.Input(shape=input_shape)
x = inputs
for _ in range(num_transformer_blocks):
x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)
x = layers.GlobalAveragePooling1D(data_format="channels_first")(x)
for dim in mlp_units:
x = layers.Dense(dim, activation="relu")(x)
x = layers.Dropout(mlp_dropout)(x)
outputs = layers.Dense(n_classes, activation="softmax")(x)
return keras.Model(inputs, outputs)
archive/model_classification_keras.py
-134
View File
@@ -1,134 +0,0 @@
#%% Import all the stuff, load data, define constants
from sklearn.utils import shuffle
from data_loader.load_data import load_files, create_target_classes
import pandas as pd
from tensorflow import keras
from utils.normalize import normalize
import tensorflow as tf
from utils.visualize import visualize_loss
from sklearn.preprocessing import MinMaxScaler
from utils.evaluate import print_classification_metrics
import numpy as np
from utils.rolling import rolling_window
from keras_models.classification import create_basic_cnn_model, create_basic_lstm_model, create_resnet_cnn_model
from keras_models.classification_transformer import create_basic_transformer_model
data = load_files(path='data/',
own_asset='BTC_ETH',
own_asset_lags=[1,2,3,4,5,6,8,10,15],
load_non_target_asset=True,
other_asset_lags=[1,2,3,4],
log_returns=False,
add_date_features=True,
own_technical_features='level2',
other_technical_features='level2',
exogenous_features='none',
index_column='int'
)
target_col = 'target'
data = create_target_classes(data, 'BTC_ETH_returns', 1, 'two')
num_classes = 2
learning_rate = 0.002
batch_size = 64
epochs = 100
split_fraction = 0.8
train_split = int(split_fraction * int(data.shape[0]))
past = 60
future = 10
start = past + future
end = start + train_split
#%% split data into training - validation sets
train_data = data.loc[0 : train_split - 1]
val_data = data.loc[train_split:]
#%% create features and target for training set & keras dataset
feature_scaler = MinMaxScaler(feature_range= (-1, 1))
x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std
y_train = keras.utils.to_categorical(data.iloc[start:end][target_col].values)
dataset_train = keras.preprocessing.timeseries_dataset_from_array(
x_train,
y_train,
sequence_length=past,
batch_size=batch_size,
)
#%% create features and target for validation set & keras dataset
x_end = len(val_data) - past - future
label_start = train_split + past + future
x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std
y_val = keras.utils.to_categorical(data.iloc[label_start:][target_col].values)
dataset_val = keras.utils.timeseries_dataset_from_array(
x_val,
y_val,
sequence_length=past,
batch_size=batch_size,
shuffle=False,
)
#%%
for batch in dataset_train.take(1):
batch_inputs, batch_targets = batch
print("Input shape:", batch_inputs.shape)
print("Target shape:", batch_targets.shape)
n_timestamps = batch_inputs.shape[1]
n_features = batch_inputs.shape[2]
# print(batch_inputs)
# print(batch_targets)
# %%
# model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
# model = create_basic_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
# model = create_basic_transformer_model(
# input_shape=(n_timestamps, n_features),
# n_classes=num_classes,
# head_size=64,
# num_heads=4,
# ff_dim=4,
# num_transformer_blocks=4,
# mlp_units=[64],
# mlp_dropout=0.4,
# dropout=0.25,
# )
optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
loss = keras.losses.CategoricalCrossentropy(from_logits=True)
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
model.summary()
# %%
path_checkpoint = "model_checkpoint.h5"
history = model.fit(
dataset_train,
epochs=epochs,
validation_data=dataset_val,
)
#%%
# pred = model.predict(rolling_window(x_val, 11))
# pred = pred.reshape(pred.shape[0], 1)
#%%
# print_classification_metrics(y_val, pred)
visualize_loss(history, "Training and Validation Loss")
archive/model_classification_sktime.py
-97
View File
@@ -1,97 +0,0 @@
#%% Import all the stuff, load data, define constants
from data_loader.load_data import load_files, create_target_classes
import pandas as pd
import numpy as np
from utils.sktime import from_df_to_sktime_data
from sktime.utils.plotting import plot_series
from sktime.forecasting.model_selection import temporal_train_test_split
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline
from sktime.classification.interval_based import (
TimeSeriesForestClassifier,
SupervisedTimeSeriesForest,
)
from sktime.forecasting.model_selection import SlidingWindowSplitter
from sktime.forecasting.model_selection import ForecastingRandomizedSearchCV
from sktime.forecasting.compose import make_reduction
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import StandardScaler
from sktime.forecasting.model_selection import (
SlidingWindowSplitter,
ForecastingGridSearchCV,
)
from utils.evaluate import print_classification_metrics, format_data_for_backtest
from sklearn.ensemble import RandomForestRegressor
from sklearnex import patch_sklearn
patch_sklearn()
ticket_to_predict = 'BTC_USD'
print('Predicting: ', ticket_to_predict)
data = load_files(path='data/',
own_asset=ticket_to_predict,
own_asset_lags=[1,2,3,4,5,6,8,10,15],
load_non_target_asset=False,
other_asset_lags=[1,2,3,4],
log_returns=True,
add_date_features=True,
own_technical_features='level2',
other_technical_features='none',
exogenous_features='none',
index_column='int'
)
target_col = 'target'
returns_col = ticket_to_predict + '_returns'
data = create_target_classes(data, returns_col, 1, 'two')
X = data.drop(columns=[target_col])
y = data[target_col]
X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2)
X_train = from_df_to_sktime_data(X_train)
X_test = from_df_to_sktime_data(X_test)
#%%
# pipe = RecursiveTabularRegressionForecaster(steps=[
# # ("deseasonalizer", OptionalPassthrough(Deseasonalizer())),
# ("scaler", StandardScaler()),
# ("classifier", TimeSeriesForestClassifier(n_estimators=200, random_state=1)),
# ])
# pipe.fit(X_train, y_train)
# regressor = RandomForestRegressor(n_estimators=20)
# model = DecisionTreeClassifier(random_state=1)
model = TimeSeriesForestClassifier(n_estimators=50, random_state=1)
model.fit(y = y_train, X = X_train)
# forecaster = make_reduction(model, scitype="tabular-regressor")
# nested_params = {"window_length": list(range(2,30)),
# "estimator__max_depth": list(range(5,16))}
# # "estimator__n_estimators": list(range(10,200))}
#%%
# cv = SlidingWindowSplitter(initial_window=40, window_length=30)
# nrcv = ForecastingRandomizedSearchCV(forecaster, strategy="refit", cv=cv,
# param_distributions=nested_params,
# n_iter=5, random_state=42)
# nrcv.fit(y = y_train, X = X_train, fh=np.array([1]))
# print(nrcv.best_params_)
# print(nrcv.best_score_)
# model = DecisionTreeClassifier(random_state=1)
# model.fit(X_train, y_train)
# preds = nrcv.best_forecaster_.predict( X=X_test)
# # print(preds)
preds = model.predict(X_test)
print(print_classification_metrics(y_test, preds))
# backtest_data = format_data_for_backtest(data, returns_col, X_test, preds)
# print(backtest_data)
# %%
config/config.py
+2 -2
View File
@@ -22,7 +22,7 @@ def get_dev_config() -> tuple[dict, dict, dict]:
forecasting_horizon = 1,
own_features = ['level_2', 'date_days'],
other_features = ['single_mom'],
exogenous_features = ['standard_scaling'],
exogenous_features = ['z_score'],
index_column= 'int',
method= 'classification',
no_of_classes= 'two',
@@ -65,7 +65,7 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]:
forecasting_horizon = 1,
own_features = ['level_2', 'date_days', 'lags_up_to_5'],
other_features = ['level_2', 'lags_up_to_5'],
exogenous_features = ['standard_scaling'],
exogenous_features = ['z_score'],
index_column= 'int',
method= 'classification',
no_of_classes= 'two',
feature_extractors/feature_extractor_presets.py
+2 -2
View File
@@ -1,4 +1,4 @@
from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_standard_scaling, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_expanding_zscore, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
from utils.types import FeatureExtractorConfig
from utils.helpers import flatten
from feature_extractors.fractional_differentiation import feature_fractional_differentiation, feature_fractional_differentiation_log
@@ -25,7 +25,7 @@ __presets = dict(
stok = [('stok', feature_STOK, [10, 30, 200])],
fracdiff = [('fracdiff', feature_fractional_differentiation, [10, 30])],
fracdiff_log = [('fracdiff_log', feature_fractional_differentiation_log, [10, 30])],
standard_scaling = [('standard_scaling', feature_standard_scaling, [0])],
z_score = [('z_score', feature_expanding_zscore, [10])],
)
presets = __presets | dict(
feature_extractors/feature_extractors.py
+3 -4
View File
@@ -2,7 +2,6 @@ import pandas as pd
import numpy as np
from feature_extractors.utils import get_close_low_high
from feature_extractors.utils import apply_log_if_necessary_series
from sklearn.preprocessing import StandardScaler
def feature_debug_future_lookahead(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
return df['returns'].shift(-period)
@@ -11,9 +10,9 @@ def feature_lag(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series
assert period > 0
return df['returns'].shift(period)
def feature_standard_scaling(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
scaler = StandardScaler()
return pd.Series(scaler.fit_transform(df['close'].to_numpy().reshape(-1, 1)).squeeze(), index = df.index)
def feature_expanding_zscore(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
close = df['close']
return (close - close.expanding(period).mean()) / close.expanding(period).std()
def feature_day_of_week(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.DataFrame:
return pd.get_dummies(pd.DatetimeIndex(df.index).dayofweek, drop_first=True, prefix="date_day_week").set_index(df.index)
sweep_ensemble.yaml
+1 -1
View File
@@ -52,4 +52,4 @@ parameters:
other_features:
value: ['level_2', 'lags_up_to_5']
exogenous_features:
value: ['standard_scaling']
value: ['z_score']
sweep_primary_models.yaml
+1 -1
View File
@@ -54,4 +54,4 @@ parameters:
other_features:
value: ['level_2', 'lags_up_to_5']
exogenous_features:
value: ['standard_scaling']
value: ['z_score']