From 1d6eed3a9400498bb76debfde8dc8d8d5dffd78d Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Tue, 16 Nov 2021 10:02:02 +0100 Subject: [PATCH] feat(Core): added the first classification model & the feature necessary (#4) * feat(Core): added the first classification model & the feature necessary * feat(Models): added basic transformers model --- keras_models/classification.py | 102 +++++++++++++++ keras_models/classification_transformer.py | 41 ++++++ load_data.py | 25 ++++ model_classification_keras.py | 122 ++++++++++++++++++ ...ssion_lstm.py => model_regression_keras.py | 4 +- utils/evaluate.py | 12 +- 6 files changed, 300 insertions(+), 6 deletions(-) create mode 100644 keras_models/classification.py create mode 100644 keras_models/classification_transformer.py create mode 100644 model_classification_keras.py rename model_regression_lstm.py => model_regression_keras.py (97%) diff --git a/keras_models/classification.py b/keras_models/classification.py new file mode 100644 index 0000000..e524c18 --- /dev/null +++ b/keras_models/classification.py @@ -0,0 +1,102 @@ +import keras + +def create_basic_lstm_model(input_shape, num_classes): + model = keras.Sequential() + model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=input_shape)) + model.add(keras.layers.Flatten()) + model.add(keras.layers.Dropout(0.3)) + model.add(keras.layers.Dense(units = 64, activation = 'sigmoid')) + model.add(keras.layers.Dropout(0.3)) + model.add(keras.layers.Dense(units = 32, activation = 'sigmoid')) + model.add(keras.layers.Dropout(0.3)) + model.add(keras.layers.Dense(units = num_classes, activation = 'softmax')) + return model + +def create_basic_cnn_model(input_shape, num_classes): + model = keras.Sequential() + model.add(keras.layers.Conv1D(filters=64, kernel_size=3, padding="same", input_shape=input_shape)) + model.add(keras.layers.BatchNormalization()) + model.add(keras.layers.ReLU()) + model.add(keras.layers.Conv1D(filters=64, kernel_size=3, padding="same")) + model.add(keras.layers.BatchNormalization()) + model.add(keras.layers.ReLU()) + model.add(keras.layers.Conv1D(filters=64, kernel_size=3, padding="same")) + model.add(keras.layers.BatchNormalization()) + model.add(keras.layers.ReLU()) + model.add(keras.layers.GlobalAveragePooling1D()) + model.add(keras.layers.Dense(num_classes, activation="softmax")) + return model + + +def create_resnet_cnn_model(input_shape, num_classes): + n_feature_maps = 64 + + input_layer = keras.layers.Input(input_shape) + + + conv_x = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=8, padding='same')(input_layer) + conv_x = keras.layers.BatchNormalization()(conv_x) + conv_x = keras.layers.Activation('relu')(conv_x) + + conv_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=5, padding='same')(conv_x) + conv_y = keras.layers.BatchNormalization()(conv_y) + conv_y = keras.layers.Activation('relu')(conv_y) + + conv_z = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=3, padding='same')(conv_y) + conv_z = keras.layers.BatchNormalization()(conv_z) + + # expand channels for the sum + shortcut_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=1, padding='same')(input_layer) + shortcut_y = keras.layers.BatchNormalization()(shortcut_y) + + output_block_1 = keras.layers.add([shortcut_y, conv_z]) + output_block_1 = keras.layers.Activation('relu')(output_block_1) + + # BLOCK 2 + + conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_1) + conv_x = keras.layers.BatchNormalization()(conv_x) + conv_x = keras.layers.Activation('relu')(conv_x) + + conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x) + conv_y = keras.layers.BatchNormalization()(conv_y) + conv_y = keras.layers.Activation('relu')(conv_y) + + conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y) + conv_z = keras.layers.BatchNormalization()(conv_z) + + # expand channels for the sum + shortcut_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=1, padding='same')(output_block_1) + shortcut_y = keras.layers.BatchNormalization()(shortcut_y) + + output_block_2 = keras.layers.add([shortcut_y, conv_z]) + output_block_2 = keras.layers.Activation('relu')(output_block_2) + + # BLOCK 3 + + conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_2) + conv_x = keras.layers.BatchNormalization()(conv_x) + conv_x = keras.layers.Activation('relu')(conv_x) + + conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x) + conv_y = keras.layers.BatchNormalization()(conv_y) + conv_y = keras.layers.Activation('relu')(conv_y) + + conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y) + conv_z = keras.layers.BatchNormalization()(conv_z) + + # no need to expand channels because they are equal + shortcut_y = keras.layers.BatchNormalization()(output_block_2) + + output_block_3 = keras.layers.add([shortcut_y, conv_z]) + output_block_3 = keras.layers.Activation('relu')(output_block_3) + + # FINAL + + gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3) + + output_layer = keras.layers.Dense(num_classes, activation='softmax')(gap_layer) + model = keras.models.Model(inputs=input_layer, outputs=output_layer) + + + return model \ No newline at end of file diff --git a/keras_models/classification_transformer.py b/keras_models/classification_transformer.py new file mode 100644 index 0000000..49d855e --- /dev/null +++ b/keras_models/classification_transformer.py @@ -0,0 +1,41 @@ +from keras import layers +import keras + +def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0): + # Normalization and Attention + x = layers.LayerNormalization(epsilon=1e-6)(inputs) + x = layers.MultiHeadAttention( + key_dim=head_size, num_heads=num_heads, dropout=dropout + )(x, x) + x = layers.Dropout(dropout)(x) + res = x + inputs + + # Feed Forward Part + x = layers.LayerNormalization(epsilon=1e-6)(res) + x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation="relu")(x) + x = layers.Dropout(dropout)(x) + x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x) + return x + res + +def create_basic_transformer_model( + input_shape, + n_classes, + head_size, + num_heads, + ff_dim, + num_transformer_blocks, + mlp_units, + dropout=0, + mlp_dropout=0, +): + inputs = keras.Input(shape=input_shape) + x = inputs + for _ in range(num_transformer_blocks): + x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout) + + x = layers.GlobalAveragePooling1D(data_format="channels_first")(x) + for dim in mlp_units: + x = layers.Dense(dim, activation="relu")(x) + x = layers.Dropout(mlp_dropout)(x) + outputs = layers.Dense(n_classes, activation="softmax")(x) + return keras.Model(inputs, outputs) \ No newline at end of file diff --git a/load_data.py b/load_data.py index dc38ef8..9ca07f2 100644 --- a/load_data.py +++ b/load_data.py @@ -65,4 +65,29 @@ def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool, nar def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: df['target'] = df[source_column].diff(period).shift(-period) df = df.iloc[:-period] + return df + + +#%% +def create_target_pos_neg_classes(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: + df['target'] = df[source_column].diff(period).shift(-period) + df['target'] = df['target'].map(lambda x: 0 if x <= 0.0 else 1) + df = df.iloc[:-period] + return df + +def create_target_four_classes(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: + def __get_class(x): + treshold = 0.08 + if x <= -treshold: + return 0 + elif x > -treshold and x <= 0: + return 1 + elif x > 0 and x <= treshold: + return 2 + else: + return 3 + + df['target'] = df[source_column].diff(period).shift(-period) + df['target'] = df['target'].map(__get_class) + df = df.iloc[:-period] return df \ No newline at end of file diff --git a/model_classification_keras.py b/model_classification_keras.py new file mode 100644 index 0000000..dab1ab0 --- /dev/null +++ b/model_classification_keras.py @@ -0,0 +1,122 @@ +#%% Import all the stuff, load data, define constants +from sklearn.utils import shuffle +from load_data import load_files, create_target_pos_neg_classes +import pandas as pd +from tensorflow import keras +from utils.normalize import normalize +import tensorflow as tf +from utils.visualize import visualize_loss +from sklearn.preprocessing import MinMaxScaler +from utils.evaluate import print_classification_metrics +import numpy as np +from utils.rolling import rolling_window +from keras_models.classification import create_basic_cnn_model, create_basic_lstm_model, create_resnet_cnn_model +from keras_models.classification_transformer import create_basic_transformer_model + +data = load_files('data/', add_features=True, log_returns=False) +data.reset_index(drop=True, inplace=True) +data = data[[column for column in data.columns if not column.endswith('volume')]] +# data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]] + +target_col = 'target' +data = create_target_pos_neg_classes(data, 'ETH_returns', 30) + +num_classes = 2 +learning_rate = 0.002 +batch_size = 64 +epochs = 100 + +split_fraction = 0.8 +train_split = int(split_fraction * int(data.shape[0])) + +past = 10 +future = 1 + +start = past + future +end = start + train_split + +#%% split data into training - validation sets +train_data = data.loc[0 : train_split - 1] +val_data = data.loc[train_split:] + +#%% create features and target for training set & keras dataset +feature_scaler = MinMaxScaler(feature_range= (-1, 1)) + +x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std +y_train = keras.utils.to_categorical(data.iloc[start:end][target_col].values) + +dataset_train = keras.preprocessing.timeseries_dataset_from_array( + x_train, + y_train, + sequence_length=past, + batch_size=batch_size, +) + + +#%% create features and target for validation set & keras dataset +x_end = len(val_data) - past - future +label_start = train_split + past + future + +x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std +y_val = keras.utils.to_categorical(data.iloc[label_start:][target_col].values) + +dataset_val = keras.utils.timeseries_dataset_from_array( + x_val, + y_val, + sequence_length=past, + batch_size=batch_size, + shuffle=False, +) + + +#%% + +for batch in dataset_train.take(10): + batch_inputs, batch_targets = batch + +print("Input shape:", batch_inputs.shape) +print("Target shape:", batch_targets.shape) + +n_timestamps = batch_inputs.shape[1] +n_features = batch_inputs.shape[2] +# print(batch_inputs) +# print(batch_targets) + +# %% +# model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) +model = create_basic_transformer_model( + input_shape=(n_timestamps, n_features), + n_classes=num_classes, + head_size=64, + num_heads=4, + ff_dim=4, + num_transformer_blocks=4, + mlp_units=[64], + mlp_dropout=0.4, + dropout=0.25, +) + +optimizer = keras.optimizers.Adam(learning_rate=learning_rate) +model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy']) +model.summary() + +# %% + +path_checkpoint = "model_checkpoint.h5" + +history = model.fit( + dataset_train, + epochs=epochs, + validation_data=dataset_val, +) + +#%% + +# pred = model.predict(rolling_window(x_val, 11)) +# pred = pred.reshape(pred.shape[0], 1) + + +#%% +# print_classification_metrics(y_val, pred) + +visualize_loss(history, "Training and Validation Loss") \ No newline at end of file diff --git a/model_regression_lstm.py b/model_regression_keras.py similarity index 97% rename from model_regression_lstm.py rename to model_regression_keras.py index 9afead2..065c1ce 100644 --- a/model_regression_lstm.py +++ b/model_regression_keras.py @@ -7,7 +7,7 @@ from utils.normalize import normalize import tensorflow as tf from utils.visualize import visualize_loss from sklearn.preprocessing import MinMaxScaler -from utils.evaluate import print_metrics +from utils.evaluate import print_regression_metrics import numpy as np from utils.rolling import rolling_window @@ -109,7 +109,7 @@ pred = model.predict(rolling_window(x_val, 11)) pred = pred.reshape(pred.shape[0], 1) pred = target_scaler.inverse_transform(pred) -print_metrics(y_val, pred) +print_regression_metrics(y_val, pred) #%% diff --git a/utils/evaluate.py b/utils/evaluate.py index 7b04729..cafa565 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -1,9 +1,13 @@ from math import sqrt -from sklearn.metrics import mean_squared_error, mean_absolute_error - -def print_metrics(y_true, y_pred): +from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score +from sklearn.metrics import confusion_matrix +def print_regression_metrics(y_true, y_pred): rmse = sqrt(mean_squared_error(y_true, y_pred)) print("RMSE: %.2f" % rmse) mae = mean_absolute_error(y_true, y_pred) - print("MAE: %.2f" % mae) \ No newline at end of file + print("MAE: %.2f" % mae) + +def print_classification_metrics(y_true, y_pred): + print("Accuracy: %.2f" % accuracy_score(y_true, y_pred)) + print("Confusion Matrix: \n", confusion_matrix(y_true, y_pred)) \ No newline at end of file