feat(Core): added the first classification model & the feature necessary (#4)

* feat(Core): added the first classification model & the feature necessary

* feat(Models): added basic transformers model

keras_models/classification.py

@@ -0,0 +1,102 @@
+import keras
+
+def create_basic_lstm_model(input_shape, num_classes):
+    model = keras.Sequential()
+    model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=input_shape))
+    model.add(keras.layers.Flatten())
+    model.add(keras.layers.Dropout(0.3))
+    model.add(keras.layers.Dense(units = 64, activation = 'sigmoid'))
+    model.add(keras.layers.Dropout(0.3))
+    model.add(keras.layers.Dense(units = 32, activation = 'sigmoid'))
+    model.add(keras.layers.Dropout(0.3))
+    model.add(keras.layers.Dense(units = num_classes, activation = 'softmax'))
+    return model
+
+def create_basic_cnn_model(input_shape, num_classes):
+    model = keras.Sequential()
+    model.add(keras.layers.Conv1D(filters=64, kernel_size=3, padding="same", input_shape=input_shape))
+    model.add(keras.layers.BatchNormalization())
+    model.add(keras.layers.ReLU())
+    model.add(keras.layers.Conv1D(filters=64, kernel_size=3, padding="same"))
+    model.add(keras.layers.BatchNormalization())
+    model.add(keras.layers.ReLU())
+    model.add(keras.layers.Conv1D(filters=64, kernel_size=3, padding="same"))
+    model.add(keras.layers.BatchNormalization())
+    model.add(keras.layers.ReLU())
+    model.add(keras.layers.GlobalAveragePooling1D())
+    model.add(keras.layers.Dense(num_classes, activation="softmax"))
+    return model
+
+
+def create_resnet_cnn_model(input_shape, num_classes):
+    n_feature_maps = 64
+
+    input_layer = keras.layers.Input(input_shape)
+
+
+    conv_x = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=8, padding='same')(input_layer)
+    conv_x = keras.layers.BatchNormalization()(conv_x)
+    conv_x = keras.layers.Activation('relu')(conv_x)
+
+    conv_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=5, padding='same')(conv_x)
+    conv_y = keras.layers.BatchNormalization()(conv_y)
+    conv_y = keras.layers.Activation('relu')(conv_y)
+
+    conv_z = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=3, padding='same')(conv_y)
+    conv_z = keras.layers.BatchNormalization()(conv_z)
+
+    # expand channels for the sum
+    shortcut_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=1, padding='same')(input_layer)
+    shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
+
+    output_block_1 = keras.layers.add([shortcut_y, conv_z])
+    output_block_1 = keras.layers.Activation('relu')(output_block_1)
+
+    # BLOCK 2
+
+    conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_1)
+    conv_x = keras.layers.BatchNormalization()(conv_x)
+    conv_x = keras.layers.Activation('relu')(conv_x)
+
+    conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)
+    conv_y = keras.layers.BatchNormalization()(conv_y)
+    conv_y = keras.layers.Activation('relu')(conv_y)
+
+    conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)
+    conv_z = keras.layers.BatchNormalization()(conv_z)
+
+    # expand channels for the sum
+    shortcut_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=1, padding='same')(output_block_1)
+    shortcut_y = keras.layers.BatchNormalization()(shortcut_y)
+
+    output_block_2 = keras.layers.add([shortcut_y, conv_z])
+    output_block_2 = keras.layers.Activation('relu')(output_block_2)
+
+    # BLOCK 3
+
+    conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_2)
+    conv_x = keras.layers.BatchNormalization()(conv_x)
+    conv_x = keras.layers.Activation('relu')(conv_x)
+
+    conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)
+    conv_y = keras.layers.BatchNormalization()(conv_y)
+    conv_y = keras.layers.Activation('relu')(conv_y)
+
+    conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)
+    conv_z = keras.layers.BatchNormalization()(conv_z)
+
+    # no need to expand channels because they are equal
+    shortcut_y = keras.layers.BatchNormalization()(output_block_2)
+
+    output_block_3 = keras.layers.add([shortcut_y, conv_z])
+    output_block_3 = keras.layers.Activation('relu')(output_block_3)
+
+    # FINAL
+
+    gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3)
+
+    output_layer = keras.layers.Dense(num_classes, activation='softmax')(gap_layer)
+    model = keras.models.Model(inputs=input_layer, outputs=output_layer)
+
+
+    return model

keras_models/classification_transformer.py

@@ -0,0 +1,41 @@
+from keras import layers
+import keras
+
+def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):
+    # Normalization and Attention
+    x = layers.LayerNormalization(epsilon=1e-6)(inputs)
+    x = layers.MultiHeadAttention(
+        key_dim=head_size, num_heads=num_heads, dropout=dropout
+    )(x, x)
+    x = layers.Dropout(dropout)(x)
+    res = x + inputs
+
+    # Feed Forward Part
+    x = layers.LayerNormalization(epsilon=1e-6)(res)
+    x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation="relu")(x)
+    x = layers.Dropout(dropout)(x)
+    x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)
+    return x + res
+
+def create_basic_transformer_model(
+    input_shape,
+    n_classes,
+    head_size,
+    num_heads,
+    ff_dim,
+    num_transformer_blocks,
+    mlp_units,
+    dropout=0,
+    mlp_dropout=0,
+):
+    inputs = keras.Input(shape=input_shape)
+    x = inputs
+    for _ in range(num_transformer_blocks):
+        x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)
+
+    x = layers.GlobalAveragePooling1D(data_format="channels_first")(x)
+    for dim in mlp_units:
+        x = layers.Dense(dim, activation="relu")(x)
+        x = layers.Dropout(mlp_dropout)(x)
+    outputs = layers.Dense(n_classes, activation="softmax")(x)
+    return keras.Model(inputs, outputs)

load_data.py

@@ -66,3 +66,28 @@ def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, peri
     df['target'] = df[source_column].diff(period).shift(-period)
     df = df.iloc[:-period]
     return df
+
+
+#%%
+def create_target_pos_neg_classes(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
+    df['target'] = df[source_column].diff(period).shift(-period)
+    df['target'] = df['target'].map(lambda x: 0 if x <= 0.0 else 1)
+    df = df.iloc[:-period]
+    return df
+
+def create_target_four_classes(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
+    def __get_class(x):
+        treshold = 0.08
+        if x <= -treshold:
+            return 0
+        elif x > -treshold and x <= 0:
+            return 1
+        elif x > 0 and x <= treshold:
+            return 2
+        else:
+            return 3
+
+    df['target'] = df[source_column].diff(period).shift(-period)
+    df['target'] = df['target'].map(__get_class)
+    df = df.iloc[:-period]
+    return df

model_classification_keras.py

@@ -0,0 +1,122 @@
+#%% Import all the stuff, load data, define constants
+from sklearn.utils import shuffle
+from load_data import load_files, create_target_pos_neg_classes
+import pandas as pd
+from tensorflow import keras
+from utils.normalize import normalize
+import tensorflow as tf
+from utils.visualize import visualize_loss
+from sklearn.preprocessing import MinMaxScaler
+from utils.evaluate import print_classification_metrics
+import numpy as np
+from utils.rolling import rolling_window
+from keras_models.classification import create_basic_cnn_model, create_basic_lstm_model, create_resnet_cnn_model
+from keras_models.classification_transformer import create_basic_transformer_model
+
+data = load_files('data/', add_features=True, log_returns=False)
+data.reset_index(drop=True, inplace=True)
+data = data[[column for column in data.columns if not column.endswith('volume')]]
+# data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]]
+
+target_col = 'target'
+data = create_target_pos_neg_classes(data, 'ETH_returns', 30)
+
+num_classes = 2
+learning_rate = 0.002
+batch_size = 64
+epochs = 100
+
+split_fraction = 0.8
+train_split = int(split_fraction * int(data.shape[0]))
+
+past = 10
+future = 1
+
+start = past + future
+end = start + train_split
+
+#%% split data into training - validation sets
+train_data = data.loc[0 : train_split - 1]
+val_data = data.loc[train_split:]
+
+#%% create features and target for training set & keras dataset
+feature_scaler = MinMaxScaler(feature_range= (-1, 1))
+
+x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std
+y_train = keras.utils.to_categorical(data.iloc[start:end][target_col].values)
+
+dataset_train = keras.preprocessing.timeseries_dataset_from_array(
+    x_train,
+    y_train,
+    sequence_length=past,
+    batch_size=batch_size,
+)
+
+
+#%% create features and target for validation set & keras dataset
+x_end = len(val_data) - past - future
+label_start = train_split + past + future
+
+x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std
+y_val = keras.utils.to_categorical(data.iloc[label_start:][target_col].values)
+
+dataset_val = keras.utils.timeseries_dataset_from_array(
+    x_val,
+    y_val,
+    sequence_length=past,
+    batch_size=batch_size,
+    shuffle=False,
+)
+
+
+#%%
+
+for batch in dataset_train.take(10):
+    batch_inputs, batch_targets = batch
+
+print("Input shape:", batch_inputs.shape)
+print("Target shape:", batch_targets.shape)
+
+n_timestamps = batch_inputs.shape[1]
+n_features = batch_inputs.shape[2]
+# print(batch_inputs)
+# print(batch_targets)
+
+# %%
+# model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
+model = create_basic_transformer_model(
+    input_shape=(n_timestamps, n_features),
+    n_classes=num_classes,
+    head_size=64,
+    num_heads=4,
+    ff_dim=4,
+    num_transformer_blocks=4,
+    mlp_units=[64],
+    mlp_dropout=0.4,
+    dropout=0.25,
+)
+
+optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
+model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy'])
+model.summary()
+
+# %%
+
+path_checkpoint = "model_checkpoint.h5"
+
+history = model.fit(
+    dataset_train,
+    epochs=epochs,
+    validation_data=dataset_val,
+)
+
+#%%
+
+# pred = model.predict(rolling_window(x_val, 11))
+# pred = pred.reshape(pred.shape[0], 1)
+
+
+#%%
+# print_classification_metrics(y_val, pred)
+
+visualize_loss(history, "Training and Validation Loss")

model_regression_keras.py

@@ -7,7 +7,7 @@ from utils.normalize import normalize
 import tensorflow as tf
 from utils.visualize import visualize_loss
 from sklearn.preprocessing import MinMaxScaler
-from utils.evaluate import print_metrics
+from utils.evaluate import print_regression_metrics
 import numpy as np
 from utils.rolling import rolling_window
 
@@ -109,7 +109,7 @@ pred = model.predict(rolling_window(x_val, 11))
 pred = pred.reshape(pred.shape[0], 1)
 pred = target_scaler.inverse_transform(pred)
 
-print_metrics(y_val, pred)
+print_regression_metrics(y_val, pred)
 
 #%%
 

utils/evaluate.py

@@ -1,9 +1,13 @@
 from math import sqrt
-from sklearn.metrics import mean_squared_error, mean_absolute_error
+from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score
-
+from sklearn.metrics import confusion_matrix
-def print_metrics(y_true, y_pred):
+def print_regression_metrics(y_true, y_pred):
     rmse = sqrt(mean_squared_error(y_true, y_pred))
     print("RMSE: %.2f" % rmse)
 
     mae = mean_absolute_error(y_true, y_pred)
     print("MAE: %.2f" % mae)
+
+def print_classification_metrics(y_true, y_pred):
+    print("Accuracy: %.2f" % accuracy_score(y_true, y_pred))
+    print("Confusion Matrix: \n", confusion_matrix(y_true, y_pred))