Merge pull request #3 from applied-exploration/feature/feature-engineering

feat(Data): added new date-related features, and ability to train on "n days in advance" returns

load_data.py

@@ -1,31 +1,59 @@
 #%%
 import pandas as pd
 import os
+import numpy as np
 
 #%%
 
-def load_files(path, add_features):
+def load_files(path: str, add_features: bool, log_returns: bool) -> pd.DataFrame:
-    dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
+    dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features, log_returns) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
     dfs = pd.concat(dfs, axis=1).fillna(0.)
+
+    dfs.index = pd.DatetimeIndex(dfs.index)
+
+    if add_features:
+        dfs['day_month'] = dfs.index.day
+        dfs['day_week'] = dfs.index.dayofweek
+        dfs['month'] = dfs.index.month
+
     return dfs.drop(index=dfs.index[0], axis=0)
 
-def __load_df(path, prefix, add_features):
+def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool) -> pd.DataFrame:
     df = pd.read_csv(path, header=0, index_col=0).fillna(0)
-    df['returns'] = df['close'].pct_change()
+
+    if log_returns:
+        df['returns'] = np.log(df['close']).diff(1)
+    else:
+        df['returns'] = df['close'].pct_change()
 
     if add_features:
         # volatility (10, 20, 30 days)
         df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5)
         df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5)
         df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5)
+        df['vol_60'] = df['returns'].rolling(30).std()*(252**0.5)
 
         # momentum (10, 20, 30, 60, 90 days)
-        df['mom_10'] = df['close'].pct_change(10)
+        if log_returns:
-        df['mom_20'] = df['close'].pct_change(20)
+            df['mom_10'] = np.log(df['close']).diff(10)
-        df['mom_30'] = df['close'].pct_change(30)
+            df['mom_20'] = np.log(df['close']).diff(20)
-        df['mom_60'] = df['close'].pct_change(60)
+            df['mom_30'] = np.log(df['close']).diff(30)
-        df['mom_90'] = df['close'].pct_change(90)
+            df['mom_60'] = np.log(df['close']).diff(60)
+            df['mom_90'] = np.log(df['close']).diff(90)
+        else:
+            df['mom_10'] = df['close'].pct_change(10)
+            df['mom_20'] = df['close'].pct_change(20)
+            df['mom_30'] = df['close'].pct_change(30)
+            df['mom_60'] = df['close'].pct_change(60)
+            df['mom_90'] = df['close'].pct_change(90)
 
+    df = df.replace([np.inf, -np.inf], 0.)
     df = df.drop(columns=['open', 'high', 'low', 'close'])
     df.columns = [prefix + "_" + c for c in df.columns]
     return df
+
+# %%
+def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
+    df['target'] = df[source_column].diff(period).shift(-period)
+    df = df.iloc[:-period]
+    return df

model_ff.py

@@ -1,96 +0,0 @@
-#%% Import all the stuff, load data, define constants
-from load_data import load_files
-import pandas as pd
-from tensorflow import keras
-from utils.normalize import normalize
-import tensorflow as tf
-from utils.visualize import visualize_loss
-
-data = load_files('data/', False)
-data.reset_index(drop=True, inplace=True)
-data = data[[column for column in data.columns if not column.endswith('volume')]]
-# data = data[["ETH_returns", "BTC_returns"]]
-
-ticker_to_predict = 'ETH_returns'
-
-learning_rate = 0.002
-batch_size = 64
-epochs = 100
-
-split_fraction = 0.715
-train_split = int(split_fraction * int(data.shape[0]))
-
-past = 10
-future = 1
-
-start = past + future
-end = start + train_split
-
-#%% split data into training - validation sets
-train_data = data.loc[0 : train_split - 1]
-val_data = data.loc[train_split:]
-
-#%% create features and target for training set & keras dataset
-
-x_train = normalize(train_data).values
-# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values
-y_train = normalize(data).iloc[start:end][ticker_to_predict].values
-
-dataset_train = keras.preprocessing.timeseries_dataset_from_array(
-    x_train,
-    y_train,
-    sequence_length=past,
-    batch_size=batch_size,
-)
-
-
-#%% create features and target for validation set & keras dataset
-x_end = len(val_data) - past - future
-label_start = train_split + past + future
-
-x_val = normalize(val_data).iloc[:x_end].values
-# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values
-y_val = normalize(data).iloc[label_start:][ticker_to_predict].values
-
-dataset_val = keras.utils.timeseries_dataset_from_array(
-    x_val,
-    y_val,
-    sequence_length=past,
-    batch_size=batch_size,
-)
-
-
-#%%
-
-for batch in dataset_train.take(10):
-    batch_inputs, batch_targets = batch
-
-print("Input shape:", batch_inputs.shape)
-print("Target shape:", batch_targets.shape)
-
-print(batch_inputs)
-print(batch_targets)
-
-# %%
-model = keras.Sequential()
-model.add(keras.layers.Dense(units = 10, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
-model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 4, activation = 'sigmoid'))
-model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 1))
-
-optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0)
-model.compile(optimizer=optimizer, loss="mean_squared_error")
-model.summary()
-
-# %%
-path_checkpoint = "model_checkpoint.h5"
-
-history = model.fit(
-    dataset_train,
-    epochs=epochs,
-    validation_data=dataset_val,
-)
-# %%
-
-visualize_loss(history, "Training and Validation Loss")

model_lstm.py

@@ -1,96 +0,0 @@
-#%% Import all the stuff, load data, define constants
-from load_data import load_files
-import pandas as pd
-from tensorflow import keras
-from utils.normalize import normalize
-import tensorflow as tf
-from utils.visualize import visualize_loss
-
-data = load_files('data/', False)
-data.reset_index(drop=True, inplace=True)
-data = data[[column for column in data.columns if not column.endswith('volume')]]
-# data = data[["ETH_returns", "BTC_returns"]]
-
-ticker_to_predict = 'ETH_returns'
-
-learning_rate = 0.002
-batch_size = 64
-epochs = 100
-
-split_fraction = 0.715
-train_split = int(split_fraction * int(data.shape[0]))
-
-past = 10
-future = 1
-
-start = past + future
-end = start + train_split
-
-#%% split data into training - validation sets
-train_data = data.loc[0 : train_split - 1]
-val_data = data.loc[train_split:]
-
-#%% create features and target for training set & keras dataset
-
-x_train = normalize(train_data).values
-# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values
-y_train = normalize(data).iloc[start:end][ticker_to_predict].values
-
-dataset_train = keras.preprocessing.timeseries_dataset_from_array(
-    x_train,
-    y_train,
-    sequence_length=past,
-    batch_size=batch_size,
-)
-
-
-#%% create features and target for validation set & keras dataset
-x_end = len(val_data) - past - future
-label_start = train_split + past + future
-
-x_val = normalize(val_data).iloc[:x_end].values
-# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values
-y_val = normalize(data).iloc[label_start:][ticker_to_predict].values
-
-dataset_val = keras.utils.timeseries_dataset_from_array(
-    x_val,
-    y_val,
-    sequence_length=past,
-    batch_size=batch_size,
-)
-
-
-#%%
-
-for batch in dataset_train.take(10):
-    batch_inputs, batch_targets = batch
-
-print("Input shape:", batch_inputs.shape)
-print("Target shape:", batch_targets.shape)
-
-print(batch_inputs)
-print(batch_targets)
-
-# %%
-model = keras.Sequential()
-model.add(keras.layers.LSTM(units = 32, return_sequences = True, activation = 'relu', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
-model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 10, activation = 'relu'))
-model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 1))
-
-optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0)
-model.compile(optimizer=optimizer, loss="mean_squared_error")
-model.summary()
-
-# %%
-path_checkpoint = "model_checkpoint.h5"
-
-history = model.fit(
-    dataset_train,
-    epochs=epochs,
-    validation_data=dataset_val,
-)
-# %%
-
-visualize_loss(history, "Training and Validation Loss")

model_regression_lstm.py

@@ -1,27 +1,34 @@
 #%% Import all the stuff, load data, define constants
-from load_data import load_files
+from sklearn.utils import shuffle
+from load_data import load_files, create_target_cum_forward_returns
 import pandas as pd
 from tensorflow import keras
 from utils.normalize import normalize
 import tensorflow as tf
 from utils.visualize import visualize_loss
+from sklearn.preprocessing import MinMaxScaler
+from utils.evaluate import print_metrics
+import numpy as np
+from utils.rolling import rolling_window
 
-data = load_files('data/', True)
+
+data = load_files('data/', add_features=True, log_returns=False)
 data.reset_index(drop=True, inplace=True)
 data = data[[column for column in data.columns if not column.endswith('volume')]]
-data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_vol_10", "BTC_mom_20", "BTC_vol_20"]]
+data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]]
 
-ticker_to_predict = 'BTC_mom_10'
+target_col = 'target'
+data = create_target_cum_forward_returns(data, 'BTC_returns', 10)
 
 learning_rate = 0.002
-batch_size = 128
+batch_size = 64
 epochs = 100
 
 split_fraction = 0.715
 train_split = int(split_fraction * int(data.shape[0]))
 
-past = 100
+past = 10
-future = 11
+future = 1
 
 start = past + future
 end = start + train_split
@@ -31,10 +38,11 @@ train_data = data.loc[0 : train_split - 1]
 val_data = data.loc[train_split:]
 
 #%% create features and target for training set & keras dataset
+feature_scaler = MinMaxScaler(feature_range= (-1, 1))
+target_scaler = MinMaxScaler(feature_range= (-1, 1))
 
-x_train = normalize(train_data).values
+x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std
-# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values
+y_train = target_scaler.fit_transform(data.iloc[start:end][target_col].values.reshape(-1, 1))
-y_train = normalize(data).iloc[start:end][ticker_to_predict].values
 
 dataset_train = keras.preprocessing.timeseries_dataset_from_array(
     x_train,
@@ -48,15 +56,15 @@ dataset_train = keras.preprocessing.timeseries_dataset_from_array(
 x_end = len(val_data) - past - future
 label_start = train_split + past + future
 
-x_val = normalize(val_data).iloc[:x_end].values
+x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std
-# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values
+y_val = target_scaler.transform(data.iloc[label_start:][target_col].values.reshape(-1, 1))
-y_val = normalize(data).iloc[label_start:][ticker_to_predict].values
 
 dataset_val = keras.utils.timeseries_dataset_from_array(
     x_val,
     y_val,
     sequence_length=past,
     batch_size=batch_size,
+    shuffle=False,
 )
 
 
@@ -68,23 +76,25 @@ for batch in dataset_train.take(10):
 print("Input shape:", batch_inputs.shape)
 print("Target shape:", batch_targets.shape)
 
-print(batch_inputs)
+# print(batch_inputs)
-print(batch_targets)
+# print(batch_targets)
 
 # %%
 model = keras.Sequential()
-model.add(keras.layers.Dense(units = 50, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
+model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
 model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 10, activation = 'sigmoid'))
+model.add(keras.layers.Dense(units = 64, activation = 'sigmoid'))
+model.add(keras.layers.Dropout(0.4))
+model.add(keras.layers.Dense(units = 32, activation = 'sigmoid'))
 model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 3, activation = 'sigmoid'))
 model.add(keras.layers.Dense(units = 1))
 
-optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0)
+optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(optimizer=optimizer, loss="mean_squared_error")
 model.summary()
 
 # %%
+
 path_checkpoint = "model_checkpoint.h5"
 
 history = model.fit(
@@ -92,6 +102,15 @@ history = model.fit(
     epochs=epochs,
     validation_data=dataset_val,
 )
-# %%
+
+#%%
+
+pred = model.predict(rolling_window(x_val, 11))
+pred = pred.reshape(pred.shape[0], 1)
+pred = target_scaler.inverse_transform(pred)
+
+print_metrics(y_val, pred)
+
+#%%
 
 visualize_loss(history, "Training and Validation Loss")

tutorial_lstm.py

@@ -1,110 +0,0 @@
-#%%
-from math import sqrt
-from numpy import concatenate
-import numpy as np
-from matplotlib import pyplot
-from pandas import read_csv
-from pandas import DataFrame
-from pandas import concat
-from sklearn.preprocessing import MinMaxScaler
-from sklearn.preprocessing import LabelEncoder
-from sklearn.metrics import mean_squared_error
-from keras.models import Sequential
-from keras.layers import Dense
-from keras.layers import LSTM
-#%%
-
-# convert series to supervised learning
-def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
-	n_vars = 1 if type(data) is list else data.shape[1]
-	df = DataFrame(data)
-	cols, names = list(), list()
-	# input sequence (t-n, ... t-1)
-	for i in range(n_in, 0, -1):
-		cols.append(df.shift(i))
-		names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
-	# forecast sequence (t, t+1, ... t+n)
-	for i in range(0, n_out):
-		cols.append(df.shift(-i))
-		if i == 0:
-			names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
-		else:
-			names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
-	# put it all together
-	agg = concat(cols, axis=1)
-	agg.columns = names
-	# drop rows with NaN values
-	if dropnan:
-		agg.dropna(inplace=True)
-	return agg
- 
-#%% load dataset
-from load_data import load_files
-data = load_files('data/', True)
-data.reset_index(drop=True, inplace=True)
-data = data[[column for column in data.columns if not column.endswith('volume')]]
-data = data[["BTC_returns", "BTC_vol_10"]]
-
-values = data.values
-# ensure all data is float
-# values = values.astype('float32')
-
-#%%
-np.isposinf(values).sum()
-
-#%% normalize features
-scaler = MinMaxScaler(feature_range=(-1, 1))
-scaled = scaler.fit_transform(values)
-
-
-# specify the number of lag hours
-past = 10
-n_features = 8
-#%% frame as supervised learning
-reframed = series_to_supervised(scaled, past, 1)
-print(reframed.shape)
- 
-#%% split into train and test sets
-values = reframed.values
-n_train_hours = 365 * 24
-train = values[:n_train_hours, :]
-test = values[n_train_hours:, :]
-# split into input and outputs
-n_obs = past * n_features
-train_X, train_y = train[:, :n_obs], train[:, -n_features]
-test_X, test_y = test[:, :n_obs], test[:, -n_features]
-print(train_X.shape, len(train_X), train_y.shape)
-# reshape input to be 3D [samples, timesteps, features]
-train_X = train_X.reshape((train_X.shape[0], past, n_features))
-test_X = test_X.reshape((test_X.shape[0], past, n_features))
-print(train_X.shape, train_y.shape, test_X.shape, test_y.shape)
- 
-#%% design network
-model = Sequential()
-model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2])))
-model.add(Dense(1))
-model.compile(loss='mae', optimizer='adam')
-
-#%% fit network
-history = model.fit(train_X, train_y, epochs=50, batch_size=72, validation_data=(test_X, test_y), verbose=2, shuffle=False)
-# plot history
-pyplot.plot(history.history['loss'], label='train')
-pyplot.plot(history.history['val_loss'], label='test')
-pyplot.legend()
-pyplot.show()
- 
-# make a prediction
-yhat = model.predict(test_X)
-test_X = test_X.reshape((test_X.shape[0], n_hours*n_features))
-# invert scaling for forecast
-inv_yhat = concatenate((yhat, test_X[:, -7:]), axis=1)
-inv_yhat = scaler.inverse_transform(inv_yhat)
-inv_yhat = inv_yhat[:,0]
-# invert scaling for actual
-test_y = test_y.reshape((len(test_y), 1))
-inv_y = concatenate((test_y, test_X[:, -7:]), axis=1)
-inv_y = scaler.inverse_transform(inv_y)
-inv_y = inv_y[:,0]
-# calculate RMSE
-rmse = sqrt(mean_squared_error(inv_y, inv_yhat))
-print('Test RMSE: %.3f' % rmse)

utils/evaluate.py

@@ -0,0 +1,9 @@
+from math import sqrt
+from sklearn.metrics import mean_squared_error, mean_absolute_error
+
+def print_metrics(y_true, y_pred):
+    rmse = sqrt(mean_squared_error(y_true, y_pred))
+    print("RMSE: %.2f" % rmse)
+
+    mae = mean_absolute_error(y_true, y_pred)
+    print("MAE: %.2f" % mae)

utils/rolling.py

@@ -0,0 +1,6 @@
+import numpy as np
+
+def rolling_window(a, window):
+    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
+    strides = a.strides + (a.strides[-1],)
+    return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)