diff --git a/load_data.py b/load_data.py index 9308b97..c9a5709 100644 --- a/load_data.py +++ b/load_data.py @@ -1,31 +1,59 @@ #%% import pandas as pd import os +import numpy as np #%% -def load_files(path, add_features): - dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))] +def load_files(path: str, add_features: bool, log_returns: bool) -> pd.DataFrame: + dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features, log_returns) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))] dfs = pd.concat(dfs, axis=1).fillna(0.) + + dfs.index = pd.DatetimeIndex(dfs.index) + + if add_features: + dfs['day_month'] = dfs.index.day + dfs['day_week'] = dfs.index.dayofweek + dfs['month'] = dfs.index.month + return dfs.drop(index=dfs.index[0], axis=0) -def __load_df(path, prefix, add_features): +def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool) -> pd.DataFrame: df = pd.read_csv(path, header=0, index_col=0).fillna(0) - df['returns'] = df['close'].pct_change() - + + if log_returns: + df['returns'] = np.log(df['close']).diff(1) + else: + df['returns'] = df['close'].pct_change() + if add_features: # volatility (10, 20, 30 days) df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5) df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5) df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5) + df['vol_60'] = df['returns'].rolling(30).std()*(252**0.5) # momentum (10, 20, 30, 60, 90 days) - df['mom_10'] = df['close'].pct_change(10) - df['mom_20'] = df['close'].pct_change(20) - df['mom_30'] = df['close'].pct_change(30) - df['mom_60'] = df['close'].pct_change(60) - df['mom_90'] = df['close'].pct_change(90) + if log_returns: + df['mom_10'] = np.log(df['close']).diff(10) + df['mom_20'] = np.log(df['close']).diff(20) + df['mom_30'] = np.log(df['close']).diff(30) + df['mom_60'] = np.log(df['close']).diff(60) + df['mom_90'] = np.log(df['close']).diff(90) + else: + df['mom_10'] = df['close'].pct_change(10) + df['mom_20'] = df['close'].pct_change(20) + df['mom_30'] = df['close'].pct_change(30) + df['mom_60'] = df['close'].pct_change(60) + df['mom_90'] = df['close'].pct_change(90) + df = df.replace([np.inf, -np.inf], 0.) df = df.drop(columns=['open', 'high', 'low', 'close']) df.columns = [prefix + "_" + c for c in df.columns] return df + +# %% +def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: + df['target'] = df[source_column].diff(period).shift(-period) + df = df.iloc[:-period] + return df \ No newline at end of file diff --git a/model_ff.py b/model_ff.py deleted file mode 100644 index d6794b3..0000000 --- a/model_ff.py +++ /dev/null @@ -1,96 +0,0 @@ -#%% Import all the stuff, load data, define constants -from load_data import load_files -import pandas as pd -from tensorflow import keras -from utils.normalize import normalize -import tensorflow as tf -from utils.visualize import visualize_loss - -data = load_files('data/', False) -data.reset_index(drop=True, inplace=True) -data = data[[column for column in data.columns if not column.endswith('volume')]] -# data = data[["ETH_returns", "BTC_returns"]] - -ticker_to_predict = 'ETH_returns' - -learning_rate = 0.002 -batch_size = 64 -epochs = 100 - -split_fraction = 0.715 -train_split = int(split_fraction * int(data.shape[0])) - -past = 10 -future = 1 - -start = past + future -end = start + train_split - -#%% split data into training - validation sets -train_data = data.loc[0 : train_split - 1] -val_data = data.loc[train_split:] - -#%% create features and target for training set & keras dataset - -x_train = normalize(train_data).values -# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values -y_train = normalize(data).iloc[start:end][ticker_to_predict].values - -dataset_train = keras.preprocessing.timeseries_dataset_from_array( - x_train, - y_train, - sequence_length=past, - batch_size=batch_size, -) - - -#%% create features and target for validation set & keras dataset -x_end = len(val_data) - past - future -label_start = train_split + past + future - -x_val = normalize(val_data).iloc[:x_end].values -# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values -y_val = normalize(data).iloc[label_start:][ticker_to_predict].values - -dataset_val = keras.utils.timeseries_dataset_from_array( - x_val, - y_val, - sequence_length=past, - batch_size=batch_size, -) - - -#%% - -for batch in dataset_train.take(10): - batch_inputs, batch_targets = batch - -print("Input shape:", batch_inputs.shape) -print("Target shape:", batch_targets.shape) - -print(batch_inputs) -print(batch_targets) - -# %% -model = keras.Sequential() -model.add(keras.layers.Dense(units = 10, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2]))) -model.add(keras.layers.Dropout(0.4)) -model.add(keras.layers.Dense(units = 4, activation = 'sigmoid')) -model.add(keras.layers.Dropout(0.4)) -model.add(keras.layers.Dense(units = 1)) - -optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0) -model.compile(optimizer=optimizer, loss="mean_squared_error") -model.summary() - -# %% -path_checkpoint = "model_checkpoint.h5" - -history = model.fit( - dataset_train, - epochs=epochs, - validation_data=dataset_val, -) -# %% - -visualize_loss(history, "Training and Validation Loss") \ No newline at end of file diff --git a/model_lstm.py b/model_lstm.py deleted file mode 100644 index f17e385..0000000 --- a/model_lstm.py +++ /dev/null @@ -1,96 +0,0 @@ -#%% Import all the stuff, load data, define constants -from load_data import load_files -import pandas as pd -from tensorflow import keras -from utils.normalize import normalize -import tensorflow as tf -from utils.visualize import visualize_loss - -data = load_files('data/', False) -data.reset_index(drop=True, inplace=True) -data = data[[column for column in data.columns if not column.endswith('volume')]] -# data = data[["ETH_returns", "BTC_returns"]] - -ticker_to_predict = 'ETH_returns' - -learning_rate = 0.002 -batch_size = 64 -epochs = 100 - -split_fraction = 0.715 -train_split = int(split_fraction * int(data.shape[0])) - -past = 10 -future = 1 - -start = past + future -end = start + train_split - -#%% split data into training - validation sets -train_data = data.loc[0 : train_split - 1] -val_data = data.loc[train_split:] - -#%% create features and target for training set & keras dataset - -x_train = normalize(train_data).values -# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values -y_train = normalize(data).iloc[start:end][ticker_to_predict].values - -dataset_train = keras.preprocessing.timeseries_dataset_from_array( - x_train, - y_train, - sequence_length=past, - batch_size=batch_size, -) - - -#%% create features and target for validation set & keras dataset -x_end = len(val_data) - past - future -label_start = train_split + past + future - -x_val = normalize(val_data).iloc[:x_end].values -# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values -y_val = normalize(data).iloc[label_start:][ticker_to_predict].values - -dataset_val = keras.utils.timeseries_dataset_from_array( - x_val, - y_val, - sequence_length=past, - batch_size=batch_size, -) - - -#%% - -for batch in dataset_train.take(10): - batch_inputs, batch_targets = batch - -print("Input shape:", batch_inputs.shape) -print("Target shape:", batch_targets.shape) - -print(batch_inputs) -print(batch_targets) - -# %% -model = keras.Sequential() -model.add(keras.layers.LSTM(units = 32, return_sequences = True, activation = 'relu', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2]))) -model.add(keras.layers.Dropout(0.4)) -model.add(keras.layers.Dense(units = 10, activation = 'relu')) -model.add(keras.layers.Dropout(0.4)) -model.add(keras.layers.Dense(units = 1)) - -optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0) -model.compile(optimizer=optimizer, loss="mean_squared_error") -model.summary() - -# %% -path_checkpoint = "model_checkpoint.h5" - -history = model.fit( - dataset_train, - epochs=epochs, - validation_data=dataset_val, -) -# %% - -visualize_loss(history, "Training and Validation Loss") \ No newline at end of file diff --git a/model_ff_10days.py b/model_regression_lstm.py similarity index 51% rename from model_ff_10days.py rename to model_regression_lstm.py index bc97cac..9afead2 100644 --- a/model_ff_10days.py +++ b/model_regression_lstm.py @@ -1,27 +1,34 @@ #%% Import all the stuff, load data, define constants -from load_data import load_files +from sklearn.utils import shuffle +from load_data import load_files, create_target_cum_forward_returns import pandas as pd from tensorflow import keras from utils.normalize import normalize import tensorflow as tf from utils.visualize import visualize_loss +from sklearn.preprocessing import MinMaxScaler +from utils.evaluate import print_metrics +import numpy as np +from utils.rolling import rolling_window -data = load_files('data/', True) + +data = load_files('data/', add_features=True, log_returns=False) data.reset_index(drop=True, inplace=True) data = data[[column for column in data.columns if not column.endswith('volume')]] -data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_vol_10", "BTC_mom_20", "BTC_vol_20"]] +data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]] -ticker_to_predict = 'BTC_mom_10' +target_col = 'target' +data = create_target_cum_forward_returns(data, 'BTC_returns', 10) learning_rate = 0.002 -batch_size = 128 +batch_size = 64 epochs = 100 split_fraction = 0.715 train_split = int(split_fraction * int(data.shape[0])) -past = 100 -future = 11 +past = 10 +future = 1 start = past + future end = start + train_split @@ -31,10 +38,11 @@ train_data = data.loc[0 : train_split - 1] val_data = data.loc[train_split:] #%% create features and target for training set & keras dataset +feature_scaler = MinMaxScaler(feature_range= (-1, 1)) +target_scaler = MinMaxScaler(feature_range= (-1, 1)) -x_train = normalize(train_data).values -# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values -y_train = normalize(data).iloc[start:end][ticker_to_predict].values +x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std +y_train = target_scaler.fit_transform(data.iloc[start:end][target_col].values.reshape(-1, 1)) dataset_train = keras.preprocessing.timeseries_dataset_from_array( x_train, @@ -48,15 +56,15 @@ dataset_train = keras.preprocessing.timeseries_dataset_from_array( x_end = len(val_data) - past - future label_start = train_split + past + future -x_val = normalize(val_data).iloc[:x_end].values -# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values -y_val = normalize(data).iloc[label_start:][ticker_to_predict].values +x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std +y_val = target_scaler.transform(data.iloc[label_start:][target_col].values.reshape(-1, 1)) dataset_val = keras.utils.timeseries_dataset_from_array( x_val, y_val, sequence_length=past, batch_size=batch_size, + shuffle=False, ) @@ -68,23 +76,25 @@ for batch in dataset_train.take(10): print("Input shape:", batch_inputs.shape) print("Target shape:", batch_targets.shape) -print(batch_inputs) -print(batch_targets) +# print(batch_inputs) +# print(batch_targets) # %% model = keras.Sequential() -model.add(keras.layers.Dense(units = 50, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2]))) +model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2]))) model.add(keras.layers.Dropout(0.4)) -model.add(keras.layers.Dense(units = 10, activation = 'sigmoid')) +model.add(keras.layers.Dense(units = 64, activation = 'sigmoid')) +model.add(keras.layers.Dropout(0.4)) +model.add(keras.layers.Dense(units = 32, activation = 'sigmoid')) model.add(keras.layers.Dropout(0.4)) -model.add(keras.layers.Dense(units = 3, activation = 'sigmoid')) model.add(keras.layers.Dense(units = 1)) -optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0) +optimizer = keras.optimizers.Adam(learning_rate=learning_rate) model.compile(optimizer=optimizer, loss="mean_squared_error") model.summary() # %% + path_checkpoint = "model_checkpoint.h5" history = model.fit( @@ -92,6 +102,15 @@ history = model.fit( epochs=epochs, validation_data=dataset_val, ) -# %% + +#%% + +pred = model.predict(rolling_window(x_val, 11)) +pred = pred.reshape(pred.shape[0], 1) +pred = target_scaler.inverse_transform(pred) + +print_metrics(y_val, pred) + +#%% visualize_loss(history, "Training and Validation Loss") \ No newline at end of file diff --git a/tutorial_lstm.py b/tutorial_lstm.py deleted file mode 100644 index e4e02fe..0000000 --- a/tutorial_lstm.py +++ /dev/null @@ -1,110 +0,0 @@ -#%% -from math import sqrt -from numpy import concatenate -import numpy as np -from matplotlib import pyplot -from pandas import read_csv -from pandas import DataFrame -from pandas import concat -from sklearn.preprocessing import MinMaxScaler -from sklearn.preprocessing import LabelEncoder -from sklearn.metrics import mean_squared_error -from keras.models import Sequential -from keras.layers import Dense -from keras.layers import LSTM -#%% - -# convert series to supervised learning -def series_to_supervised(data, n_in=1, n_out=1, dropnan=True): - n_vars = 1 if type(data) is list else data.shape[1] - df = DataFrame(data) - cols, names = list(), list() - # input sequence (t-n, ... t-1) - for i in range(n_in, 0, -1): - cols.append(df.shift(i)) - names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)] - # forecast sequence (t, t+1, ... t+n) - for i in range(0, n_out): - cols.append(df.shift(-i)) - if i == 0: - names += [('var%d(t)' % (j+1)) for j in range(n_vars)] - else: - names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)] - # put it all together - agg = concat(cols, axis=1) - agg.columns = names - # drop rows with NaN values - if dropnan: - agg.dropna(inplace=True) - return agg - -#%% load dataset -from load_data import load_files -data = load_files('data/', True) -data.reset_index(drop=True, inplace=True) -data = data[[column for column in data.columns if not column.endswith('volume')]] -data = data[["BTC_returns", "BTC_vol_10"]] - -values = data.values -# ensure all data is float -# values = values.astype('float32') - -#%% -np.isposinf(values).sum() - -#%% normalize features -scaler = MinMaxScaler(feature_range=(-1, 1)) -scaled = scaler.fit_transform(values) - - -# specify the number of lag hours -past = 10 -n_features = 8 -#%% frame as supervised learning -reframed = series_to_supervised(scaled, past, 1) -print(reframed.shape) - -#%% split into train and test sets -values = reframed.values -n_train_hours = 365 * 24 -train = values[:n_train_hours, :] -test = values[n_train_hours:, :] -# split into input and outputs -n_obs = past * n_features -train_X, train_y = train[:, :n_obs], train[:, -n_features] -test_X, test_y = test[:, :n_obs], test[:, -n_features] -print(train_X.shape, len(train_X), train_y.shape) -# reshape input to be 3D [samples, timesteps, features] -train_X = train_X.reshape((train_X.shape[0], past, n_features)) -test_X = test_X.reshape((test_X.shape[0], past, n_features)) -print(train_X.shape, train_y.shape, test_X.shape, test_y.shape) - -#%% design network -model = Sequential() -model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2]))) -model.add(Dense(1)) -model.compile(loss='mae', optimizer='adam') - -#%% fit network -history = model.fit(train_X, train_y, epochs=50, batch_size=72, validation_data=(test_X, test_y), verbose=2, shuffle=False) -# plot history -pyplot.plot(history.history['loss'], label='train') -pyplot.plot(history.history['val_loss'], label='test') -pyplot.legend() -pyplot.show() - -# make a prediction -yhat = model.predict(test_X) -test_X = test_X.reshape((test_X.shape[0], n_hours*n_features)) -# invert scaling for forecast -inv_yhat = concatenate((yhat, test_X[:, -7:]), axis=1) -inv_yhat = scaler.inverse_transform(inv_yhat) -inv_yhat = inv_yhat[:,0] -# invert scaling for actual -test_y = test_y.reshape((len(test_y), 1)) -inv_y = concatenate((test_y, test_X[:, -7:]), axis=1) -inv_y = scaler.inverse_transform(inv_y) -inv_y = inv_y[:,0] -# calculate RMSE -rmse = sqrt(mean_squared_error(inv_y, inv_yhat)) -print('Test RMSE: %.3f' % rmse) \ No newline at end of file diff --git a/utils/evaluate.py b/utils/evaluate.py new file mode 100644 index 0000000..7b04729 --- /dev/null +++ b/utils/evaluate.py @@ -0,0 +1,9 @@ +from math import sqrt +from sklearn.metrics import mean_squared_error, mean_absolute_error + +def print_metrics(y_true, y_pred): + rmse = sqrt(mean_squared_error(y_true, y_pred)) + print("RMSE: %.2f" % rmse) + + mae = mean_absolute_error(y_true, y_pred) + print("MAE: %.2f" % mae) \ No newline at end of file diff --git a/utils/rolling.py b/utils/rolling.py new file mode 100644 index 0000000..e080174 --- /dev/null +++ b/utils/rolling.py @@ -0,0 +1,6 @@ +import numpy as np + +def rolling_window(a, window): + shape = a.shape[:-1] + (a.shape[-1] - window + 1, window) + strides = a.strides + (a.strides[-1],) + return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides) \ No newline at end of file