mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-29 03:37:45 +00:00
Merge pull request #3 from applied-exploration/feature/feature-engineering
feat(Data): added new date-related features, and ability to train on "n days in advance" returns
This commit is contained in:
Daniel Szemerey
+38
-10
@@ -1,31 +1,59 @@
|
||||
#%%
|
||||
import pandas as pd
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
#%%
|
||||
|
||||
def load_files(path, add_features):
|
||||
dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
|
||||
def load_files(path: str, add_features: bool, log_returns: bool) -> pd.DataFrame:
|
||||
dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features, log_returns) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
|
||||
dfs = pd.concat(dfs, axis=1).fillna(0.)
|
||||
|
||||
dfs.index = pd.DatetimeIndex(dfs.index)
|
||||
|
||||
if add_features:
|
||||
dfs['day_month'] = dfs.index.day
|
||||
dfs['day_week'] = dfs.index.dayofweek
|
||||
dfs['month'] = dfs.index.month
|
||||
|
||||
return dfs.drop(index=dfs.index[0], axis=0)
|
||||
|
||||
def __load_df(path, prefix, add_features):
|
||||
def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool) -> pd.DataFrame:
|
||||
df = pd.read_csv(path, header=0, index_col=0).fillna(0)
|
||||
df['returns'] = df['close'].pct_change()
|
||||
|
||||
|
||||
if log_returns:
|
||||
df['returns'] = np.log(df['close']).diff(1)
|
||||
else:
|
||||
df['returns'] = df['close'].pct_change()
|
||||
|
||||
if add_features:
|
||||
# volatility (10, 20, 30 days)
|
||||
df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5)
|
||||
df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5)
|
||||
df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5)
|
||||
df['vol_60'] = df['returns'].rolling(30).std()*(252**0.5)
|
||||
|
||||
# momentum (10, 20, 30, 60, 90 days)
|
||||
df['mom_10'] = df['close'].pct_change(10)
|
||||
df['mom_20'] = df['close'].pct_change(20)
|
||||
df['mom_30'] = df['close'].pct_change(30)
|
||||
df['mom_60'] = df['close'].pct_change(60)
|
||||
df['mom_90'] = df['close'].pct_change(90)
|
||||
if log_returns:
|
||||
df['mom_10'] = np.log(df['close']).diff(10)
|
||||
df['mom_20'] = np.log(df['close']).diff(20)
|
||||
df['mom_30'] = np.log(df['close']).diff(30)
|
||||
df['mom_60'] = np.log(df['close']).diff(60)
|
||||
df['mom_90'] = np.log(df['close']).diff(90)
|
||||
else:
|
||||
df['mom_10'] = df['close'].pct_change(10)
|
||||
df['mom_20'] = df['close'].pct_change(20)
|
||||
df['mom_30'] = df['close'].pct_change(30)
|
||||
df['mom_60'] = df['close'].pct_change(60)
|
||||
df['mom_90'] = df['close'].pct_change(90)
|
||||
|
||||
df = df.replace([np.inf, -np.inf], 0.)
|
||||
df = df.drop(columns=['open', 'high', 'low', 'close'])
|
||||
df.columns = [prefix + "_" + c for c in df.columns]
|
||||
return df
|
||||
|
||||
# %%
|
||||
def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
|
||||
df['target'] = df[source_column].diff(period).shift(-period)
|
||||
df = df.iloc[:-period]
|
||||
return df
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
#%% Import all the stuff, load data, define constants
|
||||
from load_data import load_files
|
||||
import pandas as pd
|
||||
from tensorflow import keras
|
||||
from utils.normalize import normalize
|
||||
import tensorflow as tf
|
||||
from utils.visualize import visualize_loss
|
||||
|
||||
data = load_files('data/', False)
|
||||
data.reset_index(drop=True, inplace=True)
|
||||
data = data[[column for column in data.columns if not column.endswith('volume')]]
|
||||
# data = data[["ETH_returns", "BTC_returns"]]
|
||||
|
||||
ticker_to_predict = 'ETH_returns'
|
||||
|
||||
learning_rate = 0.002
|
||||
batch_size = 64
|
||||
epochs = 100
|
||||
|
||||
split_fraction = 0.715
|
||||
train_split = int(split_fraction * int(data.shape[0]))
|
||||
|
||||
past = 10
|
||||
future = 1
|
||||
|
||||
start = past + future
|
||||
end = start + train_split
|
||||
|
||||
#%% split data into training - validation sets
|
||||
train_data = data.loc[0 : train_split - 1]
|
||||
val_data = data.loc[train_split:]
|
||||
|
||||
#%% create features and target for training set & keras dataset
|
||||
|
||||
x_train = normalize(train_data).values
|
||||
# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values
|
||||
y_train = normalize(data).iloc[start:end][ticker_to_predict].values
|
||||
|
||||
dataset_train = keras.preprocessing.timeseries_dataset_from_array(
|
||||
x_train,
|
||||
y_train,
|
||||
sequence_length=past,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
|
||||
#%% create features and target for validation set & keras dataset
|
||||
x_end = len(val_data) - past - future
|
||||
label_start = train_split + past + future
|
||||
|
||||
x_val = normalize(val_data).iloc[:x_end].values
|
||||
# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values
|
||||
y_val = normalize(data).iloc[label_start:][ticker_to_predict].values
|
||||
|
||||
dataset_val = keras.utils.timeseries_dataset_from_array(
|
||||
x_val,
|
||||
y_val,
|
||||
sequence_length=past,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
for batch in dataset_train.take(10):
|
||||
batch_inputs, batch_targets = batch
|
||||
|
||||
print("Input shape:", batch_inputs.shape)
|
||||
print("Target shape:", batch_targets.shape)
|
||||
|
||||
print(batch_inputs)
|
||||
print(batch_targets)
|
||||
|
||||
# %%
|
||||
model = keras.Sequential()
|
||||
model.add(keras.layers.Dense(units = 10, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 4, activation = 'sigmoid'))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 1))
|
||||
|
||||
optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0)
|
||||
model.compile(optimizer=optimizer, loss="mean_squared_error")
|
||||
model.summary()
|
||||
|
||||
# %%
|
||||
path_checkpoint = "model_checkpoint.h5"
|
||||
|
||||
history = model.fit(
|
||||
dataset_train,
|
||||
epochs=epochs,
|
||||
validation_data=dataset_val,
|
||||
)
|
||||
# %%
|
||||
|
||||
visualize_loss(history, "Training and Validation Loss")
|
||||
@@ -1,96 +0,0 @@
|
||||
#%% Import all the stuff, load data, define constants
|
||||
from load_data import load_files
|
||||
import pandas as pd
|
||||
from tensorflow import keras
|
||||
from utils.normalize import normalize
|
||||
import tensorflow as tf
|
||||
from utils.visualize import visualize_loss
|
||||
|
||||
data = load_files('data/', False)
|
||||
data.reset_index(drop=True, inplace=True)
|
||||
data = data[[column for column in data.columns if not column.endswith('volume')]]
|
||||
# data = data[["ETH_returns", "BTC_returns"]]
|
||||
|
||||
ticker_to_predict = 'ETH_returns'
|
||||
|
||||
learning_rate = 0.002
|
||||
batch_size = 64
|
||||
epochs = 100
|
||||
|
||||
split_fraction = 0.715
|
||||
train_split = int(split_fraction * int(data.shape[0]))
|
||||
|
||||
past = 10
|
||||
future = 1
|
||||
|
||||
start = past + future
|
||||
end = start + train_split
|
||||
|
||||
#%% split data into training - validation sets
|
||||
train_data = data.loc[0 : train_split - 1]
|
||||
val_data = data.loc[train_split:]
|
||||
|
||||
#%% create features and target for training set & keras dataset
|
||||
|
||||
x_train = normalize(train_data).values
|
||||
# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values
|
||||
y_train = normalize(data).iloc[start:end][ticker_to_predict].values
|
||||
|
||||
dataset_train = keras.preprocessing.timeseries_dataset_from_array(
|
||||
x_train,
|
||||
y_train,
|
||||
sequence_length=past,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
|
||||
#%% create features and target for validation set & keras dataset
|
||||
x_end = len(val_data) - past - future
|
||||
label_start = train_split + past + future
|
||||
|
||||
x_val = normalize(val_data).iloc[:x_end].values
|
||||
# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values
|
||||
y_val = normalize(data).iloc[label_start:][ticker_to_predict].values
|
||||
|
||||
dataset_val = keras.utils.timeseries_dataset_from_array(
|
||||
x_val,
|
||||
y_val,
|
||||
sequence_length=past,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
for batch in dataset_train.take(10):
|
||||
batch_inputs, batch_targets = batch
|
||||
|
||||
print("Input shape:", batch_inputs.shape)
|
||||
print("Target shape:", batch_targets.shape)
|
||||
|
||||
print(batch_inputs)
|
||||
print(batch_targets)
|
||||
|
||||
# %%
|
||||
model = keras.Sequential()
|
||||
model.add(keras.layers.LSTM(units = 32, return_sequences = True, activation = 'relu', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 10, activation = 'relu'))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 1))
|
||||
|
||||
optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0)
|
||||
model.compile(optimizer=optimizer, loss="mean_squared_error")
|
||||
model.summary()
|
||||
|
||||
# %%
|
||||
path_checkpoint = "model_checkpoint.h5"
|
||||
|
||||
history = model.fit(
|
||||
dataset_train,
|
||||
epochs=epochs,
|
||||
validation_data=dataset_val,
|
||||
)
|
||||
# %%
|
||||
|
||||
visualize_loss(history, "Training and Validation Loss")
|
||||
@@ -1,27 +1,34 @@
|
||||
#%% Import all the stuff, load data, define constants
|
||||
from load_data import load_files
|
||||
from sklearn.utils import shuffle
|
||||
from load_data import load_files, create_target_cum_forward_returns
|
||||
import pandas as pd
|
||||
from tensorflow import keras
|
||||
from utils.normalize import normalize
|
||||
import tensorflow as tf
|
||||
from utils.visualize import visualize_loss
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from utils.evaluate import print_metrics
|
||||
import numpy as np
|
||||
from utils.rolling import rolling_window
|
||||
|
||||
data = load_files('data/', True)
|
||||
|
||||
data = load_files('data/', add_features=True, log_returns=False)
|
||||
data.reset_index(drop=True, inplace=True)
|
||||
data = data[[column for column in data.columns if not column.endswith('volume')]]
|
||||
data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_vol_10", "BTC_mom_20", "BTC_vol_20"]]
|
||||
data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]]
|
||||
|
||||
ticker_to_predict = 'BTC_mom_10'
|
||||
target_col = 'target'
|
||||
data = create_target_cum_forward_returns(data, 'BTC_returns', 10)
|
||||
|
||||
learning_rate = 0.002
|
||||
batch_size = 128
|
||||
batch_size = 64
|
||||
epochs = 100
|
||||
|
||||
split_fraction = 0.715
|
||||
train_split = int(split_fraction * int(data.shape[0]))
|
||||
|
||||
past = 100
|
||||
future = 11
|
||||
past = 10
|
||||
future = 1
|
||||
|
||||
start = past + future
|
||||
end = start + train_split
|
||||
@@ -31,10 +38,11 @@ train_data = data.loc[0 : train_split - 1]
|
||||
val_data = data.loc[train_split:]
|
||||
|
||||
#%% create features and target for training set & keras dataset
|
||||
feature_scaler = MinMaxScaler(feature_range= (-1, 1))
|
||||
target_scaler = MinMaxScaler(feature_range= (-1, 1))
|
||||
|
||||
x_train = normalize(train_data).values
|
||||
# x_train = normalize(train_data).drop(ticker_to_predict, axis=1).values
|
||||
y_train = normalize(data).iloc[start:end][ticker_to_predict].values
|
||||
x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std
|
||||
y_train = target_scaler.fit_transform(data.iloc[start:end][target_col].values.reshape(-1, 1))
|
||||
|
||||
dataset_train = keras.preprocessing.timeseries_dataset_from_array(
|
||||
x_train,
|
||||
@@ -48,15 +56,15 @@ dataset_train = keras.preprocessing.timeseries_dataset_from_array(
|
||||
x_end = len(val_data) - past - future
|
||||
label_start = train_split + past + future
|
||||
|
||||
x_val = normalize(val_data).iloc[:x_end].values
|
||||
# x_val = normalize(val_data).iloc[:x_end].drop(ticker_to_predict, axis=1).values
|
||||
y_val = normalize(data).iloc[label_start:][ticker_to_predict].values
|
||||
x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std
|
||||
y_val = target_scaler.transform(data.iloc[label_start:][target_col].values.reshape(-1, 1))
|
||||
|
||||
dataset_val = keras.utils.timeseries_dataset_from_array(
|
||||
x_val,
|
||||
y_val,
|
||||
sequence_length=past,
|
||||
batch_size=batch_size,
|
||||
shuffle=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -68,23 +76,25 @@ for batch in dataset_train.take(10):
|
||||
print("Input shape:", batch_inputs.shape)
|
||||
print("Target shape:", batch_targets.shape)
|
||||
|
||||
print(batch_inputs)
|
||||
print(batch_targets)
|
||||
# print(batch_inputs)
|
||||
# print(batch_targets)
|
||||
|
||||
# %%
|
||||
model = keras.Sequential()
|
||||
model.add(keras.layers.Dense(units = 50, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
|
||||
model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 10, activation = 'sigmoid'))
|
||||
model.add(keras.layers.Dense(units = 64, activation = 'sigmoid'))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 32, activation = 'sigmoid'))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 3, activation = 'sigmoid'))
|
||||
model.add(keras.layers.Dense(units = 1))
|
||||
|
||||
optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm=1.0)
|
||||
optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
|
||||
model.compile(optimizer=optimizer, loss="mean_squared_error")
|
||||
model.summary()
|
||||
|
||||
# %%
|
||||
|
||||
path_checkpoint = "model_checkpoint.h5"
|
||||
|
||||
history = model.fit(
|
||||
@@ -92,6 +102,15 @@ history = model.fit(
|
||||
epochs=epochs,
|
||||
validation_data=dataset_val,
|
||||
)
|
||||
# %%
|
||||
|
||||
#%%
|
||||
|
||||
pred = model.predict(rolling_window(x_val, 11))
|
||||
pred = pred.reshape(pred.shape[0], 1)
|
||||
pred = target_scaler.inverse_transform(pred)
|
||||
|
||||
print_metrics(y_val, pred)
|
||||
|
||||
#%%
|
||||
|
||||
visualize_loss(history, "Training and Validation Loss")
|
||||
@@ -1,110 +0,0 @@
|
||||
#%%
|
||||
from math import sqrt
|
||||
from numpy import concatenate
|
||||
import numpy as np
|
||||
from matplotlib import pyplot
|
||||
from pandas import read_csv
|
||||
from pandas import DataFrame
|
||||
from pandas import concat
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from sklearn.metrics import mean_squared_error
|
||||
from keras.models import Sequential
|
||||
from keras.layers import Dense
|
||||
from keras.layers import LSTM
|
||||
#%%
|
||||
|
||||
# convert series to supervised learning
|
||||
def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
|
||||
n_vars = 1 if type(data) is list else data.shape[1]
|
||||
df = DataFrame(data)
|
||||
cols, names = list(), list()
|
||||
# input sequence (t-n, ... t-1)
|
||||
for i in range(n_in, 0, -1):
|
||||
cols.append(df.shift(i))
|
||||
names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
|
||||
# forecast sequence (t, t+1, ... t+n)
|
||||
for i in range(0, n_out):
|
||||
cols.append(df.shift(-i))
|
||||
if i == 0:
|
||||
names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
|
||||
else:
|
||||
names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
|
||||
# put it all together
|
||||
agg = concat(cols, axis=1)
|
||||
agg.columns = names
|
||||
# drop rows with NaN values
|
||||
if dropnan:
|
||||
agg.dropna(inplace=True)
|
||||
return agg
|
||||
|
||||
#%% load dataset
|
||||
from load_data import load_files
|
||||
data = load_files('data/', True)
|
||||
data.reset_index(drop=True, inplace=True)
|
||||
data = data[[column for column in data.columns if not column.endswith('volume')]]
|
||||
data = data[["BTC_returns", "BTC_vol_10"]]
|
||||
|
||||
values = data.values
|
||||
# ensure all data is float
|
||||
# values = values.astype('float32')
|
||||
|
||||
#%%
|
||||
np.isposinf(values).sum()
|
||||
|
||||
#%% normalize features
|
||||
scaler = MinMaxScaler(feature_range=(-1, 1))
|
||||
scaled = scaler.fit_transform(values)
|
||||
|
||||
|
||||
# specify the number of lag hours
|
||||
past = 10
|
||||
n_features = 8
|
||||
#%% frame as supervised learning
|
||||
reframed = series_to_supervised(scaled, past, 1)
|
||||
print(reframed.shape)
|
||||
|
||||
#%% split into train and test sets
|
||||
values = reframed.values
|
||||
n_train_hours = 365 * 24
|
||||
train = values[:n_train_hours, :]
|
||||
test = values[n_train_hours:, :]
|
||||
# split into input and outputs
|
||||
n_obs = past * n_features
|
||||
train_X, train_y = train[:, :n_obs], train[:, -n_features]
|
||||
test_X, test_y = test[:, :n_obs], test[:, -n_features]
|
||||
print(train_X.shape, len(train_X), train_y.shape)
|
||||
# reshape input to be 3D [samples, timesteps, features]
|
||||
train_X = train_X.reshape((train_X.shape[0], past, n_features))
|
||||
test_X = test_X.reshape((test_X.shape[0], past, n_features))
|
||||
print(train_X.shape, train_y.shape, test_X.shape, test_y.shape)
|
||||
|
||||
#%% design network
|
||||
model = Sequential()
|
||||
model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2])))
|
||||
model.add(Dense(1))
|
||||
model.compile(loss='mae', optimizer='adam')
|
||||
|
||||
#%% fit network
|
||||
history = model.fit(train_X, train_y, epochs=50, batch_size=72, validation_data=(test_X, test_y), verbose=2, shuffle=False)
|
||||
# plot history
|
||||
pyplot.plot(history.history['loss'], label='train')
|
||||
pyplot.plot(history.history['val_loss'], label='test')
|
||||
pyplot.legend()
|
||||
pyplot.show()
|
||||
|
||||
# make a prediction
|
||||
yhat = model.predict(test_X)
|
||||
test_X = test_X.reshape((test_X.shape[0], n_hours*n_features))
|
||||
# invert scaling for forecast
|
||||
inv_yhat = concatenate((yhat, test_X[:, -7:]), axis=1)
|
||||
inv_yhat = scaler.inverse_transform(inv_yhat)
|
||||
inv_yhat = inv_yhat[:,0]
|
||||
# invert scaling for actual
|
||||
test_y = test_y.reshape((len(test_y), 1))
|
||||
inv_y = concatenate((test_y, test_X[:, -7:]), axis=1)
|
||||
inv_y = scaler.inverse_transform(inv_y)
|
||||
inv_y = inv_y[:,0]
|
||||
# calculate RMSE
|
||||
rmse = sqrt(mean_squared_error(inv_y, inv_yhat))
|
||||
print('Test RMSE: %.3f' % rmse)
|
||||
@@ -0,0 +1,9 @@
|
||||
from math import sqrt
|
||||
from sklearn.metrics import mean_squared_error, mean_absolute_error
|
||||
|
||||
def print_metrics(y_true, y_pred):
|
||||
rmse = sqrt(mean_squared_error(y_true, y_pred))
|
||||
print("RMSE: %.2f" % rmse)
|
||||
|
||||
mae = mean_absolute_error(y_true, y_pred)
|
||||
print("MAE: %.2f" % mae)
|
||||
@@ -0,0 +1,6 @@
|
||||
import numpy as np
|
||||
|
||||
def rolling_window(a, window):
|
||||
shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
|
||||
strides = a.strides + (a.strides[-1],)
|
||||
return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)
|
||||
Reference in New Issue
Block a user