gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-22 15:28:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Model): trying to get to a model that can train

Browse Source
This commit is contained in:
Mark Aron Szulyovszky
2021-11-11 17:53:24 +01:00
parent 0e5fe227f7
commit 78b3c07420
5 changed files with 308 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5 -1
View File
@@ -1 +1,5 @@
# real-estate-forecasting # Price prediction models
## Installation
Make sure you use keras `2.6.0`, otherwise you won't be able to train models.
exploration.ipynb
+240 -17
View File
@@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@@ -10,33 +10,33 @@
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"<class 'pandas.core.frame.DataFrame'>\n", "<class 'pandas.core.frame.DataFrame'>\n",
"Index: 1501 entries, 2017-10-02 to 2021-11-10\n", "Index: 1500 entries, 2017-10-03 to 2021-11-10\n",
"Data columns (total 21 columns):\n", "Data columns (total 21 columns):\n",
" # Column Non-Null Count Dtype \n", " # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n", "--- ------ -------------- ----- \n",
" 0 BTC_returns 1500 non-null float64\n", " 0 BTC_returns 1500 non-null float64\n",
" 1 TRX_returns 1500 non-null float64\n", " 1 TRX_returns 1500 non-null float64\n",
" 2 XRP_returns 1500 non-null float64\n", " 2 XRP_returns 1500 non-null float64\n",
" 3 IEF_volume 1006 non-null float64\n", " 3 IEF_volume 1500 non-null float64\n",
" 4 IEF_returns 1005 non-null float64\n", " 4 IEF_returns 1500 non-null float64\n",
" 5 QQQ_volume 1006 non-null float64\n", " 5 QQQ_volume 1500 non-null float64\n",
" 6 QQQ_returns 1005 non-null float64\n", " 6 QQQ_returns 1500 non-null float64\n",
" 7 FIL_returns 504 non-null float64\n", " 7 FIL_returns 1500 non-null float64\n",
" 8 GLD_volume 1006 non-null float64\n", " 8 GLD_volume 1500 non-null float64\n",
" 9 GLD_returns 1005 non-null float64\n", " 9 GLD_returns 1500 non-null float64\n",
" 10 TLT_volume 1006 non-null float64\n", " 10 TLT_volume 1500 non-null float64\n",
" 11 TLT_returns 1005 non-null float64\n", " 11 TLT_returns 1500 non-null float64\n",
" 12 SPY_volume 1006 non-null float64\n", " 12 SPY_volume 1500 non-null float64\n",
" 13 SPY_returns 1005 non-null float64\n", " 13 SPY_returns 1500 non-null float64\n",
" 14 ETH_returns 1500 non-null float64\n", " 14 ETH_returns 1500 non-null float64\n",
" 15 UNI_returns 420 non-null float64\n", " 15 UNI_returns 1500 non-null float64\n",
" 16 DOT_returns 447 non-null float64\n", " 16 DOT_returns 1500 non-null float64\n",
" 17 ADA_returns 1500 non-null float64\n", " 17 ADA_returns 1500 non-null float64\n",
" 18 BNB_returns 1500 non-null float64\n", " 18 BNB_returns 1500 non-null float64\n",
" 19 LTC_returns 1500 non-null float64\n", " 19 LTC_returns 1500 non-null float64\n",
" 20 SOL_returns 580 non-null float64\n", " 20 SOL_returns 1500 non-null float64\n",
"dtypes: float64(21)\n", "dtypes: float64(21)\n",
"memory usage: 258.0+ KB\n" "memory usage: 257.8+ KB\n"
] ]
} }
], ],
@@ -51,6 +51,229 @@
"data.info()" "data.info()"
] ]
}, },
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>BTC_returns</th>\n",
" <th>TRX_returns</th>\n",
" <th>XRP_returns</th>\n",
" <th>IEF_volume</th>\n",
" <th>IEF_returns</th>\n",
" <th>QQQ_volume</th>\n",
" <th>QQQ_returns</th>\n",
" <th>FIL_returns</th>\n",
" <th>GLD_volume</th>\n",
" <th>GLD_returns</th>\n",
" <th>...</th>\n",
" <th>TLT_returns</th>\n",
" <th>SPY_volume</th>\n",
" <th>SPY_returns</th>\n",
" <th>ETH_returns</th>\n",
" <th>UNI_returns</th>\n",
" <th>DOT_returns</th>\n",
" <th>ADA_returns</th>\n",
" <th>BNB_returns</th>\n",
" <th>LTC_returns</th>\n",
" <th>SOL_returns</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2017-10-03</th>\n",
" <td>-0.019799</td>\n",
" <td>-0.134043</td>\n",
" <td>0.000985</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>-0.016846</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>-0.194745</td>\n",
" <td>-0.068931</td>\n",
" <td>-0.021392</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2017-10-04</th>\n",
" <td>-0.022141</td>\n",
" <td>0.372236</td>\n",
" <td>0.050172</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>-0.000445</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.050384</td>\n",
" <td>-0.117039</td>\n",
" <td>-0.017833</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2017-10-05</th>\n",
" <td>0.024363</td>\n",
" <td>-0.074008</td>\n",
" <td>0.114754</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.011348</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>-0.016903</td>\n",
" <td>0.032059</td>\n",
" <td>0.008200</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2017-10-06</th>\n",
" <td>0.011686</td>\n",
" <td>-0.030293</td>\n",
" <td>-0.019328</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.045222</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>-0.140799</td>\n",
" <td>-0.070490</td>\n",
" <td>0.008521</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2017-10-07</th>\n",
" <td>0.014609</td>\n",
" <td>-0.202393</td>\n",
" <td>0.026992</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.009503</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.124932</td>\n",
" <td>0.006427</td>\n",
" <td>0.009409</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 21 columns</p>\n",
"</div>"
],
"text/plain": [
" BTC_returns TRX_returns XRP_returns IEF_volume IEF_returns \\\n",
"2017-10-03 -0.019799 -0.134043 0.000985 0.0 0.0 \n",
"2017-10-04 -0.022141 0.372236 0.050172 0.0 0.0 \n",
"2017-10-05 0.024363 -0.074008 0.114754 0.0 0.0 \n",
"2017-10-06 0.011686 -0.030293 -0.019328 0.0 0.0 \n",
"2017-10-07 0.014609 -0.202393 0.026992 0.0 0.0 \n",
"\n",
" QQQ_volume QQQ_returns FIL_returns GLD_volume GLD_returns \\\n",
"2017-10-03 0.0 0.0 0.0 0.0 0.0 \n",
"2017-10-04 0.0 0.0 0.0 0.0 0.0 \n",
"2017-10-05 0.0 0.0 0.0 0.0 0.0 \n",
"2017-10-06 0.0 0.0 0.0 0.0 0.0 \n",
"2017-10-07 0.0 0.0 0.0 0.0 0.0 \n",
"\n",
" ... TLT_returns SPY_volume SPY_returns ETH_returns \\\n",
"2017-10-03 ... 0.0 0.0 0.0 -0.016846 \n",
"2017-10-04 ... 0.0 0.0 0.0 -0.000445 \n",
"2017-10-05 ... 0.0 0.0 0.0 0.011348 \n",
"2017-10-06 ... 0.0 0.0 0.0 0.045222 \n",
"2017-10-07 ... 0.0 0.0 0.0 0.009503 \n",
"\n",
" UNI_returns DOT_returns ADA_returns BNB_returns LTC_returns \\\n",
"2017-10-03 0.0 0.0 -0.194745 -0.068931 -0.021392 \n",
"2017-10-04 0.0 0.0 0.050384 -0.117039 -0.017833 \n",
"2017-10-05 0.0 0.0 -0.016903 0.032059 0.008200 \n",
"2017-10-06 0.0 0.0 -0.140799 -0.070490 0.008521 \n",
"2017-10-07 0.0 0.0 0.124932 0.006427 0.009409 \n",
"\n",
" SOL_returns \n",
"2017-10-03 0.0 \n",
"2017-10-04 0.0 \n",
"2017-10-05 0.0 \n",
"2017-10-06 0.0 \n",
"2017-10-07 0.0 \n",
"\n",
"[5 rows x 21 columns]"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.head(5)"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 2,
load_data.py
+2 -1
View File
@@ -6,7 +6,8 @@ import os
def load_files(path, add_features): def load_files(path, add_features):
dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))] dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
return pd.concat(dfs, axis=1) dfs = pd.concat(dfs, axis=1).fillna(0.)
return dfs.drop(index=dfs.index[0], axis=0)
def __load_df(path, prefix, add_features): def __load_df(path, prefix, add_features):
df = pd.read_csv(path, header=0, index_col=0).fillna(0) df = pd.read_csv(path, header=0, index_col=0).fillna(0)
model_lstm.py
+57 -26
View File
@@ -1,74 +1,105 @@
#%% Import all the stuff, load data, define constants #%% Import all the stuff, load data, define constants
from load_data import load_files from load_data import load_files
import pandas as pd import pandas as pd
import keras from tensorflow import keras
from utils.normalize import normalize from utils.normalize import normalize
import tensorflow as tf
data = load_files('data/', False) data = load_files('data/', False)
data.reset_index(drop=True, inplace=True) data.reset_index(drop=True, inplace=True)
data = data[[column for column in data.columns if not column.endswith('volume')]]
ticker_to_predict = 'ETH_returns' ticker_to_predict = 'ETH_returns'
learning_rate = 0.001 learning_rate = 0.001
batch_size = 256 batch_size = 128
epochs = 10 epochs = 30
split_fraction = 0.715 split_fraction = 0.715
train_split = int(split_fraction * int(data.shape[0])) train_split = int(split_fraction * int(data.shape[0]))
past = 720 past = 10
future = 72 future = 1
start = past + future start = past + future
end = start + train_split end = start + train_split
#%% #%% split data into training - validation sets
train_data = data.loc[0 : train_split - 1]
val_data = data.loc[train_split:]
x_train = data.loc[0 : train_split - 1].drop(ticker_to_predict, axis=1).values #%% create features and target for training set & keras dataset
y_train = data.iloc[start:end][ticker_to_predict]
#%% x_train = normalize(train_data).values
# x_train = train_data.drop(ticker_to_predict, axis=1).values
y_train = normalize(data).iloc[start:end][ticker_to_predict].values
dataset_train = keras.preprocessing.timeseries_dataset_from_array( dataset_train = keras.preprocessing.timeseries_dataset_from_array(
x_train, x_train,
y_train, y_train,
sequence_length=past, sequence_length=past,
sampling_rate=1, batch_size=batch_size,
batch_size=32,
) )
#%% create features and target for validation set & keras dataset
x_end = len(val_data) - past - future
label_start = train_split + past + future
x_val = normalize(val_data).iloc[:x_end].values
# x_val = val_data.iloc[:x_end].drop(ticker_to_predict, axis=1).values
y_val = normalize(data).iloc[label_start:][ticker_to_predict].values
dataset_val = keras.utils.timeseries_dataset_from_array(
x_val,
y_val,
sequence_length=past,
batch_size=batch_size,
)
#%%
for batch in dataset_train.take(1): for batch in dataset_train.take(1):
inputs, targets = batch batch_inputs, batch_targets = batch
print("Input shape:", batch_inputs.shape)
print("Target shape:", batch_targets.shape)
print("Input shape:", inputs.numpy().shape)
print("Target shape:", targets.numpy().shape)
inputs
# %% # %%
inputs = keras.layers.Input(shape=(inputs.shape[1], inputs.shape[2])) # model = keras.Sequential()
# model.add(keras.layers.LSTM(units = 50, return_sequences = True, input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
# model.add(keras.layers.Dropout(0.2))
# model.add(keras.layers.Dense(units = 1))
inputs = keras.layers.Input(shape=(batch_inputs.shape[1], batch_inputs.shape[2]))
lstm_out = keras.layers.LSTM(32)(inputs) lstm_out = keras.layers.LSTM(32)(inputs)
outputs = keras.layers.Dense(1)(lstm_out) outputs = keras.layers.Dense(1)(lstm_out)
model = keras.Model(inputs=inputs, outputs=outputs) model = keras.Model(inputs=inputs, outputs=outputs)
model.compile(optimizer=keras.optimizers.Adam(learning_rate=learning_rate), loss="mse")
optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm = 1.)
model.compile(optimizer=optimizer, loss="mean_squared_error")
model.summary() model.summary()
# %% # %%
path_checkpoint = "model_checkpoint.h5" path_checkpoint = "model_checkpoint.h5"
es_callback = keras.callbacks.EarlyStopping(monitor="val_loss", min_delta=0, patience=5) # es_callback = keras.callbacks.EarlyStopping(monitor="val_loss", min_delta=0, patience=5)
modelckpt_callback = keras.callbacks.ModelCheckpoint( # modelckpt_callback = keras.callbacks.ModelCheckpoint(
monitor="val_loss", # monitor="val_loss",
filepath=path_checkpoint, # filepath=path_checkpoint,
verbose=1, # verbose=1,
save_weights_only=True, # save_weights_only=True,
save_best_only=True, # save_best_only=True,
# )
tf.debugging.enable_check_numerics(
stack_height_limit=30, path_length_limit=50
) )
history = model.fit( history = model.fit(
dataset_train, dataset_train,
epochs=epochs, epochs=epochs,
validation_data=dataset_val, validation_data=dataset_val,
callbacks=[es_callback, modelckpt_callback], # callbacks=[modelckpt_callback],
) )
# %% # %%
utils/normalize.py
+4 -3
View File
@@ -1,5 +1,6 @@
import pandas as pd
def normalize(data, train_split): def normalize(data: pd.DataFrame) -> pd.DataFrame:
data_mean = data[:train_split].mean(axis=0) data_mean = data.mean(axis=0)
data_std = data[:train_split].std(axis=0) data_std = data.std(axis=0)
return (data - data_mean) / data_std return (data - data_mean) / data_std