From 78b3c074207c651263143539e928cacf7d454835 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Thu, 11 Nov 2021 17:53:24 +0100 Subject: [PATCH] feat(Model): trying to get to a model that can train --- README.md | 6 +- exploration.ipynb | 257 ++++++++++++++++++++++++++++++++++++++++++--- load_data.py | 3 +- model_lstm.py | 85 ++++++++++----- utils/normalize.py | 7 +- 5 files changed, 309 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 9d5cbf5..3fe1661 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -# real-estate-forecasting \ No newline at end of file +# Price prediction models + +## Installation + +Make sure you use keras `2.6.0`, otherwise you won't be able to train models. \ No newline at end of file diff --git a/exploration.ipynb b/exploration.ipynb index a6498d3..6d48142 100644 --- a/exploration.ipynb +++ b/exploration.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -10,33 +10,33 @@ "output_type": "stream", "text": [ "\n", - "Index: 1501 entries, 2017-10-02 to 2021-11-10\n", + "Index: 1500 entries, 2017-10-03 to 2021-11-10\n", "Data columns (total 21 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 BTC_returns 1500 non-null float64\n", " 1 TRX_returns 1500 non-null float64\n", " 2 XRP_returns 1500 non-null float64\n", - " 3 IEF_volume 1006 non-null float64\n", - " 4 IEF_returns 1005 non-null float64\n", - " 5 QQQ_volume 1006 non-null float64\n", - " 6 QQQ_returns 1005 non-null float64\n", - " 7 FIL_returns 504 non-null float64\n", - " 8 GLD_volume 1006 non-null float64\n", - " 9 GLD_returns 1005 non-null float64\n", - " 10 TLT_volume 1006 non-null float64\n", - " 11 TLT_returns 1005 non-null float64\n", - " 12 SPY_volume 1006 non-null float64\n", - " 13 SPY_returns 1005 non-null float64\n", + " 3 IEF_volume 1500 non-null float64\n", + " 4 IEF_returns 1500 non-null float64\n", + " 5 QQQ_volume 1500 non-null float64\n", + " 6 QQQ_returns 1500 non-null float64\n", + " 7 FIL_returns 1500 non-null float64\n", + " 8 GLD_volume 1500 non-null float64\n", + " 9 GLD_returns 1500 non-null float64\n", + " 10 TLT_volume 1500 non-null float64\n", + " 11 TLT_returns 1500 non-null float64\n", + " 12 SPY_volume 1500 non-null float64\n", + " 13 SPY_returns 1500 non-null float64\n", " 14 ETH_returns 1500 non-null float64\n", - " 15 UNI_returns 420 non-null float64\n", - " 16 DOT_returns 447 non-null float64\n", + " 15 UNI_returns 1500 non-null float64\n", + " 16 DOT_returns 1500 non-null float64\n", " 17 ADA_returns 1500 non-null float64\n", " 18 BNB_returns 1500 non-null float64\n", " 19 LTC_returns 1500 non-null float64\n", - " 20 SOL_returns 580 non-null float64\n", + " 20 SOL_returns 1500 non-null float64\n", "dtypes: float64(21)\n", - "memory usage: 258.0+ KB\n" + "memory usage: 257.8+ KB\n" ] } ], @@ -51,6 +51,229 @@ "data.info()" ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BTC_returnsTRX_returnsXRP_returnsIEF_volumeIEF_returnsQQQ_volumeQQQ_returnsFIL_returnsGLD_volumeGLD_returns...TLT_returnsSPY_volumeSPY_returnsETH_returnsUNI_returnsDOT_returnsADA_returnsBNB_returnsLTC_returnsSOL_returns
2017-10-03-0.019799-0.1340430.0009850.00.00.00.00.00.00.0...0.00.00.0-0.0168460.00.0-0.194745-0.068931-0.0213920.0
2017-10-04-0.0221410.3722360.0501720.00.00.00.00.00.00.0...0.00.00.0-0.0004450.00.00.050384-0.117039-0.0178330.0
2017-10-050.024363-0.0740080.1147540.00.00.00.00.00.00.0...0.00.00.00.0113480.00.0-0.0169030.0320590.0082000.0
2017-10-060.011686-0.030293-0.0193280.00.00.00.00.00.00.0...0.00.00.00.0452220.00.0-0.140799-0.0704900.0085210.0
2017-10-070.014609-0.2023930.0269920.00.00.00.00.00.00.0...0.00.00.00.0095030.00.00.1249320.0064270.0094090.0
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " BTC_returns TRX_returns XRP_returns IEF_volume IEF_returns \\\n", + "2017-10-03 -0.019799 -0.134043 0.000985 0.0 0.0 \n", + "2017-10-04 -0.022141 0.372236 0.050172 0.0 0.0 \n", + "2017-10-05 0.024363 -0.074008 0.114754 0.0 0.0 \n", + "2017-10-06 0.011686 -0.030293 -0.019328 0.0 0.0 \n", + "2017-10-07 0.014609 -0.202393 0.026992 0.0 0.0 \n", + "\n", + " QQQ_volume QQQ_returns FIL_returns GLD_volume GLD_returns \\\n", + "2017-10-03 0.0 0.0 0.0 0.0 0.0 \n", + "2017-10-04 0.0 0.0 0.0 0.0 0.0 \n", + "2017-10-05 0.0 0.0 0.0 0.0 0.0 \n", + "2017-10-06 0.0 0.0 0.0 0.0 0.0 \n", + "2017-10-07 0.0 0.0 0.0 0.0 0.0 \n", + "\n", + " ... TLT_returns SPY_volume SPY_returns ETH_returns \\\n", + "2017-10-03 ... 0.0 0.0 0.0 -0.016846 \n", + "2017-10-04 ... 0.0 0.0 0.0 -0.000445 \n", + "2017-10-05 ... 0.0 0.0 0.0 0.011348 \n", + "2017-10-06 ... 0.0 0.0 0.0 0.045222 \n", + "2017-10-07 ... 0.0 0.0 0.0 0.009503 \n", + "\n", + " UNI_returns DOT_returns ADA_returns BNB_returns LTC_returns \\\n", + "2017-10-03 0.0 0.0 -0.194745 -0.068931 -0.021392 \n", + "2017-10-04 0.0 0.0 0.050384 -0.117039 -0.017833 \n", + "2017-10-05 0.0 0.0 -0.016903 0.032059 0.008200 \n", + "2017-10-06 0.0 0.0 -0.140799 -0.070490 0.008521 \n", + "2017-10-07 0.0 0.0 0.124932 0.006427 0.009409 \n", + "\n", + " SOL_returns \n", + "2017-10-03 0.0 \n", + "2017-10-04 0.0 \n", + "2017-10-05 0.0 \n", + "2017-10-06 0.0 \n", + "2017-10-07 0.0 \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.head(5)" + ] + }, { "cell_type": "code", "execution_count": 2, diff --git a/load_data.py b/load_data.py index 3c05baf..9308b97 100644 --- a/load_data.py +++ b/load_data.py @@ -6,7 +6,8 @@ import os def load_files(path, add_features): dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))] - return pd.concat(dfs, axis=1) + dfs = pd.concat(dfs, axis=1).fillna(0.) + return dfs.drop(index=dfs.index[0], axis=0) def __load_df(path, prefix, add_features): df = pd.read_csv(path, header=0, index_col=0).fillna(0) diff --git a/model_lstm.py b/model_lstm.py index 82929d8..58389f2 100644 --- a/model_lstm.py +++ b/model_lstm.py @@ -1,74 +1,105 @@ #%% Import all the stuff, load data, define constants from load_data import load_files import pandas as pd -import keras +from tensorflow import keras from utils.normalize import normalize +import tensorflow as tf data = load_files('data/', False) data.reset_index(drop=True, inplace=True) +data = data[[column for column in data.columns if not column.endswith('volume')]] ticker_to_predict = 'ETH_returns' learning_rate = 0.001 -batch_size = 256 -epochs = 10 +batch_size = 128 +epochs = 30 split_fraction = 0.715 train_split = int(split_fraction * int(data.shape[0])) -past = 720 -future = 72 +past = 10 +future = 1 start = past + future end = start + train_split -#%% +#%% split data into training - validation sets +train_data = data.loc[0 : train_split - 1] +val_data = data.loc[train_split:] -x_train = data.loc[0 : train_split - 1].drop(ticker_to_predict, axis=1).values -y_train = data.iloc[start:end][ticker_to_predict] +#%% create features and target for training set & keras dataset -#%% +x_train = normalize(train_data).values +# x_train = train_data.drop(ticker_to_predict, axis=1).values +y_train = normalize(data).iloc[start:end][ticker_to_predict].values dataset_train = keras.preprocessing.timeseries_dataset_from_array( x_train, y_train, sequence_length=past, - sampling_rate=1, - batch_size=32, + batch_size=batch_size, ) -for batch in dataset_train.take(1): - inputs, targets = batch +#%% create features and target for validation set & keras dataset +x_end = len(val_data) - past - future +label_start = train_split + past + future + +x_val = normalize(val_data).iloc[:x_end].values +# x_val = val_data.iloc[:x_end].drop(ticker_to_predict, axis=1).values +y_val = normalize(data).iloc[label_start:][ticker_to_predict].values + +dataset_val = keras.utils.timeseries_dataset_from_array( + x_val, + y_val, + sequence_length=past, + batch_size=batch_size, +) + + +#%% + +for batch in dataset_train.take(1): + batch_inputs, batch_targets = batch + +print("Input shape:", batch_inputs.shape) +print("Target shape:", batch_targets.shape) -print("Input shape:", inputs.numpy().shape) -print("Target shape:", targets.numpy().shape) -inputs # %% -inputs = keras.layers.Input(shape=(inputs.shape[1], inputs.shape[2])) +# model = keras.Sequential() +# model.add(keras.layers.LSTM(units = 50, return_sequences = True, input_shape=(batch_inputs.shape[1], batch_inputs.shape[2]))) +# model.add(keras.layers.Dropout(0.2)) +# model.add(keras.layers.Dense(units = 1)) + +inputs = keras.layers.Input(shape=(batch_inputs.shape[1], batch_inputs.shape[2])) lstm_out = keras.layers.LSTM(32)(inputs) outputs = keras.layers.Dense(1)(lstm_out) - model = keras.Model(inputs=inputs, outputs=outputs) -model.compile(optimizer=keras.optimizers.Adam(learning_rate=learning_rate), loss="mse") + +optimizer = keras.optimizers.Adam(learning_rate=learning_rate, clipnorm = 1.) +model.compile(optimizer=optimizer, loss="mean_squared_error") model.summary() # %% path_checkpoint = "model_checkpoint.h5" -es_callback = keras.callbacks.EarlyStopping(monitor="val_loss", min_delta=0, patience=5) +# es_callback = keras.callbacks.EarlyStopping(monitor="val_loss", min_delta=0, patience=5) -modelckpt_callback = keras.callbacks.ModelCheckpoint( - monitor="val_loss", - filepath=path_checkpoint, - verbose=1, - save_weights_only=True, - save_best_only=True, +# modelckpt_callback = keras.callbacks.ModelCheckpoint( +# monitor="val_loss", +# filepath=path_checkpoint, +# verbose=1, +# save_weights_only=True, +# save_best_only=True, +# ) +tf.debugging.enable_check_numerics( + stack_height_limit=30, path_length_limit=50 ) history = model.fit( dataset_train, epochs=epochs, validation_data=dataset_val, - callbacks=[es_callback, modelckpt_callback], + # callbacks=[modelckpt_callback], ) # %% diff --git a/utils/normalize.py b/utils/normalize.py index a8dcf94..8bd4661 100644 --- a/utils/normalize.py +++ b/utils/normalize.py @@ -1,5 +1,6 @@ +import pandas as pd -def normalize(data, train_split): - data_mean = data[:train_split].mean(axis=0) - data_std = data[:train_split].std(axis=0) +def normalize(data: pd.DataFrame) -> pd.DataFrame: + data_mean = data.mean(axis=0) + data_std = data.std(axis=0) return (data - data_mean) / data_std