feat(Model): continued with the keras model scaffolding

fetch_data.py

@@ -7,8 +7,6 @@ crypto_tickers = ["BTC", "ETH", "BNB", "ADA", "SOL", "XRP", "DOT", "LTC", "UNI",
 etf_tickers = ["GLD", "IEF", "TLT", "SPY", "QQQ"]
 tickers = crypto_tickers + etf_tickers
 
-add_features = True
-
 #%%
 for ticker in tickers:
     print("Fetching ", ticker)
@@ -16,20 +14,6 @@ for ticker in tickers:
         df = get_crypto_price_crypto_compare(ticker, "USD", 1500)
     else:
         df = get_stock_price_av(ticker, "2017-11-10")
-    df['returns'] = df['close'].pct_change()
-    
-    if add_features:
-        # volatility (10, 20, 30 days)
-        df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5)
-        df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5)
-        df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5)
-
-        # momentum (10, 20, 30, 60, 90 days)
-        df['mom_10'] = df['close'].pct_change(10)
-        df['mom_20'] = df['close'].pct_change(20)
-        df['mom_30'] = df['close'].pct_change(30)
-        df['mom_60'] = df['close'].pct_change(60)
-        df['mom_90'] = df['close'].pct_change(90)
 
     df.to_csv(f"data/{ticker}.csv", index=True)
 

load_data.py

@@ -2,18 +2,29 @@
 import pandas as pd
 import os
 
-
 #%%
 
-def load_files(path):
-    dfs = [__load_df(os.path.join(path,f), f.split('.')[0]) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
+def load_files(path, add_features):
+    dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
     return pd.concat(dfs, axis=1)
 
-def __load_df(path, prefix):
+def __load_df(path, prefix, add_features):
     df = pd.read_csv(path, header=0, index_col=0).fillna(0)
-    df = df.drop(columns=['open', 'high', 'low'])
+    df['returns'] = df['close'].pct_change()
+    
+    if add_features:
+        # volatility (10, 20, 30 days)
+        df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5)
+        df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5)
+        df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5)
+
+        # momentum (10, 20, 30, 60, 90 days)
+        df['mom_10'] = df['close'].pct_change(10)
+        df['mom_20'] = df['close'].pct_change(20)
+        df['mom_30'] = df['close'].pct_change(30)
+        df['mom_60'] = df['close'].pct_change(60)
+        df['mom_90'] = df['close'].pct_change(90)
+
+    df = df.drop(columns=['open', 'high', 'low', 'close'])
     df.columns = [prefix + "_" + c for c in df.columns]
     return df
-
-# %%
-load_files("data/")

model_lstm.py

@@ -1,5 +1,72 @@
+#%% Import all the stuff, load data, define constants
 from load_data import load_files
 import pandas as pd
+import keras
+from utils.normalize import normalize
 
-data = load_files()
+data = load_files('data/', False)
 
+ticker_to_predict = 'ETH_returns'
+
+learning_rate = 0.001
+batch_size = 256
+epochs = 10
+
+split_fraction = 0.715
+train_split = int(split_fraction * int(data.shape[0]))
+
+past = 720
+future = 72
+
+start = past + future
+end = start + train_split
+
+#%%
+
+x_train = data.loc[0 : train_split - 1].drop(ticker_to_predict, axis=1).values
+y_train = data.iloc[start:end][ticker_to_predict]
+
+
+dataset_train = keras.preprocessing.timeseries_dataset_from_array(
+    x_train,
+    y_train,
+    sequence_length=past,
+    sampling_rate=1,
+    batch_size=32,
+)
+
+for batch in dataset_train.take(1):
+    inputs, targets = batch
+
+print("Input shape:", inputs.numpy().shape)
+print("Target shape:", targets.numpy().shape)
+inputs
+
+# %%
+inputs = keras.layers.Input(shape=(inputs.shape[1], inputs.shape[2]))
+lstm_out = keras.layers.LSTM(32)(inputs)
+outputs = keras.layers.Dense(1)(lstm_out)
+
+model = keras.Model(inputs=inputs, outputs=outputs)
+model.compile(optimizer=keras.optimizers.Adam(learning_rate=learning_rate), loss="mse")
+model.summary()
+
+# %%
+path_checkpoint = "model_checkpoint.h5"
+es_callback = keras.callbacks.EarlyStopping(monitor="val_loss", min_delta=0, patience=5)
+
+modelckpt_callback = keras.callbacks.ModelCheckpoint(
+    monitor="val_loss",
+    filepath=path_checkpoint,
+    verbose=1,
+    save_weights_only=True,
+    save_best_only=True,
+)
+
+history = model.fit(
+    dataset_train,
+    epochs=epochs,
+    validation_data=dataset_val,
+    callbacks=[es_callback, modelckpt_callback],
+)
+# %%

utils/get_prices.py

@@ -38,11 +38,11 @@ def get_stock_price_av(symbol: str, start_date: str = None) -> pd.DataFrame:
     api_url = f'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&outputsize=full&apikey={AV_API_KEY}'
     raw_df = requests.get(api_url).json()
     df = pd.DataFrame(raw_df['Time Series (Daily)']).T
-    df = df.rename(columns = {'1. open': 'open', '2. high': 'high', '3. low': 'low', '4. close': 'close', '5. adjusted close': 'adj_close', '6. volume': 'volume'})
+    df = df.rename(columns = {'1. open': 'open', '2. high': 'high', '3. low': 'low', '5. adjusted close': 'close', '6. volume': 'volume'})
     for i in df.columns:
         df[i] = df[i].astype(float)
     df.index = pd.to_datetime(df.index)
-    df = df.iloc[::-1].drop(['7. dividend amount', '8. split coefficient'], axis = 1)
+    df = df.iloc[::-1].drop(['4. close', '7. dividend amount', '8. split coefficient'], axis = 1)
     if start_date:
         df = df[df.index >= start_date]
     df.sort_index(inplace = True, ascending= True)

utils/normalize.py

@@ -0,0 +1,5 @@
+
+def normalize(data, train_split):
+    data_mean = data[:train_split].mean(axis=0)
+    data_std = data[:train_split].std(axis=0)
+    return (data - data_mean) / data_std