From 9d342d6039465c659c2742ba54055f6118b8c6ae Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Sun, 14 Nov 2021 11:59:59 +0100 Subject: [PATCH] feat(Data): added day_month, month, day_week columns as features --- load_data.py | 42 ++++++++++++++++++++++++++++++++---------- model_ff.py | 2 +- model_ff_10days.py | 2 +- model_lstm.py | 2 +- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/load_data.py b/load_data.py index 9308b97..420d73c 100644 --- a/load_data.py +++ b/load_data.py @@ -1,18 +1,31 @@ #%% import pandas as pd import os +import numpy as np #%% -def load_files(path, add_features): - dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))] +def load_files(path: str, add_features: bool, log_returns: bool) -> pd.DataFrame: + dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features, log_returns) for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))] dfs = pd.concat(dfs, axis=1).fillna(0.) + + dfs.index = pd.DatetimeIndex(dfs.index) + + if add_features: + dfs['day_month'] = dfs.index.day + dfs['day_week'] = dfs.index.dayofweek + dfs['month'] = dfs.index.month + return dfs.drop(index=dfs.index[0], axis=0) -def __load_df(path, prefix, add_features): +def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool) -> pd.DataFrame: df = pd.read_csv(path, header=0, index_col=0).fillna(0) - df['returns'] = df['close'].pct_change() - + + if log_returns: + df['returns'] = np.log(df['close']).diff(1) + else: + df['returns'] = df['close'].pct_change() + if add_features: # volatility (10, 20, 30 days) df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5) @@ -20,12 +33,21 @@ def __load_df(path, prefix, add_features): df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5) # momentum (10, 20, 30, 60, 90 days) - df['mom_10'] = df['close'].pct_change(10) - df['mom_20'] = df['close'].pct_change(20) - df['mom_30'] = df['close'].pct_change(30) - df['mom_60'] = df['close'].pct_change(60) - df['mom_90'] = df['close'].pct_change(90) + if log_returns: + df['mom_10'] = np.log(df['close']).diff(10) + df['mom_20'] = np.log(df['close']).diff(20) + df['mom_30'] = np.log(df['close']).diff(30) + df['mom_60'] = np.log(df['close']).diff(60) + df['mom_90'] = np.log(df['close']).diff(90) + else: + df['mom_10'] = df['close'].pct_change(10) + df['mom_20'] = df['close'].pct_change(20) + df['mom_30'] = df['close'].pct_change(30) + df['mom_60'] = df['close'].pct_change(60) + df['mom_90'] = df['close'].pct_change(90) df = df.drop(columns=['open', 'high', 'low', 'close']) df.columns = [prefix + "_" + c for c in df.columns] return df + +# %% diff --git a/model_ff.py b/model_ff.py index d6794b3..ee34cc4 100644 --- a/model_ff.py +++ b/model_ff.py @@ -6,7 +6,7 @@ from utils.normalize import normalize import tensorflow as tf from utils.visualize import visualize_loss -data = load_files('data/', False) +data = load_files('data/', add_features=False, log_returns=False) data.reset_index(drop=True, inplace=True) data = data[[column for column in data.columns if not column.endswith('volume')]] # data = data[["ETH_returns", "BTC_returns"]] diff --git a/model_ff_10days.py b/model_ff_10days.py index bc97cac..cf6349e 100644 --- a/model_ff_10days.py +++ b/model_ff_10days.py @@ -6,7 +6,7 @@ from utils.normalize import normalize import tensorflow as tf from utils.visualize import visualize_loss -data = load_files('data/', True) +data = load_files('data/', add_features=True, log_returns=True) data.reset_index(drop=True, inplace=True) data = data[[column for column in data.columns if not column.endswith('volume')]] data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_vol_10", "BTC_mom_20", "BTC_vol_20"]] diff --git a/model_lstm.py b/model_lstm.py index d88a414..2ba9948 100644 --- a/model_lstm.py +++ b/model_lstm.py @@ -7,7 +7,7 @@ import tensorflow as tf from utils.visualize import visualize_loss from sklearn.preprocessing import StandardScaler -data = load_files('data/', True) +data = load_files('data/', add_features=True, log_returns=True) data.reset_index(drop=True, inplace=True) data = data[[column for column in data.columns if not column.endswith('volume')]] data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_vol_10", "BTC_mom_20", "BTC_vol_20"]]