mirror of
https://github.com/webclinic017/drift.git
synced 2026-08-21 14:58:11 +00:00
feat(Data): create DataSource, DataCollection, added hourly crypto data (#96)
* feat(Data): create DataSource, DataCollection, added hourly crypto data * fix(Data): hourly data format, loading & config
This commit is contained in:
Mark Aron Szulyovszky
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,7 @@ from pytorch_forecasting.metrics import QuantileLoss
|
||||
import sys
|
||||
sys.path.insert(0, '..')
|
||||
|
||||
from utils.load_data import load_files
|
||||
from data_loader.load_data import load_files
|
||||
|
||||
print("success")
|
||||
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
#%%
|
||||
import pandas as pd
|
||||
|
||||
oecd_housing = pd.read_csv('data/oecd_housing_prices.csv')
|
||||
# we're only interested in "real" prices
|
||||
oecd_housing = oecd_housing[oecd_housing['SUBJECT'] == 'REAL']
|
||||
# we're only interested annual data
|
||||
oecd_housing = oecd_housing[oecd_housing['FREQUENCY'] == 'A']
|
||||
oecd_housing
|
||||
# %%
|
||||
countries = oecd_housing['LOCATION'].unique()
|
||||
countries
|
||||
|
||||
# %%
|
||||
oecd_housing[oecd_housing['LOCATION'] == 'HUN'].plot(x = "TIME", y= "Value")
|
||||
# %%
|
||||
|
||||
# %%
|
||||
@@ -8,7 +8,7 @@ from pytorch_forecasting.metrics import QuantileLoss
|
||||
import sys
|
||||
sys.path.insert(0, '..')
|
||||
|
||||
from utils.load_data import load_files
|
||||
from data_loader.load_data import load_files
|
||||
|
||||
print("success")
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ warnings.filterwarnings("ignore")
|
||||
import sys
|
||||
sys.path.insert(0, '..')
|
||||
|
||||
from utils.load_data import load_files
|
||||
from data_loader.load_data import load_files
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#%% Import all the stuff, load data, define constants
|
||||
from sklearn.utils import shuffle
|
||||
from utils.load_data import load_files, create_target_classes
|
||||
from data_loader.load_data import load_files, create_target_classes
|
||||
import pandas as pd
|
||||
from tensorflow import keras
|
||||
from utils.normalize import normalize
|
||||
@@ -16,7 +16,7 @@ from keras_models.classification_transformer import create_basic_transformer_mod
|
||||
data = load_files(path='data/',
|
||||
own_asset='BTC_ETH',
|
||||
own_asset_lags=[1,2,3,4,5,6,8,10,15],
|
||||
load_other_assets=True,
|
||||
load_non_target_asset=True,
|
||||
other_asset_lags=[1,2,3,4],
|
||||
log_returns=False,
|
||||
add_date_features=True,
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
#%% Import all the stuff, load data, define constants
|
||||
from sklearnex import patch_sklearn
|
||||
patch_sklearn()
|
||||
|
||||
from utils.load_data import create_target_cum_forward_returns, create_target_classes, load_files
|
||||
from sktime.forecasting.model_selection import temporal_train_test_split
|
||||
from sklearn.metrics import accuracy_score
|
||||
from sklearn.metrics import confusion_matrix
|
||||
# from utils.evaluate import print_classification_metrics, format_data_for_backtest
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.neighbors import KNeighborsClassifier
|
||||
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
|
||||
from sklearn.naive_bayes import GaussianNB
|
||||
from sklearn.svm import SVC
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
|
||||
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
|
||||
from utils.sliding_window import sliding_window_and_flatten
|
||||
|
||||
ticket_to_predict = 'BTC_ETH'
|
||||
print('Predicting: ', ticket_to_predict)
|
||||
|
||||
data = load_files(path='data/',
|
||||
own_asset=ticket_to_predict,
|
||||
own_asset_lags=[1,2,3,4,5,6,8,10,15],
|
||||
load_other_assets=False,
|
||||
other_asset_lags=[1,2,3,4],
|
||||
log_returns=True,
|
||||
add_date_features=True,
|
||||
own_technical_features='level2',
|
||||
other_technical_features='none',
|
||||
exogenous_features='none',
|
||||
index_column='int'
|
||||
)
|
||||
|
||||
target_col = 'target'
|
||||
returns_col = ticket_to_predict + '_returns'
|
||||
data = create_target_classes(data, returns_col, 1, 'three')
|
||||
|
||||
X = data.drop(columns=['target'])
|
||||
y = data[target_col]
|
||||
|
||||
X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2)
|
||||
feature_scaler = MinMaxScaler(feature_range= (-1, 1))
|
||||
X_test_orig = X_test.copy()
|
||||
X_train = feature_scaler.fit_transform(X_train)
|
||||
X_test = feature_scaler.transform(X_test)
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
sliding_window_size = 10
|
||||
X_train = sliding_window_and_flatten(X_train, sliding_window_size)
|
||||
X_test = sliding_window_and_flatten(X_test, sliding_window_size)
|
||||
X_test_orig = X_test_orig.iloc[sliding_window_size-1:]
|
||||
y_train = y_train[sliding_window_size-1:]
|
||||
y_test = y_test[sliding_window_size-1:]
|
||||
|
||||
assert X_train.shape[0] == y_train.shape[0]
|
||||
assert X_test.shape[0] == y_test.shape[0]
|
||||
scoring = 'accuracy'
|
||||
|
||||
# %%
|
||||
num_folds = 10
|
||||
|
||||
models = []
|
||||
models.append(('LR', LogisticRegression(n_jobs=-1)))
|
||||
models.append(('LDA', LinearDiscriminantAnalysis()))
|
||||
models.append(('KNN', KNeighborsClassifier()))
|
||||
models.append(('CART', DecisionTreeClassifier()))
|
||||
models.append(('NB', GaussianNB()))
|
||||
# models.append(('NN', MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False)))
|
||||
models.append(('AB', AdaBoostClassifier()))
|
||||
# models.append(('GBM', GradientBoostingClassifier()))
|
||||
models.append(('RF', RandomForestClassifier(n_jobs=-1)))
|
||||
|
||||
results = []
|
||||
names = []
|
||||
for name, model in models:
|
||||
kfold = KFold(n_splits=num_folds, shuffle=False)
|
||||
cv_results = cross_val_score(model, X_train, y_train, cv=kfold, scoring=scoring)
|
||||
results.append(cv_results)
|
||||
names.append(name)
|
||||
msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
|
||||
print(msg)
|
||||
|
||||
# # compare algorithms
|
||||
# fig = plt.figure()
|
||||
# fig.suptitle('Algorithm Comparison')
|
||||
# ax = fig.add_subplot(111)
|
||||
# plt.boxplot(results)
|
||||
# ax.set_xticklabels(names)
|
||||
# fig.set_size_inches(15,8)
|
||||
# plt.show()
|
||||
|
||||
#%%
|
||||
# n_estimators = [20,80]
|
||||
# max_depth= [5,10, 15]
|
||||
# criterion = ["gini","entropy"]
|
||||
# param_grid = dict(n_estimators=n_estimators, max_depth=max_depth, criterion = criterion )
|
||||
# model = RandomForestClassifier(n_jobs=-1)
|
||||
# kfold = KFold(n_splits=10, shuffle=False)
|
||||
# grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scoring, cv=kfold)
|
||||
# grid_result = grid.fit(X_train, y_train)
|
||||
|
||||
# #Print Results
|
||||
# print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
|
||||
# means = grid_result.cv_results_['mean_test_score']
|
||||
# stds = grid_result.cv_results_['std_test_score']
|
||||
# params = grid_result.cv_results_['params']
|
||||
# ranks = grid_result.cv_results_['rank_test_score']
|
||||
# for mean, stdev, param, rank in zip(means, stds, params, ranks):
|
||||
# print("#%d %f (%f) with: %r" % (rank, mean, stdev, param))
|
||||
|
||||
|
||||
#%% prepare model
|
||||
model = RandomForestClassifier(criterion='entropy', n_estimators=80, max_depth=5, n_jobs=-1)
|
||||
# model = LogisticRegression()
|
||||
# model = MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False, max_iter=1000)
|
||||
model = GaussianNB()
|
||||
model.fit(X_train, y_train)
|
||||
|
||||
|
||||
#%%
|
||||
# estimate accuracy on validation set
|
||||
predictions = model.predict(X_test)
|
||||
print(accuracy_score(y_test, predictions))
|
||||
print(confusion_matrix(y_test, predictions))
|
||||
print(classification_report(y_test, predictions))
|
||||
|
||||
|
||||
#%%
|
||||
# feat_importance = pd.DataFrame({'Importance':model.feature_importances_*100}, index=X.columns)
|
||||
# feat_importance.sort_values('Importance', axis=0, ascending=True)
|
||||
# feat_importance.plot(kind='barh', color='r' )
|
||||
# plt.xlabel('Variable Importance')
|
||||
# print(feat_importance)
|
||||
|
||||
#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day
|
||||
backtestdata = pd.DataFrame(index= X_test_orig.index)
|
||||
backtestdata['signal_pred'] = predictions
|
||||
backtestdata['signal_actual'] = y_test
|
||||
backtestdata['returns'] = X_test_orig[returns_col]
|
||||
backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1)
|
||||
backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1)
|
||||
|
||||
# %%
|
||||
print(backtestdata.cumsum().apply(np.exp).tail(1))
|
||||
|
||||
# %%
|
||||
@@ -1,142 +0,0 @@
|
||||
#%% Import all the stuff, load data, define constants
|
||||
from sklearnex import patch_sklearn
|
||||
patch_sklearn()
|
||||
|
||||
from utils.load_data import create_target_classes, load_files
|
||||
from sktime.forecasting.model_selection import temporal_train_test_split
|
||||
from sklearn.metrics import accuracy_score
|
||||
from sklearn.metrics import confusion_matrix
|
||||
# from utils.evaluate import print_classification_metrics, format_data_for_backtest
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.neighbors import KNeighborsClassifier
|
||||
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
|
||||
from sklearn.naive_bayes import GaussianNB
|
||||
from sklearn.svm import SVC
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
|
||||
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
|
||||
from utils.sliding_window import sliding_window_and_flatten
|
||||
|
||||
ticket_to_predict = 'BTC_ETH'
|
||||
print('Predicting: ', ticket_to_predict)
|
||||
|
||||
data = load_files(path='data/',
|
||||
own_asset=ticket_to_predict,
|
||||
own_asset_lags=[1,2,3,4,5,6,8,10,15],
|
||||
load_other_assets=False,
|
||||
other_asset_lags=[1,2,3,4],
|
||||
log_returns=True,
|
||||
add_date_features=True,
|
||||
own_technical_features='level1',
|
||||
other_technical_features='none',
|
||||
exogenous_features='none',
|
||||
index_column='int'
|
||||
)
|
||||
|
||||
target_col = 'target'
|
||||
returns_col = ticket_to_predict + '_returns'
|
||||
data = create_target_classes(data, returns_col, 1, 'two')
|
||||
|
||||
X = data.drop(columns=[target_col])
|
||||
y = data[target_col]
|
||||
|
||||
X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.1)
|
||||
feature_scaler = MinMaxScaler(feature_range= (-1, 1))
|
||||
X_test_orig = X_test.copy()
|
||||
X_train = feature_scaler.fit_transform(X_train)
|
||||
X_test = feature_scaler.transform(X_test)
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
sliding_window_size = 120
|
||||
retrain_every = 60
|
||||
X_train = sliding_window_and_flatten(X_train, sliding_window_size)
|
||||
# X_test = sliding_window_and_flatten(X_test, sliding_window_size)
|
||||
X_test_orig = X_test_orig.iloc[sliding_window_size-1:]
|
||||
y_train = y_train[sliding_window_size-1:]
|
||||
# y_test = y_test[sliding_window_size-1:]
|
||||
|
||||
|
||||
|
||||
|
||||
def evaluate_predictions(model_name: str, y: pd.Series, preds: pd.Series, sliding_window_size: int):
|
||||
print("Model: ", model_name)
|
||||
evaluate_from = sliding_window_size*2
|
||||
print(accuracy_score(y[evaluate_from:-1], preds[evaluate_from:]))
|
||||
print(confusion_matrix(y[evaluate_from:-1], preds[evaluate_from:]))
|
||||
print(classification_report(y[evaluate_from:-1], preds[evaluate_from:]))
|
||||
|
||||
|
||||
def walk_forward_train_test(
|
||||
create_model,
|
||||
X_train: pd.DataFrame,
|
||||
y_train: pd.Series,
|
||||
window_size: int,
|
||||
retrain_every: int
|
||||
):
|
||||
predictions = [None] * (len(y_train)-1)
|
||||
models = [None] * len(predictions)
|
||||
|
||||
train_from = sliding_window_size+1
|
||||
train_till = len(y_train)-2
|
||||
|
||||
iterations_since_retrain = 0
|
||||
|
||||
for i in range(train_from, train_till):
|
||||
# if i % 20 == 0: print('Fold: ', i)
|
||||
iterations_since_retrain += 1
|
||||
window_start = i - window_size
|
||||
window_end = i
|
||||
X_train_slice = X_train[window_start:window_end]
|
||||
y_train_slice = y_train[window_start:window_end]
|
||||
|
||||
if iterations_since_retrain >= retrain_every or models[i-1] is None:
|
||||
model = create_model()
|
||||
model.fit(X_train_slice, y_train_slice)
|
||||
iterations_since_retrain = 0
|
||||
else:
|
||||
model = models[i-1]
|
||||
models[window_end] = model
|
||||
|
||||
predictions[window_end+1] = model.predict(X_train[window_end+1].reshape(1, -1)).item()
|
||||
return models, predictions
|
||||
|
||||
|
||||
#%%
|
||||
for model_name, create_model in models_to_try:
|
||||
|
||||
model_over_time, preds = walk_forward_train_test(
|
||||
create_model = create_model,
|
||||
X_train = X_train,
|
||||
y_train = y_train,
|
||||
window_size = sliding_window_size,
|
||||
retrain_every = retrain_every
|
||||
)
|
||||
|
||||
evaluate_predictions(model_name, y_train, preds)
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
|
||||
#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day
|
||||
# backtestdata = pd.DataFrame(index= X_test_orig.index)
|
||||
# backtestdata['signal_pred'] = predictions
|
||||
# backtestdata['signal_actual'] = y_test
|
||||
# backtestdata['returns'] = X_test_orig[returns_col]
|
||||
# backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1)
|
||||
# backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1)
|
||||
|
||||
# %%
|
||||
# print(backtestdata.cumsum().apply(np.exp).tail(1))
|
||||
|
||||
# %%
|
||||
@@ -1,5 +1,5 @@
|
||||
#%% Import all the stuff, load data, define constants
|
||||
from utils.load_data import load_files, create_target_classes
|
||||
from data_loader.load_data import load_files, create_target_classes
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from utils.sktime import from_df_to_sktime_data
|
||||
@@ -36,7 +36,7 @@ print('Predicting: ', ticket_to_predict)
|
||||
data = load_files(path='data/',
|
||||
own_asset=ticket_to_predict,
|
||||
own_asset_lags=[1,2,3,4,5,6,8,10,15],
|
||||
load_other_assets=False,
|
||||
load_non_target_asset=False,
|
||||
other_asset_lags=[1,2,3,4],
|
||||
log_returns=True,
|
||||
add_date_features=True,
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
#%% Import all the stuff, load data, define constants
|
||||
from sklearn.utils import shuffle
|
||||
from utils.load_data import load_files, create_target_cum_forward_returns
|
||||
import pandas as pd
|
||||
from tensorflow import keras
|
||||
from utils.normalize import normalize
|
||||
import tensorflow as tf
|
||||
from utils.visualize import visualize_loss
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from utils.evaluate import print_regression_metrics
|
||||
import numpy as np
|
||||
from utils.rolling import rolling_window
|
||||
|
||||
|
||||
data = load_files('data/', add_features=True, log_returns=False)
|
||||
data.reset_index(drop=True, inplace=True)
|
||||
data = data[[column for column in data.columns if not column.endswith('volume')]]
|
||||
data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]]
|
||||
|
||||
target_col = 'target'
|
||||
data = create_target_cum_forward_returns(data, 'BTC_returns', 10)
|
||||
|
||||
learning_rate = 0.002
|
||||
batch_size = 64
|
||||
epochs = 100
|
||||
|
||||
split_fraction = 0.715
|
||||
train_split = int(split_fraction * int(data.shape[0]))
|
||||
|
||||
past = 10
|
||||
future = 1
|
||||
|
||||
start = past + future
|
||||
end = start + train_split
|
||||
|
||||
#%% split data into training - validation sets
|
||||
train_data = data.loc[0 : train_split - 1]
|
||||
val_data = data.loc[train_split:]
|
||||
|
||||
#%% create features and target for training set & keras dataset
|
||||
feature_scaler = MinMaxScaler(feature_range= (-1, 1))
|
||||
target_scaler = MinMaxScaler(feature_range= (-1, 1))
|
||||
|
||||
x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std
|
||||
y_train = target_scaler.fit_transform(data.iloc[start:end][target_col].values.reshape(-1, 1))
|
||||
|
||||
dataset_train = keras.preprocessing.timeseries_dataset_from_array(
|
||||
x_train,
|
||||
y_train,
|
||||
sequence_length=past,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
|
||||
#%% create features and target for validation set & keras dataset
|
||||
x_end = len(val_data) - past - future
|
||||
label_start = train_split + past + future
|
||||
|
||||
x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std
|
||||
y_val = target_scaler.transform(data.iloc[label_start:][target_col].values.reshape(-1, 1))
|
||||
|
||||
dataset_val = keras.utils.timeseries_dataset_from_array(
|
||||
x_val,
|
||||
y_val,
|
||||
sequence_length=past,
|
||||
batch_size=batch_size,
|
||||
shuffle=False,
|
||||
)
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
for batch in dataset_train.take(10):
|
||||
batch_inputs, batch_targets = batch
|
||||
|
||||
print("Input shape:", batch_inputs.shape)
|
||||
print("Target shape:", batch_targets.shape)
|
||||
|
||||
# print(batch_inputs)
|
||||
# print(batch_targets)
|
||||
|
||||
# %%
|
||||
model = keras.Sequential()
|
||||
model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 64, activation = 'sigmoid'))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 32, activation = 'sigmoid'))
|
||||
model.add(keras.layers.Dropout(0.4))
|
||||
model.add(keras.layers.Dense(units = 1))
|
||||
|
||||
optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
|
||||
model.compile(optimizer=optimizer, loss="mean_squared_error")
|
||||
model.summary()
|
||||
|
||||
# %%
|
||||
|
||||
path_checkpoint = "model_checkpoint.h5"
|
||||
|
||||
history = model.fit(
|
||||
dataset_train,
|
||||
epochs=epochs,
|
||||
validation_data=dataset_val,
|
||||
)
|
||||
|
||||
#%%
|
||||
|
||||
pred = model.predict(rolling_window(x_val, 11))
|
||||
pred = pred.reshape(pred.shape[0], 1)
|
||||
pred = target_scaler.inverse_transform(pred)
|
||||
|
||||
print_regression_metrics(y_val, pred)
|
||||
|
||||
#%%
|
||||
|
||||
visualize_loss(history, "Training and Validation Loss")
|
||||
@@ -1,4 +1,4 @@
|
||||
from utils.load_data import load_files
|
||||
from data_loader.load_data import load_files
|
||||
import pandas as pd
|
||||
# from tensorflow import keras
|
||||
from utils.normalize import normalize
|
||||
|
||||
Reference in New Issue
Block a user