feat(Data): added various data loading config options, walk forward method draft (#9)

* feat(Eval): added format_data_for_backtest()

* feat(Data): added many configurable parameters to load_files to reduce boilerplate and prepare for HPO

* feat(Core): added walk forward method of training/testing

* fix(Model): remove the unnecessary softmax activation from the keras models

* feat(Core): added walk_forward_train_test()

model_classification_sklearn_walk_forward.py

@@ -0,0 +1,164 @@
+#%% Import all the stuff, load data, define constants
+from sklearnex import patch_sklearn
+patch_sklearn()
+
+from load_data import create_target_cum_forward_returns, create_target_classes, load_files
+from sktime.forecasting.model_selection import temporal_train_test_split
+from sklearn.metrics import accuracy_score
+from sklearn.metrics import confusion_matrix
+# from utils.evaluate import print_classification_metrics, format_data_for_backtest
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
+from sklearn.naive_bayes import GaussianNB
+from sklearn.svm import SVC
+from sklearn.neural_network import MLPClassifier
+from sklearn.pipeline import Pipeline
+from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
+from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
+from sklearn.preprocessing import MinMaxScaler
+
+from utils.sliding_window import sliding_window_and_flatten
+
+ticket_to_predict = 'BTC_ETH'
+print('Predicting: ', ticket_to_predict)
+
+data = load_files(path='data/',
+    own_asset=ticket_to_predict,
+    load_other_assets=False,
+    log_returns=True,
+    add_date_features=True,
+    own_technical_features='level1',
+    other_technical_features='none',
+    exogenous_features='none',
+    index_column='int'
+)
+
+target_col = 'target'
+returns_col = ticket_to_predict + '_returns'
+data = create_target_classes(data, returns_col, 1, 'two')
+
+X = data.drop(columns=[target_col])
+y = data[target_col]
+
+X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.1)
+feature_scaler = MinMaxScaler(feature_range= (-1, 1))
+X_test_orig = X_test.copy()
+X_train = feature_scaler.fit_transform(X_train)
+X_test = feature_scaler.transform(X_test)
+
+
+#%%
+
+sliding_window_size = 120
+X_train = sliding_window_and_flatten(X_train, sliding_window_size)
+# X_test = sliding_window_and_flatten(X_test, sliding_window_size)
+X_test_orig = X_test_orig.iloc[sliding_window_size-1:]
+y_train = y_train[sliding_window_size-1:]
+# y_test = y_test[sliding_window_size-1:]
+
+
+def walk_forward_train_test(
+        create_model,
+        X_train: pd.DataFrame,
+        y_train: pd.Series,
+        window_size: int,
+        retrain_every: int
+    ):
+    predictions = [None] * (len(y_train)-1)
+    models = [None] * (len(y_train)-1)
+
+    train_from = sliding_window_size+1
+    train_till = len(y_train)-2
+    
+    iterations_since_retrain = 0
+
+    for i in range(train_from, train_till):
+        if i % 10 == 0: print('Fold: ', i)
+        iterations_since_retrain += 1
+        window_start = i - window_size
+        window_end = i
+        X_train_slice = X_train[window_start:window_end]
+        y_train_slice = y_train[window_start:window_end]
+
+        if iterations_since_retrain >= retrain_every or models[i-1] is None:
+            model = create_model()
+            model.fit(X_train_slice, y_train_slice)
+            models.append(model)
+        else:
+            model = models[i-1]
+        models[window_end] = model
+
+        predictions[window_end+1] = model.predict(X_train[window_end+1].reshape(1, -1)).item()
+    return models, predictions
+
+
+#%%
+models, preds = walk_forward_train_test(lambda : GaussianNB(), X_train, y_train, 120, 10)
+
+# %%
+
+models = []
+models.append(('LR', LogisticRegression(n_jobs=-1)))
+models.append(('LDA', LinearDiscriminantAnalysis()))
+models.append(('KNN', KNeighborsClassifier()))
+models.append(('CART', DecisionTreeClassifier()))
+models.append(('NB', GaussianNB()))
+# models.append(('NN', MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False)))
+models.append(('AB', AdaBoostClassifier()))
+# models.append(('GBM', GradientBoostingClassifier()))
+models.append(('RF', RandomForestClassifier(n_jobs=-1)))
+# model = RandomForestClassifier(n_jobs=-1)
+# model = KNeighborsClassifier()
+model = GaussianNB()
+# model = AdaBoostClassifier()
+
+# re-train the model every n steps
+# store the model
+# iterate over all potential folds
+# retrieve model for the range
+# predict on the input data
+# store the predictions
+
+predictions = [0] * (len(y_train)-1)
+
+
+for i in range(sliding_window_size+1, len(y_train)-2):
+    if i % 10 == 0: print('Fold: ', i)
+    X_train_up_to_i = X_train[i-sliding_window_size:i]
+    y_train_up_to_i = y_train[i-sliding_window_size:i]
+    model.fit(X_train_up_to_i, y_train_up_to_i)
+    predictions[i+1] = model.predict(X_train[i+1].reshape(1, -1)).item()
+
+
+#%%
+
+print(accuracy_score(y_train[500:-1], predictions[500:]))
+print(confusion_matrix(y_train[500:-1], predictions[500:]))
+print(classification_report(y_train[500:-1], predictions[500:]))
+
+
+#%%
+# feat_importance = pd.DataFrame({'Importance':model.feature_importances_*100}, index=X.columns)
+# feat_importance.sort_values('Importance', axis=0, ascending=True)
+# feat_importance.plot(kind='barh', color='r' )
+# plt.xlabel('Variable Importance')
+# print(feat_importance)
+
+#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day
+# backtestdata = pd.DataFrame(index= X_test_orig.index)
+# backtestdata['signal_pred'] = predictions
+# backtestdata['signal_actual'] = y_test
+# backtestdata['returns'] = X_test_orig[returns_col]
+# backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1)
+# backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1)
+
+# %%
+# print(backtestdata.cumsum().apply(np.exp).tail(1))
+
+# %%