feat(Data): added various data loading config options, walk forward method draft (#9)

* feat(Eval): added format_data_for_backtest()

* feat(Data): added many configurable parameters to load_files to reduce boilerplate and prepare for HPO

* feat(Core): added walk forward method of training/testing

* fix(Model): remove the unnecessary softmax activation from the keras models

* feat(Core): added walk_forward_train_test()

keras_models/classification.py

@@ -9,7 +9,7 @@ def create_basic_lstm_model(input_shape, num_classes):
     model.add(keras.layers.Dropout(0.3))
     model.add(keras.layers.Dense(units = 32, activation = 'sigmoid'))
     model.add(keras.layers.Dropout(0.3))
-    model.add(keras.layers.Dense(units = num_classes, activation = 'softmax'))
+    model.add(keras.layers.Dense(units = num_classes, activation = 'linear'))
     return model
 
 def create_basic_cnn_model(input_shape, num_classes):
@@ -24,7 +24,7 @@ def create_basic_cnn_model(input_shape, num_classes):
     model.add(keras.layers.BatchNormalization())
     model.add(keras.layers.ReLU())
     model.add(keras.layers.GlobalAveragePooling1D())
-    model.add(keras.layers.Dense(num_classes, activation="softmax"))
+    model.add(keras.layers.Dense(num_classes, activation="linear"))
     return model
 
 
@@ -95,7 +95,7 @@ def create_resnet_cnn_model(input_shape, num_classes):
 
     gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3)
 
-    output_layer = keras.layers.Dense(num_classes, activation='softmax')(gap_layer)
+    output_layer = keras.layers.Dense(num_classes, activation='linear')(gap_layer)
     model = keras.models.Model(inputs=input_layer, outputs=output_layer)
 
 

load_data.py

@@ -2,13 +2,28 @@
 import pandas as pd
 import os
 import numpy as np
+from pandas.core.frame import DataFrame
 from utils.technical_indicators import ROC, RSI, STOK, STOD
+from typing import Literal
 
 #%%
 
-def load_files(path: str, add_features: bool, log_returns: bool, narrow_format: bool = False) -> pd.DataFrame:
+def load_files(path: str,
+            own_asset: str,
+            load_other_assets: bool,
+            log_returns: bool,
+            add_date_features: bool,
+            own_technical_features: Literal['none', 'level1', 'level2'],
+            other_technical_features: Literal['none', 'level1', 'level2'],
+            exogenous_features: Literal['none', 'level1'],
+            index_column: Literal['date', 'int'],
+            narrow_format: bool = False
+            ) -> pd.DataFrame:
+    
     files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
-    dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features, log_returns, narrow_format) for f in files]
+    files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(own_asset))]
+    def is_own_asset(own_asset: str, file: str): return file.split('.')[0].startswith(own_asset)
+    dfs = [__load_df(path=os.path.join(path,f), prefix=f.split('.')[0], log_returns=log_returns, technical_features=own_technical_features if is_own_asset(own_asset, f) else other_technical_features, narrow_format=narrow_format) for f in files]
     if narrow_format:
         dfs = pd.concat(dfs, axis=0).fillna(0.)
     else:
@@ -16,17 +31,20 @@ def load_files(path: str, add_features: bool, log_returns: bool, narrow_format:
 
     dfs.index = pd.DatetimeIndex(dfs.index)
 
-    if add_features:
+    if add_date_features:
         dfs['day_month'] = dfs.index.day
         dfs['day_week'] = dfs.index.dayofweek
         dfs['month'] = dfs.index.month
 
+    if index_column == 'int':
+        dfs.reset_index(drop=True, inplace=True)
+
     if narrow_format:
         return dfs
     else:
         return dfs.drop(index=dfs.index[0], axis=0)
 
-def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool, narrow_format: bool = False) -> pd.DataFrame:
+def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Literal['none', 'level1', 'level2'], narrow_format: bool = False) -> pd.DataFrame:
     df = pd.read_csv(path, header=0, index_col=0).fillna(0)
 
     if log_returns:
@@ -34,7 +52,22 @@ def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool, nar
     else:
         df['returns'] = df['close'].pct_change()
 
-    if add_features:
+    df = __augment_derived_features(df, log_returns=log_returns, technical_features=technical_features)
+
+    df = df.replace([np.inf, -np.inf], 0.)
+    df = df.drop(columns=['open', 'high', 'low', 'close'])
+    # we're not ready for this just yet
+    if 'volume' in df.columns:
+        df = df.drop(columns=['volume'])
+    
+    if narrow_format:
+        df["ticker"] = np.repeat(prefix, df.shape[0])
+    else: 
+        df.columns = [prefix + "_" + c for c in df.columns]
+    return df
+
+def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_features: Literal['none', 'level1', 'level2']) -> pd.DataFrame:
+    if technical_features == 'level1' or technical_features == 'level2':
         # volatility (10, 20, 30 days)
         df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5)
         df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5)
@@ -55,30 +88,23 @@ def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool, nar
             df['mom_60'] = df['close'].pct_change(60)
             df['mom_90'] = df['close'].pct_change(90)
 
+    if technical_features == 'level2':
+        df['roc_10'] = ROC(df['close'], 10)
+        df['roc_30'] = ROC(df['close'], 30)
 
-    df['roc_10'] = ROC(df['close'], 10)
-    df['roc_30'] = ROC(df['close'], 30)
+        df['rsi_10'] = RSI(df['close'], 10)
+        df['rsi_30'] = RSI(df['close'], 30)
+        df['rsi_100'] = RSI(df['close'], 30)
 
-    df['rsi_10'] = RSI(df['close'], 10)
-    df['rsi_30'] = RSI(df['close'], 30)
-    df['rsi_100'] = RSI(df['close'], 30)
-
-    df['stok_10'] = STOK(df['close'], df['low'], df['high'], 10)
-    df['stod_10'] = STOD(df['close'], df['low'], df['high'], 10)
-    df['stok_30'] = STOK(df['close'], df['low'], df['high'], 30)
-    df['stod_30'] = STOD(df['close'], df['low'], df['high'], 30)
-    df['stok_200'] = STOK(df['close'], df['low'], df['high'], 200)
-    df['stod_200'] = STOD(df['close'], df['low'], df['high'], 200)
-
-
-    df = df.replace([np.inf, -np.inf], 0.)
-    df = df.drop(columns=['open', 'high', 'low', 'close'])
-    if narrow_format:
-        df["ticker"] = np.repeat(prefix, df.shape[0])
-    else: 
-        df.columns = [prefix + "_" + c for c in df.columns]
+        df['stok_10'] = STOK(df['close'], df['low'], df['high'], 10)
+        df['stod_10'] = STOD(df['close'], df['low'], df['high'], 10)
+        df['stok_30'] = STOK(df['close'], df['low'], df['high'], 30)
+        df['stod_30'] = STOD(df['close'], df['low'], df['high'], 30)
+        df['stok_200'] = STOK(df['close'], df['low'], df['high'], 200)
+        df['stod_200'] = STOD(df['close'], df['low'], df['high'], 200)
     return df
 
+
 # %%
 def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
     df['target'] = df[source_column].diff(period).shift(-period)
@@ -86,34 +112,32 @@ def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, peri
     return df
 
 
-#%%
-def create_target_pos_neg_classes(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
-    if period > 0:
-        df['target'] = df[source_column].diff(period).shift(-period)
-    else:
-        df['target'] = df[source_column].diff(period)
-    df['target'] = df['target'].map(lambda x: 0 if x <= 0.0 else 1)
-    if period > 0:
-        df = df.iloc[:-period]
-    return df
+def create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.DataFrame:
 
-def create_target_four_classes(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
-    def __get_class(x):
-        treshold = 0.08
-        if x <= -treshold:
+    def get_class_binary(x):
+        return 0 if x <= 0.0 else 1
+
+    def get_class_threeway(x):
+        bins = pd.qcut(df[source_column], 4, duplicates='raise', retbins=True)[1]
+        lower_threshold = bins[1]
+        upper_threshold = bins[3]
+        if x <= lower_threshold:
+            return -1
+        elif x > lower_threshold and x < upper_threshold:
             return 0
-        elif x > -treshold and x <= 0:
-            return 1
-        elif x > 0 and x <= treshold:
-            return 2
         else:
-            return 3
+            return 1
 
     if period > 0:
-        df['target'] = df[source_column].diff(period).shift(-period)
+        df['target'] = df[source_column].shift(-period)
     else:
-        df['target'] = df[source_column].diff(period)
-    df['target'] = df['target'].map(__get_class)
+        df['target'] = df[source_column]
+    
+    get_class_function = get_class_binary
+    if no_of_classes == "three":
+        get_class_function = get_class_threeway
+
+    df['target'] = df['target'].map(get_class_function)
     if period > 0:
         df = df.iloc[:-period]
     return df

model_classification_keras.py

@@ -1,6 +1,6 @@
 #%% Import all the stuff, load data, define constants
 from sklearn.utils import shuffle
-from load_data import load_files, create_target_pos_neg_classes
+from load_data import load_files, create_target_classes
 import pandas as pd
 from tensorflow import keras
 from utils.normalize import normalize
@@ -13,13 +13,20 @@ from utils.rolling import rolling_window
 from keras_models.classification import create_basic_cnn_model, create_basic_lstm_model, create_resnet_cnn_model
 from keras_models.classification_transformer import create_basic_transformer_model
 
-data = load_files('data/', add_features=True, log_returns=False)
-data.reset_index(drop=True, inplace=True)
-data = data[[column for column in data.columns if not column.endswith('volume')]]
-# data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]]
+data = load_files(path='data/',
+    own_asset='BTC_ETH',
+    load_other_assets=True,
+    log_returns=False,
+    add_date_features=True,
+    own_technical_features='level2',
+    other_technical_features='level2',
+    exogenous_features='none',
+    index_column='int'
+)
+
 
 target_col = 'target'
-data = create_target_pos_neg_classes(data, 'BTC_ETH_returns', 1)
+data = create_target_classes(data, 'BTC_ETH_returns', 1, 'two')
 
 num_classes = 2
 learning_rate = 0.002
@@ -71,7 +78,7 @@ dataset_val = keras.utils.timeseries_dataset_from_array(
 
 #%%
 
-for batch in dataset_train.take(10):
+for batch in dataset_train.take(1):
     batch_inputs, batch_targets = batch
 
 print("Input shape:", batch_inputs.shape)
@@ -83,9 +90,9 @@ n_features = batch_inputs.shape[2]
 # print(batch_targets)
 
 # %%
-model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
+# model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
 # model = create_basic_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
-# model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
+model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes)
 # model = create_basic_transformer_model(
 #     input_shape=(n_timestamps, n_features),
 #     n_classes=num_classes,
@@ -99,7 +106,8 @@ model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_clas
 # )
 
 optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
-model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy'])
+loss = keras.losses.CategoricalCrossentropy(from_logits=True)
+model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
 model.summary()
 
 # %%

model_classification_sklearn.py

@@ -0,0 +1,156 @@
+#%% Import all the stuff, load data, define constants
+from sklearnex import patch_sklearn
+patch_sklearn()
+
+from load_data import create_target_cum_forward_returns, create_target_classes, load_files
+from sktime.forecasting.model_selection import temporal_train_test_split
+from sklearn.metrics import accuracy_score
+from sklearn.metrics import confusion_matrix
+# from utils.evaluate import print_classification_metrics, format_data_for_backtest
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
+from sklearn.naive_bayes import GaussianNB
+from sklearn.svm import SVC
+from sklearn.neural_network import MLPClassifier
+from sklearn.pipeline import Pipeline
+from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
+from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
+from sklearn.preprocessing import MinMaxScaler
+
+from utils.sliding_window import sliding_window_and_flatten
+
+ticket_to_predict = 'BTC_ETH'
+print('Predicting: ', ticket_to_predict)
+
+data = load_files(path='data/',
+    own_asset=ticket_to_predict,
+    load_other_assets=True,
+    log_returns=True,
+    add_date_features=True,
+    own_technical_features='level2',
+    other_technical_features='none',
+    exogenous_features='none',
+    index_column='int'
+)
+
+target_col = 'target'
+returns_col = ticket_to_predict + '_returns'
+data = create_target_classes(data, returns_col, 1, 'three')
+
+X = data.drop(columns=['target'])
+y = data[target_col]
+
+X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2)
+feature_scaler = MinMaxScaler(feature_range= (-1, 1))
+X_test_orig = X_test.copy()
+X_train = feature_scaler.fit_transform(X_train)
+X_test = feature_scaler.transform(X_test)
+
+
+#%%
+
+sliding_window_size = 10
+X_train = sliding_window_and_flatten(X_train, sliding_window_size)
+X_test = sliding_window_and_flatten(X_test, sliding_window_size)
+X_test_orig = X_test_orig.iloc[sliding_window_size-1:]
+y_train = y_train[sliding_window_size-1:]
+y_test = y_test[sliding_window_size-1:]
+
+assert X_train.shape[0] == y_train.shape[0]
+assert X_test.shape[0] == y_test.shape[0]
+scoring = 'accuracy'
+
+# %%
+num_folds = 10
+
+models = []
+models.append(('LR', LogisticRegression(n_jobs=-1)))
+models.append(('LDA', LinearDiscriminantAnalysis()))
+models.append(('KNN', KNeighborsClassifier()))
+models.append(('CART', DecisionTreeClassifier()))
+models.append(('NB', GaussianNB()))
+# models.append(('NN', MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False)))
+models.append(('AB', AdaBoostClassifier()))
+# models.append(('GBM', GradientBoostingClassifier()))
+models.append(('RF', RandomForestClassifier(n_jobs=-1)))
+
+results = []
+names = []
+for name, model in models:
+    kfold = KFold(n_splits=num_folds, shuffle=False)
+    cv_results = cross_val_score(model, X_train, y_train, cv=kfold, scoring=scoring)
+    results.append(cv_results)
+    names.append(name)
+    msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
+    print(msg)
+
+# # compare algorithms
+# fig = plt.figure()
+# fig.suptitle('Algorithm Comparison')
+# ax = fig.add_subplot(111)
+# plt.boxplot(results)
+# ax.set_xticklabels(names)
+# fig.set_size_inches(15,8)
+# plt.show()
+
+#%%
+# n_estimators = [20,80]
+# max_depth= [5,10, 15]
+# criterion = ["gini","entropy"]
+# param_grid = dict(n_estimators=n_estimators, max_depth=max_depth, criterion = criterion )
+# model = RandomForestClassifier(n_jobs=-1)
+# kfold = KFold(n_splits=10, shuffle=False)
+# grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scoring, cv=kfold)
+# grid_result = grid.fit(X_train, y_train)
+
+# #Print Results
+# print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
+# means = grid_result.cv_results_['mean_test_score']
+# stds = grid_result.cv_results_['std_test_score']
+# params = grid_result.cv_results_['params']
+# ranks = grid_result.cv_results_['rank_test_score']
+# for mean, stdev, param, rank in zip(means, stds, params, ranks):
+#     print("#%d %f (%f) with: %r" % (rank, mean, stdev, param))
+
+
+#%% prepare model
+model = RandomForestClassifier(criterion='entropy', n_estimators=80, max_depth=5, n_jobs=-1) 
+# model = LogisticRegression() 
+# model = MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False, max_iter=1000)
+model = GaussianNB()
+model.fit(X_train, y_train)
+
+
+#%%
+# estimate accuracy on validation set
+predictions = model.predict(X_test)
+print(accuracy_score(y_test, predictions))
+print(confusion_matrix(y_test, predictions))
+print(classification_report(y_test, predictions))
+
+
+#%%
+# feat_importance = pd.DataFrame({'Importance':model.feature_importances_*100}, index=X.columns)
+# feat_importance.sort_values('Importance', axis=0, ascending=True)
+# feat_importance.plot(kind='barh', color='r' )
+# plt.xlabel('Variable Importance')
+# print(feat_importance)
+
+#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day
+backtestdata = pd.DataFrame(index= X_test_orig.index)
+backtestdata['signal_pred'] = predictions
+backtestdata['signal_actual'] = y_test
+backtestdata['returns'] = X_test_orig[returns_col]
+backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1)
+backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1)
+
+# %%
+print(backtestdata.cumsum().apply(np.exp).tail(1))
+
+# %%

model_classification_sklearn_walk_forward.py

@@ -0,0 +1,164 @@
+#%% Import all the stuff, load data, define constants
+from sklearnex import patch_sklearn
+patch_sklearn()
+
+from load_data import create_target_cum_forward_returns, create_target_classes, load_files
+from sktime.forecasting.model_selection import temporal_train_test_split
+from sklearn.metrics import accuracy_score
+from sklearn.metrics import confusion_matrix
+# from utils.evaluate import print_classification_metrics, format_data_for_backtest
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
+from sklearn.naive_bayes import GaussianNB
+from sklearn.svm import SVC
+from sklearn.neural_network import MLPClassifier
+from sklearn.pipeline import Pipeline
+from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
+from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
+from sklearn.preprocessing import MinMaxScaler
+
+from utils.sliding_window import sliding_window_and_flatten
+
+ticket_to_predict = 'BTC_ETH'
+print('Predicting: ', ticket_to_predict)
+
+data = load_files(path='data/',
+    own_asset=ticket_to_predict,
+    load_other_assets=False,
+    log_returns=True,
+    add_date_features=True,
+    own_technical_features='level1',
+    other_technical_features='none',
+    exogenous_features='none',
+    index_column='int'
+)
+
+target_col = 'target'
+returns_col = ticket_to_predict + '_returns'
+data = create_target_classes(data, returns_col, 1, 'two')
+
+X = data.drop(columns=[target_col])
+y = data[target_col]
+
+X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.1)
+feature_scaler = MinMaxScaler(feature_range= (-1, 1))
+X_test_orig = X_test.copy()
+X_train = feature_scaler.fit_transform(X_train)
+X_test = feature_scaler.transform(X_test)
+
+
+#%%
+
+sliding_window_size = 120
+X_train = sliding_window_and_flatten(X_train, sliding_window_size)
+# X_test = sliding_window_and_flatten(X_test, sliding_window_size)
+X_test_orig = X_test_orig.iloc[sliding_window_size-1:]
+y_train = y_train[sliding_window_size-1:]
+# y_test = y_test[sliding_window_size-1:]
+
+
+def walk_forward_train_test(
+        create_model,
+        X_train: pd.DataFrame,
+        y_train: pd.Series,
+        window_size: int,
+        retrain_every: int
+    ):
+    predictions = [None] * (len(y_train)-1)
+    models = [None] * (len(y_train)-1)
+
+    train_from = sliding_window_size+1
+    train_till = len(y_train)-2
+    
+    iterations_since_retrain = 0
+
+    for i in range(train_from, train_till):
+        if i % 10 == 0: print('Fold: ', i)
+        iterations_since_retrain += 1
+        window_start = i - window_size
+        window_end = i
+        X_train_slice = X_train[window_start:window_end]
+        y_train_slice = y_train[window_start:window_end]
+
+        if iterations_since_retrain >= retrain_every or models[i-1] is None:
+            model = create_model()
+            model.fit(X_train_slice, y_train_slice)
+            models.append(model)
+        else:
+            model = models[i-1]
+        models[window_end] = model
+
+        predictions[window_end+1] = model.predict(X_train[window_end+1].reshape(1, -1)).item()
+    return models, predictions
+
+
+#%%
+models, preds = walk_forward_train_test(lambda : GaussianNB(), X_train, y_train, 120, 10)
+
+# %%
+
+models = []
+models.append(('LR', LogisticRegression(n_jobs=-1)))
+models.append(('LDA', LinearDiscriminantAnalysis()))
+models.append(('KNN', KNeighborsClassifier()))
+models.append(('CART', DecisionTreeClassifier()))
+models.append(('NB', GaussianNB()))
+# models.append(('NN', MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False)))
+models.append(('AB', AdaBoostClassifier()))
+# models.append(('GBM', GradientBoostingClassifier()))
+models.append(('RF', RandomForestClassifier(n_jobs=-1)))
+# model = RandomForestClassifier(n_jobs=-1)
+# model = KNeighborsClassifier()
+model = GaussianNB()
+# model = AdaBoostClassifier()
+
+# re-train the model every n steps
+# store the model
+# iterate over all potential folds
+# retrieve model for the range
+# predict on the input data
+# store the predictions
+
+predictions = [0] * (len(y_train)-1)
+
+
+for i in range(sliding_window_size+1, len(y_train)-2):
+    if i % 10 == 0: print('Fold: ', i)
+    X_train_up_to_i = X_train[i-sliding_window_size:i]
+    y_train_up_to_i = y_train[i-sliding_window_size:i]
+    model.fit(X_train_up_to_i, y_train_up_to_i)
+    predictions[i+1] = model.predict(X_train[i+1].reshape(1, -1)).item()
+
+
+#%%
+
+print(accuracy_score(y_train[500:-1], predictions[500:]))
+print(confusion_matrix(y_train[500:-1], predictions[500:]))
+print(classification_report(y_train[500:-1], predictions[500:]))
+
+
+#%%
+# feat_importance = pd.DataFrame({'Importance':model.feature_importances_*100}, index=X.columns)
+# feat_importance.sort_values('Importance', axis=0, ascending=True)
+# feat_importance.plot(kind='barh', color='r' )
+# plt.xlabel('Variable Importance')
+# print(feat_importance)
+
+#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day
+# backtestdata = pd.DataFrame(index= X_test_orig.index)
+# backtestdata['signal_pred'] = predictions
+# backtestdata['signal_actual'] = y_test
+# backtestdata['returns'] = X_test_orig[returns_col]
+# backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1)
+# backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1)
+
+# %%
+# print(backtestdata.cumsum().apply(np.exp).tail(1))
+
+# %%

model_classification_sktime.py

@@ -1,5 +1,5 @@
 #%% Import all the stuff, load data, define constants
-from load_data import create_target_cum_forward_returns, create_target_pos_neg_classes, load_files, create_target_four_classes
+from load_data import load_files, create_target_classes
 import pandas as pd
 import numpy as np
 from utils.sktime import from_df_to_sktime_data
@@ -9,27 +9,46 @@ from sklearn.metrics import accuracy_score
 from sklearn.pipeline import Pipeline
 from sktime.classification.interval_based import (
     TimeSeriesForestClassifier,
+    SupervisedTimeSeriesForest,
+
 )
+from sktime.forecasting.model_selection import SlidingWindowSplitter
+from sktime.forecasting.model_selection import ForecastingRandomizedSearchCV
+
+from sktime.forecasting.compose import make_reduction
+
+from sklearn.tree import DecisionTreeClassifier
 from sklearn.metrics import confusion_matrix
 from sklearn.preprocessing import StandardScaler
 from sktime.forecasting.model_selection import (
     SlidingWindowSplitter,
     ForecastingGridSearchCV,
 )
-# from sktime.forecasting.model_evaluation import 
-# from sklearnex import patch_sklearn
-# patch_sklearn()
+from utils.evaluate import print_classification_metrics, format_data_for_backtest
+from sklearn.ensemble import RandomForestRegressor
+from sklearnex import patch_sklearn
+patch_sklearn()
 
 
+ticket_to_predict = 'BTC_USD'
+print('Predicting: ', ticket_to_predict)
+
+data = load_files(path='data/',
+    own_asset=ticket_to_predict,
+    load_other_assets=True,
+    log_returns=True,
+    add_date_features=True,
+    own_technical_features='level2',
+    other_technical_features='none',
+    exogenous_features='none',
+    index_column='int'
+)
 
-data = load_files('data/', add_features=True, log_returns=True, narrow_format=False)
-data.reset_index(drop=True, inplace=True)
-data = data[[column for column in data.columns if not column.endswith('volume')]]
-data = data[[column for column in data.columns if column.startswith('BTC_ETH_')]]
 target_col = 'target'
-data = create_target_pos_neg_classes(data, 'BTC_ETH_returns', 1)
+returns_col = ticket_to_predict + '_returns'
+data = create_target_classes(data, returns_col, 1, 'two')
 
-X = data
+X = data.drop(columns=[target_col])
 y = data[target_col]
 
 X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2)
@@ -45,11 +64,32 @@ X_test = from_df_to_sktime_data(X_test)
 # ])
 
 # pipe.fit(X_train, y_train)
+# regressor = RandomForestRegressor(n_estimators=20)
+# model = DecisionTreeClassifier(random_state=1)
+model = TimeSeriesForestClassifier(n_estimators=50, random_state=1)
+model.fit(y = y_train, X = X_train)
+# forecaster = make_reduction(model, scitype="tabular-regressor")
+# nested_params = {"window_length": list(range(2,30)), 
+#                  "estimator__max_depth": list(range(5,16))}
+#                 # "estimator__n_estimators": list(range(10,200))}
+#%%
 
-model = TimeSeriesForestClassifier(n_estimators=200, random_state=1)
-model.fit(X_train, y_train)
+# cv = SlidingWindowSplitter(initial_window=40, window_length=30)
+# nrcv = ForecastingRandomizedSearchCV(forecaster, strategy="refit", cv=cv, 
+#                                      param_distributions=nested_params, 
+#                                      n_iter=5, random_state=42)
+# nrcv.fit(y = y_train, X = X_train, fh=np.array([1]))
+# print(nrcv.best_params_)
+# print(nrcv.best_score_)
 
+# model = DecisionTreeClassifier(random_state=1)
+# model.fit(X_train, y_train)
+
+# preds = nrcv.best_forecaster_.predict( X=X_test)
+# # print(preds)
 preds = model.predict(X_test)
-print(model.score(X_test, y_test))
-print(confusion_matrix(y_test, preds))
+print(print_classification_metrics(y_test, preds))
 
+# backtest_data = format_data_for_backtest(data, returns_col, X_test, preds)
+# print(backtest_data)
+# %%

utils/evaluate.py

@@ -1,6 +1,8 @@
 from math import sqrt
 from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score
 from sklearn.metrics import confusion_matrix
+import pandas as pd
+
 def print_regression_metrics(y_true, y_pred):
     rmse = sqrt(mean_squared_error(y_true, y_pred))
     print("RMSE: %.2f" % rmse)
@@ -10,4 +12,13 @@ def print_regression_metrics(y_true, y_pred):
 
 def print_classification_metrics(y_true, y_pred):
     print("Accuracy: %.2f" % accuracy_score(y_true, y_pred))
-    print("Confusion Matrix: \n", confusion_matrix(y_true, y_pred))
+    print("Confusion Matrix: \n", confusion_matrix(y_true, y_pred))
+
+def format_data_for_backtest(aggregated_data: pd.DataFrame, returns_col: str, only_test_data: pd.DataFrame, preds) -> pd.DataFrame:
+    backtest_data = aggregated_data.iloc[-only_test_data.shape[0]:].copy()[returns_col]
+    assert backtest_data.shape[0] == only_test_data.shape[0]
+    backtest_data = backtest_data.reset_index(drop=True)    
+    return pd.concat([backtest_data, pd.Series(preds)], axis='columns')
+
+
+# def backtest()

utils/sliding_window.py

@@ -0,0 +1,200 @@
+import numpy as np
+from numpy.lib.stride_tricks import as_strided
+from numpy.core.numeric import normalize_axis_tuple
+
+def sliding_window_and_flatten(x: np.array, window_size: int) -> np.array:
+    n_features = x.shape[1]
+    x = np.squeeze(__numpy_sliding_window_view(x, (window_size, n_features)), axis = 1)
+    return x.reshape(x.shape[0], -1)
+
+
+def __numpy_sliding_window_view(x, window_shape, axis=None, *,
+                        subok=False, writeable=False):
+    """
+    Create a sliding window view into the array with the given window shape.
+    Also known as rolling or moving window, the window slides across all
+    dimensions of the array and extracts subsets of the array at all window
+    positions.
+    
+    .. versionadded:: 1.20.0
+    Parameters
+    ----------
+    x : array_like
+        Array to create the sliding window view from.
+    window_shape : int or tuple of int
+        Size of window over each axis that takes part in the sliding window.
+        If `axis` is not present, must have same length as the number of input
+        array dimensions. Single integers `i` are treated as if they were the
+        tuple `(i,)`.
+    axis : int or tuple of int, optional
+        Axis or axes along which the sliding window is applied.
+        By default, the sliding window is applied to all axes and
+        `window_shape[i]` will refer to axis `i` of `x`.
+        If `axis` is given as a `tuple of int`, `window_shape[i]` will refer to
+        the axis `axis[i]` of `x`.
+        Single integers `i` are treated as if they were the tuple `(i,)`.
+    subok : bool, optional
+        If True, sub-classes will be passed-through, otherwise the returned
+        array will be forced to be a base-class array (default).
+    writeable : bool, optional
+        When true, allow writing to the returned view. The default is false,
+        as this should be used with caution: the returned view contains the
+        same memory location multiple times, so writing to one location will
+        cause others to change.
+    Returns
+    -------
+    view : ndarray
+        Sliding window view of the array. The sliding window dimensions are
+        inserted at the end, and the original dimensions are trimmed as
+        required by the size of the sliding window.
+        That is, ``view.shape = x_shape_trimmed + window_shape``, where
+        ``x_shape_trimmed`` is ``x.shape`` with every entry reduced by one less
+        than the corresponding window size.
+    See Also
+    --------
+    lib.stride_tricks.as_strided: A lower-level and less safe routine for
+        creating arbitrary views from custom shape and strides.
+    broadcast_to: broadcast an array to a given shape.
+    Notes
+    -----
+    For many applications using a sliding window view can be convenient, but
+    potentially very slow. Often specialized solutions exist, for example:
+    - `scipy.signal.fftconvolve`
+    - filtering functions in `scipy.ndimage`
+    - moving window functions provided by
+      `bottleneck <https://github.com/pydata/bottleneck>`_.
+    As a rough estimate, a sliding window approach with an input size of `N`
+    and a window size of `W` will scale as `O(N*W)` where frequently a special
+    algorithm can achieve `O(N)`. That means that the sliding window variant
+    for a window size of 100 can be a 100 times slower than a more specialized
+    version.
+    Nevertheless, for small window sizes, when no custom algorithm exists, or
+    as a prototyping and developing tool, this function can be a good solution.
+    Examples
+    --------
+    >>> x = np.arange(6)
+    >>> x.shape
+    (6,)
+    >>> v = sliding_window_view(x, 3)
+    >>> v.shape
+    (4, 3)
+    >>> v
+    array([[0, 1, 2],
+           [1, 2, 3],
+           [2, 3, 4],
+           [3, 4, 5]])
+    This also works in more dimensions, e.g.
+    >>> i, j = np.ogrid[:3, :4]
+    >>> x = 10*i + j
+    >>> x.shape
+    (3, 4)
+    >>> x
+    array([[ 0,  1,  2,  3],
+           [10, 11, 12, 13],
+           [20, 21, 22, 23]])
+    >>> shape = (2,2)
+    >>> v = sliding_window_view(x, shape)
+    >>> v.shape
+    (2, 3, 2, 2)
+    >>> v
+    array([[[[ 0,  1],
+             [10, 11]],
+            [[ 1,  2],
+             [11, 12]],
+            [[ 2,  3],
+             [12, 13]]],
+           [[[10, 11],
+             [20, 21]],
+            [[11, 12],
+             [21, 22]],
+            [[12, 13],
+             [22, 23]]]])
+    The axis can be specified explicitly:
+    >>> v = sliding_window_view(x, 3, 0)
+    >>> v.shape
+    (1, 4, 3)
+    >>> v
+    array([[[ 0, 10, 20],
+            [ 1, 11, 21],
+            [ 2, 12, 22],
+            [ 3, 13, 23]]])
+    The same axis can be used several times. In that case, every use reduces
+    the corresponding original dimension:
+    >>> v = sliding_window_view(x, (2, 3), (1, 1))
+    >>> v.shape
+    (3, 1, 2, 3)
+    >>> v
+    array([[[[ 0,  1,  2],
+             [ 1,  2,  3]]],
+           [[[10, 11, 12],
+             [11, 12, 13]]],
+           [[[20, 21, 22],
+             [21, 22, 23]]]])
+    Combining with stepped slicing (`::step`), this can be used to take sliding
+    views which skip elements:
+    >>> x = np.arange(7)
+    >>> sliding_window_view(x, 5)[:, ::2]
+    array([[0, 2, 4],
+           [1, 3, 5],
+           [2, 4, 6]])
+    or views which move by multiple elements
+    >>> x = np.arange(7)
+    >>> sliding_window_view(x, 3)[::2, :]
+    array([[0, 1, 2],
+           [2, 3, 4],
+           [4, 5, 6]])
+    A common application of `sliding_window_view` is the calculation of running
+    statistics. The simplest example is the
+    `moving average <https://en.wikipedia.org/wiki/Moving_average>`_:
+    >>> x = np.arange(6)
+    >>> x.shape
+    (6,)
+    >>> v = sliding_window_view(x, 3)
+    >>> v.shape
+    (4, 3)
+    >>> v
+    array([[0, 1, 2],
+           [1, 2, 3],
+           [2, 3, 4],
+           [3, 4, 5]])
+    >>> moving_average = v.mean(axis=-1)
+    >>> moving_average
+    array([1., 2., 3., 4.])
+    Note that a sliding window approach is often **not** optimal (see Notes).
+    """
+    window_shape = (tuple(window_shape)
+                    if np.iterable(window_shape)
+                    else (window_shape,))
+    # first convert input to array, possibly keeping subclass
+    x = np.array(x, copy=False, subok=subok)
+
+    window_shape_array = np.array(window_shape)
+    if np.any(window_shape_array < 0):
+        raise ValueError('`window_shape` cannot contain negative values')
+
+    if axis is None:
+        axis = tuple(range(x.ndim))
+        if len(window_shape) != len(axis):
+            raise ValueError(f'Since axis is `None`, must provide '
+                             f'window_shape for all dimensions of `x`; '
+                             f'got {len(window_shape)} window_shape elements '
+                             f'and `x.ndim` is {x.ndim}.')
+    else:
+        axis = normalize_axis_tuple(axis, x.ndim, allow_duplicate=True)
+        if len(window_shape) != len(axis):
+            raise ValueError(f'Must provide matching length window_shape and '
+                             f'axis; got {len(window_shape)} window_shape '
+                             f'elements and {len(axis)} axes elements.')
+
+    out_strides = x.strides + tuple(x.strides[ax] for ax in axis)
+
+    # note: same axis can be windowed repeatedly
+    x_shape_trimmed = list(x.shape)
+    for ax, dim in zip(axis, window_shape):
+        if x_shape_trimmed[ax] < dim:
+            raise ValueError(
+                'window shape cannot be larger than input array shape')
+        x_shape_trimmed[ax] -= dim - 1
+    out_shape = tuple(x_shape_trimmed) + window_shape
+    return as_strided(x, strides=out_strides, shape=out_shape,
+                      subok=subok, writeable=writeable)