From 7aedb910697701b07128317b6005938e9cc36db7 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Wed, 1 Dec 2021 09:28:24 +0100 Subject: [PATCH] feat(Data): added various data loading config options, walk forward method draft (#9) * feat(Eval): added format_data_for_backtest() * feat(Data): added many configurable parameters to load_files to reduce boilerplate and prepare for HPO * feat(Core): added walk forward method of training/testing * fix(Model): remove the unnecessary softmax activation from the keras models * feat(Core): added walk_forward_train_test() --- keras_models/classification.py | 6 +- load_data.py | 118 ++++++----- model_classification_keras.py | 28 ++- model_classification_sklearn.py | 156 +++++++++++++++ model_classification_sklearn_walk_forward.py | 164 +++++++++++++++ model_classification_sktime.py | 68 +++++-- utils/evaluate.py | 13 +- utils/sliding_window.py | 200 +++++++++++++++++++ 8 files changed, 678 insertions(+), 75 deletions(-) create mode 100644 model_classification_sklearn.py create mode 100644 model_classification_sklearn_walk_forward.py create mode 100644 utils/sliding_window.py diff --git a/keras_models/classification.py b/keras_models/classification.py index 0bc5e93..b840b3e 100644 --- a/keras_models/classification.py +++ b/keras_models/classification.py @@ -9,7 +9,7 @@ def create_basic_lstm_model(input_shape, num_classes): model.add(keras.layers.Dropout(0.3)) model.add(keras.layers.Dense(units = 32, activation = 'sigmoid')) model.add(keras.layers.Dropout(0.3)) - model.add(keras.layers.Dense(units = num_classes, activation = 'softmax')) + model.add(keras.layers.Dense(units = num_classes, activation = 'linear')) return model def create_basic_cnn_model(input_shape, num_classes): @@ -24,7 +24,7 @@ def create_basic_cnn_model(input_shape, num_classes): model.add(keras.layers.BatchNormalization()) model.add(keras.layers.ReLU()) model.add(keras.layers.GlobalAveragePooling1D()) - model.add(keras.layers.Dense(num_classes, activation="softmax")) + model.add(keras.layers.Dense(num_classes, activation="linear")) return model @@ -95,7 +95,7 @@ def create_resnet_cnn_model(input_shape, num_classes): gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3) - output_layer = keras.layers.Dense(num_classes, activation='softmax')(gap_layer) + output_layer = keras.layers.Dense(num_classes, activation='linear')(gap_layer) model = keras.models.Model(inputs=input_layer, outputs=output_layer) diff --git a/load_data.py b/load_data.py index 2962021..acbf176 100644 --- a/load_data.py +++ b/load_data.py @@ -2,13 +2,28 @@ import pandas as pd import os import numpy as np +from pandas.core.frame import DataFrame from utils.technical_indicators import ROC, RSI, STOK, STOD +from typing import Literal #%% -def load_files(path: str, add_features: bool, log_returns: bool, narrow_format: bool = False) -> pd.DataFrame: +def load_files(path: str, + own_asset: str, + load_other_assets: bool, + log_returns: bool, + add_date_features: bool, + own_technical_features: Literal['none', 'level1', 'level2'], + other_technical_features: Literal['none', 'level1', 'level2'], + exogenous_features: Literal['none', 'level1'], + index_column: Literal['date', 'int'], + narrow_format: bool = False + ) -> pd.DataFrame: + files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')] - dfs = [__load_df(os.path.join(path,f), f.split('.')[0], add_features, log_returns, narrow_format) for f in files] + files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(own_asset))] + def is_own_asset(own_asset: str, file: str): return file.split('.')[0].startswith(own_asset) + dfs = [__load_df(path=os.path.join(path,f), prefix=f.split('.')[0], log_returns=log_returns, technical_features=own_technical_features if is_own_asset(own_asset, f) else other_technical_features, narrow_format=narrow_format) for f in files] if narrow_format: dfs = pd.concat(dfs, axis=0).fillna(0.) else: @@ -16,17 +31,20 @@ def load_files(path: str, add_features: bool, log_returns: bool, narrow_format: dfs.index = pd.DatetimeIndex(dfs.index) - if add_features: + if add_date_features: dfs['day_month'] = dfs.index.day dfs['day_week'] = dfs.index.dayofweek dfs['month'] = dfs.index.month + if index_column == 'int': + dfs.reset_index(drop=True, inplace=True) + if narrow_format: return dfs else: return dfs.drop(index=dfs.index[0], axis=0) -def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool, narrow_format: bool = False) -> pd.DataFrame: +def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Literal['none', 'level1', 'level2'], narrow_format: bool = False) -> pd.DataFrame: df = pd.read_csv(path, header=0, index_col=0).fillna(0) if log_returns: @@ -34,7 +52,22 @@ def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool, nar else: df['returns'] = df['close'].pct_change() - if add_features: + df = __augment_derived_features(df, log_returns=log_returns, technical_features=technical_features) + + df = df.replace([np.inf, -np.inf], 0.) + df = df.drop(columns=['open', 'high', 'low', 'close']) + # we're not ready for this just yet + if 'volume' in df.columns: + df = df.drop(columns=['volume']) + + if narrow_format: + df["ticker"] = np.repeat(prefix, df.shape[0]) + else: + df.columns = [prefix + "_" + c for c in df.columns] + return df + +def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_features: Literal['none', 'level1', 'level2']) -> pd.DataFrame: + if technical_features == 'level1' or technical_features == 'level2': # volatility (10, 20, 30 days) df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5) df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5) @@ -55,30 +88,23 @@ def __load_df(path: str, prefix: str, add_features: bool, log_returns: bool, nar df['mom_60'] = df['close'].pct_change(60) df['mom_90'] = df['close'].pct_change(90) + if technical_features == 'level2': + df['roc_10'] = ROC(df['close'], 10) + df['roc_30'] = ROC(df['close'], 30) - df['roc_10'] = ROC(df['close'], 10) - df['roc_30'] = ROC(df['close'], 30) + df['rsi_10'] = RSI(df['close'], 10) + df['rsi_30'] = RSI(df['close'], 30) + df['rsi_100'] = RSI(df['close'], 30) - df['rsi_10'] = RSI(df['close'], 10) - df['rsi_30'] = RSI(df['close'], 30) - df['rsi_100'] = RSI(df['close'], 30) - - df['stok_10'] = STOK(df['close'], df['low'], df['high'], 10) - df['stod_10'] = STOD(df['close'], df['low'], df['high'], 10) - df['stok_30'] = STOK(df['close'], df['low'], df['high'], 30) - df['stod_30'] = STOD(df['close'], df['low'], df['high'], 30) - df['stok_200'] = STOK(df['close'], df['low'], df['high'], 200) - df['stod_200'] = STOD(df['close'], df['low'], df['high'], 200) - - - df = df.replace([np.inf, -np.inf], 0.) - df = df.drop(columns=['open', 'high', 'low', 'close']) - if narrow_format: - df["ticker"] = np.repeat(prefix, df.shape[0]) - else: - df.columns = [prefix + "_" + c for c in df.columns] + df['stok_10'] = STOK(df['close'], df['low'], df['high'], 10) + df['stod_10'] = STOD(df['close'], df['low'], df['high'], 10) + df['stok_30'] = STOK(df['close'], df['low'], df['high'], 30) + df['stod_30'] = STOD(df['close'], df['low'], df['high'], 30) + df['stok_200'] = STOK(df['close'], df['low'], df['high'], 200) + df['stod_200'] = STOD(df['close'], df['low'], df['high'], 200) return df + # %% def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: df['target'] = df[source_column].diff(period).shift(-period) @@ -86,34 +112,32 @@ def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, peri return df -#%% -def create_target_pos_neg_classes(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: - if period > 0: - df['target'] = df[source_column].diff(period).shift(-period) - else: - df['target'] = df[source_column].diff(period) - df['target'] = df['target'].map(lambda x: 0 if x <= 0.0 else 1) - if period > 0: - df = df.iloc[:-period] - return df +def create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.DataFrame: -def create_target_four_classes(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: - def __get_class(x): - treshold = 0.08 - if x <= -treshold: + def get_class_binary(x): + return 0 if x <= 0.0 else 1 + + def get_class_threeway(x): + bins = pd.qcut(df[source_column], 4, duplicates='raise', retbins=True)[1] + lower_threshold = bins[1] + upper_threshold = bins[3] + if x <= lower_threshold: + return -1 + elif x > lower_threshold and x < upper_threshold: return 0 - elif x > -treshold and x <= 0: - return 1 - elif x > 0 and x <= treshold: - return 2 else: - return 3 + return 1 if period > 0: - df['target'] = df[source_column].diff(period).shift(-period) + df['target'] = df[source_column].shift(-period) else: - df['target'] = df[source_column].diff(period) - df['target'] = df['target'].map(__get_class) + df['target'] = df[source_column] + + get_class_function = get_class_binary + if no_of_classes == "three": + get_class_function = get_class_threeway + + df['target'] = df['target'].map(get_class_function) if period > 0: df = df.iloc[:-period] return df \ No newline at end of file diff --git a/model_classification_keras.py b/model_classification_keras.py index 999bc85..fc4af00 100644 --- a/model_classification_keras.py +++ b/model_classification_keras.py @@ -1,6 +1,6 @@ #%% Import all the stuff, load data, define constants from sklearn.utils import shuffle -from load_data import load_files, create_target_pos_neg_classes +from load_data import load_files, create_target_classes import pandas as pd from tensorflow import keras from utils.normalize import normalize @@ -13,13 +13,20 @@ from utils.rolling import rolling_window from keras_models.classification import create_basic_cnn_model, create_basic_lstm_model, create_resnet_cnn_model from keras_models.classification_transformer import create_basic_transformer_model -data = load_files('data/', add_features=True, log_returns=False) -data.reset_index(drop=True, inplace=True) -data = data[[column for column in data.columns if not column.endswith('volume')]] -# data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]] +data = load_files(path='data/', + own_asset='BTC_ETH', + load_other_assets=True, + log_returns=False, + add_date_features=True, + own_technical_features='level2', + other_technical_features='level2', + exogenous_features='none', + index_column='int' +) + target_col = 'target' -data = create_target_pos_neg_classes(data, 'BTC_ETH_returns', 1) +data = create_target_classes(data, 'BTC_ETH_returns', 1, 'two') num_classes = 2 learning_rate = 0.002 @@ -71,7 +78,7 @@ dataset_val = keras.utils.timeseries_dataset_from_array( #%% -for batch in dataset_train.take(10): +for batch in dataset_train.take(1): batch_inputs, batch_targets = batch print("Input shape:", batch_inputs.shape) @@ -83,9 +90,9 @@ n_features = batch_inputs.shape[2] # print(batch_targets) # %% -model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) +# model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) # model = create_basic_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) -# model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) +model = create_resnet_cnn_model(input_shape=(n_timestamps, n_features), num_classes=num_classes) # model = create_basic_transformer_model( # input_shape=(n_timestamps, n_features), # n_classes=num_classes, @@ -99,7 +106,8 @@ model = create_basic_lstm_model(input_shape=(n_timestamps, n_features), num_clas # ) optimizer = keras.optimizers.Adam(learning_rate=learning_rate) -model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy']) +loss = keras.losses.CategoricalCrossentropy(from_logits=True) +model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy']) model.summary() # %% diff --git a/model_classification_sklearn.py b/model_classification_sklearn.py new file mode 100644 index 0000000..6581dd0 --- /dev/null +++ b/model_classification_sklearn.py @@ -0,0 +1,156 @@ +#%% Import all the stuff, load data, define constants +from sklearnex import patch_sklearn +patch_sklearn() + +from load_data import create_target_cum_forward_returns, create_target_classes, load_files +from sktime.forecasting.model_selection import temporal_train_test_split +from sklearn.metrics import accuracy_score +from sklearn.metrics import confusion_matrix +# from utils.evaluate import print_classification_metrics, format_data_for_backtest + +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV +from sklearn.linear_model import LogisticRegression +from sklearn.tree import DecisionTreeClassifier +from sklearn.neighbors import KNeighborsClassifier +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis +from sklearn.naive_bayes import GaussianNB +from sklearn.svm import SVC +from sklearn.neural_network import MLPClassifier +from sklearn.pipeline import Pipeline +from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier +from sklearn.metrics import classification_report, confusion_matrix, accuracy_score +from sklearn.preprocessing import MinMaxScaler + +from utils.sliding_window import sliding_window_and_flatten + +ticket_to_predict = 'BTC_ETH' +print('Predicting: ', ticket_to_predict) + +data = load_files(path='data/', + own_asset=ticket_to_predict, + load_other_assets=True, + log_returns=True, + add_date_features=True, + own_technical_features='level2', + other_technical_features='none', + exogenous_features='none', + index_column='int' +) + +target_col = 'target' +returns_col = ticket_to_predict + '_returns' +data = create_target_classes(data, returns_col, 1, 'three') + +X = data.drop(columns=['target']) +y = data[target_col] + +X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2) +feature_scaler = MinMaxScaler(feature_range= (-1, 1)) +X_test_orig = X_test.copy() +X_train = feature_scaler.fit_transform(X_train) +X_test = feature_scaler.transform(X_test) + + +#%% + +sliding_window_size = 10 +X_train = sliding_window_and_flatten(X_train, sliding_window_size) +X_test = sliding_window_and_flatten(X_test, sliding_window_size) +X_test_orig = X_test_orig.iloc[sliding_window_size-1:] +y_train = y_train[sliding_window_size-1:] +y_test = y_test[sliding_window_size-1:] + +assert X_train.shape[0] == y_train.shape[0] +assert X_test.shape[0] == y_test.shape[0] +scoring = 'accuracy' + +# %% +num_folds = 10 + +models = [] +models.append(('LR', LogisticRegression(n_jobs=-1))) +models.append(('LDA', LinearDiscriminantAnalysis())) +models.append(('KNN', KNeighborsClassifier())) +models.append(('CART', DecisionTreeClassifier())) +models.append(('NB', GaussianNB())) +# models.append(('NN', MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False))) +models.append(('AB', AdaBoostClassifier())) +# models.append(('GBM', GradientBoostingClassifier())) +models.append(('RF', RandomForestClassifier(n_jobs=-1))) + +results = [] +names = [] +for name, model in models: + kfold = KFold(n_splits=num_folds, shuffle=False) + cv_results = cross_val_score(model, X_train, y_train, cv=kfold, scoring=scoring) + results.append(cv_results) + names.append(name) + msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std()) + print(msg) + +# # compare algorithms +# fig = plt.figure() +# fig.suptitle('Algorithm Comparison') +# ax = fig.add_subplot(111) +# plt.boxplot(results) +# ax.set_xticklabels(names) +# fig.set_size_inches(15,8) +# plt.show() + +#%% +# n_estimators = [20,80] +# max_depth= [5,10, 15] +# criterion = ["gini","entropy"] +# param_grid = dict(n_estimators=n_estimators, max_depth=max_depth, criterion = criterion ) +# model = RandomForestClassifier(n_jobs=-1) +# kfold = KFold(n_splits=10, shuffle=False) +# grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scoring, cv=kfold) +# grid_result = grid.fit(X_train, y_train) + +# #Print Results +# print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) +# means = grid_result.cv_results_['mean_test_score'] +# stds = grid_result.cv_results_['std_test_score'] +# params = grid_result.cv_results_['params'] +# ranks = grid_result.cv_results_['rank_test_score'] +# for mean, stdev, param, rank in zip(means, stds, params, ranks): +# print("#%d %f (%f) with: %r" % (rank, mean, stdev, param)) + + +#%% prepare model +model = RandomForestClassifier(criterion='entropy', n_estimators=80, max_depth=5, n_jobs=-1) +# model = LogisticRegression() +# model = MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False, max_iter=1000) +model = GaussianNB() +model.fit(X_train, y_train) + + +#%% +# estimate accuracy on validation set +predictions = model.predict(X_test) +print(accuracy_score(y_test, predictions)) +print(confusion_matrix(y_test, predictions)) +print(classification_report(y_test, predictions)) + + +#%% +# feat_importance = pd.DataFrame({'Importance':model.feature_importances_*100}, index=X.columns) +# feat_importance.sort_values('Importance', axis=0, ascending=True) +# feat_importance.plot(kind='barh', color='r' ) +# plt.xlabel('Variable Importance') +# print(feat_importance) + +#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day +backtestdata = pd.DataFrame(index= X_test_orig.index) +backtestdata['signal_pred'] = predictions +backtestdata['signal_actual'] = y_test +backtestdata['returns'] = X_test_orig[returns_col] +backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1) +backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1) + +# %% +print(backtestdata.cumsum().apply(np.exp).tail(1)) + +# %% diff --git a/model_classification_sklearn_walk_forward.py b/model_classification_sklearn_walk_forward.py new file mode 100644 index 0000000..0d65cf2 --- /dev/null +++ b/model_classification_sklearn_walk_forward.py @@ -0,0 +1,164 @@ +#%% Import all the stuff, load data, define constants +from sklearnex import patch_sklearn +patch_sklearn() + +from load_data import create_target_cum_forward_returns, create_target_classes, load_files +from sktime.forecasting.model_selection import temporal_train_test_split +from sklearn.metrics import accuracy_score +from sklearn.metrics import confusion_matrix +# from utils.evaluate import print_classification_metrics, format_data_for_backtest + +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV +from sklearn.linear_model import LogisticRegression +from sklearn.tree import DecisionTreeClassifier +from sklearn.neighbors import KNeighborsClassifier +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis +from sklearn.naive_bayes import GaussianNB +from sklearn.svm import SVC +from sklearn.neural_network import MLPClassifier +from sklearn.pipeline import Pipeline +from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier +from sklearn.metrics import classification_report, confusion_matrix, accuracy_score +from sklearn.preprocessing import MinMaxScaler + +from utils.sliding_window import sliding_window_and_flatten + +ticket_to_predict = 'BTC_ETH' +print('Predicting: ', ticket_to_predict) + +data = load_files(path='data/', + own_asset=ticket_to_predict, + load_other_assets=False, + log_returns=True, + add_date_features=True, + own_technical_features='level1', + other_technical_features='none', + exogenous_features='none', + index_column='int' +) + +target_col = 'target' +returns_col = ticket_to_predict + '_returns' +data = create_target_classes(data, returns_col, 1, 'two') + +X = data.drop(columns=[target_col]) +y = data[target_col] + +X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.1) +feature_scaler = MinMaxScaler(feature_range= (-1, 1)) +X_test_orig = X_test.copy() +X_train = feature_scaler.fit_transform(X_train) +X_test = feature_scaler.transform(X_test) + + +#%% + +sliding_window_size = 120 +X_train = sliding_window_and_flatten(X_train, sliding_window_size) +# X_test = sliding_window_and_flatten(X_test, sliding_window_size) +X_test_orig = X_test_orig.iloc[sliding_window_size-1:] +y_train = y_train[sliding_window_size-1:] +# y_test = y_test[sliding_window_size-1:] + + +def walk_forward_train_test( + create_model, + X_train: pd.DataFrame, + y_train: pd.Series, + window_size: int, + retrain_every: int + ): + predictions = [None] * (len(y_train)-1) + models = [None] * (len(y_train)-1) + + train_from = sliding_window_size+1 + train_till = len(y_train)-2 + + iterations_since_retrain = 0 + + for i in range(train_from, train_till): + if i % 10 == 0: print('Fold: ', i) + iterations_since_retrain += 1 + window_start = i - window_size + window_end = i + X_train_slice = X_train[window_start:window_end] + y_train_slice = y_train[window_start:window_end] + + if iterations_since_retrain >= retrain_every or models[i-1] is None: + model = create_model() + model.fit(X_train_slice, y_train_slice) + models.append(model) + else: + model = models[i-1] + models[window_end] = model + + predictions[window_end+1] = model.predict(X_train[window_end+1].reshape(1, -1)).item() + return models, predictions + + +#%% +models, preds = walk_forward_train_test(lambda : GaussianNB(), X_train, y_train, 120, 10) + +# %% + +models = [] +models.append(('LR', LogisticRegression(n_jobs=-1))) +models.append(('LDA', LinearDiscriminantAnalysis())) +models.append(('KNN', KNeighborsClassifier())) +models.append(('CART', DecisionTreeClassifier())) +models.append(('NB', GaussianNB())) +# models.append(('NN', MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False))) +models.append(('AB', AdaBoostClassifier())) +# models.append(('GBM', GradientBoostingClassifier())) +models.append(('RF', RandomForestClassifier(n_jobs=-1))) +# model = RandomForestClassifier(n_jobs=-1) +# model = KNeighborsClassifier() +model = GaussianNB() +# model = AdaBoostClassifier() + +# re-train the model every n steps +# store the model +# iterate over all potential folds +# retrieve model for the range +# predict on the input data +# store the predictions + +predictions = [0] * (len(y_train)-1) + + +for i in range(sliding_window_size+1, len(y_train)-2): + if i % 10 == 0: print('Fold: ', i) + X_train_up_to_i = X_train[i-sliding_window_size:i] + y_train_up_to_i = y_train[i-sliding_window_size:i] + model.fit(X_train_up_to_i, y_train_up_to_i) + predictions[i+1] = model.predict(X_train[i+1].reshape(1, -1)).item() + + +#%% + +print(accuracy_score(y_train[500:-1], predictions[500:])) +print(confusion_matrix(y_train[500:-1], predictions[500:])) +print(classification_report(y_train[500:-1], predictions[500:])) + + +#%% +# feat_importance = pd.DataFrame({'Importance':model.feature_importances_*100}, index=X.columns) +# feat_importance.sort_values('Importance', axis=0, ascending=True) +# feat_importance.plot(kind='barh', color='r' ) +# plt.xlabel('Variable Importance') +# print(feat_importance) + +#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day +# backtestdata = pd.DataFrame(index= X_test_orig.index) +# backtestdata['signal_pred'] = predictions +# backtestdata['signal_actual'] = y_test +# backtestdata['returns'] = X_test_orig[returns_col] +# backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1) +# backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1) + +# %% +# print(backtestdata.cumsum().apply(np.exp).tail(1)) + +# %% diff --git a/model_classification_sktime.py b/model_classification_sktime.py index 1ff6548..ccee083 100644 --- a/model_classification_sktime.py +++ b/model_classification_sktime.py @@ -1,5 +1,5 @@ #%% Import all the stuff, load data, define constants -from load_data import create_target_cum_forward_returns, create_target_pos_neg_classes, load_files, create_target_four_classes +from load_data import load_files, create_target_classes import pandas as pd import numpy as np from utils.sktime import from_df_to_sktime_data @@ -9,27 +9,46 @@ from sklearn.metrics import accuracy_score from sklearn.pipeline import Pipeline from sktime.classification.interval_based import ( TimeSeriesForestClassifier, + SupervisedTimeSeriesForest, + ) +from sktime.forecasting.model_selection import SlidingWindowSplitter +from sktime.forecasting.model_selection import ForecastingRandomizedSearchCV + +from sktime.forecasting.compose import make_reduction + +from sklearn.tree import DecisionTreeClassifier from sklearn.metrics import confusion_matrix from sklearn.preprocessing import StandardScaler from sktime.forecasting.model_selection import ( SlidingWindowSplitter, ForecastingGridSearchCV, ) -# from sktime.forecasting.model_evaluation import -# from sklearnex import patch_sklearn -# patch_sklearn() +from utils.evaluate import print_classification_metrics, format_data_for_backtest +from sklearn.ensemble import RandomForestRegressor +from sklearnex import patch_sklearn +patch_sklearn() +ticket_to_predict = 'BTC_USD' +print('Predicting: ', ticket_to_predict) + +data = load_files(path='data/', + own_asset=ticket_to_predict, + load_other_assets=True, + log_returns=True, + add_date_features=True, + own_technical_features='level2', + other_technical_features='none', + exogenous_features='none', + index_column='int' +) -data = load_files('data/', add_features=True, log_returns=True, narrow_format=False) -data.reset_index(drop=True, inplace=True) -data = data[[column for column in data.columns if not column.endswith('volume')]] -data = data[[column for column in data.columns if column.startswith('BTC_ETH_')]] target_col = 'target' -data = create_target_pos_neg_classes(data, 'BTC_ETH_returns', 1) +returns_col = ticket_to_predict + '_returns' +data = create_target_classes(data, returns_col, 1, 'two') -X = data +X = data.drop(columns=[target_col]) y = data[target_col] X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2) @@ -45,11 +64,32 @@ X_test = from_df_to_sktime_data(X_test) # ]) # pipe.fit(X_train, y_train) +# regressor = RandomForestRegressor(n_estimators=20) +# model = DecisionTreeClassifier(random_state=1) +model = TimeSeriesForestClassifier(n_estimators=50, random_state=1) +model.fit(y = y_train, X = X_train) +# forecaster = make_reduction(model, scitype="tabular-regressor") +# nested_params = {"window_length": list(range(2,30)), +# "estimator__max_depth": list(range(5,16))} +# # "estimator__n_estimators": list(range(10,200))} +#%% -model = TimeSeriesForestClassifier(n_estimators=200, random_state=1) -model.fit(X_train, y_train) +# cv = SlidingWindowSplitter(initial_window=40, window_length=30) +# nrcv = ForecastingRandomizedSearchCV(forecaster, strategy="refit", cv=cv, +# param_distributions=nested_params, +# n_iter=5, random_state=42) +# nrcv.fit(y = y_train, X = X_train, fh=np.array([1])) +# print(nrcv.best_params_) +# print(nrcv.best_score_) +# model = DecisionTreeClassifier(random_state=1) +# model.fit(X_train, y_train) + +# preds = nrcv.best_forecaster_.predict( X=X_test) +# # print(preds) preds = model.predict(X_test) -print(model.score(X_test, y_test)) -print(confusion_matrix(y_test, preds)) +print(print_classification_metrics(y_test, preds)) +# backtest_data = format_data_for_backtest(data, returns_col, X_test, preds) +# print(backtest_data) +# %% diff --git a/utils/evaluate.py b/utils/evaluate.py index cafa565..d1aeb66 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -1,6 +1,8 @@ from math import sqrt from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score from sklearn.metrics import confusion_matrix +import pandas as pd + def print_regression_metrics(y_true, y_pred): rmse = sqrt(mean_squared_error(y_true, y_pred)) print("RMSE: %.2f" % rmse) @@ -10,4 +12,13 @@ def print_regression_metrics(y_true, y_pred): def print_classification_metrics(y_true, y_pred): print("Accuracy: %.2f" % accuracy_score(y_true, y_pred)) - print("Confusion Matrix: \n", confusion_matrix(y_true, y_pred)) \ No newline at end of file + print("Confusion Matrix: \n", confusion_matrix(y_true, y_pred)) + +def format_data_for_backtest(aggregated_data: pd.DataFrame, returns_col: str, only_test_data: pd.DataFrame, preds) -> pd.DataFrame: + backtest_data = aggregated_data.iloc[-only_test_data.shape[0]:].copy()[returns_col] + assert backtest_data.shape[0] == only_test_data.shape[0] + backtest_data = backtest_data.reset_index(drop=True) + return pd.concat([backtest_data, pd.Series(preds)], axis='columns') + + +# def backtest() \ No newline at end of file diff --git a/utils/sliding_window.py b/utils/sliding_window.py new file mode 100644 index 0000000..005be8f --- /dev/null +++ b/utils/sliding_window.py @@ -0,0 +1,200 @@ +import numpy as np +from numpy.lib.stride_tricks import as_strided +from numpy.core.numeric import normalize_axis_tuple + +def sliding_window_and_flatten(x: np.array, window_size: int) -> np.array: + n_features = x.shape[1] + x = np.squeeze(__numpy_sliding_window_view(x, (window_size, n_features)), axis = 1) + return x.reshape(x.shape[0], -1) + + +def __numpy_sliding_window_view(x, window_shape, axis=None, *, + subok=False, writeable=False): + """ + Create a sliding window view into the array with the given window shape. + Also known as rolling or moving window, the window slides across all + dimensions of the array and extracts subsets of the array at all window + positions. + + .. versionadded:: 1.20.0 + Parameters + ---------- + x : array_like + Array to create the sliding window view from. + window_shape : int or tuple of int + Size of window over each axis that takes part in the sliding window. + If `axis` is not present, must have same length as the number of input + array dimensions. Single integers `i` are treated as if they were the + tuple `(i,)`. + axis : int or tuple of int, optional + Axis or axes along which the sliding window is applied. + By default, the sliding window is applied to all axes and + `window_shape[i]` will refer to axis `i` of `x`. + If `axis` is given as a `tuple of int`, `window_shape[i]` will refer to + the axis `axis[i]` of `x`. + Single integers `i` are treated as if they were the tuple `(i,)`. + subok : bool, optional + If True, sub-classes will be passed-through, otherwise the returned + array will be forced to be a base-class array (default). + writeable : bool, optional + When true, allow writing to the returned view. The default is false, + as this should be used with caution: the returned view contains the + same memory location multiple times, so writing to one location will + cause others to change. + Returns + ------- + view : ndarray + Sliding window view of the array. The sliding window dimensions are + inserted at the end, and the original dimensions are trimmed as + required by the size of the sliding window. + That is, ``view.shape = x_shape_trimmed + window_shape``, where + ``x_shape_trimmed`` is ``x.shape`` with every entry reduced by one less + than the corresponding window size. + See Also + -------- + lib.stride_tricks.as_strided: A lower-level and less safe routine for + creating arbitrary views from custom shape and strides. + broadcast_to: broadcast an array to a given shape. + Notes + ----- + For many applications using a sliding window view can be convenient, but + potentially very slow. Often specialized solutions exist, for example: + - `scipy.signal.fftconvolve` + - filtering functions in `scipy.ndimage` + - moving window functions provided by + `bottleneck `_. + As a rough estimate, a sliding window approach with an input size of `N` + and a window size of `W` will scale as `O(N*W)` where frequently a special + algorithm can achieve `O(N)`. That means that the sliding window variant + for a window size of 100 can be a 100 times slower than a more specialized + version. + Nevertheless, for small window sizes, when no custom algorithm exists, or + as a prototyping and developing tool, this function can be a good solution. + Examples + -------- + >>> x = np.arange(6) + >>> x.shape + (6,) + >>> v = sliding_window_view(x, 3) + >>> v.shape + (4, 3) + >>> v + array([[0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5]]) + This also works in more dimensions, e.g. + >>> i, j = np.ogrid[:3, :4] + >>> x = 10*i + j + >>> x.shape + (3, 4) + >>> x + array([[ 0, 1, 2, 3], + [10, 11, 12, 13], + [20, 21, 22, 23]]) + >>> shape = (2,2) + >>> v = sliding_window_view(x, shape) + >>> v.shape + (2, 3, 2, 2) + >>> v + array([[[[ 0, 1], + [10, 11]], + [[ 1, 2], + [11, 12]], + [[ 2, 3], + [12, 13]]], + [[[10, 11], + [20, 21]], + [[11, 12], + [21, 22]], + [[12, 13], + [22, 23]]]]) + The axis can be specified explicitly: + >>> v = sliding_window_view(x, 3, 0) + >>> v.shape + (1, 4, 3) + >>> v + array([[[ 0, 10, 20], + [ 1, 11, 21], + [ 2, 12, 22], + [ 3, 13, 23]]]) + The same axis can be used several times. In that case, every use reduces + the corresponding original dimension: + >>> v = sliding_window_view(x, (2, 3), (1, 1)) + >>> v.shape + (3, 1, 2, 3) + >>> v + array([[[[ 0, 1, 2], + [ 1, 2, 3]]], + [[[10, 11, 12], + [11, 12, 13]]], + [[[20, 21, 22], + [21, 22, 23]]]]) + Combining with stepped slicing (`::step`), this can be used to take sliding + views which skip elements: + >>> x = np.arange(7) + >>> sliding_window_view(x, 5)[:, ::2] + array([[0, 2, 4], + [1, 3, 5], + [2, 4, 6]]) + or views which move by multiple elements + >>> x = np.arange(7) + >>> sliding_window_view(x, 3)[::2, :] + array([[0, 1, 2], + [2, 3, 4], + [4, 5, 6]]) + A common application of `sliding_window_view` is the calculation of running + statistics. The simplest example is the + `moving average `_: + >>> x = np.arange(6) + >>> x.shape + (6,) + >>> v = sliding_window_view(x, 3) + >>> v.shape + (4, 3) + >>> v + array([[0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5]]) + >>> moving_average = v.mean(axis=-1) + >>> moving_average + array([1., 2., 3., 4.]) + Note that a sliding window approach is often **not** optimal (see Notes). + """ + window_shape = (tuple(window_shape) + if np.iterable(window_shape) + else (window_shape,)) + # first convert input to array, possibly keeping subclass + x = np.array(x, copy=False, subok=subok) + + window_shape_array = np.array(window_shape) + if np.any(window_shape_array < 0): + raise ValueError('`window_shape` cannot contain negative values') + + if axis is None: + axis = tuple(range(x.ndim)) + if len(window_shape) != len(axis): + raise ValueError(f'Since axis is `None`, must provide ' + f'window_shape for all dimensions of `x`; ' + f'got {len(window_shape)} window_shape elements ' + f'and `x.ndim` is {x.ndim}.') + else: + axis = normalize_axis_tuple(axis, x.ndim, allow_duplicate=True) + if len(window_shape) != len(axis): + raise ValueError(f'Must provide matching length window_shape and ' + f'axis; got {len(window_shape)} window_shape ' + f'elements and {len(axis)} axes elements.') + + out_strides = x.strides + tuple(x.strides[ax] for ax in axis) + + # note: same axis can be windowed repeatedly + x_shape_trimmed = list(x.shape) + for ax, dim in zip(axis, window_shape): + if x_shape_trimmed[ax] < dim: + raise ValueError( + 'window shape cannot be larger than input array shape') + x_shape_trimmed[ax] -= dim - 1 + out_shape = tuple(x_shape_trimmed) + window_shape + return as_strided(x, strides=out_strides, shape=out_shape, + subok=subok, writeable=writeable) \ No newline at end of file