From 1eaba0c22148357b4c35a6d0d5cca8d140c913f2 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Wed, 15 Dec 2021 21:11:12 +0100 Subject: [PATCH] fix(Evaluate): ignore empty data at evaluation time, add backtesting metrics (sharpe, etc), fixed crash when predicting `0.0` (#23) * fix(Evaluate): ignore empty data at evaluation time, so we don't inflate the model's performance * refactor(Pipeline): pass in data_loader arguments to the pipeline * feat(Evaluation): added sharpe, sortino, etc * fix: Took out the method to fill NaN numbers with 0s. This way in evaluation we can ignore NaN values. * fix: Fix of the fix added fillna back. Either we root out NaN lines in the very beginning or we stick with the method you created. Co-authored-by: Daniel Szemerey --- load_data.py | 7 ++++-- run_pipeline.py | 52 +++++++++++++++++++++++++++---------------- utils/evaluate.py | 46 +++++++++++++++++++++++++++----------- utils/walk_forward.py | 7 +++++- 4 files changed, 77 insertions(+), 35 deletions(-) diff --git a/load_data.py b/load_data.py index 40e6d74..6bc9c98 100644 --- a/load_data.py +++ b/load_data.py @@ -9,8 +9,11 @@ from sklearn.preprocessing import OneHotEncoder #%% -def get_all_assets(path: str) -> list[str]: - return [f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')] +def get_crypto_assets(path: str) -> list[str]: + return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' in f and not f.startswith('.')]) + +def get_etf_assets(path: str) -> list[str]: + return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' not in f and not f.startswith('.')]) def load_data(path: str, diff --git a/run_pipeline.py b/run_pipeline.py index 6033dcc..46fb241 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -1,12 +1,13 @@ +from logging import log from typing import Literal from sklearnex import patch_sklearn patch_sklearn() -from load_data import get_all_assets, load_data +from load_data import get_crypto_assets, get_etf_assets, load_data from utils.evaluate import evaluate_predictions import pandas as pd -from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression +from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression, Ridge from sklearn.tree import DecisionTreeClassifier from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier from sklearn.discriminant_analysis import LinearDiscriminantAnalysis @@ -19,7 +20,9 @@ from sklearn.preprocessing import MinMaxScaler from utils.walk_forward import walk_forward_train_test regression_models = [ - ('LR', LinearRegression(n_jobs=-1)), + # ('LR', LinearRegression(n_jobs=-1)), + ('Lasso', Lasso(alpha=0.1, max_iter=10000)), + ('Ridge', Ridge(alpha=1.0)), ('BayesianRidge', BayesianRidge()), ('KNN', KNeighborsRegressor(n_neighbors=15)), # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)), @@ -41,6 +44,7 @@ classification_models = [ def run_whole_pipeline( ticker_to_predict: str, + load_data_args: dict, models, method: Literal['regression', 'classification'], sliding_window_size: int, @@ -49,20 +53,7 @@ def run_whole_pipeline( ): print('--------\nPredicting: ', ticker_to_predict) - X, y = load_data(path='data/', - target_asset=ticker_to_predict, - target_asset_lags=[1,2,3,4,5,6,8,10,15], - load_other_assets=False, - other_asset_lags=[], - log_returns=True, - add_date_features=True, - own_technical_features='level2', - other_technical_features='none', - exogenous_features='none', - index_column='int', - method=method, - ) - + X, y = load_data(**load_data_args) if scaling: # TODO: should move scaling to an expanding window compomenent, probably worth not turning it on for now @@ -82,19 +73,42 @@ def run_whole_pipeline( window_size = sliding_window_size, retrain_every = retrain_every ) - result = evaluate_predictions(model_name, y, preds, sliding_window_size, method) + result = evaluate_predictions( + model_name = model_name, + y_true = y, + y_pred = preds, + sliding_window_size = sliding_window_size, + method = method, + ) column_name = ticker_to_predict + "_" + model_name results[column_name] = result return results results = pd.DataFrame() -all_assets = get_all_assets('data/') +all_assets = get_crypto_assets('data/') + + for asset in all_assets: for method in ['regression']: + load_data_args = dict(path='data/', + target_asset= asset, + target_asset_lags= [1,2,3,4,5,6,8,10,15], + load_other_assets= False, + other_asset_lags= [], + log_returns= True, + add_date_features= True, + own_technical_features= 'level2', + other_technical_features= 'none', + exogenous_features= 'none', + index_column= 'int', + method= method, + ) + current_result = run_whole_pipeline( ticker_to_predict = asset, + load_data_args = load_data_args, models = regression_models if method == 'regression' else classification_models, method = method, sliding_window_size = 120, diff --git a/utils/evaluate.py b/utils/evaluate.py index 0d03d15..a2b4b11 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -1,14 +1,13 @@ from typing import Literal -from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, r2_score, classification_report -from sklearn.metrics import confusion_matrix +from sklearn.metrics import mean_absolute_error, accuracy_score, r2_score, f1_score, precision_score, recall_score +from quantstats.stats import expected_return, sharpe, skew, sortino import pandas as pd import numpy as np -def format_data_for_backtest(aggregated_data: pd.DataFrame, returns_col: str, only_test_data: pd.DataFrame, preds) -> pd.DataFrame: - backtest_data = aggregated_data.iloc[-only_test_data.shape[0]:].copy()[returns_col] - assert backtest_data.shape[0] == only_test_data.shape[0] - backtest_data = backtest_data.reset_index(drop=True) - return pd.concat([backtest_data, pd.Series(preds)], axis='columns') +def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.02) -> pd.Series: + delta_pos = signal.diff(1).abs().fillna(0.) + costs = transaction_cost * delta_pos + return (signal * returns) - costs def __preprocess(y_true: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']): @@ -26,33 +25,54 @@ def __preprocess(y_true: pd.Series, y_pred: pd.Series, method: Literal['classifi df['is_incorrect'] = 0 df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong df['is_predicted'] = df.is_correct + df.is_incorrect - df['result'] = df.sign_pred * df.y_true + df['result'] = backtest(df.y_true, df.sign_pred) + return df def evaluate_predictions(model_name: str, y_true: pd.Series, y_pred: pd.Series, sliding_window_size: int, method: Literal['classification', 'regression']): - evaluate_from = sliding_window_size+1 + # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size) + first_nonzero_return = np.where(y_true != 0)[0][0] + evaluate_from = first_nonzero_return + sliding_window_size + 1 + y_true = pd.Series(y_true[evaluate_from:]) + # if there are lots of zeros in the ground truth returns, probably something is wrong, but we can tolerate a couple of days of missing data. + is_zero = y_true[y_true == 0] + assert len(is_zero) < 5 + # we can't deal with 0 returns, so we'll just remap the few examples to 1 + y_true = y_true.apply(lambda x: 1 if x == 0 else x) y_pred = pd.Series(y_pred[evaluate_from:]) df = __preprocess(y_true, y_pred, method) scorecard = pd.Series() if method == 'regression': - scorecard.loc['RSQ'] = r2_score(df.y_true,df.y_pred) - scorecard.loc['MAE'] = mean_absolute_error(df.y_true,df.y_pred) + scorecard.loc['RSQ'] = r2_score(df.y_true, df.y_pred) + scorecard.loc['MAE'] = mean_absolute_error(df.y_true, df.y_pred) elif method == 'classification': scorecard.loc['RSQ'] = 0. scorecard.loc['MAE Matrix'] = 0. - scorecard.loc['directional_accuracy'] = df.is_correct.sum()*1. / (df.is_predicted.sum()*1.)*100 + sign_true = df.sign_true.astype(int) + sign_pred = df.sign_pred.astype(int) + + scorecard.loc['sharpe'] = sharpe(df.result) + scorecard.loc['sortino'] = sortino(df.result) + scorecard.loc['skew'] = skew(df.result) + + scorecard.loc['accuracy'] = accuracy_score(sign_true, sign_pred) * 100 + scorecard.loc['recall'] = recall_score(sign_true, sign_pred, labels = [1, -1]) + scorecard.loc['precision'] = precision_score(sign_true, sign_pred, labels = [1, -1]) + scorecard.loc['f1_score'] = f1_score(sign_true, sign_pred, labels = [1, -1]) scorecard.loc['edge'] = df.result.mean() scorecard.loc['noise'] = df.y_pred.diff().abs().mean() scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise'] + + if method == 'regression': scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE'] elif method == 'classification': scorecard.loc['edge_to_mae'] = 0. - # TODO: add confusion matrix, f1 score, precision, recall + scorecard = scorecard.round(3) print("Model name: ", model_name) print(scorecard) return scorecard diff --git a/utils/walk_forward.py b/utils/walk_forward.py index dd3a7a4..740b6b8 100644 --- a/utils/walk_forward.py +++ b/utils/walk_forward.py @@ -38,6 +38,11 @@ def walk_forward_train_test( models[window_end] = current_model next_timestep = X.iloc[window_end+1].to_numpy().reshape(1, -1) - predictions[window_end+1] = current_model.predict(next_timestep).item() + prediction = current_model.predict(next_timestep).item() + if prediction == 0.: + # TODO: we shouldn't feed in zeros to the model, and skip training / predicting when everything is 0 + # print("Warning: model predicted 0., overriding it with 0.0001") + prediction = 0.0001 + predictions[window_end+1] = prediction return models, predictions