From beb281fc3aae771c81837a865cdc3df6db8fbfc8 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Tue, 14 Dec 2021 21:25:43 +0100 Subject: [PATCH] feat(Evaluation): created a unified evaluation framework for both regression / classification --- load_data.py | 8 +++--- model_walk_forward.py | 21 +++++--------- utils/evaluate.py | 64 +++++++++++++++++++++++-------------------- utils/walk_forward.py | 3 +- 4 files changed, 46 insertions(+), 50 deletions(-) diff --git a/load_data.py b/load_data.py index c64d9d9..d9f9a9e 100644 --- a/load_data.py +++ b/load_data.py @@ -56,9 +56,9 @@ def load_data(path: str, target_col = 'target' returns_col = target_asset + '_returns' if method == 'regression': - dfs = create_target_cum_forward_returns(dfs, returns_col, 1) + dfs = __create_target_cum_forward_returns(dfs, returns_col, 1) elif method == 'classification': - dfs = create_target_classes(dfs, returns_col, 1, 'two') + dfs = __create_target_classes(dfs, returns_col, 1, 'two') X = dfs.drop(columns=[target_col]) y = dfs[target_col] @@ -130,13 +130,13 @@ def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_fe # %% -def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: +def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: df['target'] = df[source_column].diff(period).shift(-period) df = df.iloc[:-period] return df -def create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.DataFrame: +def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.DataFrame: def get_class_binary(x): return 0 if x <= 0.0 else 1 diff --git a/model_walk_forward.py b/model_walk_forward.py index f3fe5f5..e3854b5 100644 --- a/model_walk_forward.py +++ b/model_walk_forward.py @@ -1,15 +1,11 @@ -#%% Import all the stuff, load data, define constants from typing import Literal from sklearnex import patch_sklearn patch_sklearn() -from load_data import create_target_cum_forward_returns, load_data, create_target_classes -from sktime.forecasting.model_selection import temporal_train_test_split -from utils.evaluate import evaluate_predictions_regression, evaluate_predictions_classification +from load_data import load_data +from utils.evaluate import evaluate_predictions -import numpy as np import pandas as pd -from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression from sklearn.tree import DecisionTreeClassifier from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier @@ -18,7 +14,6 @@ from sklearn.svm import SVR from sklearn.naive_bayes import GaussianNB from sklearn.neural_network import MLPRegressor, MLPClassifier from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier -from sklearn.metrics import r2_score, mean_absolute_error, confusion_matrix, classification_report, accuracy_score from sklearn.preprocessing import MinMaxScaler from utils.walk_forward import walk_forward_train_test @@ -68,8 +63,9 @@ def run_whole_pipeline( method=method, ) + if scaling: - # TODO: should move scaling to an expanding window compomenent + # TODO: should move scaling to an expanding window compomenent, probably worth not turning it on for now feature_scaler = MinMaxScaler(feature_range= (-1, 1)) X = pd.DataFrame(feature_scaler.fit_transform(X), columns=X.columns, index=X.index) # TODO: should scale y as well probably @@ -84,10 +80,7 @@ def run_whole_pipeline( window_size = sliding_window_size, retrain_every = retrain_every ) - if method == 'regression': - evaluate_predictions_regression(model_name, y, preds, sliding_window_size) - elif method == 'classification': - evaluate_predictions_classification(model_name, y, preds, sliding_window_size) + evaluate_predictions(model_name, y, preds, sliding_window_size, method) ticker_to_predict = 'BTC_USD' @@ -97,7 +90,7 @@ run_whole_pipeline( method = 'regression', sliding_window_size = 120, retrain_every = 50, - scaling=False + scaling = False ) run_whole_pipeline( ticker_to_predict = ticker_to_predict, @@ -105,5 +98,5 @@ run_whole_pipeline( method = 'classification', sliding_window_size = 120, retrain_every = 50, - scaling=False + scaling = False ) \ No newline at end of file diff --git a/utils/evaluate.py b/utils/evaluate.py index a261529..0d03d15 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -1,3 +1,4 @@ +from typing import Literal from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, r2_score, classification_report from sklearn.metrics import confusion_matrix import pandas as pd @@ -10,46 +11,49 @@ def format_data_for_backtest(aggregated_data: pd.DataFrame, returns_col: str, on return pd.concat([backtest_data, pd.Series(preds)], axis='columns') -def evaluate_predictions_regression(model_name: str, y_true, y_pred, sliding_window_size: int): +def __preprocess(y_true: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']): + y_pred.name = 'y_pred' + y_true.name = 'y_true' + df = pd.concat([y_pred, y_true],axis=1).dropna() + + if method == 'regression': + df['sign_pred'] = df.y_pred.apply(np.sign) + else: + df['sign_pred'] = df.y_pred.apply(lambda x: 1 if x>0 else -1) + df['sign_true'] = df.y_true.apply(np.sign) + df['is_correct'] = 0 + df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct + df['is_incorrect'] = 0 + df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong + df['is_predicted'] = df.is_correct + df.is_incorrect + df['result'] = df.sign_pred * df.y_true + return df + +def evaluate_predictions(model_name: str, y_true: pd.Series, y_pred: pd.Series, sliding_window_size: int, method: Literal['classification', 'regression']): evaluate_from = sliding_window_size+1 - y_true = pd.Series(y_true[evaluate_from:-1]) + y_true = pd.Series(y_true[evaluate_from:]) y_pred = pd.Series(y_pred[evaluate_from:]) - def preprocess(y_true, y_pred): - y_pred.name = 'y_pred' - y_true.name = 'y_true' - df = pd.concat([y_pred, y_true],axis=1).dropna() - - df['sign_pred'] = df.y_pred.apply(np.sign) - df['sign_true'] = df.y_true.apply(np.sign) - df['is_correct'] = 0 - df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct - df['is_incorrect'] = 0 - df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong - df['is_predicted'] = df.is_correct + df.is_incorrect - df['result'] = df.sign_pred * df.y_true - return df - - df = preprocess(y_true, y_pred) + df = __preprocess(y_true, y_pred, method) scorecard = pd.Series() - scorecard.loc['RSQ'] = r2_score(df.y_true,df.y_pred) - scorecard.loc['MAE'] = mean_absolute_error(df.y_true,df.y_pred) + if method == 'regression': + scorecard.loc['RSQ'] = r2_score(df.y_true,df.y_pred) + scorecard.loc['MAE'] = mean_absolute_error(df.y_true,df.y_pred) + elif method == 'classification': + scorecard.loc['RSQ'] = 0. + scorecard.loc['MAE Matrix'] = 0. scorecard.loc['directional_accuracy'] = df.is_correct.sum()*1. / (df.is_predicted.sum()*1.)*100 scorecard.loc['edge'] = df.result.mean() scorecard.loc['noise'] = df.y_pred.diff().abs().mean() scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise'] - scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE'] + if method == 'regression': + scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE'] + elif method == 'classification': + scorecard.loc['edge_to_mae'] = 0. + + # TODO: add confusion matrix, f1 score, precision, recall print("Model name: ", model_name) print(scorecard) return scorecard -def evaluate_predictions_classification(model_name: str, y, preds, sliding_window_size: int): - print("Model: ", model_name) - evaluate_from = sliding_window_size+1 - y = pd.Series(y[evaluate_from:]) - preds = pd.Series(preds[evaluate_from:]) - print(accuracy_score(y, preds)) - print(confusion_matrix(y, preds)) - print(classification_report(y, preds)) - diff --git a/utils/walk_forward.py b/utils/walk_forward.py index 6c0f0ce..34b689b 100644 --- a/utils/walk_forward.py +++ b/utils/walk_forward.py @@ -10,9 +10,8 @@ def walk_forward_train_test( y: pd.Series, window_size: int, retrain_every: int - ) -> tuple[list[SKLearnModel], list[float]]: + ) -> tuple[pd.Series, pd.Series]: - print("Training: ", model_name) predictions = pd.Series(index=y.index) models = pd.Series(index=y.index)