feat(Evaluation): created a unified evaluation framework for both regression / classification

load_data.py

@@ -56,9 +56,9 @@ def load_data(path: str,
     target_col = 'target'
     returns_col = target_asset + '_returns'
     if method == 'regression':
-        dfs = create_target_cum_forward_returns(dfs, returns_col, 1)
+        dfs = __create_target_cum_forward_returns(dfs, returns_col, 1)
     elif method == 'classification':
-        dfs = create_target_classes(dfs, returns_col, 1, 'two')
+        dfs = __create_target_classes(dfs, returns_col, 1, 'two')
         
     X = dfs.drop(columns=[target_col])
     y = dfs[target_col]
@@ -130,13 +130,13 @@ def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_fe
 
 
 # %%
-def create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
+def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
     df['target'] = df[source_column].diff(period).shift(-period)
     df = df.iloc[:-period]
     return df
 
 
-def create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.DataFrame:
+def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.DataFrame:
 
     def get_class_binary(x):
         return 0 if x <= 0.0 else 1

model_walk_forward.py

@@ -1,15 +1,11 @@
-#%% Import all the stuff, load data, define constants
 from typing import Literal
 from sklearnex import patch_sklearn
 patch_sklearn()
 
-from load_data import create_target_cum_forward_returns, load_data, create_target_classes
-from sktime.forecasting.model_selection import temporal_train_test_split
-from utils.evaluate import evaluate_predictions_regression, evaluate_predictions_classification
+from load_data import load_data
+from utils.evaluate import evaluate_predictions
 
-import numpy as np
 import pandas as pd
-from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
 from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier
@@ -18,7 +14,6 @@ from sklearn.svm import SVR
 from sklearn.naive_bayes import GaussianNB
 from sklearn.neural_network import MLPRegressor, MLPClassifier
 from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
-from sklearn.metrics import r2_score, mean_absolute_error, confusion_matrix, classification_report, accuracy_score
 from sklearn.preprocessing import MinMaxScaler
 
 from utils.walk_forward import walk_forward_train_test
@@ -68,8 +63,9 @@ def run_whole_pipeline(
         method=method,
     )
 
+
     if scaling:
-        # TODO: should move scaling to an expanding window compomenent
+        # TODO: should move scaling to an expanding window compomenent, probably worth not turning it on for now
         feature_scaler = MinMaxScaler(feature_range= (-1, 1))
         X = pd.DataFrame(feature_scaler.fit_transform(X), columns=X.columns, index=X.index)
         # TODO: should scale y as well probably
@@ -84,10 +80,7 @@ def run_whole_pipeline(
             window_size = sliding_window_size,
             retrain_every = retrain_every
         )
-        if method == 'regression':
-            evaluate_predictions_regression(model_name, y, preds, sliding_window_size)
-        elif method == 'classification':
-            evaluate_predictions_classification(model_name, y, preds, sliding_window_size)
+        evaluate_predictions(model_name, y, preds, sliding_window_size, method)
 
 
 ticker_to_predict = 'BTC_USD'
@@ -97,7 +90,7 @@ run_whole_pipeline(
     method = 'regression',
     sliding_window_size = 120,
     retrain_every = 50,
-    scaling=False
+    scaling = False
 )
 run_whole_pipeline(
     ticker_to_predict = ticker_to_predict,
@@ -105,5 +98,5 @@ run_whole_pipeline(
     method = 'classification',
     sliding_window_size = 120,
     retrain_every = 50,
-    scaling=False
+    scaling = False
 )

utils/evaluate.py

@@ -1,3 +1,4 @@
+from typing import Literal
 from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, r2_score, classification_report
 from sklearn.metrics import confusion_matrix
 import pandas as pd
@@ -10,46 +11,49 @@ def format_data_for_backtest(aggregated_data: pd.DataFrame, returns_col: str, on
     return pd.concat([backtest_data, pd.Series(preds)], axis='columns')
 
 
-def evaluate_predictions_regression(model_name: str, y_true, y_pred, sliding_window_size: int):
+def __preprocess(y_true: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']):
+    y_pred.name = 'y_pred'
+    y_true.name = 'y_true'
+    df = pd.concat([y_pred, y_true],axis=1).dropna()
+
+    if method == 'regression':
+        df['sign_pred'] = df.y_pred.apply(np.sign)
+    else:
+        df['sign_pred'] = df.y_pred.apply(lambda x: 1 if x>0 else -1)
+    df['sign_true'] = df.y_true.apply(np.sign)
+    df['is_correct'] = 0
+    df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct
+    df['is_incorrect'] = 0
+    df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong
+    df['is_predicted'] = df.is_correct + df.is_incorrect
+    df['result'] = df.sign_pred * df.y_true 
+    return df
+
+def evaluate_predictions(model_name: str, y_true: pd.Series, y_pred: pd.Series, sliding_window_size: int, method: Literal['classification', 'regression']):
     evaluate_from = sliding_window_size+1
-    y_true = pd.Series(y_true[evaluate_from:-1])
+    y_true = pd.Series(y_true[evaluate_from:])
     y_pred = pd.Series(y_pred[evaluate_from:])
 
-    def preprocess(y_true, y_pred):
-        y_pred.name = 'y_pred'
-        y_true.name = 'y_true'
-        df = pd.concat([y_pred, y_true],axis=1).dropna()
-
-        df['sign_pred'] = df.y_pred.apply(np.sign)
-        df['sign_true'] = df.y_true.apply(np.sign)
-        df['is_correct'] = 0
-        df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct
-        df['is_incorrect'] = 0
-        df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong
-        df['is_predicted'] = df.is_correct + df.is_incorrect
-        df['result'] = df.sign_pred * df.y_true 
-        return df
-    
-    df = preprocess(y_true, y_pred)
+    df = __preprocess(y_true, y_pred, method)
     
     scorecard = pd.Series()
-    scorecard.loc['RSQ'] = r2_score(df.y_true,df.y_pred)
-    scorecard.loc['MAE'] = mean_absolute_error(df.y_true,df.y_pred)
+    if method == 'regression':
+        scorecard.loc['RSQ'] = r2_score(df.y_true,df.y_pred)
+        scorecard.loc['MAE'] = mean_absolute_error(df.y_true,df.y_pred)
+    elif method == 'classification':
+        scorecard.loc['RSQ'] = 0.
+        scorecard.loc['MAE Matrix'] = 0.
     scorecard.loc['directional_accuracy'] = df.is_correct.sum()*1. / (df.is_predicted.sum()*1.)*100
     scorecard.loc['edge'] = df.result.mean()
     scorecard.loc['noise'] = df.y_pred.diff().abs().mean()
     scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise']
-    scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE']
+    if method == 'regression':
+        scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE']
+    elif method == 'classification':
+        scorecard.loc['edge_to_mae'] = 0.
+
+    # TODO: add confusion matrix, f1 score, precision, recall
     print("Model name: ", model_name)
     print(scorecard)
     return scorecard  
 
-def evaluate_predictions_classification(model_name: str, y, preds, sliding_window_size: int):
-    print("Model: ", model_name)
-    evaluate_from = sliding_window_size+1
-    y = pd.Series(y[evaluate_from:])
-    preds = pd.Series(preds[evaluate_from:])
-    print(accuracy_score(y, preds))
-    print(confusion_matrix(y, preds))
-    print(classification_report(y, preds))
-

utils/walk_forward.py

@@ -10,9 +10,8 @@ def walk_forward_train_test(
                             y: pd.Series,
                             window_size: int,
                             retrain_every: int
-                        ) -> tuple[list[SKLearnModel], list[float]]:
+                        ) -> tuple[pd.Series, pd.Series]:
                         
-    print("Training: ", model_name)
     predictions = pd.Series(index=y.index)
     models = pd.Series(index=y.index)