feat(Evaluation): created a unified evaluation framework for both regression / classification

model_walk_forward.py

@@ -1,15 +1,11 @@
-#%% Import all the stuff, load data, define constants
 from typing import Literal
 from sklearnex import patch_sklearn
 patch_sklearn()
 
-from load_data import create_target_cum_forward_returns, load_data, create_target_classes
-from sktime.forecasting.model_selection import temporal_train_test_split
-from utils.evaluate import evaluate_predictions_regression, evaluate_predictions_classification
+from load_data import load_data
+from utils.evaluate import evaluate_predictions
 
-import numpy as np
 import pandas as pd
-from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
 from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier
@@ -18,7 +14,6 @@ from sklearn.svm import SVR
 from sklearn.naive_bayes import GaussianNB
 from sklearn.neural_network import MLPRegressor, MLPClassifier
 from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
-from sklearn.metrics import r2_score, mean_absolute_error, confusion_matrix, classification_report, accuracy_score
 from sklearn.preprocessing import MinMaxScaler
 
 from utils.walk_forward import walk_forward_train_test
@@ -68,8 +63,9 @@ def run_whole_pipeline(
         method=method,
     )
 
+
     if scaling:
-        # TODO: should move scaling to an expanding window compomenent
+        # TODO: should move scaling to an expanding window compomenent, probably worth not turning it on for now
         feature_scaler = MinMaxScaler(feature_range= (-1, 1))
         X = pd.DataFrame(feature_scaler.fit_transform(X), columns=X.columns, index=X.index)
         # TODO: should scale y as well probably
@@ -84,10 +80,7 @@ def run_whole_pipeline(
             window_size = sliding_window_size,
             retrain_every = retrain_every
         )
-        if method == 'regression':
-            evaluate_predictions_regression(model_name, y, preds, sliding_window_size)
-        elif method == 'classification':
-            evaluate_predictions_classification(model_name, y, preds, sliding_window_size)
+        evaluate_predictions(model_name, y, preds, sliding_window_size, method)
 
 
 ticker_to_predict = 'BTC_USD'
@@ -97,7 +90,7 @@ run_whole_pipeline(
     method = 'regression',
     sliding_window_size = 120,
     retrain_every = 50,
-    scaling=False
+    scaling = False
 )
 run_whole_pipeline(
     ticker_to_predict = ticker_to_predict,
@@ -105,5 +98,5 @@ run_whole_pipeline(
     method = 'classification',
     sliding_window_size = 120,
     retrain_every = 50,
-    scaling=False
+    scaling = False
 )