feat(Data): add option to predict 3 classes (#79)

* feat(Data): add option to predict 3 classes

* feat(Evaluation): added ability to evaluate 3 class predictions

* chore(Config): set sensible config for regression models

* feat(Data): added option to use balanced or imbalanced three-class data

* feat(Evaluate): correctly track "no_of_samples" now that we have three classes

* chore(Sweep): remove probably not useful scaler values from sweep

config.py

@@ -6,7 +6,7 @@ from models.model_map import model_names_classification, model_names_regression
 def get_default_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
-        expanding_window = True,
+        expanding_window = False,
         sliding_window_size = 200,
         retrain_every = 100,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
@@ -23,13 +23,14 @@ def get_default_config() -> tuple[dict, dict, dict]:
         other_features = [],
         index_column= 'int',
         method= 'classification',
+        no_of_classes= 'two'
     )
 
     # regression_models = ["Lasso", "Ridge", "BayesianRidge", "KNN", "AB", "LR", "MLP", "RF", "SVR"]
-    regression_models = model_names_regression
+    regression_models = ["Lasso", "KNN", "RF"]
     regression_ensemble_models = ['Ensemble_Average']
-    # classification_models = ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"]
-    classification_models = model_names_classification        
+    classification_models = ["LR", "LDA", "KNN", "CART", "RF"]
+    # classification_models = model_names_classification        
     classification_ensemble_models = ['Ensemble_Average']
 
     model_config = dict(

models/model_map.py

@@ -22,18 +22,18 @@ model_map = {
         KNN = SKLearnModel(KNeighborsRegressor(n_neighbors=25)),
         AB = SKLearnModel(AdaBoostRegressor(random_state=1)),
         MLP = SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
-        RF = SKLearnModel(RandomForestRegressor(n_jobs=-1)),
+        RF = SKLearnModel(RandomForestRegressor(n_jobs=-1, random_state=1)),
         SVR = SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1)),
         StaticNaive = StaticNaiveModel(),
     ),
     "classification_models": dict(
-        LR= SKLearnModel(LogisticRegression(solver='liblinear', C=10, max_iter=1000)),
+        LR= SKLearnModel(LogisticRegression(C=10, random_state=1, max_iter=1000)),
         LDA= SKLearnModel(LinearDiscriminantAnalysis()),
         KNN= SKLearnModel(KNeighborsClassifier()),
         CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)),
         NB= SKLearnModel(GaussianNB()),
         AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)),
-        RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20)),
+        RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)),
         StaticMom= StaticMomentumModel(allow_short=True),
     ),     
     "classification_ensemble_models": dict(

run_pipeline.py

@@ -46,9 +46,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
             sliding_window_size = training_config['sliding_window_size'],
             retrain_every =  training_config['retrain_every'],
             scaler =  training_config['scaler'],
-            wandb = wandb,
-            project_name=project_name,
-            sweep=sweep
+            no_of_classes = data_config['no_of_classes']
         )
         results = pd.concat([results, current_result], axis=1)
         all_predictions = pd.concat([all_predictions, current_predictions], axis=1)
@@ -71,9 +69,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
                 sliding_window_size = training_config['sliding_window_size'],
                 retrain_every = training_config['retrain_every'],
                 scaler = training_config['scaler'],
-                wandb = wandb,
-                project_name=project_name,
-                sweep=sweep
+                no_of_classes = data_config['no_of_classes']
             )
 
             results = pd.concat([results, ensemble_result], axis=1)
@@ -86,8 +82,9 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
     level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]
     ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]]
 
-    print("Mean Sharpe ratio for Level-1 models: ", level1_columns.loc['sharpe'].mean())
-    print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", ensemble_columns.loc['sharpe'].mean())
+    print("Mean no of samples: ", results.loc['no_of_samples'].mean())
+    print("Mean Sharpe ratio for Level-1 models: ", round(level1_columns.loc['sharpe'].mean(), 3))
+    print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_columns.loc['sharpe'].mean(), 3))
 
     if sweep:
         if wandb.run is not None:

sweep.yaml

@@ -21,22 +21,23 @@ parameters:
       values: [10, 20, 30]
       distribution: categorical
     scaler:
-      values: ['minmax', 'normalize', 'minmax', 'standardize', 'none']
+      values: ['minmax', 'none']
       distribution: categorical
     include_original_data_in_ensemble:
-      values: [True, False]
-      distribution: categorical
+      values: False
     method: 
       value: 'classification'
+    no_of_classes:
+      values: ['two', 'three-balanced', 'three-imbalanced']
+      distribution: categorical
     forecasting_horizon: 
-      values: [1,2,3,4,5,6,7,8,9,10]
+      values: 1
       distribution: categorical
     load_other_assets: 
       values: [True, False]
       distribution: categorical
     log_returns: 
-      values: [True, False]
-      distribution: categorical
+      value: True
     index_column: 
       value: 'int'
     level_1_models: 
@@ -44,7 +45,6 @@ parameters:
       distribution: categorical
     level_2_models: 
       value: []
-      distribution: constant
     own_features:
       values: [['only_mom', 'date_days'], [], ['level_1', 'date_days'], ['level_1', 'date_days', 'level_2']]
       distribution: categorical

tests/test_evaluation.py

@@ -85,7 +85,9 @@ def test_evaluation():
         model_name='test',
         target_returns=fake_target_returns,
         y_pred=processed_predictions_to_match_returns,
-        method='classification'
+        y_true=y,
+        method='classification',
+        no_of_classes='two'
     )
 
     assert result['accuracy'] == 100.0

training/training.py

@@ -26,10 +26,8 @@ def run_single_asset_trainig(
                     sliding_window_size: int,
                     retrain_every: int,
                     scaler: Literal['normalize', 'minmax', 'standardize', 'none'],
-                    wandb,
-                    project_name:str,
-                    sweep:bool
-                    ) -> tuple[pd.DataFrame, pd.DataFrame]:
+                    no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
+    ) -> tuple[pd.DataFrame, pd.DataFrame]:
 
 
     scaler = __get_scaler(scaler)
@@ -37,8 +35,6 @@ def run_single_asset_trainig(
     results = pd.DataFrame()
     predictions = pd.DataFrame()
     
-    wandb_active = type(wandb) is not type(None)
-
     for model_name, model in models:
         model_over_time, preds = walk_forward_train_test(
             model_name=model_name,
@@ -56,7 +52,9 @@ def run_single_asset_trainig(
             model_name = model_name,
             target_returns = target_returns,
             y_pred = preds,
+            y_true = y,
             method = method,
+            no_of_classes=no_of_classes
         )
         column_name = ticker_to_predict + "_" + model_name
         results[column_name] = result

utils/evaluate.py

@@ -11,23 +11,22 @@ def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.00) ->
     return (signal * returns) - costs
 
 
-def __preprocess(target_returns: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']) -> pd.DataFrame:
+def __preprocess(target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, method: Literal['classification', 'regression'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']) -> pd.DataFrame:
     y_pred.name = 'y_pred'
     target_returns.name = 'target_returns'
     df = pd.concat([y_pred, target_returns],axis=1).dropna()
 
-    df['sign_pred'] = df.y_pred.apply(lambda x: 1 if x>0 else -1)
-    def sign_true(x):
-        if x > 0:
-            return 1
-        else:
-            return -1
-    df['sign_true'] = df.target_returns.apply(sign_true)
-    df['is_correct'] = 0
-    df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct
-    df['is_incorrect'] = 0
-    df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong
-    df['is_predicted'] = df.is_correct + df.is_incorrect
+    def categorize_binary(x): return 1 if x > 0 else -1
+    def categorize_threeway(x): return 0 if x == 0 else 1 if x > 0 else -1
+    categorize = categorize_binary if no_of_classes == 'two' else categorize_threeway
+    # make sure that we evaluate binary/three-way predictions even if the model is a regression
+    if method == 'regression':
+        df['sign_pred'] = df.y_pred.apply(categorize)
+        df['sign_true'] = df.target_returns.apply(categorize)
+    else:
+        df['sign_pred'] = df.y_pred.apply(categorize)
+        df['sign_true'] = y_true
+
     df['result'] = backtest(df.target_returns, df.sign_pred)
 
     return df
@@ -36,7 +35,9 @@ def evaluate_predictions(
                         model_name: str,
                         target_returns: pd.Series,
                         y_pred: pd.Series,
-                        method: Literal['classification', 'regression']
+                        y_true: pd.Series,
+                        method: Literal['classification', 'regression'],
+                        no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                         ) -> pd.Series:
     # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
     first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(y_pred))
@@ -50,7 +51,7 @@ def evaluate_predictions(
 
     y_pred = pd.Series(y_pred[evaluate_from:])
 
-    df = __preprocess(target_returns, y_pred, method)
+    df = __preprocess(target_returns, y_pred, y_true, method, no_of_classes)
     
     scorecard = pd.Series()
     if method == 'regression':
@@ -62,15 +63,20 @@ def evaluate_predictions(
     sign_true = df.sign_true.astype(int)
     sign_pred = df.sign_pred.astype(int)
     
-    scorecard.loc['no_of_samples'] = len(target_returns) - evaluate_from
+    def count_non_zero(series: pd.Series) -> int:
+        return len(series[series != 0])
+    scorecard.loc['no_of_samples'] = count_non_zero(y_pred[evaluate_from:])
     scorecard.loc['sharpe'] = sharpe(df.result)
     scorecard.loc['sortino'] = sortino(df.result)
     scorecard.loc['skew'] = skew(df.result)
 
+    labels = [1, -1] if no_of_classes == 'two' else [1, -1, 0]
+    avg_type = 'weighted' if no_of_classes == 'two' else 'macro'
+
     scorecard.loc['accuracy'] = accuracy_score(sign_true, sign_pred) * 100
-    scorecard.loc['recall'] = recall_score(sign_true, sign_pred, labels = [1, -1])
-    scorecard.loc['precision'] = precision_score(sign_true, sign_pred, labels = [1, -1])
-    scorecard.loc['f1_score'] = f1_score(sign_true, sign_pred, labels = [1, -1])
+    scorecard.loc['recall'] = recall_score(sign_true, sign_pred, labels = labels, average=avg_type)
+    scorecard.loc['precision'] = precision_score(sign_true, sign_pred, labels = labels, average=avg_type)
+    scorecard.loc['f1_score'] = f1_score(sign_true, sign_pred, labels = labels, average=avg_type)
     scorecard.loc['edge'] = df.result.mean()
     scorecard.loc['noise'] = df.y_pred.diff().abs().mean()
     scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise']
@@ -82,11 +88,6 @@ def evaluate_predictions(
         scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(sign_pred)
     
 
-
-    # scorecard.loc['ratio_of_classes_y'] = ' / '.join([str(index) + " " + str(round(row / len(sign_true), 2)) for index, row in sign_true.value_counts().iteritems()])
-    # scorecard.loc['ratio_of_classes_pred'] = ' / '.join([str(round(row / len(sign_pred), 2)) for index, row in sign_pred.value_counts().iteritems()])
-
-
     if method == 'regression':
         scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE']
     elif method == 'classification':

utils/load_data.py

@@ -2,11 +2,8 @@
 import pandas as pd
 import os
 import numpy as np
-from pandas.core.frame import DataFrame
-from utils.technical_indicators import ROC, RSI, STOK, STOD
 from utils.typing import FeatureExtractor
-from typing import Callable, Literal
-from sklearn.preprocessing import OneHotEncoder
+from typing import Literal
 
 #%%
 
@@ -26,6 +23,7 @@ def load_data(path: str,
             other_features: list[tuple[str, FeatureExtractor, list[int]]],
             index_column: Literal['date', 'int'],
             method: Literal['regression', 'classification'],
+            no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
             narrow_format: bool = False,
             all_assets:list=[]
         ) -> tuple[pd.DataFrame, pd.Series, pd.Series]:
@@ -68,7 +66,7 @@ def load_data(path: str,
     if method == 'regression':
         dfs[target_col] = forward_returns
     elif method == 'classification':
-        dfs[target_col] = __create_target_classes(dfs, returns_col, forecasting_horizon, 'two')
+        dfs[target_col] = __create_target_classes(dfs, returns_col, forecasting_horizon, no_of_classes)
     # we need to drop the last row, because we forward-shift the target (see what happens if you call .shift[-1] on a pd.Series) 
     dfs = dfs.iloc[:-forecasting_horizon]
     forward_returns = forward_returns.iloc[:-forecasting_horizon]
@@ -78,29 +76,6 @@ def load_data(path: str,
 
     return X, y, forward_returns
 
-# %%
-
-def load_crypto_only_returns(path: str, index_column: Literal['date', 'int'], returns: Literal['price', 'returns']) -> pd.DataFrame:
-    files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')]
-    dfs = [__load_df(
-        path=os.path.join(path,f),
-        prefix=f.split('.')[0],
-        returns=returns,
-        feature_extractors=[],
-        narrow_format=False,
-    ) for f in files]
-    dfs = pd.concat(dfs, axis=1)
-    dfs = dfs.applymap(lambda x: np.nan if x == 0 else x)
-    dfs.index = pd.DatetimeIndex(dfs.index)
-    dfs.columns = [column.split('_')[0] for column in dfs.columns]
-    if index_column == 'int':
-        dfs.reset_index(drop=True, inplace=True)
-
-    return dfs
-
-def load_crypto_assets_availability(path: str, index_column: Literal['date', 'int']) -> pd.DataFrame:
-    return load_crypto_only_returns(path, index_column, 'returns').applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1)
-
 
 def __load_df(path: str,
             prefix: str,
@@ -156,24 +131,88 @@ def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, pe
 def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.Series:
     assert period > 0
 
-    def get_class_binary(x):
+    def get_class_binary(x: float) -> int:
         return -1 if x <= 0.0 else 1
 
-    def get_class_threeway(x):
-        bins = pd.qcut(df[source_column], 4, duplicates='raise', retbins=True)[1]
-        lower_threshold = bins[1]
-        upper_threshold = bins[3]
-        if x <= lower_threshold:
-            return -1
-        elif x > lower_threshold and x < upper_threshold:
-            return 0
-        else:
-            return 1
+    def get_class_threeway_balanced(series: pd.Series) -> pd.Series:
+
+        def get_bins_threeway(x):
+            bins = pd.qcut(df[source_column], 3, retbins=True, duplicates = 'drop')[1]
+            
+            if len(bins) != 4:
+                # if we don't have enough data for the quantiles, we'll need to add hard-coded values
+                lower_bound = bins[0]
+                upper_bound = bins[-1]
+                bins = [lower_bound] + [-0.02, 0.02] + [upper_bound]
+            return bins
+        bins = get_bins_threeway(series)
+
+        def map_class_threeway(current_value):
+            lower_threshold = bins[1]
+            upper_threshold = bins[2]
+            if current_value <= lower_threshold:
+                return -1
+            elif current_value > lower_threshold and current_value < upper_threshold:
+                return 0
+            else:
+                return 1
+        return series.map(map_class_threeway)
+
+    def get_class_threeway_imbalanced(series: pd.Series) -> pd.Series:
+
+        def get_bins_threeway(x):
+            bins = pd.qcut(df[source_column], 4, retbins=True, duplicates = 'drop')[1]
+            
+            if len(bins) != 5:
+                # if we don't have enough data for the quantiles, we'll need to add hard-coded values
+                lower_bound = bins[0]
+                upper_bound = bins[-1]
+                bins = [lower_bound] + [-0.02, 0.0, 0.02] + [upper_bound]
+            return bins
+        bins = get_bins_threeway(series)
+
+        def map_class_threeway(current_value):
+            lower_threshold = bins[1]
+            upper_threshold = bins[3]
+            if current_value <= lower_threshold:
+                return -1
+            elif current_value > lower_threshold and current_value < upper_threshold:
+                return 0
+            else:
+                return 1
+        return series.map(map_class_threeway)
 
     target_column = df[source_column].shift(-period)
     
-    get_class_function = get_class_binary
-    if no_of_classes == "three":
-        get_class_function = get_class_threeway
+    if no_of_classes == "three-balanced":
+        return get_class_threeway_balanced(target_column)
+    elif no_of_classes == "three-imbalanced":
+        return get_class_threeway_imbalanced(target_column)
+    else:
+        return target_column.map(get_class_binary)
+
+
+
+
+# These are needed for the portfolio feature, maybe we can do this in a more elegant way
+# def load_crypto_only_returns(path: str, index_column: Literal['date', 'int'], returns: Literal['price', 'returns']) -> pd.DataFrame:
+#     files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')]
+#     dfs = [__load_df(
+#         path=os.path.join(path,f),
+#         prefix=f.split('.')[0],
+#         returns=returns,
+#         feature_extractors=[],
+#         narrow_format=False,
+#     ) for f in files]
+#     dfs = pd.concat(dfs, axis=1)
+#     dfs = dfs.applymap(lambda x: np.nan if x == 0 else x)
+#     dfs.index = pd.DatetimeIndex(dfs.index)
+#     dfs.columns = [column.split('_')[0] for column in dfs.columns]
+#     if index_column == 'int':
+#         dfs.reset_index(drop=True, inplace=True)
+
+#     return dfs
+
+# def load_crypto_assets_availability(path: str, index_column: Literal['date', 'int']) -> pd.DataFrame:
+#     return load_crypto_only_returns(path, index_column, 'returns').applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1)
 
-    return target_column.map(get_class_function)