refactor(Project): removed regression method (we can still use regression models, but we'll need map them to classification later) (#182)

* refactor(Project): removed regression method (we can still use regression models, but we'll need map them to classes later)

* fix(Training): removed mistakenly left in `method` parameter

config/config.py

@@ -25,7 +25,6 @@ def get_dev_config() -> tuple[dict, dict, dict]:
         other_features = ['single_mom'],
         exogenous_features = ['z_score'],
         index_column= 'int',
-        method= 'classification',
         no_of_classes= 'two',
         narrow_format = False,
     )
@@ -34,7 +33,7 @@ def get_dev_config() -> tuple[dict, dict, dict]:
     classification_models = ["LogisticRegression_two_class"]
 
     model_config = dict(
-        primary_models = regression_models if data_config['method'] == 'regression' else classification_models,
+        primary_models = classification_models,
         meta_labeling_models = [],
         ensemble_model = None
     )
@@ -69,7 +68,6 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]:
         other_features = ['level_2', 'lags_up_to_5'],
         exogenous_features = ['z_score'],
         index_column= 'int',
-        method= 'classification',
         no_of_classes= 'two',
         narrow_format = False,
     )
@@ -80,7 +78,7 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]:
     ensemble_model = 'Average'
 
     model_config = dict(
-        primary_models = regression_models if data_config['method'] == 'regression' else classification_models,
+        primary_models = classification_models,
         meta_labeling_models = meta_labeling_models,
         ensemble_model = ensemble_model
     )
@@ -115,7 +113,6 @@ def get_lightweight_ensemble_config() -> tuple[dict, dict, dict]:
         other_features = ['level_2'],
         exogenous_features = ['z_score'],
         index_column= 'int',
-        method= 'classification',
         no_of_classes= 'two',
         narrow_format = False,
     )
@@ -126,7 +123,7 @@ def get_lightweight_ensemble_config() -> tuple[dict, dict, dict]:
     ensemble_model = 'Average'
 
     model_config = dict(
-        primary_models = regression_models if data_config['method'] == 'regression' else classification_models,
+        primary_models = classification_models,
         meta_labeling_models = meta_labeling_models,
         ensemble_model = ensemble_model
     )

config/hashing.py

@@ -18,7 +18,6 @@ def hash_data_config(data_config: dict) -> str:
         hash_feature_extractors(data_config['other_features']),
         hash_feature_extractors(data_config['exogenous_features']),
         data_config['index_column'],
-        data_config['method'],
         data_config['no_of_classes'],
         data_config['narrow_format']
     ]))

config/preprocess.py

@@ -5,7 +5,7 @@ from models.model_map import get_model_map
 from data_loader.collections import data_collections
 
 def preprocess_config(model_config:dict, training_config:dict, data_config:dict) -> tuple[dict, dict, dict]:
-    model_config = __preprocess_model_config(model_config, data_config['method'])
+    model_config = __preprocess_model_config(model_config)
     data_config = __preprocess_feature_extractors_config(data_config)
     data_config = __preprocess_data_collections_config(data_config)
 
@@ -20,7 +20,7 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
         data_dict[key] = flatten([feature_extractor_presets[preset_name] for preset_name in preset_names])
     return data_dict
 
-def __preprocess_model_config(model_config:dict, method:str) -> dict:
+def __preprocess_model_config(model_config:dict) -> dict:
     model_map = get_model_map(model_config)
     model_config['primary_models'] = [(model_name, model_map['primary_models'][model_name]) for model_name in model_config['primary_models']]
     if len(model_config['meta_labeling_models']) > 0:

config/sweep_ensemble.yaml

@@ -32,8 +32,6 @@ parameters:
       value: 20
     scaler:
       value: 'minmax'
-    method: 
-      value: 'classification'
     no_of_classes:
       value: 'two'
     forecasting_horizon: 

config/sweep_meta_labeling.yaml

@@ -36,8 +36,6 @@ parameters:
       distribution: categorical
     scaler:
       value: 'minmax'
-    method: 
-      value: 'classification'
     no_of_classes:
       values: ['two', 'three-balanced', 'three-imbalanced']
       distribution: categorical

config/sweep_primary_models.yaml

@@ -32,8 +32,6 @@ parameters:
       distribution: categorical
     scaler:
       value: 'minmax'
-    method: 
-      value: 'classification'
     no_of_classes:
       value: 'two'
     forecasting_horizon: 

data_loader/load_data.py

@@ -30,7 +30,6 @@ def __load_data(assets: DataCollection,
             other_features: list[tuple[str, FeatureExtractor, list[int]]],
             exogenous_features: list[tuple[str, FeatureExtractor, list[int]]],
             index_column: Literal['date', 'int'],
-            method: Literal['regression', 'classification'],
             no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
             narrow_format: bool = False
         ) -> tuple[pd.DataFrame, pd.Series, pd.Series]:
@@ -105,10 +104,7 @@ def __load_data(assets: DataCollection,
     target_col = 'target'
     returns_col = target_asset[1] + '_returns'
     forward_returns = __create_target_cum_forward_returns(df_target_asset_only_returns, returns_col, forecasting_horizon)
-    if method == 'regression':
-        dfs[target_col] = forward_returns
-    elif method == 'classification':
-        dfs[target_col] = __create_target_classes(dfs, returns_col, forecasting_horizon, no_of_classes)
+    dfs[target_col] = __create_target_classes(dfs, returns_col, forecasting_horizon, no_of_classes)
     # we need to drop the last row, because we forward-shift the target (see what happens if you call .shift[-1] on a pd.Series) 
     dfs = dfs.iloc[:-forecasting_horizon]
     forward_returns = forward_returns.iloc[:-forecasting_horizon]

tests/test_evaluation.py

@@ -102,7 +102,6 @@ def test_evaluation():
         target_returns=fake_target_returns,
         y_pred=processed_predictions_to_match_returns,
         y_true=y,
-        method='classification',
         no_of_classes='two',
         print_results = False,
         discretize=True

training/meta_labeling.py

@@ -2,7 +2,6 @@ from utils.evaluate import discretize_threeway_threshold, evaluate_predictions
 from utils.helpers import equal_except_nan
 from training.primary_model import train_primary_model
 import pandas as pd
-from models.model_map import default_feature_selector_classification, default_feature_selector_regression
 from models.base import Model
 from reporting.types import Reporting
 from typing import Union, Optional
@@ -35,7 +34,6 @@ def train_meta_labeling_model(
         y = meta_y,
         target_returns = target_returns,
         models = models,
-        method = 'classification',
         expanding_window = training_config['expanding_window_meta_labeling'],
         sliding_window_size = training_config['sliding_window_size_meta_labeling'],
         retrain_every = training_config['retrain_every'],
@@ -59,7 +57,6 @@ def train_meta_labeling_model(
         target_returns = target_returns,
         y_pred = avg_predictions_with_sizing,
         y_true = y,
-        method = 'classification',
         no_of_classes = 'two',
         print_results = True,
         discretize=False

training/primary_model.py

@@ -15,7 +15,6 @@ def train_primary_model(
                     y: pd.Series,
                     target_returns: pd.Series,
                     models: list[tuple[str, Model]],
-                    method: Literal['regression', 'classification'],
                     expanding_window: bool,
                     sliding_window_size: int,
                     retrain_every: int,
@@ -80,7 +79,6 @@ def train_primary_model(
             target_returns = target_returns,
             y_pred = preds,
             y_true = y,
-            method = method,
             no_of_classes=no_of_classes,
             print_results = print_results,
             discretize=True

training/training_steps.py

@@ -28,7 +28,6 @@ def primary_step(
         y = y,
         target_returns = target_returns,
         models = model_config['primary_models'],
-        method = data_config['method'],
         expanding_window = training_config['expanding_window_primary'],
         sliding_window_size = training_config['sliding_window_size_primary'],
         retrain_every =  training_config['retrain_every'],
@@ -94,9 +93,7 @@ def secondary_step(
             X = current_predictions,
             y = y,
             target_returns = target_returns,
-            models = [model_config['ensemble_model']],
-            method = data_config['method'],
-            expanding_window = False,
+            models = [model_config['ensemble_model']],            expanding_window = False,
             sliding_window_size = 1,
             retrain_every = training_config['retrain_every'],
             from_index = from_index,

utils/evaluate.py

@@ -12,19 +12,15 @@ def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.002) ->
     return (signal * returns) - costs
 
 
-def __preprocess(target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, method: Literal['classification', 'regression'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame:
+def __preprocess(target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame:
     y_pred.name = 'y_pred'
     target_returns.name = 'target_returns'
     df = pd.concat([y_pred, target_returns],axis=1).dropna()
 
     discretize_func = get_discretize_function(no_of_classes)
     # make sure that we evaluate binary/three-way predictions even if the model is a regression
-    if method == 'regression':
-        df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
-        df['sign_true'] = df.target_returns.apply(discretize_func)
-    else:
-        df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
-        df['sign_true'] = y_true
+    df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
+    df['sign_true'] = y_true
 
     df['result'] = backtest(df.target_returns, df.sign_pred)
 
@@ -35,7 +31,6 @@ def evaluate_predictions(
                         target_returns: pd.Series,
                         y_pred: pd.Series,
                         y_true: pd.Series,
-                        method: Literal['classification', 'regression'],
                         no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                         print_results: bool,
                         discretize: bool = False,
@@ -45,14 +40,9 @@ def evaluate_predictions(
     evaluate_from = first_nonzero_return + 1
     
     target_returns = pd.Series(target_returns[evaluate_from:])
-    if method == 'regression':
-        # if there are lots of zeros in the ground truth returns, probably something is wrong, but we can tolerate a couple of days of missing data.
-        is_zero = target_returns[target_returns == 0]
-        assert len(is_zero) < 15
-
     y_pred = pd.Series(y_pred[evaluate_from:])
 
-    df = __preprocess(target_returns, y_pred, y_true, method, no_of_classes, discretize)
+    df = __preprocess(target_returns, y_pred, y_true, no_of_classes, discretize)
     
     scorecard = pd.Series()