fix(Selection): dynamic step size for feature selection (#123)

* fix(Selection): dynamic step size for feature selection

* refactor(Pipeline): type definition

* chore(Cache): renamed clear_cache script

* feat(Config): dynamic feature selection is now a toggleable feature

* fix(Training): not passing in necessary parameter

config/config.py

@@ -5,6 +5,7 @@ def get_default_level_1_daily_config() -> tuple[dict, dict, dict]:
         meta_labeling_lvl_1 = False,
         dimensionality_reduction = True,
         n_features_to_select = 30,
+        dynamic_feature_selection = True,
         expanding_window_level1 = False,
         expanding_window_level2 = False,
         sliding_window_size_level1 = 380,
@@ -46,6 +47,7 @@ def get_default_level_2_hourly_config() -> tuple[dict, dict, dict]:
         meta_labeling_lvl_1 = True,
         dimensionality_reduction = True,
         n_features_to_select = 30,
+        dynamic_feature_selection = True,
         expanding_window_level1 = True,
         expanding_window_level2 = False,
         sliding_window_size_level1 = 2480,
@@ -89,6 +91,7 @@ def get_default_level_2_daily_config() -> tuple[dict, dict, dict]:
         meta_labeling_lvl_1 = True,
         dimensionality_reduction = True,
         n_features_to_select = 30,
+        dynamic_feature_selection = True,
         expanding_window_level1 = False,
         expanding_window_level2 = True,
         sliding_window_size_level1 = 380,

feature_selection/feature_selection.py

@@ -9,7 +9,7 @@ from diskcache import Cache
 cache = Cache(".cachedir/feature_selection")
 
 def select_features(**kwargs) -> pd.DataFrame:
-    hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling']
+    hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling'] + str(kwargs['dynamic_feature_selection'])
     if hashed in cache:
         return cache.get(hashed)
     else:
@@ -17,7 +17,7 @@ def select_features(**kwargs) -> pd.DataFrame:
         cache[hashed] = return_value
         return return_value
 
-def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel, scaling: ScalerTypes, data_config_hash: str) -> pd.DataFrame:
+def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel, scaling: ScalerTypes, dynamic_feature_selection: bool, data_config_hash: str) -> pd.DataFrame:
     ''' Select features using RFECV, returns a pd.DataFrame (X) with only the selected features.'''
     if model.model_type != 'ml': return X
 
@@ -33,7 +33,8 @@ def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to
         feat_selector_model = backup_model.model
 
 #     selector = RFECV(feat_selector_model, cv = cv, step=5, min_features_to_select=min_features_to_select)
-    selector = RFE(feat_selector_model, n_features_to_select= n_features_to_select, step=5)
+    step = 0.05 if dynamic_feature_selection else 5
+    selector = RFE(feat_selector_model, n_features_to_select= n_features_to_select, step=step)
     selector = selector.fit(X_scaled, y)
     print("Kept %d features out of %d" % (selector.n_features_, X_scaled.shape[1]))
 

run_pipeline.py

@@ -12,6 +12,7 @@ from feature_selection.dim_reduction import reduce_dimensionality
 from training.meta_labeling import run_meta_labeling_training
 from training.averaged import average_and_evaluate_predictions
 from reporting.reporting import report_results
+from typing import Callable, Optional
 import ray
 ray.init()
 
@@ -22,7 +23,7 @@ def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:obj
     report_results(results, all_predictions, model_config, wandb, sweep, project_name)
     
 
-def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:object):
+def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config: Callable) -> tuple[Optional[object], dict, dict, dict]:
     model_config, training_config, data_config = get_config() 
     wandb = None
     if with_wandb: 
@@ -66,7 +67,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
         print("Feature Selection started")
         # TODO: this needs to be done per model!
         backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
-        X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = hash_data_config(data_params))
+        X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], dynamic_feature_selection = training_config['dynamic_feature_selection'], data_config_hash = hash_data_config(data_params))
 
         # 3. Train Level-1 models
         current_result, current_predictions, current_probabilities, all_models_for_single_asset = run_single_asset_trainig(

training/meta_labeling.py

@@ -24,7 +24,7 @@ def run_meta_labeling_training(
     print("Feature Selection started")
     backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
     meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X_pca, meta_y)
-    feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = random_string(10))
+    feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], dynamic_feature_selection = training_config['dynamic_feature_selection'], data_config_hash = random_string(10))
     meta_selected_features_X = X_pca[feature_selection_output.columns]
 
     meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1)
@@ -59,4 +59,4 @@ def run_meta_labeling_training(
     )
     meta_result.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True)
 
-    return meta_result, avg_predictions_with_sizing, meta_probabilities, all_models_single_asset
+    return meta_result, avg_predictions_with_sizing, meta_probabilities, all_models_single_asset