From 57f63f1e93d43861a3b693f81ddde282efaf8d70 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Fri, 7 Jan 2022 18:45:03 +0100 Subject: [PATCH] fix(Selection): dynamic step size for feature selection (#123) * fix(Selection): dynamic step size for feature selection * refactor(Pipeline): type definition * chore(Cache): renamed clear_cache script * feat(Config): dynamic feature selection is now a toggleable feature * fix(Training): not passing in necessary parameter --- config/config.py | 3 +++ feature_selection/feature_selection.py | 7 ++++--- run_clean_cache.py => run_clear_cache.py | 0 run_pipeline.py | 5 +++-- training/meta_labeling.py | 4 ++-- 5 files changed, 12 insertions(+), 7 deletions(-) rename run_clean_cache.py => run_clear_cache.py (100%) diff --git a/config/config.py b/config/config.py index 111e66a..c6f0f56 100644 --- a/config/config.py +++ b/config/config.py @@ -5,6 +5,7 @@ def get_default_level_1_daily_config() -> tuple[dict, dict, dict]: meta_labeling_lvl_1 = False, dimensionality_reduction = True, n_features_to_select = 30, + dynamic_feature_selection = True, expanding_window_level1 = False, expanding_window_level2 = False, sliding_window_size_level1 = 380, @@ -46,6 +47,7 @@ def get_default_level_2_hourly_config() -> tuple[dict, dict, dict]: meta_labeling_lvl_1 = True, dimensionality_reduction = True, n_features_to_select = 30, + dynamic_feature_selection = True, expanding_window_level1 = True, expanding_window_level2 = False, sliding_window_size_level1 = 2480, @@ -89,6 +91,7 @@ def get_default_level_2_daily_config() -> tuple[dict, dict, dict]: meta_labeling_lvl_1 = True, dimensionality_reduction = True, n_features_to_select = 30, + dynamic_feature_selection = True, expanding_window_level1 = False, expanding_window_level2 = True, sliding_window_size_level1 = 380, diff --git a/feature_selection/feature_selection.py b/feature_selection/feature_selection.py index 2b46d5e..d02512e 100644 --- a/feature_selection/feature_selection.py +++ b/feature_selection/feature_selection.py @@ -9,7 +9,7 @@ from diskcache import Cache cache = Cache(".cachedir/feature_selection") def select_features(**kwargs) -> pd.DataFrame: - hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling'] + hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling'] + str(kwargs['dynamic_feature_selection']) if hashed in cache: return cache.get(hashed) else: @@ -17,7 +17,7 @@ def select_features(**kwargs) -> pd.DataFrame: cache[hashed] = return_value return return_value -def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel, scaling: ScalerTypes, data_config_hash: str) -> pd.DataFrame: +def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel, scaling: ScalerTypes, dynamic_feature_selection: bool, data_config_hash: str) -> pd.DataFrame: ''' Select features using RFECV, returns a pd.DataFrame (X) with only the selected features.''' if model.model_type != 'ml': return X @@ -33,7 +33,8 @@ def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to feat_selector_model = backup_model.model # selector = RFECV(feat_selector_model, cv = cv, step=5, min_features_to_select=min_features_to_select) - selector = RFE(feat_selector_model, n_features_to_select= n_features_to_select, step=5) + step = 0.05 if dynamic_feature_selection else 5 + selector = RFE(feat_selector_model, n_features_to_select= n_features_to_select, step=step) selector = selector.fit(X_scaled, y) print("Kept %d features out of %d" % (selector.n_features_, X_scaled.shape[1])) diff --git a/run_clean_cache.py b/run_clear_cache.py similarity index 100% rename from run_clean_cache.py rename to run_clear_cache.py diff --git a/run_pipeline.py b/run_pipeline.py index 71e2e03..a16bb73 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -12,6 +12,7 @@ from feature_selection.dim_reduction import reduce_dimensionality from training.meta_labeling import run_meta_labeling_training from training.averaged import average_and_evaluate_predictions from reporting.reporting import report_results +from typing import Callable, Optional import ray ray.init() @@ -22,7 +23,7 @@ def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:obj report_results(results, all_predictions, model_config, wandb, sweep, project_name) -def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:object): +def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config: Callable) -> tuple[Optional[object], dict, dict, dict]: model_config, training_config, data_config = get_config() wandb = None if with_wandb: @@ -66,7 +67,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): print("Feature Selection started") # TODO: this needs to be done per model! backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification - X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = hash_data_config(data_params)) + X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], dynamic_feature_selection = training_config['dynamic_feature_selection'], data_config_hash = hash_data_config(data_params)) # 3. Train Level-1 models current_result, current_predictions, current_probabilities, all_models_for_single_asset = run_single_asset_trainig( diff --git a/training/meta_labeling.py b/training/meta_labeling.py index 0844ac2..90f1107 100644 --- a/training/meta_labeling.py +++ b/training/meta_labeling.py @@ -24,7 +24,7 @@ def run_meta_labeling_training( print("Feature Selection started") backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X_pca, meta_y) - feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = random_string(10)) + feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], dynamic_feature_selection = training_config['dynamic_feature_selection'], data_config_hash = random_string(10)) meta_selected_features_X = X_pca[feature_selection_output.columns] meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1) @@ -59,4 +59,4 @@ def run_meta_labeling_training( ) meta_result.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True) - return meta_result, avg_predictions_with_sizing, meta_probabilities, all_models_single_asset \ No newline at end of file + return meta_result, avg_predictions_with_sizing, meta_probabilities, all_models_single_asset