diff --git a/data_loader/process_data.py b/data_loader/process_data.py index e5c32e0..ae1f764 100644 --- a/data_loader/process_data.py +++ b/data_loader/process_data.py @@ -10,18 +10,11 @@ import warnings -def process_data(X:pd.DataFrame, y:pd.Series, configs: dict) -> tuple[pd.DataFrame,pd.DataFrame,pd.DataFrame]: +def process_data(X:pd.DataFrame, y:pd.Series, configs: dict) -> tuple[pd.DataFrame,pd.DataFrame]: model_config, training_config, data_config = itemgetter('model_config', 'training_config', 'data_config')(configs) original_X = X.copy() - - # 2a. Dimensionality Reduction (optional) - if training_config['dimensionality_reduction']: - X_pca = reduce_dimensionality(X, int(len(X.columns) / 2)) - X = X_pca.copy() - else: - X_pca = X.copy() # 2b. Feature Selection print("Feature Selection started") @@ -29,7 +22,7 @@ def process_data(X:pd.DataFrame, y:pd.Series, configs: dict) -> tuple[pd.DataFra backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification X = select_features(X = X, y = y, model = model_config['primary_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler']) - return X, original_X, X_pca + return X, original_X def check_data(X:pd.DataFrame, y:pd.Series, training_config:dict): """ Returns True if data is valid, else returns False.""" diff --git a/run_pipeline.py b/run_pipeline.py index 1f390a1..5e59937 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -56,13 +56,13 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): # 1. Load data, check for validity and process data (feature selection, dimensionality reduction, etc.) X, y, target_returns = load_data(**configs['data_config']) if check_data(X, y, training_config) is False: continue - X, original_X, X_pca = process_data(X, y, configs) + X, original_X = process_data(X, y, configs) # 2. Train a Primary model with optional metalabeling for each asset - training_step_primary, current_predictions = primary_step(X, y, original_X, X_pca, asset, target_returns, configs, reporting) + training_step_primary, current_predictions = primary_step(X, y, original_X, asset, target_returns, configs, reporting) # 3. Train an Ensemble model with optional metalabeling for each asset - training_step_secondary = secondary_step(X, y, original_X, X_pca, current_predictions, asset, target_returns, configs, reporting) + training_step_secondary = secondary_step(X, y, original_X, current_predictions, asset, target_returns, configs, reporting) # 4. Save the models reporting.all_assets.append(Reporting.Asset(ticker=asset[1], primary=training_step_primary, secondary=training_step_secondary)) diff --git a/training/meta_labeling.py b/training/meta_labeling.py index 6a9c67a..a76afea 100644 --- a/training/meta_labeling.py +++ b/training/meta_labeling.py @@ -10,7 +10,7 @@ from reporting.types import Reporting def train_meta_labeling_model( target_asset: str, - X_pca: pd.DataFrame, + X: pd.DataFrame, input_predictions: pd.Series, y: pd.Series, target_returns: pd.Series, @@ -28,9 +28,9 @@ def train_meta_labeling_model( print("Feature Selection started") backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification - meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X_pca, meta_y) + meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X, meta_y) feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = models[0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler']) - meta_selected_features_X = X_pca[feature_selection_output.columns] + meta_selected_features_X = X[feature_selection_output.columns] meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1) diff --git a/training/training_steps.py b/training/training_steps.py index fdabc08..ff2bdd5 100644 --- a/training/training_steps.py +++ b/training/training_steps.py @@ -11,7 +11,6 @@ def primary_step( X: pd.DataFrame, y:pd.Series, original_X:pd.DataFrame, - X_pca:pd.DataFrame, asset:list, target_returns:pd.Series, configs: dict, @@ -46,7 +45,7 @@ def primary_step( primary_model_predictions = current_predictions[model_name] primary_meta_result, primary_meta_preds, primary_meta_probabilities, meta_labeling_models = train_meta_labeling_model( target_asset=asset[1], - X_pca = X_pca, + X = original_X, input_predictions= primary_model_predictions, y = y, target_returns = target_returns, @@ -73,7 +72,6 @@ def secondary_step( X:pd.DataFrame, y:pd.Series, original_X:pd.DataFrame, - X_pca:pd.DataFrame, current_predictions:pd.DataFrame, asset:list, target_returns:pd.Series, @@ -114,7 +112,7 @@ def secondary_step( # 3. Train a Meta-labeling model on the averaged level-1 model predictions ensemble_meta_result, ensemble_meta_predictions, ensemble_meta_probabilities, ensemble_meta_labeling_models = train_meta_labeling_model( target_asset=asset[1], - X_pca = X_pca, + X = original_X, input_predictions= ensemble_predictions, y = y, target_returns = target_returns,