diff --git a/config/config.py b/config/config.py index 7532791..111e66a 100644 --- a/config/config.py +++ b/config/config.py @@ -2,7 +2,7 @@ def get_default_level_1_daily_config() -> tuple[dict, dict, dict]: training_config = dict( - meta_labeling_lvl_1 = True, + meta_labeling_lvl_1 = False, dimensionality_reduction = True, n_features_to_select = 30, expanding_window_level1 = False, diff --git a/run_model_dev.py b/run_model_dev.py new file mode 100644 index 0000000..3b2b971 --- /dev/null +++ b/run_model_dev.py @@ -0,0 +1,5 @@ +from run_pipeline import run_pipeline +from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config + + +run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_level_1_daily_config) diff --git a/run_pipeline.py b/run_pipeline.py index f2378d5..71e2e03 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -16,14 +16,14 @@ import ray ray.init() -def run_pipeline(project_name:str, with_wandb: bool, sweep: bool): - wandb, model_config, training_config, data_config = __setup_pipeline(project_name, with_wandb, sweep) +def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:object): + wandb, model_config, training_config, data_config = __setup_pipeline(project_name, with_wandb, sweep, get_config) results, all_predictions, all_probabilities = __run_training(model_config, training_config, data_config) report_results(results, all_predictions, model_config, wandb, sweep, project_name) -def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): - model_config, training_config, data_config = get_default_level_2_daily_config() +def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:object): + model_config, training_config, data_config = get_config() wandb = None if with_wandb: wandb = launch_wandb(project_name=project_name, default_config=dict(**model_config, **training_config, **data_config), sweep=sweep) @@ -37,6 +37,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): results = pd.DataFrame() all_predictions = pd.DataFrame() all_probabilities = pd.DataFrame() + all_models_for_all_assets = dict() validate_config(model_config, training_config, data_config) for asset in data_config['assets']: @@ -68,7 +69,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = hash_data_config(data_params)) # 3. Train Level-1 models - current_result, current_predictions, current_probabilities = run_single_asset_trainig( + current_result, current_predictions, current_probabilities, all_models_for_single_asset = run_single_asset_trainig( ticker_to_predict = asset[1], original_X = original_X, X = X, @@ -84,12 +85,16 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): level = 1 ) + all_models_for_all_assets[asset[1]] = dict( + name=asset[1], + models=all_models_for_single_asset) + # 4. Train a Meta-Labeling model for each Level-1 model and replace its predictions with the meta-labeling predictions if training_config['meta_labeling_lvl_1'] == True: - for column in current_result.columns: - lvl1_model_predictions = current_predictions[column] - prev_sharpe = current_result[column]['sharpe'] - lvl1_meta_result, lvl1_meta_preds, lvl1_meta_probabilities = run_meta_labeling_training( + for model_name in current_result.columns: + lvl1_model_predictions = current_predictions[model_name] + prev_sharpe = current_result[model_name]['sharpe'] + lvl1_meta_result, lvl1_meta_preds, lvl1_meta_probabilities, meta_labeling_models = run_meta_labeling_training( target_asset=asset[1], X_pca = X_pca, input_predictions= lvl1_model_predictions, @@ -101,9 +106,11 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): ) new_sharpe = lvl1_meta_result['sharpe'] print("Improvement in sharpe for the meta model: ", ((new_sharpe / prev_sharpe) - 1) * 100, "%") - current_result[column] = lvl1_meta_result - current_predictions[column] = lvl1_meta_preds + current_result[model_name] = lvl1_meta_result + current_predictions[model_name] = lvl1_meta_preds + all_models_for_all_assets[asset[1]][model_name] = meta_labeling_models + results = pd.concat([results, current_result], axis=1) # With static models, because of the lag in the indicator, the first prediction is NA, so we fill it with zero. all_predictions = pd.concat([all_predictions, current_predictions], axis=1).fillna(0.) @@ -115,7 +122,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): averaged_predictions, averaged_results = average_and_evaluate_predictions(current_predictions, y, target_returns, data_config) # 3. Train a Meta-labeling model on the averaged level-1 model predictions - meta_result, avg_predictions_with_sizing, meta_probabilities = run_meta_labeling_training( + meta_result, avg_predictions_with_sizing, meta_probabilities, meta_labeling_models = run_meta_labeling_training( target_asset=asset[1], X_pca = X_pca, input_predictions= averaged_predictions, @@ -136,4 +143,4 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): if __name__ == '__main__': - run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False) \ No newline at end of file + run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_level_2_daily_config) \ No newline at end of file diff --git a/run_sweep.py b/run_sweep.py index 6a3ce0a..703c2d1 100644 --- a/run_sweep.py +++ b/run_sweep.py @@ -1,3 +1,4 @@ from run_pipeline import run_pipeline +from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config -run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True) \ No newline at end of file +run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True, get_config= get_default_level_2_daily_config) \ No newline at end of file diff --git a/training/meta_labeling.py b/training/meta_labeling.py index 9976228..0844ac2 100644 --- a/training/meta_labeling.py +++ b/training/meta_labeling.py @@ -15,7 +15,7 @@ def run_meta_labeling_training( data_config: dict, model_config: dict, training_config: dict - ) -> tuple[pd.Series, pd.Series, pd.DataFrame]: + ) -> tuple[pd.Series, pd.Series, pd.DataFrame, dict]: discretize = discretize_threeway_threshold(0.33) discretized_predictions = input_predictions.apply(discretize) @@ -29,7 +29,7 @@ def run_meta_labeling_training( meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1) - _, meta_preds, meta_probabilities = run_single_asset_trainig( + _, meta_preds, meta_probabilities, all_models_single_asset = run_single_asset_trainig( ticker_to_predict = "prediction_correct", original_X = meta_X, X = meta_X, @@ -59,4 +59,4 @@ def run_meta_labeling_training( ) meta_result.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True) - return meta_result, avg_predictions_with_sizing, meta_probabilities \ No newline at end of file + return meta_result, avg_predictions_with_sizing, meta_probabilities, all_models_single_asset \ No newline at end of file diff --git a/training/training.py b/training/training.py index 466eca9..bc2b271 100644 --- a/training/training.py +++ b/training/training.py @@ -20,12 +20,13 @@ def run_single_asset_trainig( scaler: ScalerTypes, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], level: int - ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]: scaler = get_scaler(scaler) results = pd.DataFrame() + all_models_single_asset = dict() predictions = pd.DataFrame(index=y.index) probabilities = pd.DataFrame(index=y.index) @@ -41,6 +42,7 @@ def run_single_asset_trainig( retrain_every = retrain_every, scaler = scaler ) + assert len(preds) == len(y) result = evaluate_predictions( model_name = model_name, @@ -53,6 +55,7 @@ def run_single_asset_trainig( ) column_name = "model_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level) results[column_name] = result + all_models_single_asset[model_name] = model_over_time # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary predictions[column_name] = preds probs_column_name = "probs_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level) @@ -60,4 +63,4 @@ def run_single_asset_trainig( probabilities = pd.concat([probabilities, probs], axis=1) - return results, predictions, probabilities \ No newline at end of file + return results, predictions, probabilities, all_models_single_asset \ No newline at end of file