feat(Inference): Models are collected and structured. (#120)

* feat: Added collection of models into a dictionary.

* feat: Models are now saved in a structured way into a dictionary.

* Rename run_model_test.py to run_model_dev.py

* fix(Pipeline): missing variable statement

Co-authored-by: Mark Aron Szulyovszky <mark.szulyovszky@gmail.com>

config/config.py

@@ -2,7 +2,7 @@
 def get_default_level_1_daily_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
-        meta_labeling_lvl_1 = True,
+        meta_labeling_lvl_1 = False,
         dimensionality_reduction = True,
         n_features_to_select = 30,
         expanding_window_level1 = False,

run_model_dev.py

@@ -0,0 +1,5 @@
+from run_pipeline import run_pipeline
+from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config
+
+
+run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_level_1_daily_config)

run_pipeline.py

@@ -16,14 +16,14 @@ import ray
 ray.init()
 
 
-def run_pipeline(project_name:str, with_wandb: bool, sweep: bool):
+def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:object):
-    wandb, model_config, training_config, data_config = __setup_pipeline(project_name, with_wandb, sweep)
+    wandb, model_config, training_config, data_config = __setup_pipeline(project_name, with_wandb, sweep, get_config)
     results, all_predictions, all_probabilities = __run_training(model_config, training_config, data_config)  
     report_results(results, all_predictions, model_config, wandb, sweep, project_name)
     
 
-def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
+def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:object):
-    model_config, training_config, data_config = get_default_level_2_daily_config()
+    model_config, training_config, data_config = get_config() 
     wandb = None
     if with_wandb: 
         wandb = launch_wandb(project_name=project_name, default_config=dict(**model_config, **training_config, **data_config), sweep=sweep)
@@ -37,6 +37,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
     results = pd.DataFrame()
     all_predictions = pd.DataFrame()
     all_probabilities = pd.DataFrame()
+    all_models_for_all_assets = dict()
     validate_config(model_config, training_config, data_config)
 
     for asset in data_config['assets']:
@@ -68,7 +69,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
         X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = hash_data_config(data_params))
 
         # 3. Train Level-1 models
-        current_result, current_predictions, current_probabilities = run_single_asset_trainig(
+        current_result, current_predictions, current_probabilities, all_models_for_single_asset = run_single_asset_trainig(
             ticker_to_predict = asset[1],
             original_X = original_X,
             X = X,
@@ -84,12 +85,16 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
             level = 1
         )
         
+        all_models_for_all_assets[asset[1]] = dict(
+            name=asset[1], 
+            models=all_models_for_single_asset)
+        
         # 4. Train a Meta-Labeling model for each Level-1 model and replace its predictions with the meta-labeling predictions
         if training_config['meta_labeling_lvl_1'] == True:
-            for column in current_result.columns:
+            for model_name in current_result.columns:
-                lvl1_model_predictions = current_predictions[column]
+                lvl1_model_predictions = current_predictions[model_name]
-                prev_sharpe = current_result[column]['sharpe']
+                prev_sharpe = current_result[model_name]['sharpe']
-                lvl1_meta_result, lvl1_meta_preds, lvl1_meta_probabilities = run_meta_labeling_training(
+                lvl1_meta_result, lvl1_meta_preds, lvl1_meta_probabilities, meta_labeling_models = run_meta_labeling_training(
                     target_asset=asset[1],
                     X_pca = X_pca,
                     input_predictions= lvl1_model_predictions,
@@ -101,9 +106,11 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
                 )
                 new_sharpe = lvl1_meta_result['sharpe']
                 print("Improvement in sharpe for the meta model: ", ((new_sharpe / prev_sharpe) - 1) * 100, "%")
-                current_result[column] = lvl1_meta_result
+                current_result[model_name] = lvl1_meta_result
-                current_predictions[column] = lvl1_meta_preds
+                current_predictions[model_name] = lvl1_meta_preds
 
+                all_models_for_all_assets[asset[1]][model_name] = meta_labeling_models
+        
         results = pd.concat([results, current_result], axis=1)
         # With static models, because of the lag in the indicator, the first prediction is NA, so we fill it with zero.
         all_predictions = pd.concat([all_predictions, current_predictions], axis=1).fillna(0.)
@@ -115,7 +122,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
             averaged_predictions, averaged_results = average_and_evaluate_predictions(current_predictions, y, target_returns, data_config)
 
             # 3. Train a Meta-labeling model on the averaged level-1 model predictions
-            meta_result, avg_predictions_with_sizing, meta_probabilities = run_meta_labeling_training(
+            meta_result, avg_predictions_with_sizing, meta_probabilities, meta_labeling_models = run_meta_labeling_training(
                 target_asset=asset[1],
                 X_pca = X_pca,
                 input_predictions= averaged_predictions,
@@ -136,4 +143,4 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
 
     
 if __name__ == '__main__':
-    run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False)
+    run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_level_2_daily_config)

run_sweep.py

@@ -1,3 +1,4 @@
 from run_pipeline import run_pipeline
+from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config
 
-run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True)
+run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True, get_config= get_default_level_2_daily_config)

training/meta_labeling.py

@@ -15,7 +15,7 @@ def run_meta_labeling_training(
                             data_config: dict,
                             model_config: dict,
                             training_config: dict
-                        ) -> tuple[pd.Series, pd.Series, pd.DataFrame]:
+                        ) -> tuple[pd.Series, pd.Series, pd.DataFrame, dict]:
 
     discretize = discretize_threeway_threshold(0.33)
     discretized_predictions = input_predictions.apply(discretize)
@@ -29,7 +29,7 @@ def run_meta_labeling_training(
 
     meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1)
 
-    _, meta_preds, meta_probabilities = run_single_asset_trainig(
+    _, meta_preds, meta_probabilities, all_models_single_asset = run_single_asset_trainig(
         ticker_to_predict = "prediction_correct",
         original_X = meta_X,
         X = meta_X,
@@ -59,4 +59,4 @@ def run_meta_labeling_training(
     )
     meta_result.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True)
 
-    return meta_result, avg_predictions_with_sizing, meta_probabilities
+    return meta_result, avg_predictions_with_sizing, meta_probabilities, all_models_single_asset

training/training.py

@@ -20,12 +20,13 @@ def run_single_asset_trainig(
                     scaler: ScalerTypes,
                     no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                     level: int
-    ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]:
 
 
     scaler = get_scaler(scaler)
 
     results = pd.DataFrame()
+    all_models_single_asset = dict()
     predictions = pd.DataFrame(index=y.index)
     probabilities = pd.DataFrame(index=y.index)
     
@@ -41,6 +42,7 @@ def run_single_asset_trainig(
             retrain_every = retrain_every,
             scaler = scaler
         )
+        
         assert len(preds) == len(y)
         result = evaluate_predictions(
             model_name = model_name,
@@ -53,6 +55,7 @@ def run_single_asset_trainig(
         )
         column_name = "model_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level)
         results[column_name] = result
+        all_models_single_asset[model_name] = model_over_time
         # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary
         predictions[column_name] = preds
         probs_column_name = "probs_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level)
@@ -60,4 +63,4 @@ def run_single_asset_trainig(
         probabilities = pd.concat([probabilities, probs], axis=1)
         
 
-    return results, predictions, probabilities
+    return results, predictions, probabilities, all_models_single_asset