fix(Evaluation): correlation test should work on a per asset level, not per model level (#142)

* fix(Evaluation): correlation test should work on a per asset level, not per model level

* fix(Evaluation): correlations series initalized correctly

* fix(Reporting): don't name the run after the incorrectly supposed model_type

* fix(Reporting): put back send_report_to_wandb() into its original place

* fix(CI): sending reports again in comment

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: run-test
+name: run test and pipeline
 on: [push]
 jobs:
   run:

reporting/wandb.py

@@ -9,7 +9,7 @@ def launch_wandb(project_name:str, default_config:dict, sweep:bool=False):
         raise Exception("Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)")  
 
     elif sweep:
-        wandb.init(project=project_name,  config = default_config)             
+        wandb.init(project=project_name, config = default_config)             
         return wandb
     else:
         wandb.init(project=project_name, config = default_config, reinit=True)
@@ -33,13 +33,12 @@ def register_config_with_wandb(wandb: Optional[object], model_config:dict, train
 def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_name: str, model_name: str):
     if wandb is None: return
 
-    run = wandb.init(project=project_name, config={"model_type": model_name}, reinit=True)
-    wandb.run.name = model_name+ "-" + wandb.run.id
-    wandb.run.save()
+    run = wandb.run
+    run.save()
 
     mean_results = weighted_average(results, 'no_of_samples')
     for key, value in mean_results.iteritems():
-        run.log({"model_type": model_name, key: value })
+        run.log({ key: value })
 
     run.finish()
 

run_evaluate_robustness.py

@@ -9,9 +9,9 @@ for index in range(6):
     all_results.append(results_1)
     all_predictions.append(predictions_1)
 
-correlations = pd.Series(index = all_predictions[0].columns)
+correlations = pd.Series()
 
-for asset_name in all_predictions[0].columns:
+for asset_name in [c for c in all_predictions[0].columns if 'ensemble' in c]:
 
     predictions_for_asset = pd.concat([preds[asset_name] for preds in all_predictions], axis=1)
     correlations[asset_name] = predictions_for_asset.corr().mean()[0]

utils/evaluate.py

@@ -55,14 +55,6 @@ def evaluate_predictions(
     df = __preprocess(target_returns, y_pred, y_true, method, no_of_classes, discretize)
     
     scorecard = pd.Series()
-    # we probably will not need regression models at all
-    # if method == 'regression':
-    #     scorecard.loc['RSQ'] = r2_score(df.target_returns, df.y_pred)
-    #     scorecard.loc['MAE'] = mean_absolute_error(df.target_returns, df.y_pred)
-    # elif method == 'classification':
-    #     scorecard.loc['RSQ'] = 0.
-    #     scorecard.loc['MAE Matrix'] = 0.
-    
     
     def count_non_zero(series: pd.Series) -> int:
         return len(series[series != 0])
@@ -95,11 +87,6 @@ def evaluate_predictions(
         for index, row in df.sign_pred.value_counts().iteritems():
             scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred)
 
-    # if method == 'regression':
-    #     scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE']
-    # elif method == 'classification':
-    #     scorecard.loc['edge_to_mae'] = 0.
-
     scorecard = scorecard.round(3)
     if print_results:        
         print("Model name: ", model_name)