fix(Reporting): use weighted average (with no_of_samples as weights) and only report level-1 OR level-2 model performance (#91)

* fix(Reporting): use weighted average (with no_of_samples as weights) and only report level-1 OR level-2 model performance

* chore(Config): updated sweep config

* fix(Reporting): missing import

* fix(Evaluation): get_first_valid_return_index can deal with zero valid indexes

* fix(Training): increase threshold for skipping assets

* fix(DataLoader): target asset should be always the first column

reporting/wandb.py

@@ -1,5 +1,6 @@
 import pandas as pd
 from typing import Optional
+from utils.helpers import weighted_average
 
 def launch_wandb(project_name:str, default_config:dict, sweep:bool=False):
     from wandb_setup import get_wandb
@@ -34,7 +35,7 @@ def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_
     wandb.run.name = model_name+ "-" + wandb.run.id
     wandb.run.save()
 
-    mean_results = results.mean(axis = 1)
+    mean_results = weighted_average(results, 'no_of_samples')
     for key, value in mean_results.iteritems():
         run.log({"model_type": model_name, key: value })
 

run_pipeline.py

@@ -5,7 +5,7 @@ from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_
 from models.model_map import map_model_name_to_function
 from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config
 from config import get_default_config, validate_config, get_model_name
-from utils.helpers import get_first_valid_return_index
+from utils.helpers import get_first_valid_return_index, weighted_average
 
 def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
     model_config, training_config, data_config = get_default_config()
@@ -21,7 +21,7 @@ def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
     
 
 
-def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict ):
+def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict):
     results = pd.DataFrame()
     validate_config(model_config, training_config, data_config)
 
@@ -36,7 +36,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
         X, y, target_returns = load_data(**data_params)
         first_valid_index = get_first_valid_return_index(X.iloc[:,0])
         samples_to_train = len(y) - first_valid_index
-        if samples_to_train < training_config['sliding_window_size'] * 2.6:
+        if samples_to_train < training_config['sliding_window_size'] * 3:
             print("Not enough samples to train")
             continue
 
@@ -83,22 +83,25 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
             all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
         
     # 4. Save & report results
-    send_report_to_wandb(results, wandb, project_name, get_model_name(model_config))
     results.to_csv('results.csv')
 
-    level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]
-    ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]]
+    level1_columns = results[[column for column in results.columns if 'lvl1' in column]]
+    level2_columns = results[[column for column in results.columns if 'lvl2' in column]]
+    
+    # Only send the results of the final model to wandb
+    results_to_send = level2_columns if level2_columns.shape[1] > 0 else level1_columns
+    send_report_to_wandb(results_to_send, wandb, project_name, get_model_name(model_config))
 
     print("\n--------\n")
-    print("Benchmark buy-and-hold sharpe: ", round(results.loc['benchmark_sharpe'].mean(), 3))
+    print("Benchmark buy-and-hold sharpe: ", round(weighted_average(results, 'no_of_samples').loc['benchmark_sharpe'], 3))
 
     print("Level-1: Number of samples evaluated: ", level1_columns.loc['no_of_samples'].sum())
-    print("Mean Sharpe ratio for Level-1 models: ", round(level1_columns.loc['sharpe'].mean(), 3))
-    print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(level1_columns.loc['prob_sharpe'].mean(), 3))
+    print("Mean Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['sharpe'], 3))
+    print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
 
-    print("Level-2 (Ensemble): Number of samples evaluated: ", ensemble_columns.loc['no_of_samples'].sum())
-    print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_columns.loc['sharpe'].mean(), 3))
-    print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_columns.loc['prob_sharpe'].mean(), 3))
+    print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum())
+    print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['sharpe'].mean(), 3))
+    print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
 
     if sweep:
         if wandb.run is not None:

sweep_level_2.yaml

@@ -45,5 +45,5 @@ parameters:
       values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']]
       distribution: categorical
     other_features:
-      values: [[], ['level_1'], ['level_2']]
+      values: [[], ['level_1']]
       distribution: categorical

training/training.py

@@ -57,7 +57,7 @@ def run_single_asset_trainig(
             method = method,
             no_of_classes=no_of_classes
         )
-        column_name = ticker_to_predict + "_" + model_name + "_" + str(level)
+        column_name = ticker_to_predict + "_" + model_name + "_lvl" + str(level)
         results[column_name] = result
         # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary
         predictions["model_" + column_name] = preds

utils/helpers.py

@@ -3,7 +3,24 @@ import pandas as pd
 import numpy as np
 
 def get_first_valid_return_index(series: pd.Series) -> int:
-    return np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series))))[0][0]
+    double_nested_results = np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series))))
+    if len(double_nested_results) == 0:
+        return 0
+    nested_result = double_nested_results[0]
+    if len(nested_result) == 0:
+        return 0
+    return nested_result[0]
 
 def flatten(list_of_lists: list) -> list:
-    return [item for sublist in list_of_lists for item in sublist]
+    return [item for sublist in list_of_lists for item in sublist]
+
+def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.DataFrame:
+    mean_df = df.iloc[:,0]
+    weights = df.loc[weights_source]
+
+    for i, row in df.iterrows():
+        if i == weights_source: continue
+        mean_df.loc[i] = (row * weights).sum() / df.loc[weights_source].sum()
+
+    return mean_df
+

utils/load_data.py

@@ -35,9 +35,10 @@ def load_data(path: str,
         - Series `forward_returns` with the target asset returns shifted by 1 day
     """
 
-    
     files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
-    files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(target_asset))]
+    target_file = [f for f in files if f.startswith(target_asset)]
+    other_files = [f for f in files if load_other_assets == True and f.startswith(target_asset) == False]
+    files = target_file + other_files
     def is_target_asset(target_asset: str, file: str): return file.split('.')[0].startswith(target_asset)
     dfs = [__load_df(
         path=os.path.join(path,f),