diff --git a/reporting/wandb.py b/reporting/wandb.py index e28e775..0002ff0 100644 --- a/reporting/wandb.py +++ b/reporting/wandb.py @@ -1,5 +1,6 @@ import pandas as pd from typing import Optional +from utils.helpers import weighted_average def launch_wandb(project_name:str, default_config:dict, sweep:bool=False): from wandb_setup import get_wandb @@ -34,7 +35,7 @@ def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_ wandb.run.name = model_name+ "-" + wandb.run.id wandb.run.save() - mean_results = results.mean(axis = 1) + mean_results = weighted_average(results, 'no_of_samples') for key, value in mean_results.iteritems(): run.log({"model_type": model_name, key: value }) diff --git a/run_pipeline.py b/run_pipeline.py index c23778d..e97abb6 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -5,7 +5,7 @@ from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_ from models.model_map import map_model_name_to_function from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config from config import get_default_config, validate_config, get_model_name -from utils.helpers import get_first_valid_return_index +from utils.helpers import get_first_valid_return_index, weighted_average def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): model_config, training_config, data_config = get_default_config() @@ -21,7 +21,7 @@ def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): -def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict ): +def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict): results = pd.DataFrame() validate_config(model_config, training_config, data_config) @@ -36,7 +36,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co X, y, target_returns = load_data(**data_params) first_valid_index = get_first_valid_return_index(X.iloc[:,0]) samples_to_train = len(y) - first_valid_index - if samples_to_train < training_config['sliding_window_size'] * 2.6: + if samples_to_train < training_config['sliding_window_size'] * 3: print("Not enough samples to train") continue @@ -83,22 +83,25 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1) # 4. Save & report results - send_report_to_wandb(results, wandb, project_name, get_model_name(model_config)) results.to_csv('results.csv') - level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]] - ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]] + level1_columns = results[[column for column in results.columns if 'lvl1' in column]] + level2_columns = results[[column for column in results.columns if 'lvl2' in column]] + + # Only send the results of the final model to wandb + results_to_send = level2_columns if level2_columns.shape[1] > 0 else level1_columns + send_report_to_wandb(results_to_send, wandb, project_name, get_model_name(model_config)) print("\n--------\n") - print("Benchmark buy-and-hold sharpe: ", round(results.loc['benchmark_sharpe'].mean(), 3)) + print("Benchmark buy-and-hold sharpe: ", round(weighted_average(results, 'no_of_samples').loc['benchmark_sharpe'], 3)) print("Level-1: Number of samples evaluated: ", level1_columns.loc['no_of_samples'].sum()) - print("Mean Sharpe ratio for Level-1 models: ", round(level1_columns.loc['sharpe'].mean(), 3)) - print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(level1_columns.loc['prob_sharpe'].mean(), 3)) + print("Mean Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['sharpe'], 3)) + print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3)) - print("Level-2 (Ensemble): Number of samples evaluated: ", ensemble_columns.loc['no_of_samples'].sum()) - print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_columns.loc['sharpe'].mean(), 3)) - print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_columns.loc['prob_sharpe'].mean(), 3)) + print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum()) + print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['sharpe'].mean(), 3)) + print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3)) if sweep: if wandb.run is not None: diff --git a/sweep_level_2.yaml b/sweep_level_2.yaml index 489bc36..c164ae5 100644 --- a/sweep_level_2.yaml +++ b/sweep_level_2.yaml @@ -45,5 +45,5 @@ parameters: values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']] distribution: categorical other_features: - values: [[], ['level_1'], ['level_2']] + values: [[], ['level_1']] distribution: categorical \ No newline at end of file diff --git a/training/training.py b/training/training.py index 4afe7f9..f95e5f4 100644 --- a/training/training.py +++ b/training/training.py @@ -57,7 +57,7 @@ def run_single_asset_trainig( method = method, no_of_classes=no_of_classes ) - column_name = ticker_to_predict + "_" + model_name + "_" + str(level) + column_name = ticker_to_predict + "_" + model_name + "_lvl" + str(level) results[column_name] = result # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary predictions["model_" + column_name] = preds diff --git a/utils/helpers.py b/utils/helpers.py index b7e4868..09761ea 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -3,7 +3,24 @@ import pandas as pd import numpy as np def get_first_valid_return_index(series: pd.Series) -> int: - return np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series))))[0][0] + double_nested_results = np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series)))) + if len(double_nested_results) == 0: + return 0 + nested_result = double_nested_results[0] + if len(nested_result) == 0: + return 0 + return nested_result[0] def flatten(list_of_lists: list) -> list: - return [item for sublist in list_of_lists for item in sublist] \ No newline at end of file + return [item for sublist in list_of_lists for item in sublist] + +def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.DataFrame: + mean_df = df.iloc[:,0] + weights = df.loc[weights_source] + + for i, row in df.iterrows(): + if i == weights_source: continue + mean_df.loc[i] = (row * weights).sum() / df.loc[weights_source].sum() + + return mean_df + diff --git a/utils/load_data.py b/utils/load_data.py index c0d734a..52ab5e1 100644 --- a/utils/load_data.py +++ b/utils/load_data.py @@ -35,9 +35,10 @@ def load_data(path: str, - Series `forward_returns` with the target asset returns shifted by 1 day """ - files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')] - files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(target_asset))] + target_file = [f for f in files if f.startswith(target_asset)] + other_files = [f for f in files if load_other_assets == True and f.startswith(target_asset) == False] + files = target_file + other_files def is_target_asset(target_asset: str, file: str): return file.split('.')[0].startswith(target_asset) dfs = [__load_df( path=os.path.join(path,f),