gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-16 04:18:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(Reporting): use weighted average (with no_of_samples as weights) and only report level-1 OR level-2 model performance (#91)

Browse Source 
* fix(Reporting): use weighted average (with no_of_samples as weights) and only report level-1 OR level-2 model performance

* chore(Config): updated sweep config

* fix(Reporting): missing import

* fix(Evaluation): get_first_valid_return_index can deal with zero valid indexes

* fix(Training): increase threshold for skipping assets

* fix(DataLoader): target asset should be always the first column
This commit is contained in:
Mark Aron Szulyovszky
2021-12-26 12:15:11 +01:00
committed by GitHub
parent fc4e59a7d2
commit a9b05dbd42
6 changed files with 41 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
run_pipeline.py
+15 -12
View File
@@ -5,7 +5,7 @@ from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_
from models.model_map import map_model_name_to_function
from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config
from config import get_default_config, validate_config, get_model_name
from utils.helpers import get_first_valid_return_index
from utils.helpers import get_first_valid_return_index, weighted_average
def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
model_config, training_config, data_config = get_default_config()
@@ -21,7 +21,7 @@ def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict ):
def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict):
results = pd.DataFrame()
validate_config(model_config, training_config, data_config)
@@ -36,7 +36,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
X, y, target_returns = load_data(**data_params)
first_valid_index = get_first_valid_return_index(X.iloc[:,0])
samples_to_train = len(y) - first_valid_index
if samples_to_train < training_config['sliding_window_size'] * 2.6:
if samples_to_train < training_config['sliding_window_size'] * 3:
print("Not enough samples to train")
continue
@@ -83,22 +83,25 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
# 4. Save & report results
send_report_to_wandb(results, wandb, project_name, get_model_name(model_config))
results.to_csv('results.csv')
level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]
ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]]
level1_columns = results[[column for column in results.columns if 'lvl1' in column]]
level2_columns = results[[column for column in results.columns if 'lvl2' in column]]
# Only send the results of the final model to wandb
results_to_send = level2_columns if level2_columns.shape[1] > 0 else level1_columns
send_report_to_wandb(results_to_send, wandb, project_name, get_model_name(model_config))
print("\n--------\n")
print("Benchmark buy-and-hold sharpe: ", round(results.loc['benchmark_sharpe'].mean(), 3))
print("Benchmark buy-and-hold sharpe: ", round(weighted_average(results, 'no_of_samples').loc['benchmark_sharpe'], 3))
print("Level-1: Number of samples evaluated: ", level1_columns.loc['no_of_samples'].sum())
print("Mean Sharpe ratio for Level-1 models: ", round(level1_columns.loc['sharpe'].mean(), 3))
print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(level1_columns.loc['prob_sharpe'].mean(), 3))
print("Mean Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['sharpe'], 3))
print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
print("Level-2 (Ensemble): Number of samples evaluated: ", ensemble_columns.loc['no_of_samples'].sum())
print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_columns.loc['sharpe'].mean(), 3))
print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_columns.loc['prob_sharpe'].mean(), 3))
print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum())
print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['sharpe'].mean(), 3))
print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
if sweep:
if wandb.run is not None: