gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-16 20:38:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(Naming): use new convention, added Ensemble model parameter back, support multiple Meta-Labeling models (#132)

Browse Source 
* refactor(Naming): use `primary_models` & `meta_labeling_models`

* refactor(Naming): using primary * meta_labeling across config and in pipeline

* feat(Pipeline): added back Ensemble models

* fix(Pipeline): compiler error

* fix(Config): typo

* chore(Pipeline): removed unused averaging step

* revert the changes in discretizing

* chore(Pipeline): remove sharpe improvement logging

* fix(Pipeline): ensemble predictions should be a pd.Series instead of a DataFrame

* fix(Pipeline): discard unnecessary ensemble_probabilities

* fix(Pipeline): fixes regarding various meta-labeling ensemble bugs

* fix(Reporting): use the new naming convention

* fix(Reporting): use the right variable

* feat(Sweep): new sweep for ensemble models

* fix(Sweep): config reference

* fix(Config): simplified dev config

* fix(Models): use the faster LR model

* fix(Models): use LGBM in the meta-labeling model for speed

* fix(Selection): always use the first model for feature selection, commented out caching from select_features() as it's close to redundant in terms of speed
This commit is contained in:
Mark Aron Szulyovszky
2022-01-09 17:21:06 +01:00
committed by GitHub
parent 22b3167cb9
commit b1c04afb13
20 changed files with 202 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reporting/reporting.py
+16 -16
View File
@@ -5,36 +5,36 @@ from utils.helpers import weighted_average
def report_results(results:pd.DataFrame, all_predictions:pd.DataFrame, model_config:dict, wandb, sweep: bool, project_name:str):
level1_columns = results[[column for column in results.columns if 'lvl1' in column]]
level2_columns = results[[column for column in results.columns if 'lvl2' in column]]
primary_results = results[[column for column in results.columns if 'primary' in column]]
ensemble_results = results[[column for column in results.columns if 'ensemble' in column]]
# Only send the results of the final model to wandb
results_to_send = level2_columns if level2_columns.shape[1] > 0 else level1_columns
results_to_send = ensemble_results if ensemble_results.shape[1] > 0 else primary_results
send_report_to_wandb(results_to_send, wandb, project_name, get_model_name(model_config))
results.to_csv('output/results.csv')
level1_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl1' in column]]
level2_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl2' in column]]
predictions_to_save = level2_predictions if level2_predictions.shape[1] > 0 else level1_predictions
primary_weights = all_predictions[[column for column in all_predictions.columns if 'primary' in column]]
ensemble_weights = all_predictions[[column for column in all_predictions.columns if 'ensemble' in column]]
predictions_to_save = ensemble_weights if ensemble_weights.shape[1] > 0 else primary_weights
predictions_to_save.to_csv('output/predictions.csv')
print("\n--------\n")
all_avg_results = weighted_average(results, 'no_of_samples')
lvl1_avg_results = weighted_average(level1_columns, 'no_of_samples')
lvl2_avg_results = weighted_average(level2_columns, 'no_of_samples')
primary_avg_results = weighted_average(primary_results, 'no_of_samples')
ensemble_avg_results = weighted_average(ensemble_results, 'no_of_samples')
print("Benchmark buy-and-hold sharpe: ", round(all_avg_results.loc['benchmark_sharpe'], 3))
print("Level-1: Number of samples evaluated: ", level1_columns.loc['no_of_samples'].sum())
print("Mean Sharpe ratio for Level-1 models: ", round(lvl1_avg_results.loc['sharpe'], 3))
print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(lvl1_avg_results.loc['prob_sharpe'].mean(), 3))
print("Level-1: Number of samples evaluated: ", primary_results.loc['no_of_samples'].sum())
print("Mean Sharpe ratio for Level-1 models: ", round(primary_avg_results.loc['sharpe'], 3))
print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(primary_avg_results.loc['prob_sharpe'].mean(), 3))
if model_config['level_2_model'] is not None:
print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum())
print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(lvl2_avg_results.loc['sharpe'].mean(), 3))
print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(lvl2_avg_results.loc['prob_sharpe'].mean(), 3))
if len(model_config['meta_labeling_models']) > 0:
print("Level-2 (Ensemble): Number of samples evaluated: ", ensemble_results.loc['no_of_samples'].sum())
print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_avg_results.loc['sharpe'].mean(), 3))
print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_avg_results.loc['prob_sharpe'].mean(), 3))
lvl2_avg_results.to_csv('output/results_level2.csv')
ensemble_avg_results.to_csv('output/results_level2.csv')
if sweep:
if wandb.run is not None: