gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-21 23:08:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Evaluation): added "no_of_samples", "ratio_of_classes" metrics to aid model debugging (#56)

Browse Source
This commit is contained in:
Mark Aron Szulyovszky
2021-12-20 16:38:44 +01:00
committed by GitHub
parent a7414eac23
commit 122b7bb128
2 changed files with 12 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
run_pipeline.py
-1
View File
@@ -142,7 +142,6 @@ for asset in data_config['all_assets']:
if WANDB: if WANDB:
combined_metrics = results.mean(axis=1) combined_metrics = results.mean(axis=1)
wandb.log({'results': results}) wandb.log({'results': results})
wandb.log({'combined': combined_metrics})
if wandb.run is not None: if wandb.run is not None:
utils/evaluate.py
+12
View File
@@ -62,6 +62,7 @@ def evaluate_predictions(
sign_true = df.sign_true.astype(int) sign_true = df.sign_true.astype(int)
sign_pred = df.sign_pred.astype(int) sign_pred = df.sign_pred.astype(int)
scorecard.loc['no_of_samples'] = len(target_returns) - evaluate_from
scorecard.loc['sharpe'] = sharpe(df.result) scorecard.loc['sharpe'] = sharpe(df.result)
scorecard.loc['sortino'] = sortino(df.result) scorecard.loc['sortino'] = sortino(df.result)
scorecard.loc['skew'] = skew(df.result) scorecard.loc['skew'] = skew(df.result)
@@ -74,6 +75,17 @@ def evaluate_predictions(
scorecard.loc['noise'] = df.y_pred.diff().abs().mean() scorecard.loc['noise'] = df.y_pred.diff().abs().mean()
scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise'] scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise']
for index, row in sign_true.value_counts().iteritems():
scorecard.loc['sign_true_ratio_' + str(index)] = row / len(sign_true)
for index, row in sign_pred.value_counts().iteritems():
scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(sign_pred)
# scorecard.loc['ratio_of_classes_y'] = ' / '.join([str(index) + " " + str(round(row / len(sign_true), 2)) for index, row in sign_true.value_counts().iteritems()])
# scorecard.loc['ratio_of_classes_pred'] = ' / '.join([str(round(row / len(sign_pred), 2)) for index, row in sign_pred.value_counts().iteritems()])
if method == 'regression': if method == 'regression':
scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE'] scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE']