From 122b7bb128a083b437004a33c4aadd653d214cab Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Mon, 20 Dec 2021 16:38:44 +0100 Subject: [PATCH] feat(Evaluation): added "no_of_samples", "ratio_of_classes" metrics to aid model debugging (#56) --- run_pipeline.py | 1 - utils/evaluate.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/run_pipeline.py b/run_pipeline.py index 661b96b..bb4fe89 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -142,7 +142,6 @@ for asset in data_config['all_assets']: if WANDB: combined_metrics = results.mean(axis=1) wandb.log({'results': results}) - wandb.log({'combined': combined_metrics}) if wandb.run is not None: diff --git a/utils/evaluate.py b/utils/evaluate.py index ae92130..7d00cb4 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -62,6 +62,7 @@ def evaluate_predictions( sign_true = df.sign_true.astype(int) sign_pred = df.sign_pred.astype(int) + scorecard.loc['no_of_samples'] = len(target_returns) - evaluate_from scorecard.loc['sharpe'] = sharpe(df.result) scorecard.loc['sortino'] = sortino(df.result) scorecard.loc['skew'] = skew(df.result) @@ -73,6 +74,17 @@ def evaluate_predictions( scorecard.loc['edge'] = df.result.mean() scorecard.loc['noise'] = df.y_pred.diff().abs().mean() scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise'] + + for index, row in sign_true.value_counts().iteritems(): + scorecard.loc['sign_true_ratio_' + str(index)] = row / len(sign_true) + + for index, row in sign_pred.value_counts().iteritems(): + scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(sign_pred) + + + + # scorecard.loc['ratio_of_classes_y'] = ' / '.join([str(index) + " " + str(round(row / len(sign_true), 2)) for index, row in sign_true.value_counts().iteritems()]) + # scorecard.loc['ratio_of_classes_pred'] = ' / '.join([str(round(row / len(sign_pred), 2)) for index, row in sign_pred.value_counts().iteritems()]) if method == 'regression':