From 73cfc67336ea4477e79efc0a4b8b510b768fae18 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Mon, 10 Jan 2022 14:17:06 +0100 Subject: [PATCH] fix(Evaluation): correlation test should work on a per asset level, not per model level (#142) * fix(Evaluation): correlation test should work on a per asset level, not per model level * fix(Evaluation): correlations series initalized correctly * fix(Reporting): don't name the run after the incorrectly supposed model_type * fix(Reporting): put back send_report_to_wandb() into its original place * fix(CI): sending reports again in comment --- .github/workflows/test.yml | 2 +- reporting/wandb.py | 9 ++++----- run_evaluate_robustness.py | 4 ++-- utils/evaluate.py | 13 ------------- 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd918e3..2d734f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: run-test +name: run test and pipeline on: [push] jobs: run: diff --git a/reporting/wandb.py b/reporting/wandb.py index 129e389..27f453c 100644 --- a/reporting/wandb.py +++ b/reporting/wandb.py @@ -9,7 +9,7 @@ def launch_wandb(project_name:str, default_config:dict, sweep:bool=False): raise Exception("Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)") elif sweep: - wandb.init(project=project_name, config = default_config) + wandb.init(project=project_name, config = default_config) return wandb else: wandb.init(project=project_name, config = default_config, reinit=True) @@ -33,13 +33,12 @@ def register_config_with_wandb(wandb: Optional[object], model_config:dict, train def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_name: str, model_name: str): if wandb is None: return - run = wandb.init(project=project_name, config={"model_type": model_name}, reinit=True) - wandb.run.name = model_name+ "-" + wandb.run.id - wandb.run.save() + run = wandb.run + run.save() mean_results = weighted_average(results, 'no_of_samples') for key, value in mean_results.iteritems(): - run.log({"model_type": model_name, key: value }) + run.log({ key: value }) run.finish() diff --git a/run_evaluate_robustness.py b/run_evaluate_robustness.py index 3a6b265..2d1cc37 100644 --- a/run_evaluate_robustness.py +++ b/run_evaluate_robustness.py @@ -9,9 +9,9 @@ for index in range(6): all_results.append(results_1) all_predictions.append(predictions_1) -correlations = pd.Series(index = all_predictions[0].columns) +correlations = pd.Series() -for asset_name in all_predictions[0].columns: +for asset_name in [c for c in all_predictions[0].columns if 'ensemble' in c]: predictions_for_asset = pd.concat([preds[asset_name] for preds in all_predictions], axis=1) correlations[asset_name] = predictions_for_asset.corr().mean()[0] diff --git a/utils/evaluate.py b/utils/evaluate.py index d7bbc51..cc2434f 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -55,14 +55,6 @@ def evaluate_predictions( df = __preprocess(target_returns, y_pred, y_true, method, no_of_classes, discretize) scorecard = pd.Series() - # we probably will not need regression models at all - # if method == 'regression': - # scorecard.loc['RSQ'] = r2_score(df.target_returns, df.y_pred) - # scorecard.loc['MAE'] = mean_absolute_error(df.target_returns, df.y_pred) - # elif method == 'classification': - # scorecard.loc['RSQ'] = 0. - # scorecard.loc['MAE Matrix'] = 0. - def count_non_zero(series: pd.Series) -> int: return len(series[series != 0]) @@ -95,11 +87,6 @@ def evaluate_predictions( for index, row in df.sign_pred.value_counts().iteritems(): scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred) - # if method == 'regression': - # scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE'] - # elif method == 'classification': - # scorecard.loc['edge_to_mae'] = 0. - scorecard = scorecard.round(3) if print_results: print("Model name: ", model_name)