From f5bbc266a476ef57860ccdef6acc3ac472a52875 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Sun, 9 Jan 2022 20:00:03 +0100 Subject: [PATCH] feat(Reporting): added robustness / correlation test (#138) * feat(Evaluation): added robustness/correlation test * feat(Reporting): saving correlations * fix(Reporting): record correlations properly * fix(Model): SVC's random seed * feat(CI): store artifacts --- .github/workflows/test.yml | 4 ++++ models/model_map.py | 2 +- run_evaluate_robustness.py | 23 +++++++++++++++++++++++ run_pipeline.py | 13 ++++++------- utils/helpers.py | 5 +++++ 5 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 run_evaluate_robustness.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf8697d..fd918e3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,10 @@ jobs: shell: bash -l {0} run: | python run_pipeline.py + - uses: actions/upload-artifact@v2 + with: + name: artifacts + path: output - name: Upload Unit Test Results if: always() uses: actions/upload-artifact@v2 diff --git a/models/model_map.py b/models/model_map.py index b810f32..98b3aff 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -53,7 +53,7 @@ model_map = { NB= SKLearnModel(GaussianNB()), AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)), RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)), - SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True)), + SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True, random_state=1)), XGB_two_class= XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')), LGBM = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)), StaticMom= StaticMomentumModel(allow_short=True), diff --git a/run_evaluate_robustness.py b/run_evaluate_robustness.py new file mode 100644 index 0000000..3a6b265 --- /dev/null +++ b/run_evaluate_robustness.py @@ -0,0 +1,23 @@ +from run_pipeline import run_pipeline +from config.config import get_default_ensemble_config, get_dev_config +import pandas as pd + +all_results = [] +all_predictions = [] +for index in range(6): + results_1, predictions_1, _ = run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config= get_default_ensemble_config) + all_results.append(results_1) + all_predictions.append(predictions_1) + +correlations = pd.Series(index = all_predictions[0].columns) + +for asset_name in all_predictions[0].columns: + + predictions_for_asset = pd.concat([preds[asset_name] for preds in all_predictions], axis=1) + correlations[asset_name] = predictions_for_asset.corr().mean()[0] + print("Correlation for asset ", asset_name, ": ", correlations[asset_name]) + +correlations["Overall"] = correlations.mean() +print("Average correlation across all assests: ", correlations.mean()) + +correlations.to_csv("output/correlations.csv") diff --git a/run_pipeline.py b/run_pipeline.py index 55c5008..3ada071 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -4,7 +4,7 @@ import pandas as pd from training.primary_model import train_primary_model from reporting.wandb import launch_wandb, register_config_with_wandb from models.model_map import default_feature_selector_regression, default_feature_selector_classification -from utils.helpers import get_first_valid_return_index +from utils.helpers import has_enough_samples_to_train from config.config import get_default_ensemble_config from config.preprocess import validate_config, preprocess_config from feature_selection.feature_selection import select_features @@ -16,13 +16,14 @@ import ray ray.init() -def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:object): - wandb, model_config, training_config, data_config = __setup_pipeline(project_name, with_wandb, sweep, get_config) +def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config: Callable) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + wandb, model_config, training_config, data_config = __setup_config(project_name, with_wandb, sweep, get_config) results, all_predictions, all_probabilities = __run_training(model_config, training_config, data_config) report_results(results, all_predictions, model_config, wandb, sweep, project_name) + return results, all_predictions, all_probabilities -def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config: Callable) -> tuple[Optional[object], dict, dict, dict]: +def __setup_config(project_name:str, with_wandb: bool, sweep: bool, get_config: Callable) -> tuple[Optional[object], dict, dict, dict]: model_config, training_config, data_config = get_config() wandb = None if with_wandb: @@ -49,9 +50,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): X, y, target_returns = load_data(**data_params) original_X = X.copy() - first_valid_index = get_first_valid_return_index(X.iloc[:,0]) - samples_to_train = len(y) - first_valid_index - if samples_to_train < training_config['sliding_window_size_primary'] * 3: + if has_enough_samples_to_train(X, y, training_config) == False: print("Not enough samples to train") continue diff --git a/utils/helpers.py b/utils/helpers.py index 2833c0d..4540834 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -17,6 +17,11 @@ def get_first_valid_return_index(series: pd.Series) -> int: return 0 return nested_result[0] +def has_enough_samples_to_train(X: pd.DataFrame, y: pd.Series, training_config: dict) -> bool: + first_valid_index = get_first_valid_return_index(X.iloc[:,0]) + samples_to_train = len(y) - first_valid_index + return samples_to_train > training_config['sliding_window_size_primary'] + training_config['sliding_window_size_meta_labeling'] + 100 + def flatten(list_of_lists: list) -> list: return [item for sublist in list_of_lists for item in sublist]