feat(Reporting): added robustness / correlation test (#138)

* feat(Evaluation): added robustness/correlation test

* feat(Reporting): saving correlations

* fix(Reporting): record correlations properly

* fix(Model): SVC's random seed

* feat(CI): store artifacts

.github/workflows/test.yml

@@ -23,6 +23,10 @@ jobs:
         shell: bash -l {0}
         run: |
           python run_pipeline.py
+      - uses: actions/upload-artifact@v2
+        with:
+          name: artifacts
+          path: output
       - name: Upload Unit Test Results
         if: always()
         uses: actions/upload-artifact@v2

models/model_map.py

@@ -53,7 +53,7 @@ model_map = {
         NB= SKLearnModel(GaussianNB()),
         AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)),
         RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)),
-        SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True)),
+        SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True, random_state=1)),
         XGB_two_class= XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')),
         LGBM = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)),
         StaticMom= StaticMomentumModel(allow_short=True),

run_evaluate_robustness.py

@@ -0,0 +1,23 @@
+from run_pipeline import run_pipeline
+from config.config import get_default_ensemble_config, get_dev_config
+import pandas as pd
+
+all_results = []
+all_predictions = []
+for index in range(6):
+    results_1, predictions_1, _ = run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config= get_default_ensemble_config)
+    all_results.append(results_1)
+    all_predictions.append(predictions_1)
+
+correlations = pd.Series(index = all_predictions[0].columns)
+
+for asset_name in all_predictions[0].columns:
+
+    predictions_for_asset = pd.concat([preds[asset_name] for preds in all_predictions], axis=1)
+    correlations[asset_name] = predictions_for_asset.corr().mean()[0]
+    print("Correlation for asset ", asset_name, ": ", correlations[asset_name])
+
+correlations["Overall"] = correlations.mean()
+print("Average correlation across all assests: ", correlations.mean())
+
+correlations.to_csv("output/correlations.csv")

run_pipeline.py

@@ -4,7 +4,7 @@ import pandas as pd
 from training.primary_model import train_primary_model
 from reporting.wandb import launch_wandb, register_config_with_wandb
 from models.model_map import default_feature_selector_regression, default_feature_selector_classification
-from utils.helpers import get_first_valid_return_index
+from utils.helpers import has_enough_samples_to_train
 from config.config import get_default_ensemble_config
 from config.preprocess import validate_config, preprocess_config
 from feature_selection.feature_selection import select_features
@@ -16,13 +16,14 @@ import ray
 ray.init()
 
 
-def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config:object):
-    wandb, model_config, training_config, data_config = __setup_pipeline(project_name, with_wandb, sweep, get_config)
+def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config: Callable) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    wandb, model_config, training_config, data_config = __setup_config(project_name, with_wandb, sweep, get_config)
     results, all_predictions, all_probabilities = __run_training(model_config, training_config, data_config)  
     report_results(results, all_predictions, model_config, wandb, sweep, project_name)
+    return results, all_predictions, all_probabilities
     
 
-def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool, get_config: Callable) -> tuple[Optional[object], dict, dict, dict]:
+def __setup_config(project_name:str, with_wandb: bool, sweep: bool, get_config: Callable) -> tuple[Optional[object], dict, dict, dict]:
     model_config, training_config, data_config = get_config() 
     wandb = None
     if with_wandb: 
@@ -49,9 +50,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict):
 
         X, y, target_returns = load_data(**data_params)
         original_X = X.copy()
-        first_valid_index = get_first_valid_return_index(X.iloc[:,0])
-        samples_to_train = len(y) - first_valid_index
-        if samples_to_train < training_config['sliding_window_size_primary'] * 3:
+        if has_enough_samples_to_train(X, y, training_config) == False:
             print("Not enough samples to train")
             continue
 

utils/helpers.py

@@ -17,6 +17,11 @@ def get_first_valid_return_index(series: pd.Series) -> int:
         return 0
     return nested_result[0]
 
+def has_enough_samples_to_train(X: pd.DataFrame, y: pd.Series, training_config: dict) -> bool:
+    first_valid_index = get_first_valid_return_index(X.iloc[:,0])
+    samples_to_train = len(y) - first_valid_index
+    return samples_to_train > training_config['sliding_window_size_primary'] + training_config['sliding_window_size_meta_labeling'] + 100
+
 def flatten(list_of_lists: list) -> list:
     return [item for sublist in list_of_lists for item in sublist]