feature(MetaLabeling): replaced previous non-functional Ensembling method with Meta-labeling method available for both lvl1 and lvl2 models (#110)

* feature(MetaLabeling): added hacky prototype

* fix(MetaLabeling): drop index until first valid X & y

* fix(MetaLabeling): transform both X & y before feature selection

* fix(MetaLabeling): got feature selection to work

* fix(MetaLabeling): correct values for meta_y

* feat(MetaLabeling): created predictions multiplied by bet sizes

* feat(Pipeline): print out averaged result

* fix(Evaluation): correctly deal with non-discretized data

* fix(Pipeline): use the right column names

* refactor(Pipeline): move out meta-labeling

* refactor(Pipeline): complete refactoring

* feat(CI): post results to PR

* fix(Pipeline): use the correct filename

* chore(Config): removed now redundant feature_selection flag

* feat(Models): added SVC

* fix(Pipeline): accidentally switched two return values

* feat(Sweep): prepared sweep_meta.yaml, moved report_results() into a separate file

* fix(Pipeline): wrong function name

* fix(Sweep): yaml + run_sweep

* fix(Sweep): typo in name

* fix(Reporting): only save averaged results

* feat(MetaLabeling): use optional meta-labeling step for every lvl1 models, before averaging

* feat(Reporting): print out sharpe improvement in meta-labeling step

* fix(Sweep): adjusted config, defaulted to good defaults

* fix(Sweep): adjusted sweep

.github/workflows/test.yml

@@ -17,8 +17,6 @@ jobs:
       - uses: iterative/setup-cml@v1
       - name: Run pytest
         shell: bash -l {0}
-#         env: 
-#           WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
         run: |
           pytest --junit-xml pytest.xml
       - name: Run pipeline
@@ -37,10 +35,9 @@ jobs:
         with:
           files: pytest.xml
 
-#       - name: Write CML report
-#         env:
-#           REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-#         run: |
-#           # Post reports as comments in GitHub PRs
-#           # cat results.txt >> report.md
-#           # cml-send-comment report.md
+      - name: Write CML report
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          cat results_level2.csv >> report.md
+          cml-send-comment report.md

.gitignore

@@ -130,6 +130,7 @@ dmypy.json
 lightning/lightning_logs/
 
 results.csv
+results_level2.csv
 predictions.csv
 wandb/
 

config/config.py

@@ -2,8 +2,8 @@
 def get_default_level_1_daily_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
+        meta_labeling_lvl_1 = True,
         dimensionality_reduction = True,
-        feature_selection = True,
         n_features_to_select = 30,
         expanding_window_level1 = False,
         expanding_window_level2 = False,
@@ -11,7 +11,6 @@ def get_default_level_1_daily_config() -> tuple[dict, dict, dict]:
         sliding_window_size_level2 = 1,
         retrain_every = 20,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
-        include_original_data_in_ensemble = False,
     )
 
     data_config = dict(
@@ -44,8 +43,8 @@ def get_default_level_1_daily_config() -> tuple[dict, dict, dict]:
 def get_default_level_2_hourly_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
+        meta_labeling_lvl_1 = True,
         dimensionality_reduction = True,
-        feature_selection = True,
         n_features_to_select = 30,
         expanding_window_level1 = True,
         expanding_window_level2 = False,
@@ -53,7 +52,6 @@ def get_default_level_2_hourly_config() -> tuple[dict, dict, dict]:
         sliding_window_size_level2 = 1,
         retrain_every = 100,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
-        include_original_data_in_ensemble = False,
     )
 
     data_config = dict(
@@ -88,16 +86,15 @@ def get_default_level_2_hourly_config() -> tuple[dict, dict, dict]:
 def get_default_level_2_daily_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
+        meta_labeling_lvl_1 = True,
         dimensionality_reduction = True,
-        feature_selection = True,
         n_features_to_select = 30,
-        expanding_window_level1 = True,
-        expanding_window_level2 = False,
+        expanding_window_level1 = False,
+        expanding_window_level2 = True,
         sliding_window_size_level1 = 380,
-        sliding_window_size_level2 = 1,
+        sliding_window_size_level2 = 240,
         retrain_every = 20,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
-        include_original_data_in_ensemble = False,
     )
 
     data_config = dict(
@@ -112,14 +109,14 @@ def get_default_level_2_daily_config() -> tuple[dict, dict, dict]:
         exogenous_features = ['standard_scaling'],
         index_column= 'int',
         method= 'classification',
-        no_of_classes= 'three-balanced',
+        no_of_classes= 'two',
         narrow_format = False,
     )
 
     regression_models = ["Lasso", "KNN", "RF"]
     regression_ensemble_model = 'KNN'
-    classification_models = ['LR', 'LDA', 'KNN', 'CART', 'NB', 'AB', 'RF', 'XGB', 'StaticMom']
-    classification_ensemble_model = 'Ensemble_Average'
+    classification_models = ['SVC', 'LDA', 'KNN', 'CART', 'NB', 'AB', 'RF', 'StaticMom']
+    classification_ensemble_model = 'LDA'
 
     model_config = dict(
         level_1_models = regression_models if data_config['method'] == 'regression' else classification_models,

data_loader/load_data.py

@@ -10,7 +10,7 @@ from config.hashing import hash_data_config
 from diskcache import Cache
 cache = Cache(".cachedir/data")
 
-def load_data(**kwargs):
+def load_data(**kwargs) -> tuple[pd.DataFrame, pd.Series, pd.Series]:
     hashed = hash_data_config(kwargs)
     if hashed in cache:
         return cache.get(hashed)

feature_selection/feature_selection.py

@@ -8,7 +8,7 @@ from utils.hashing import hash_df, hash_series
 from diskcache import Cache
 cache = Cache(".cachedir/feature_selection")
 
-def select_features(**kwargs):
+def select_features(**kwargs) -> pd.DataFrame:
     hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling']
     if hashed in cache:
         return cache.get(hashed)

models/model_map.py

@@ -2,7 +2,7 @@ from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, Logisti
 from sklearn.tree import DecisionTreeClassifier
 from sklearnex.neighbors import KNeighborsRegressor, KNeighborsClassifier
 from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
-from sklearnex.svm import SVR
+from sklearnex.svm import SVR, SVC
 from sklearn.naive_bayes import GaussianNB
 from sklearn.neural_network import MLPRegressor, MLPClassifier
 from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier
@@ -44,7 +44,9 @@ model_map = {
         NB= SKLearnModel(GaussianNB()),
         AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)),
         RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)),
-        XGB= SKLearnModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, use_label_encoder=True, objective='multi:softprob', eval_metric='mlogloss')),
+        SVC = SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True)),
+        XGB_three_class= SKLearnModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, use_label_encoder=True, objective='multi:softprob', eval_metric='mlogloss')),
+        XGB_two_class= SKLearnModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', eval_metric='mlogloss')),
         StaticMom= StaticMomentumModel(allow_short=True),
         Ensemble_Average= StaticAverageModel(),
     ),

reporting/reporting.py

@@ -0,0 +1,41 @@
+from reporting.wandb import send_report_to_wandb
+import pandas as pd
+from config.preprocess import get_model_name
+from utils.helpers import weighted_average
+
+def report_results(results:pd.DataFrame, all_predictions:pd.DataFrame, model_config:dict, wandb, sweep: bool, project_name:str):
+
+    level1_columns = results[[column for column in results.columns if 'lvl1' in column]]
+    level2_columns = results[[column for column in results.columns if 'lvl2' in column]]
+
+    # Only send the results of the final model to wandb
+    results_to_send = level2_columns if level2_columns.shape[1] > 0 else level1_columns
+    send_report_to_wandb(results_to_send, wandb, project_name, get_model_name(model_config))
+    results.to_csv('results.csv')
+
+    level1_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl1' in column]]
+    level2_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl2' in column]]
+    predictions_to_save = level2_predictions if level2_predictions.shape[1] > 0 else level1_predictions
+    predictions_to_save.to_csv('predictions.csv')
+
+    print("\n--------\n")
+    all_avg_results = weighted_average(results, 'no_of_samples')
+    lvl1_avg_results = weighted_average(level1_columns, 'no_of_samples')
+    lvl2_avg_results = weighted_average(level2_columns, 'no_of_samples')
+
+    print("Benchmark buy-and-hold sharpe: ", round(all_avg_results.loc['benchmark_sharpe'], 3))
+
+    print("Level-1: Number of samples evaluated: ", level1_columns.loc['no_of_samples'].sum())
+    print("Mean Sharpe ratio for Level-1 models: ", round(lvl1_avg_results.loc['sharpe'], 3))
+    print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(lvl1_avg_results.loc['prob_sharpe'].mean(), 3))
+
+    if model_config['level_2_model'] is not None: 
+        print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum())
+        print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(lvl2_avg_results.loc['sharpe'].mean(), 3))
+        print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(lvl2_avg_results.loc['prob_sharpe'].mean(), 3))
+
+    lvl2_avg_results.to_csv('results_level2.csv')
+
+    if sweep:
+        if wandb.run is not None:
+            wandb.finish()

run_pipeline.py

@@ -4,22 +4,25 @@ import pandas as pd
 from training.training import run_single_asset_trainig
 from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_with_wandb
 from models.model_map import default_feature_selector_regression, default_feature_selector_classification
-from utils.helpers import get_first_valid_return_index, weighted_average
+from utils.helpers import get_first_valid_return_index
 from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config
-from config.preprocess import validate_config, get_model_name, preprocess_config
+from config.preprocess import validate_config, preprocess_config
 from feature_selection.feature_selection import select_features
 from feature_selection.dim_reduction import reduce_dimensionality
+from training.meta_labeling import run_meta_labeling_training
+from training.averaged import average_and_evaluate_predictions
+from reporting.reporting import report_results
 import ray
 ray.init()
 
 
 def run_pipeline(project_name:str, with_wandb: bool, sweep: bool):
-    wandb, model_config, training_config, data_config = setup_pipeline(project_name, with_wandb, sweep)
-    results, all_predictions, all_probabilities = run_training(project_name, wandb, sweep, model_config, training_config, data_config)  
-    reporting(results, all_predictions, all_probabilities, model_config, wandb, sweep, project_name)
+    wandb, model_config, training_config, data_config = __setup_pipeline(project_name, with_wandb, sweep)
+    results, all_predictions, all_probabilities = __run_training(model_config, training_config, data_config)  
+    report_results(results, all_predictions, model_config, wandb, sweep, project_name)
     
 
-def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
+def __setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
     model_config, training_config, data_config = get_default_level_2_daily_config()
     wandb = None
     if with_wandb: 
@@ -27,15 +30,10 @@ def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
         register_config_with_wandb(wandb, model_config, training_config, data_config)
     model_config, training_config, data_config = preprocess_config(model_config, training_config, data_config)
 
-
     return wandb, model_config, training_config, data_config
     
 
-
-
-
-
-def run_training(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict):
+def __run_training(model_config:dict, training_config:dict, data_config:dict):
     results = pd.DataFrame()
     all_predictions = pd.DataFrame()
     all_probabilities = pd.DataFrame()
@@ -58,15 +56,16 @@ def run_training(project_name:str, wandb, sweep:bool, model_config:dict, trainin
 
         # 2a. Dimensionality Reduction (optional)
         if training_config['dimensionality_reduction']:
-            X = reduce_dimensionality(X, int(len(X.columns) / 2))
+            X_pca = reduce_dimensionality(X, int(len(X.columns) / 2))
+            X = X_pca.copy()
+        else:
+            X_pca = X.copy()
 
-        # 2b. Feature Selection (optional)
-        if training_config['feature_selection']:
-            print("Feature Selection started")
-            # TODO: this needs to be done per model!
-            backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
-            X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = hash_data_config(data_params))
-            print("Feature Selection ended")
+        # 2b. Feature Selection
+        print("Feature Selection started")
+        # TODO: this needs to be done per model!
+        backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
+        X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = hash_data_config(data_params))
 
         # 3. Train Level-1 models
         current_result, current_predictions, current_probabilities = run_single_asset_trainig(
@@ -84,70 +83,57 @@ def run_training(project_name:str, wandb, sweep:bool, model_config:dict, trainin
             no_of_classes = data_config['no_of_classes'],
             level = 1
         )
+        
+        # 4. Train a Meta-Labeling model for each Level-1 model and replace its predictions with the meta-labeling predictions
+        if training_config['meta_labeling_lvl_1'] == True:
+            for column in current_result.columns:
+                lvl1_model_predictions = current_predictions[column]
+                prev_sharpe = current_result[column]['sharpe']
+                lvl1_meta_result, lvl1_meta_preds, lvl1_meta_probabilities = run_meta_labeling_training(
+                    target_asset=asset[1],
+                    X_pca = X_pca,
+                    input_predictions= lvl1_model_predictions,
+                    y = y,
+                    target_returns = target_returns,
+                    data_config= data_config,
+                    model_config= model_config,
+                    training_config= training_config
+                )
+                new_sharpe = lvl1_meta_result['sharpe']
+                print("Improvement in sharpe for the meta model: ", ((new_sharpe / prev_sharpe) - 1) * 100, "%")
+                current_result[column] = lvl1_meta_result
+                current_predictions[column] = lvl1_meta_preds
+
         results = pd.concat([results, current_result], axis=1)
         # With static models, because of the lag in the indicator, the first prediction is NA, so we fill it with zero.
         all_predictions = pd.concat([all_predictions, current_predictions], axis=1).fillna(0.)
         all_probabilities = pd.concat([all_probabilities, current_probabilities], axis=1).fillna(0.)
 
-        # 3. Train Level-2 (Ensemble) model (Optional)
         if model_config['level_2_model'] is not None: 
-            ensemble_X = pd.concat([all_predictions, all_probabilities], axis = 1)
-            if training_config['include_original_data_in_ensemble']:
-                ensemble_X = pd.concat([ensemble_X, X], axis=1)
 
-            ensemble_result, ensemble_preds, ensemble_probabilities = run_single_asset_trainig(
-                ticker_to_predict = asset[1],
-                original_X = ensemble_X,
-                X = ensemble_X,
+            # 3. Average the Level-1 model predictions
+            averaged_predictions, averaged_results = average_and_evaluate_predictions(current_predictions, y, target_returns, data_config)
+
+            # 3. Train a Meta-labeling model on the averaged level-1 model predictions
+            meta_result, avg_predictions_with_sizing, meta_probabilities = run_meta_labeling_training(
+                target_asset=asset[1],
+                X_pca = X_pca,
+                input_predictions= averaged_predictions,
                 y = y,
                 target_returns = target_returns,
-                models = [model_config['level_2_model']],
-                method = data_config['method'],
-                expanding_window = training_config['expanding_window_level2'],
-                sliding_window_size = training_config['sliding_window_size_level2'],
-                retrain_every = training_config['retrain_every'],
-                scaler = training_config['scaler'],
-                no_of_classes = data_config['no_of_classes'],
-                level = 2
+                data_config= data_config,
+                model_config= model_config,
+                training_config= training_config
             )
 
-            results = pd.concat([results, ensemble_result], axis=1)
-            all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
-            all_probabilities = pd.concat([all_probabilities, ensemble_probabilities], axis=1).fillna(0.)
+            results = pd.concat([results, meta_result], axis=1)
+            all_predictions = pd.concat([all_predictions, avg_predictions_with_sizing], axis=1)
+            all_probabilities = pd.concat([all_probabilities, meta_probabilities], axis=1).fillna(0.)
         
     return results, all_predictions, all_probabilities
     
     
-def reporting(results:pd.DataFrame, all_predictions:pd.DataFrame, all_probabilities:pd.DataFrame, model_config:dict, wandb, sweep: bool, project_name:str):
-    results.to_csv('results.csv')
 
-    level1_columns = results[[column for column in results.columns if 'lvl1' in column]]
-    level2_columns = results[[column for column in results.columns if 'lvl2' in column]]
-
-    # Only send the results of the final model to wandb
-    results_to_send = level2_columns if level2_columns.shape[1] > 0 else level1_columns
-    send_report_to_wandb(results_to_send, wandb, project_name, get_model_name(model_config))
-
-    level1_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl1' in column]]
-    level2_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl2' in column]]
-    predictions_to_save = level2_predictions if level2_predictions.shape[1] > 0 else level1_predictions
-    predictions_to_save.to_csv('predictions.csv')
-
-    print("\n--------\n")
-    print("Benchmark buy-and-hold sharpe: ", round(weighted_average(results, 'no_of_samples').loc['benchmark_sharpe'], 3))
-
-    print("Level-1: Number of samples evaluated: ", level1_columns.loc['no_of_samples'].sum())
-    print("Mean Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['sharpe'], 3))
-    print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
-
-    if model_config['level_2_model'] is not None: 
-        print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum())
-        print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['sharpe'].mean(), 3))
-        print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3))
-
-    if sweep:
-        if wandb.run is not None:
-            wandb.finish()
     
 if __name__ == '__main__':
     run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False)

run_sweep.py

@@ -1,3 +1,3 @@
-from run_pipeline import setup_pipeline
+from run_pipeline import run_pipeline
 
-setup_pipeline(project_name='price-prediction', with_wandb = True, sweep = True)
+run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True)

sweep_exo_data.yaml

@@ -6,6 +6,8 @@ metric:
   goal: maximize
   name: sharpe
 parameters:
+    meta_labeling_lvl_1:
+      value: True
     assets:
       value: ['daily_crypto']
     other_assets:
@@ -21,8 +23,6 @@ parameters:
       value: 380
     sliding_window_size_level2: 
       value: 1
-    feature_selection:
-      value: True
     n_features_to_select:
       value: 30
     dimensionality_reduction:
@@ -31,8 +31,6 @@ parameters:
       value: 20
     scaler:
       value: 'minmax'
-    include_original_data_in_ensemble:
-      value: False
     method: 
       value: 'classification'
     no_of_classes:

sweep_level_1.yaml

@@ -6,6 +6,8 @@ metric:
   goal: maximize
   name: sharpe
 parameters:
+    meta_labeling_lvl_1:
+      value: True
     assets:
       value: ['daily_crypto']
     other_assets:
@@ -17,8 +19,6 @@ parameters:
       distribution: categorical
     expanding_window_level2:
       value: False
-    feature_selection:
-      value: True
     n_features_to_select:
       values: [10, 20, 30]
       distribution: categorical
@@ -34,8 +34,6 @@ parameters:
       distribution: categorical
     scaler:
       value: 'minmax'
-    include_original_data_in_ensemble:
-      value: False
     method: 
       value: 'classification'
     no_of_classes:

sweep_level_2.yaml

@@ -6,6 +6,8 @@ metric:
   goal: maximize
   name: sharpe
 parameters:
+    meta_labeling_lvl_1:
+      value: True
     assets:
       value: ['daily_crypto']
     other_assets:
@@ -18,8 +20,6 @@ parameters:
     expanding_window_level2:
       values: [True, False]
       distribution: categorical
-    feature_selection:
-      value: True
     n_features_to_select:
       values: [10, 20, 30]
       distribution: categorical
@@ -36,9 +36,6 @@ parameters:
       distribution: categorical
     scaler:
       value: 'minmax'
-    include_original_data_in_ensemble:
-      values: [True, False]
-      distribution: categorical
     method: 
       value: 'classification'
     no_of_classes:

sweep_meta.yaml

@@ -0,0 +1,56 @@
+program: run_sweep.py
+method: grid
+project: price-forecasting
+name: Meta labelling
+metric:
+  goal: maximize
+  name: sharpe
+parameters:
+    meta_labeling_lvl_1:
+      values: [True, False]
+      distribution: categorical
+    assets:
+      value: ['daily_crypto']
+    other_assets:
+      value: ['daily_etf']
+    exogenous_data:
+      value: ['daily_glassnode']
+    expanding_window_level1:
+      value: False
+    expanding_window_level2:
+      value: True
+    sliding_window_size_level1: 
+      value: 380
+    sliding_window_size_level2: 
+      value: 380
+    n_features_to_select:
+      value: 30
+    dimensionality_reduction:
+      value: True
+    retrain_every: 
+      value: 20
+    scaler:
+      value: 'minmax'
+    method: 
+      value: 'classification'
+    no_of_classes:
+      value: 'two'
+    forecasting_horizon: 
+      value: 1
+    load_non_target_asset: 
+      value: True
+    log_returns: 
+      value: True
+    index_column: 
+      value: 'int'
+    level_1_models: 
+      value: ["LDA", "KNN", "SVC", "CART", "NB", "AB", "RF", "StaticMom"]
+    level_2_model: 
+      values: ["LDA", "KNN", "NB", "AB", "RF", "XGB_two_class"]
+      distribution: categorical
+    own_features:
+      value: ['date_days', 'level_2', 'lags_up_to_5']
+    other_features:
+      value: ['level_2', 'lags_up_to_5']
+    exogenous_features:
+      value: ['standard_scaling']

tests/test_evaluation.py

@@ -94,7 +94,8 @@ def test_evaluation():
         y_pred=processed_predictions_to_match_returns,
         y_true=y,
         method='classification',
-        no_of_classes='two'
+        no_of_classes='two',
+        discretize=True
     )
 
     assert result['accuracy'] == 100.0

training/averaged.py

@@ -0,0 +1,25 @@
+
+import pandas as pd
+from utils.evaluate import evaluate_predictions
+
+def average_and_evaluate_predictions(predictions: pd.DataFrame, y: pd.Series, target_returns: pd.Series, data_config: dict) -> tuple[pd.Series, pd.DataFrame]:
+    averaged_predictions = predictions.mean(axis = 1)
+    non_discretized_result = evaluate_predictions(
+        model_name = 'Averaged - Non-discrete',
+        target_returns = target_returns,
+        y_pred = averaged_predictions,
+        y_true = y,
+        method = 'classification',
+        no_of_classes = data_config['no_of_classes'],
+        discretize=False
+    )
+    discretized_result = evaluate_predictions(
+        model_name = 'Averaged - Discrete',
+        target_returns = target_returns,
+        y_pred = averaged_predictions,
+        y_true = y,
+        method = 'classification',
+        no_of_classes = data_config['no_of_classes'],
+        discretize=True
+    )
+    return averaged_predictions, pd.concat([non_discretized_result, discretized_result], axis = 1)

training/meta_labeling.py

@@ -0,0 +1,62 @@
+from utils.evaluate import discretize_threeway_threshold, evaluate_predictions
+from utils.helpers import random_string, equal_except_nan, drop_until_first_valid_index
+from training.training import run_single_asset_trainig
+from feature_selection.feature_selection import select_features
+import pandas as pd
+from models.model_map import default_feature_selector_regression, default_feature_selector_classification
+
+
+def run_meta_labeling_training(
+                            target_asset: str,
+                            X_pca: pd.DataFrame,
+                            input_predictions: pd.Series,
+                            y: pd.Series,
+                            target_returns: pd.Series,
+                            data_config: dict,
+                            model_config: dict,
+                            training_config: dict
+                        ) -> tuple[pd.Series, pd.Series, pd.DataFrame]:
+
+    discretize = discretize_threeway_threshold(0.33)
+    discretized_predictions = input_predictions.apply(discretize)
+    meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply(equal_except_nan, axis = 1)
+
+    print("Feature Selection started")
+    backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
+    meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X_pca, meta_y)
+    feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], data_config_hash = random_string(10))
+    meta_selected_features_X = X_pca[feature_selection_output.columns]
+
+    meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1)
+
+    _, meta_preds, meta_probabilities = run_single_asset_trainig(
+        ticker_to_predict = "prediction_correct",
+        original_X = meta_X,
+        X = meta_X,
+        y = meta_y,
+        target_returns = target_returns,
+        models = [model_config['level_2_model']],
+        method = data_config['method'],
+        expanding_window = training_config['expanding_window_level2'],
+        sliding_window_size = training_config['sliding_window_size_level2'],
+        retrain_every = training_config['retrain_every'],
+        scaler = training_config['scaler'],
+        no_of_classes = 'two',
+        level = 2
+    )
+    bet_size = meta_probabilities.iloc[:,1]
+    avg_predictions_with_sizing = input_predictions * bet_size
+    avg_predictions_with_sizing.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True)
+
+    meta_result = evaluate_predictions(
+        model_name = "Meta",
+        target_returns = target_returns,
+        y_pred = avg_predictions_with_sizing,
+        y_true = y,
+        method = 'classification',
+        no_of_classes = data_config['no_of_classes'],
+        discretize=False
+    )
+    meta_result.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True)
+
+    return meta_result, avg_predictions_with_sizing, meta_probabilities

training/training.py

@@ -48,7 +48,8 @@ def run_single_asset_trainig(
             y_pred = preds,
             y_true = y,
             method = method,
-            no_of_classes=no_of_classes
+            no_of_classes=no_of_classes,
+            discretize=True
         )
         column_name = "model_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level)
         results[column_name] = result

training/walk_forward.py

@@ -22,7 +22,7 @@ def walk_forward_train_test(
     probabilities = pd.DataFrame(index=y.index)
     models = pd.Series(index=y.index).rename(model_name)
 
-    first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(X.iloc[:,0]))
+    first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y))
     train_from = first_nonzero_return + window_size + 1
     train_till = len(y)
     iterations_before_retrain = 0

utils/evaluate.py

@@ -1,9 +1,10 @@
-from typing import Literal
+from typing import Literal, Callable
 from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
 from quantstats.stats import skew, sortino
 from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio
 from utils.helpers import get_first_valid_return_index
 import pandas as pd
+import numpy as np
 
 def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.00) -> pd.Series:
     delta_pos = signal.diff(1).abs().fillna(0.)
@@ -11,20 +12,18 @@ def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.00) ->
     return (signal * returns) - costs
 
 
-def __preprocess(target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, method: Literal['classification', 'regression'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']) -> pd.DataFrame:
+def __preprocess(target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, method: Literal['classification', 'regression'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame:
     y_pred.name = 'y_pred'
     target_returns.name = 'target_returns'
     df = pd.concat([y_pred, target_returns],axis=1).dropna()
 
-    def categorize_binary(x): return 1 if x > 0 else -1
-    def categorize_threeway(x): return 0 if x == 0 else 1 if x > 0 else -1
-    categorize = categorize_binary if no_of_classes == 'two' else categorize_threeway
+    discretize_func = get_discretize_function(no_of_classes)
     # make sure that we evaluate binary/three-way predictions even if the model is a regression
     if method == 'regression':
-        df['sign_pred'] = df.y_pred.apply(categorize)
-        df['sign_true'] = df.target_returns.apply(categorize)
+        df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
+        df['sign_true'] = df.target_returns.apply(discretize_func)
     else:
-        df['sign_pred'] = df.y_pred.apply(categorize)
+        df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
         df['sign_true'] = y_true
 
     df['result'] = backtest(df.target_returns, df.sign_pred)
@@ -38,6 +37,7 @@ def evaluate_predictions(
                         y_true: pd.Series,
                         method: Literal['classification', 'regression'],
                         no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
+                        discretize: bool = False,
                         ) -> pd.Series:
     # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
     first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(y_pred))
@@ -51,7 +51,7 @@ def evaluate_predictions(
 
     y_pred = pd.Series(y_pred[evaluate_from:])
 
-    df = __preprocess(target_returns, y_pred, y_true, method, no_of_classes)
+    df = __preprocess(target_returns, y_pred, y_true, method, no_of_classes, discretize)
     
     scorecard = pd.Series()
     # we probably will not need regression models at all
@@ -78,19 +78,21 @@ def evaluate_predictions(
     labels = [1, -1] if no_of_classes == 'two' else [1, -1, 0]
     avg_type = 'weighted' if no_of_classes == 'two' else 'macro'
 
-    scorecard.loc['accuracy'] = accuracy_score(df.sign_true, df.sign_pred) * 100
-    scorecard.loc['recall'] = recall_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
-    scorecard.loc['precision'] = precision_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
-    scorecard.loc['f1_score'] = f1_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
+    if discretize == True:
+        scorecard.loc['accuracy'] = accuracy_score(df.sign_true, df.sign_pred) * 100
+        scorecard.loc['recall'] = recall_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
+        scorecard.loc['precision'] = precision_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
+        scorecard.loc['f1_score'] = f1_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
     scorecard.loc['edge'] = df.result.mean()
     scorecard.loc['noise'] = df.y_pred.diff().abs().mean()
     scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise']
     
-    for index, row in df.sign_true.value_counts().iteritems():
-        scorecard.loc['sign_true_ratio_' + str(index)] = row / len(df.sign_true)
-    
-    for index, row in df.sign_pred.value_counts().iteritems():
-        scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred)
+    if discretize == True:
+        for index, row in df.sign_true.value_counts().iteritems():
+            scorecard.loc['sign_true_ratio_' + str(index)] = row / len(df.sign_true)
+        
+        for index, row in df.sign_pred.value_counts().iteritems():
+            scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred)
 
     # if method == 'regression':
     #     scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE']
@@ -102,3 +104,24 @@ def evaluate_predictions(
     print(scorecard)
     return scorecard  
 
+
+def __discretize_binary(x): return 1 if x > 0 else -1
+def __discretize_threeway(x): return 0 if x == 0 else 1 if x > 0 else -1
+def discretize_threeway_threshold(threshold: float) -> Callable:
+    def discretize(current_value):
+        lower_threshold = -threshold
+        upper_threshold = threshold
+        if np.isnan(current_value):
+            return np.nan 
+        elif current_value <= lower_threshold:
+            return -1
+        elif current_value > lower_threshold and current_value < upper_threshold:
+            return 0
+        else:
+            return 1
+    return discretize
+
+def get_discretize_function(no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']) -> Callable:
+    return __discretize_binary if no_of_classes == 'two' else __discretize_threeway
+
+

utils/helpers.py

@@ -1,12 +1,15 @@
 import pandas as pd
 import numpy as np
 import os
+import string
+import random
+from typing import Union
 
 def get_files_from_dir(path: str) -> list[str]:
     return [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
 
 def get_first_valid_return_index(series: pd.Series) -> int:
-    double_nested_results = np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series))))
+    double_nested_results = np.where(np.logical_and(series != 0, np.logical_not(pd.isna(series))))
     if len(double_nested_results) == 0:
         return 0
     nested_result = double_nested_results[0]
@@ -17,7 +20,7 @@ def get_first_valid_return_index(series: pd.Series) -> int:
 def flatten(list_of_lists: list) -> list:
     return [item for sublist in list_of_lists for item in sublist]
 
-def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.DataFrame:
+def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.Series:
     if df.shape[0] == 0:
         return df
     mean_df = df.iloc[:,0]
@@ -35,4 +38,19 @@ def drop_columns_if_exist(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     for column in columns:
         if column in df.columns:
             df = df.drop(column, axis=1)
-    return df
+    return df
+
+def random_string(n: int) -> str:
+    return ''.join(random.choices(string.ascii_uppercase + string.digits, k=n))
+
+def equal_except_nan(row: pd.Series):
+    if np.isnan(row.iloc[0]) or np.isnan(row.iloc[1]):
+        return np.nan
+    if row.iloc[0] == row.iloc[1]:
+        return 1.
+    else:
+        return 0.
+
+def drop_until_first_valid_index(df: pd.DataFrame, series: pd.Series) -> tuple[pd.DataFrame, pd.Series]:
+    first_valid_index = max(get_first_valid_return_index(df.iloc[:,0]), get_first_valid_return_index(series))
+    return df.iloc[first_valid_index:], series.iloc[first_valid_index:]