fix(MetaLabeling): previously misinterpreted meta-labeling, now also multiplying base model's prediction with the meta model's prediction (#193)

* fix(MetaLabeling): previously misinterpreted meta-labeling, now also multiplying base model's prediction with the meta model's prediction

* fix(Evaluate): print results

* fix(Evaluate): make sure we have numerical stability in returns

* fix(Inference): only output and print stats in training mode

* fix(Evaluate): don't add miniscule amount to result

training/bet_sizing.py

@@ -62,27 +62,31 @@ def bet_sizing_with_meta_models(
         from_index = from_index,
         no_of_classes = 'two',
         level = 'meta',
-        print_results = False,
+        output_stats = config.mode == 'training',
         transformations_over_time = transformations_over_time,
         models_over_time = preloaded_models,
     )
 
     # Ensemble predictions if necessary
     if len(models) > 1:
-        # meta_predictions = pd.concat([outcome.predictions for outcome in meta_outcomes]).mean(axis = 1)
+        meta_predictions = pd.concat([outcome.predictions for outcome in meta_outcomes], axis = 1).mean(axis = 1).apply(discretize_threeway_threshold(0.5))
         bet_size = pd.concat([outcome.probabilities[outcome.probabilities.columns[1::2]] for outcome in meta_outcomes], axis = 1).mean(axis = 1)
     else:
+        meta_predictions = meta_outcomes[0].predictions
         bet_size = meta_outcomes[0].probabilities.iloc[:,1]
-    avg_predictions_with_sizing = input_predictions * bet_size
+    avg_predictions_with_sizing = input_predictions * meta_predictions * bet_size
 
-    stats = evaluate_predictions(
-        forward_returns = forward_returns,
-        y_pred = avg_predictions_with_sizing,
-        y_true = y,
-        no_of_classes = 'two',
-        print_results = True,
-        discretize=False
-    )
+    if config.mode == 'training':
+        stats = evaluate_predictions(
+            forward_returns = forward_returns,
+            y_pred = avg_predictions_with_sizing,
+            y_true = y,
+            no_of_classes = 'three-balanced',
+            discretize=False
+        )
+        print(stats)
+    else:
+        stats = None
     model_id = "model_" + config.target_asset[1] + "_" + model_suffix
 
     return BetSizingWithMetaOutcome(model_id, meta_outcomes, transformations_over_time, avg_predictions_with_sizing, stats)

training/directional_training.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-from .types import DirectionalTrainingOutcome
+from .types import DirectionalTrainingOutcome, TrainingOutcome
 from training.train_model import train_model
 from training.walk_forward import walk_forward_process_transformations
 
@@ -42,7 +42,12 @@ def train_directional_models(
     else:
         transformations_over_time = preloaded_training_step.transformations
 
-    training_outcomes = [train_model(
+    def print_stats(outcome: TrainingOutcome) -> TrainingOutcome:
+        if config.mode == 'training':
+            print(outcome.stats)
+        return outcome
+
+    training_outcomes = [print_stats(train_model(
         ticker_to_predict = config.target_asset[1],
         X = X,
         y = y,
@@ -54,9 +59,9 @@ def train_directional_models(
         from_index = from_index,
         no_of_classes = config.no_of_classes,
         level = 'primary',
-        print_results= True,
+        output_stats= config.mode == 'training',
         transformations_over_time = transformations_over_time,
         model_over_time = preloaded_training_step.training[index].model_over_time if preloaded_training_step else None
-    ) for index, model in enumerate(models)]
+    )) for index, model in enumerate(models)]
     return DirectionalTrainingOutcome(training_outcomes, transformations_over_time)
 

training/ensemble.py

@@ -9,14 +9,18 @@ def ensemble_weights(
                     forward_returns: ForwardReturnSeries,
                     y: ySeries,
                     no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
+                    output_stats: bool
                 ) -> EnsembleOutcome:
     weights = pd.concat(input_weights, axis=1).mean(axis=1)
-    stats = evaluate_predictions(
-        forward_returns = forward_returns,
-        y_pred = weights,
-        y_true = y,
-        no_of_classes = no_of_classes,
-        print_results = False,
-        discretize = True,
-    )
+    if output_stats:
+        stats = evaluate_predictions(
+            forward_returns = forward_returns,
+            y_pred = weights,
+            y_true = y,
+            no_of_classes = no_of_classes,
+            discretize = True,
+        )
+        print(stats)
+    else:
+        stats = None
     return EnsembleOutcome(weights, stats)

training/train_model.py

@@ -17,11 +17,11 @@ def train_models(
                 from_index: Optional[pd.Timestamp],
                 no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                 level: str,
-                print_results: bool,
+                output_stats: bool,
                 transformations_over_time: TransformationsOverTime,
                 models_over_time: Optional[list[ModelOverTime]]
     ) -> list[TrainingOutcome]:
-    return [train_model(ticker_to_predict, X, y, forward_returns, model, expanding_window, sliding_window_size, retrain_every, from_index, no_of_classes, level, print_results, transformations_over_time, models_over_time[index] if models_over_time else None) for index, model in enumerate(models)]
+    return [train_model(ticker_to_predict, X, y, forward_returns, model, expanding_window, sliding_window_size, retrain_every, from_index, no_of_classes, level, output_stats, transformations_over_time, models_over_time[index] if models_over_time else None) for index, model in enumerate(models)]
 
 
 def train_model(
@@ -36,7 +36,7 @@ def train_model(
                 from_index: Optional[pd.Timestamp],
                 no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                 level: str,
-                print_results: bool,
+                output_stats: bool,
                 transformations_over_time: TransformationsOverTime,
                 model_over_time: Optional[ModelOverTime]
     ) -> TrainingOutcome:
@@ -73,13 +73,15 @@ def train_model(
     )
     
     assert len(predictions) == len(y)
-    stats = evaluate_predictions(
-        forward_returns = forward_returns,
-        y_pred = predictions,
-        y_true = y,
-        no_of_classes=no_of_classes,
-        print_results = print_results,
-        discretize=True
-    )
+    if output_stats:
+        stats = evaluate_predictions(
+            forward_returns = forward_returns,
+            y_pred = predictions,
+            y_true = y,
+            no_of_classes=no_of_classes,
+            discretize=True
+        )
+    else:
+        stats = None
 
     return TrainingOutcome(model_id, predictions, probabilities, stats, model_over_time)

training/types.py

@@ -16,13 +16,13 @@ class TrainingOutcome:
     model_id: str
     predictions: PredictionsSeries
     probabilities: ProbabilitiesDataFrame
-    stats: Stats
+    stats: Optional[Stats]
     model_over_time: ModelOverTime
 
 @dataclass
 class EnsembleOutcome:
     weights: WeightsSeries
-    stats: Stats
+    stats: Optional[Stats]
 
 @dataclass
 class BetSizingWithMetaOutcome:
@@ -30,7 +30,7 @@ class BetSizingWithMetaOutcome:
     meta_training: list[TrainingOutcome]
     meta_transformations: TransformationsOverTime
     weights: WeightsSeries
-    stats: Stats
+    stats: Optional[Stats]
 
 @dataclass
 class DirectionalTrainingOutcome:
@@ -47,5 +47,5 @@ class PipelineOutcome:
     def get_output_weights(self) -> WeightsSeries:
         return self.secondary_bet_sizing.weights if self.secondary_bet_sizing else self.ensemble.weights
 
-    def get_output_stats(self) -> Stats:
+    def get_output_stats(self) -> Optional[Stats]:
         return self.secondary_bet_sizing.stats if self.secondary_bet_sizing else self.ensemble.stats