fix(MetaLabeling): previously misinterpreted meta-labeling, now also multiplying base model's prediction with the meta model's prediction (#193)

* fix(MetaLabeling): previously misinterpreted meta-labeling, now also multiplying base model's prediction with the meta model's prediction

* fix(Evaluate): print results

* fix(Evaluate): make sure we have numerical stability in returns

* fix(Inference): only output and print stats in training mode

* fix(Evaluate): don't add miniscule amount to result

config/preprocess.py

@@ -15,6 +15,7 @@ def preprocess_config(raw_config: RawConfig) -> Config:
     config_dict = __preprocess_event_labeller_config(config_dict)
 
     config_dict['no_of_classes'] = 'two'
+    config_dict['mode'] = 'training'
     config = Config(**config_dict)
     validate_config(config)
     return config

config/types.py

@@ -57,6 +57,8 @@ class Config(BaseModel):
     labeling: EventLabeller
     no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
 
+    mode: Literal['training', 'inference']
+
     directional_models: list[Model]
     meta_models: list[Model]
 

run_inference.py

@@ -19,6 +19,7 @@ def run_inference(preload_models:bool, fallback_raw_config: RawConfig):
     else:
         pipeline_outcome, config = run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, raw_config=fallback_raw_config)
     
+    config.mode = 'inference'
     __inference(config, pipeline_outcome)
 
 
@@ -39,9 +40,9 @@ def __inference(config: Config, pipeline_outcome: PipelineOutcome):
 
     events, X, y, forward_returns = label_data(config.event_filter, config.labeling, X, returns, forward_returns)
 
-    inference_from: pd.Timestamp = X.index[len(X.index) - 2]
+    inference_from: pd.Timestamp = X.index[len(X.index) - 1]
 
-        # 2. Filter for significant events when we want to trade, and label data
+    # 2. Filter for significant events when we want to trade, and label data
     events, X, y, forward_returns = label_data(config.event_filter, config.labeling, X, returns, forward_returns)
 
     # 3. Train directional models
@@ -51,7 +52,7 @@ def __inference(config: Config, pipeline_outcome: PipelineOutcome):
     bet_sizing_outcomes = [bet_sizing_with_meta_models(X, training_outcome.predictions, y, forward_returns, config.meta_models, config, 'meta', from_index = inference_from, transformations_over_time = preloaded_outcome.meta_transformations, preloaded_models = [b.model_over_time for b in preloaded_outcome.meta_training]) for training_outcome, preloaded_outcome in zip(directional_training_outcome.training, pipeline_outcome.bet_sizing)]
 
     # 4. Ensemble weights
-    ensemble_outcome = ensemble_weights([o.weights for o in bet_sizing_outcomes], forward_returns, y, config.no_of_classes)
+    ensemble_outcome = ensemble_weights([o.weights for o in bet_sizing_outcomes], forward_returns, y, config.no_of_classes, config.mode == 'training')
 
     # 5. (Optional) Additional bet sizing on top of the ensembled weights
     ensemble_bet_sizing_outcome = bet_sizing_with_meta_models(X, ensemble_outcome.weights, y, forward_returns, config.meta_models, config, 'ensemble', from_index = inference_from, transformations_over_time = pipeline_outcome.secondary_bet_sizing.meta_transformations, preloaded_models= [b.model_over_time for b in pipeline_outcome.secondary_bet_sizing.meta_training]) if len(config.meta_models) > 0 else None

run_pipeline.py

@@ -69,7 +69,7 @@ def __run_training(config: Config) -> PipelineOutcome:
     bet_sizing_outcomes = [bet_sizing_with_meta_models(X, outcome.predictions, y, forward_returns, config.meta_models, config, 'meta', None, None, None) for outcome in directional_training_outcome.training]
 
     # 4. Ensemble weights
-    ensemble_outcome = ensemble_weights([o.weights for o in bet_sizing_outcomes], forward_returns, y, config.no_of_classes)
+    ensemble_outcome = ensemble_weights([o.weights for o in bet_sizing_outcomes], forward_returns, y, config.no_of_classes, config.mode == 'training')
 
     # 5. (Optional) Additional bet sizing on top of the ensembled weights
     ensemble_bet_sizing_outcome = bet_sizing_with_meta_models(X, ensemble_outcome.weights, y, forward_returns, config.meta_models, config, 'ensemble', None, None, None) if len(config.meta_models) > 0 else None

tests/test_evaluation.py

@@ -100,7 +100,6 @@ def test_evaluation():
         y_pred=processed_predictions_to_match_returns,
         y_true=y,
         no_of_classes='two',
-        print_results = False,
         discretize=True
     )
 

training/bet_sizing.py

@@ -62,27 +62,31 @@ def bet_sizing_with_meta_models(
         from_index = from_index,
         no_of_classes = 'two',
         level = 'meta',
-        print_results = False,
+        output_stats = config.mode == 'training',
         transformations_over_time = transformations_over_time,
         models_over_time = preloaded_models,
     )
 
     # Ensemble predictions if necessary
     if len(models) > 1:
-        # meta_predictions = pd.concat([outcome.predictions for outcome in meta_outcomes]).mean(axis = 1)
+        meta_predictions = pd.concat([outcome.predictions for outcome in meta_outcomes], axis = 1).mean(axis = 1).apply(discretize_threeway_threshold(0.5))
         bet_size = pd.concat([outcome.probabilities[outcome.probabilities.columns[1::2]] for outcome in meta_outcomes], axis = 1).mean(axis = 1)
     else:
+        meta_predictions = meta_outcomes[0].predictions
         bet_size = meta_outcomes[0].probabilities.iloc[:,1]
-    avg_predictions_with_sizing = input_predictions * bet_size
+    avg_predictions_with_sizing = input_predictions * meta_predictions * bet_size
 
-    stats = evaluate_predictions(
-        forward_returns = forward_returns,
-        y_pred = avg_predictions_with_sizing,
-        y_true = y,
-        no_of_classes = 'two',
-        print_results = True,
-        discretize=False
-    )
+    if config.mode == 'training':
+        stats = evaluate_predictions(
+            forward_returns = forward_returns,
+            y_pred = avg_predictions_with_sizing,
+            y_true = y,
+            no_of_classes = 'three-balanced',
+            discretize=False
+        )
+        print(stats)
+    else:
+        stats = None
     model_id = "model_" + config.target_asset[1] + "_" + model_suffix
 
     return BetSizingWithMetaOutcome(model_id, meta_outcomes, transformations_over_time, avg_predictions_with_sizing, stats)

training/directional_training.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-from .types import DirectionalTrainingOutcome
+from .types import DirectionalTrainingOutcome, TrainingOutcome
 from training.train_model import train_model
 from training.walk_forward import walk_forward_process_transformations
 
@@ -42,7 +42,12 @@ def train_directional_models(
     else:
         transformations_over_time = preloaded_training_step.transformations
 
-    training_outcomes = [train_model(
+    def print_stats(outcome: TrainingOutcome) -> TrainingOutcome:
+        if config.mode == 'training':
+            print(outcome.stats)
+        return outcome
+
+    training_outcomes = [print_stats(train_model(
         ticker_to_predict = config.target_asset[1],
         X = X,
         y = y,
@@ -54,9 +59,9 @@ def train_directional_models(
         from_index = from_index,
         no_of_classes = config.no_of_classes,
         level = 'primary',
-        print_results= True,
+        output_stats= config.mode == 'training',
         transformations_over_time = transformations_over_time,
         model_over_time = preloaded_training_step.training[index].model_over_time if preloaded_training_step else None
-    ) for index, model in enumerate(models)]
+    )) for index, model in enumerate(models)]
     return DirectionalTrainingOutcome(training_outcomes, transformations_over_time)
 

training/ensemble.py

@@ -9,14 +9,18 @@ def ensemble_weights(
                     forward_returns: ForwardReturnSeries,
                     y: ySeries,
                     no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
+                    output_stats: bool
                 ) -> EnsembleOutcome:
     weights = pd.concat(input_weights, axis=1).mean(axis=1)
-    stats = evaluate_predictions(
-        forward_returns = forward_returns,
-        y_pred = weights,
-        y_true = y,
-        no_of_classes = no_of_classes,
-        print_results = False,
-        discretize = True,
-    )
+    if output_stats:
+        stats = evaluate_predictions(
+            forward_returns = forward_returns,
+            y_pred = weights,
+            y_true = y,
+            no_of_classes = no_of_classes,
+            discretize = True,
+        )
+        print(stats)
+    else:
+        stats = None
     return EnsembleOutcome(weights, stats)

training/train_model.py

@@ -17,11 +17,11 @@ def train_models(
                 from_index: Optional[pd.Timestamp],
                 no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                 level: str,
-                print_results: bool,
+                output_stats: bool,
                 transformations_over_time: TransformationsOverTime,
                 models_over_time: Optional[list[ModelOverTime]]
     ) -> list[TrainingOutcome]:
-    return [train_model(ticker_to_predict, X, y, forward_returns, model, expanding_window, sliding_window_size, retrain_every, from_index, no_of_classes, level, print_results, transformations_over_time, models_over_time[index] if models_over_time else None) for index, model in enumerate(models)]
+    return [train_model(ticker_to_predict, X, y, forward_returns, model, expanding_window, sliding_window_size, retrain_every, from_index, no_of_classes, level, output_stats, transformations_over_time, models_over_time[index] if models_over_time else None) for index, model in enumerate(models)]
 
 
 def train_model(
@@ -36,7 +36,7 @@ def train_model(
                 from_index: Optional[pd.Timestamp],
                 no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                 level: str,
-                print_results: bool,
+                output_stats: bool,
                 transformations_over_time: TransformationsOverTime,
                 model_over_time: Optional[ModelOverTime]
     ) -> TrainingOutcome:
@@ -73,13 +73,15 @@ def train_model(
     )
     
     assert len(predictions) == len(y)
-    stats = evaluate_predictions(
-        forward_returns = forward_returns,
-        y_pred = predictions,
-        y_true = y,
-        no_of_classes=no_of_classes,
-        print_results = print_results,
-        discretize=True
-    )
+    if output_stats:
+        stats = evaluate_predictions(
+            forward_returns = forward_returns,
+            y_pred = predictions,
+            y_true = y,
+            no_of_classes=no_of_classes,
+            discretize=True
+        )
+    else:
+        stats = None
 
     return TrainingOutcome(model_id, predictions, probabilities, stats, model_over_time)

training/types.py

@@ -16,13 +16,13 @@ class TrainingOutcome:
     model_id: str
     predictions: PredictionsSeries
     probabilities: ProbabilitiesDataFrame
-    stats: Stats
+    stats: Optional[Stats]
     model_over_time: ModelOverTime
 
 @dataclass
 class EnsembleOutcome:
     weights: WeightsSeries
-    stats: Stats
+    stats: Optional[Stats]
 
 @dataclass
 class BetSizingWithMetaOutcome:
@@ -30,7 +30,7 @@ class BetSizingWithMetaOutcome:
     meta_training: list[TrainingOutcome]
     meta_transformations: TransformationsOverTime
     weights: WeightsSeries
-    stats: Stats
+    stats: Optional[Stats]
 
 @dataclass
 class DirectionalTrainingOutcome:
@@ -47,5 +47,5 @@ class PipelineOutcome:
     def get_output_weights(self) -> WeightsSeries:
         return self.secondary_bet_sizing.weights if self.secondary_bet_sizing else self.ensemble.weights
 
-    def get_output_stats(self) -> Stats:
+    def get_output_stats(self) -> Optional[Stats]:
         return self.secondary_bet_sizing.stats if self.secondary_bet_sizing else self.ensemble.stats

utils/evaluate.py

@@ -33,7 +33,6 @@ def evaluate_predictions(
                         y_pred: WeightsSeries,
                         y_true: ySeries,
                         no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
-                        print_results: bool,
                         discretize: bool = False,
                         ) -> Stats:
     # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
@@ -78,8 +77,6 @@ def evaluate_predictions(
             scorecard['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred)
 
     scorecard = {k: round(float(v), 3) for k, v in scorecard.items()}
-    if print_results:        
-        print(scorecard)
     return scorecard