feat(Labeling): purge overlapping events, sort dataframe at loading time (#226)

* feat(Labeling): purge overlapping events, sort dataframe at loading time

* fix(Linter): ran

* refactor(Labeling): moved purge_overlapping_events one abstraction level higher

* fix(Data): renamed class

* fix(Data): corrected parameter name

* fix(Config): parameters

* fix(Data): fixed path

* fix(Data): uncommented required code

* feat(EventFilters): use vol based CUSUM

* fix(Config): only retrain every 2000 samples

* fix(Config): filter out even more events

* fix(Inference): added remove_overlapping_events

* refactor(Types): simplified type hierarchy

training/bet_sizing.py

@@ -7,7 +7,12 @@ from models.base import Model
 from models.model_map import default_feature_selector_classification
 from typing import Optional
 from config.types import Config
-from .types import BetSizingWithMetaOutcome, ModelOverTime, TransformationsOverTime
+from .types import (
+    BetSizingWithMetaOutcome,
+    ModelOverTime,
+    TrainingOutcome,
+    TransformationsOverTime,
+)
 from training.walk_forward import walk_forward_process_transformations
 from transformations.base import Transformation
 
@@ -64,6 +69,9 @@ def bet_sizing_with_meta_model(
         transformations_over_time=transformations_over_time,
         model_over_time=preloaded_models,
     )
+    meta_outcome = TrainingOutcome(
+        **vars(meta_outcome), transformations=transformations_over_time
+    )
 
     meta_predictions = meta_outcome.predictions
     bet_size = meta_outcome.probabilities.iloc[:, 1]
@@ -85,7 +93,6 @@ def bet_sizing_with_meta_model(
     return BetSizingWithMetaOutcome(
         model_id,
         meta_outcome,
-        transformations_over_time,
         avg_predictions_with_sizing,
         stats,
     )

training/directional_training.py

@@ -2,7 +2,7 @@ import pandas as pd
 
 from transformations.base import Transformation
 
-from .types import DirectionalTrainingOutcome, TrainingOutcome
+from .types import TrainingOutcome
 from training.train_model import train_model
 from training.walk_forward import walk_forward_process_transformations
 
@@ -19,8 +19,8 @@ def train_directional_model(
     model: Model,
     transformations: list[Transformation],
     from_index: Optional[pd.Timestamp],
-    preloaded_training_step: Optional[DirectionalTrainingOutcome] = None,
-) -> DirectionalTrainingOutcome:
+    preloaded_training_step: Optional[TrainingOutcome] = None,
+) -> TrainingOutcome:
 
     if preloaded_training_step is None:
         print("Preprocess transformations")
@@ -49,11 +49,13 @@ def train_directional_model(
         level="primary",
         output_stats=config.mode == "training",
         transformations_over_time=transformations_over_time,
-        model_over_time=preloaded_training_step.training.model_over_time
+        model_over_time=preloaded_training_step.model_over_time
         if preloaded_training_step
         else None,
     )
     if config.mode == "training":
         print(training_outcome.stats)
 
-    return DirectionalTrainingOutcome(training_outcome, transformations_over_time)
+    return TrainingOutcome(
+        **vars(training_outcome), transformations=transformations_over_time
+    )

training/train_model.py

@@ -7,7 +7,11 @@ from training.walk_forward import (
 )
 from utils.evaluate import evaluate_predictions
 from models.base import Model
-from .types import ModelOverTime, TransformationsOverTime, TrainingOutcome
+from .types import (
+    ModelOverTime,
+    TransformationsOverTime,
+    TrainingOutcomeWithoutTransformations,
+)
 
 
 def train_model(
@@ -24,7 +28,7 @@ def train_model(
     output_stats: bool,
     transformations_over_time: TransformationsOverTime,
     model_over_time: Optional[ModelOverTime],
-) -> TrainingOutcome:
+) -> TrainingOutcomeWithoutTransformations:
 
     if model_over_time is None:
         print("Train model")
@@ -74,4 +78,6 @@ def train_model(
     else:
         stats = None
 
-    return TrainingOutcome(model_id, predictions, probabilities, stats, model_over_time)
+    return TrainingOutcomeWithoutTransformations(
+        model_id, predictions, probabilities, stats, model_over_time
+    )

training/types.py

@@ -12,7 +12,7 @@ TransformationsOverTime = list[pd.Series]
 
 
 @dataclass
-class TrainingOutcome:
+class TrainingOutcomeWithoutTransformations:
     model_id: str
     predictions: PredictionsSeries
     probabilities: ProbabilitiesDataFrame
@@ -20,24 +20,22 @@ class TrainingOutcome:
     model_over_time: ModelOverTime
 
 
+@dataclass
+class TrainingOutcome(TrainingOutcomeWithoutTransformations):
+    transformations: TransformationsOverTime
+
+
 @dataclass
 class BetSizingWithMetaOutcome:
     model_id: str
     meta_training: TrainingOutcome
-    meta_transformations: TransformationsOverTime
     weights: WeightsSeries
     stats: Optional[Stats]
 
 
-@dataclass
-class DirectionalTrainingOutcome:
-    training: TrainingOutcome
-    transformations: TransformationsOverTime
-
-
 @dataclass
 class PipelineOutcome:
-    directional_training: DirectionalTrainingOutcome
+    directional_training: TrainingOutcome
     bet_sizing: BetSizingWithMetaOutcome
 
     def get_output_weights(self) -> WeightsSeries: