refactor(Evaluate): print out accuracy, f1, etc. for the final & meta predictions, separated out evaluation step (#228)

* refactor(Evaluate): print out accuracy, f1, etc. for the final & meta predictions, separated out evaluation step

* fix(Linter): ran

* fix(Tests): syntax change

* fix(Inference): runs now again

* fix(Linter): ran

labeling/labellers/fixed_time_three_class_balanced.py

@@ -2,6 +2,7 @@ from data_loader.types import ReturnSeries
 from ..types import EventLabeller, EventsDataFrame
 import pandas as pd
 from .utils import create_forward_returns
+from typing import Callable
 
 
 class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
@@ -52,3 +53,9 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
             }
         )
         return events
+
+    def get_labels(self) -> list[int]:
+        return [-1, 0, 1]
+
+    def get_discretize_function(self) -> Callable:
+        raise NotImplementedError

labeling/labellers/fixed_time_three_class_imbalanced.py

@@ -1,6 +1,7 @@
 from ..types import EventLabeller, EventsDataFrame, ReturnSeries
 import pandas as pd
 from .utils import create_forward_returns
+from typing import Callable
 
 
 class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
@@ -51,3 +52,9 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
             }
         )
         return events
+
+    def get_labels(self) -> list[int]:
+        return [-1, 0, 1]
+
+    def get_discretize_function(self) -> Callable:
+        raise NotImplementedError

labeling/labellers/fixed_time_two_class.py

@@ -1,6 +1,8 @@
 from ..types import EventLabeller, EventsDataFrame, ReturnSeries
 import pandas as pd
 from .utils import create_forward_returns
+from typing import Callable
+from .utils import discretize_binary
 
 
 class FixedTimeHorionTwoClassEventLabeller(EventLabeller):
@@ -32,3 +34,9 @@ class FixedTimeHorionTwoClassEventLabeller(EventLabeller):
             }
         )
         return events
+
+    def get_labels(self) -> list[int]:
+        return [-1, 1]
+
+    def get_discretize_function(self) -> Callable:
+        return discretize_binary

labeling/labellers/utils.py

@@ -1,6 +1,8 @@
 import pandas as pd
 from data_loader.types import ForwardReturnSeries
 from labeling.types import EventsDataFrame
+from typing import Callable
+import numpy as np
 
 
 def create_forward_returns(series: pd.Series, period: int) -> ForwardReturnSeries:
@@ -22,3 +24,31 @@ def purge_overlapping_events(events: EventsDataFrame) -> EventsDataFrame:
             last_event_end = row["end"]
     events.drop(indicies_to_remove, inplace=True)
     return events
+
+
+def discretize_binary(x):
+    return 1 if x > 0 else -1
+
+
+def discretize_binary_zero_one(x):
+    return 1 if x > 0 else 0
+
+
+def discretize_threeway(x):
+    return 0 if x == 0 else 1 if x > 0 else -1
+
+
+def discretize_threeway_threshold(threshold: float) -> Callable:
+    def discretize(current_value):
+        lower_threshold = -threshold
+        upper_threshold = threshold
+        if np.isnan(current_value):
+            return np.nan
+        elif current_value <= lower_threshold:
+            return -1
+        elif current_value > lower_threshold and current_value < upper_threshold:
+            return 0
+        else:
+            return 1
+
+    return discretize

labeling/types.py

@@ -3,6 +3,7 @@ from abc import ABC, abstractmethod
 import pandas as pd
 import pandera as pa
 from pandera.typing import DataFrame, Series
+from typing import Callable
 
 
 class EventFilter(ABC):
@@ -27,3 +28,9 @@ class EventLabeller(ABC):
         self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
     ) -> EventsDataFrame:
         raise NotImplementedError
+
+    def get_labels(self) -> list[int]:
+        raise NotImplementedError
+
+    def get_discretize_function(self) -> Callable:
+        raise NotImplementedError

run_inference.py

@@ -75,10 +75,9 @@ def __inference(config: Config, pipeline_outcome: PipelineOutcome):
         model=config.meta_model,
         transformations=config.transformations,
         config=config,
-        model_suffix="meta",
         from_index=inference_from,
-        transformations_over_time=pipeline_outcome.bet_sizing.meta_training.transformations,
+        transformations_over_time=pipeline_outcome.bet_sizing.transformations,
-        preloaded_models=pipeline_outcome.bet_sizing.meta_training.model_over_time,
+        preloaded_models=pipeline_outcome.bet_sizing.model_over_time,
     )
 
     return PipelineOutcome(directional_training_outcome, bet_sizing_outcome)

run_pipeline.py

@@ -100,7 +100,6 @@ def run_training(config: Config) -> PipelineOutcome:
         config.meta_model,
         config.transformations,
         config,
-        "meta",
         None,
         None,
         None,

tests/test_evaluation.py

@@ -4,6 +4,7 @@ from training.walk_forward import walk_forward_train, walk_forward_inference
 from models.base import Model
 from utils.evaluate import evaluate_predictions
 from sklearn.base import BaseEstimator, ClassifierMixin
+from labeling.labellers.utils import discretize_binary
 
 no_of_rows = 100
 
@@ -97,8 +98,9 @@ def test_evaluation():
         forward_returns=fake_forward_returns,
         y_pred=processed_predictions_to_match_returns,
         y_true=y,
-        no_of_classes="two",
+        discretize_func=discretize_binary,
-        discretize=True,
+        labels=[1, -1],
+        transaction_costs=0.002,
     )
 
     assert result["accuracy"] == 100.0

training/bet_sizing.py

@@ -1,20 +1,23 @@
 from data_loader.types import ForwardReturnSeries, XDataFrame, ySeries
-from utils.evaluate import discretize_threeway_threshold, evaluate_predictions
+from utils.evaluate import evaluate_predictions
 from utils.helpers import equal_except_nan
 from .train_model import train_model
 import pandas as pd
 from models.base import Model
-from models.model_map import default_feature_selector_classification
 from typing import Optional
 from config.types import Config
 from .types import (
     BetSizingWithMetaOutcome,
     ModelOverTime,
-    TrainingOutcome,
     TransformationsOverTime,
 )
 from training.walk_forward import walk_forward_process_transformations
 from transformations.base import Transformation
+from labeling.labellers.utils import (
+    discretize_binary_zero_one,
+    discretize_threeway_threshold,
+)
+import pprint
 
 
 def bet_sizing_with_meta_model(
@@ -25,7 +28,6 @@ def bet_sizing_with_meta_model(
     model: Model,
     transformations: list[Transformation],
     config: Config,
-    model_suffix: str,
     from_index: Optional[pd.Timestamp],
     transformations_over_time: Optional[TransformationsOverTime] = None,
     preloaded_models: Optional[ModelOverTime] = None,
@@ -63,36 +65,44 @@ def bet_sizing_with_meta_model(
         sliding_window_size=config.sliding_window_size,
         retrain_every=config.retrain_every,
         from_index=from_index,
-        no_of_classes="two",
         level="meta",
-        output_stats=config.mode == "training",
         transformations_over_time=transformations_over_time,
         model_over_time=preloaded_models,
     )
-    meta_outcome = TrainingOutcome(
-        **vars(meta_outcome), transformations=transformations_over_time
-    )
 
     meta_predictions = meta_outcome.predictions
     bet_size = meta_outcome.probabilities.iloc[:, 1]
     avg_predictions_with_sizing = input_predictions * meta_predictions * bet_size
 
     if config.mode == "training":
+        pp = pprint.PrettyPrinter(depth=2)
+        meta_stats = evaluate_predictions(
+            forward_returns=forward_returns,
+            y_pred=meta_outcome.predictions,
+            y_true=meta_y,
+            discretize_func=discretize_binary_zero_one,
+            labels=[0, 1],
+            transaction_costs=config.transaction_costs,
+        )
+        pp.pprint(meta_stats)
         stats = evaluate_predictions(
             forward_returns=forward_returns,
             y_pred=avg_predictions_with_sizing,
             y_true=y,
-            no_of_classes="three-balanced",
+            discretize_func=config.labeling.get_discretize_function(),
-            discretize=False,
+            labels=config.labeling.get_labels(),
+            transaction_costs=config.transaction_costs,
         )
-        print(stats)
+        pp.pprint(stats)
     else:
         stats = None
-    model_id = "model_" + config.target_asset[1] + "_" + model_suffix
+    model_id = "model_" + config.target_asset[1] + "_meta"
 
+    outcome_dict = vars(meta_outcome)
+    outcome_dict["model_id"] = model_id
     return BetSizingWithMetaOutcome(
-        model_id,
+        **outcome_dict,
-        meta_outcome,
+        transformations=transformations_over_time,
-        avg_predictions_with_sizing,
+        weights=avg_predictions_with_sizing,
-        stats,
+        stats=stats,
     )

training/directional_training.py

@@ -1,6 +1,7 @@
 import pandas as pd
 
 from transformations.base import Transformation
+from utils.evaluate import evaluate_predictions
 
 from .types import TrainingOutcome
 from training.train_model import train_model
@@ -9,6 +10,7 @@ from training.walk_forward import walk_forward_process_transformations
 from typing import Optional
 from config.types import Config
 from models.base import Model
+import pprint
 
 
 def train_directional_model(
@@ -45,17 +47,30 @@ def train_directional_model(
         sliding_window_size=config.sliding_window_size,
         retrain_every=config.retrain_every,
         from_index=from_index,
-        no_of_classes=config.no_of_classes,
         level="primary",
-        output_stats=config.mode == "training",
         transformations_over_time=transformations_over_time,
         model_over_time=preloaded_training_step.model_over_time
         if preloaded_training_step
         else None,
     )
-    if config.mode == "training":
+
-        print(training_outcome.stats)
+    stats = (
+        evaluate_predictions(
+            forward_returns=forward_returns,
+            y_pred=training_outcome.predictions,
+            y_true=y,
+            discretize_func=config.labeling.get_discretize_function(),
+            labels=config.labeling.get_labels(),
+            transaction_costs=config.transaction_costs,
+        )
+        if config.mode == "training"
+        else None
+    )
+
+    if stats is not None:
+        pp = pprint.PrettyPrinter(depth=2)
+        pp.pprint(stats)
 
     return TrainingOutcome(
-        **vars(training_outcome), transformations=transformations_over_time
+        **vars(training_outcome), transformations=transformations_over_time, stats=stats
     )

training/train_model.py

@@ -5,12 +5,11 @@ from training.walk_forward import (
     walk_forward_inference,
     walk_forward_inference_batched,
 )
-from utils.evaluate import evaluate_predictions
 from models.base import Model
 from .types import (
     ModelOverTime,
     TransformationsOverTime,
-    TrainingOutcomeWithoutTransformations,
+    BaseTrainingOutcome,
 )
 
 
@@ -23,12 +22,17 @@ def train_model(
     sliding_window_size: int,
     retrain_every: int,
     from_index: Optional[pd.Timestamp],
-    no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
     level: str,
-    output_stats: bool,
     transformations_over_time: TransformationsOverTime,
     model_over_time: Optional[ModelOverTime],
-) -> TrainingOutcomeWithoutTransformations:
+) -> BaseTrainingOutcome:
+
+    levelname = ("_" + level) if level == "meta" else ""
+    model_id = (
+        "model_" + model.name + "_" + ticker_to_predict + levelname
+        if model_over_time is None
+        else model_over_time.name
+    )
 
     if model_over_time is None:
         print("Train model")
@@ -44,12 +48,6 @@ def train_model(
             transformations_over_time=transformations_over_time,
         )
 
-    levelname = ("_" + level) if level == "meta" else ""
-    if model_over_time is None:
-        model_id = "model_" + model.name + "_" + ticker_to_predict + levelname
-    else:
-        model_id = model_over_time.name
-
     inference_function = (
         walk_forward_inference
         if from_index is not None
@@ -67,17 +65,5 @@ def train_model(
     )
 
     assert len(predictions) == len(y)
-    if output_stats:
-        stats = evaluate_predictions(
-            forward_returns=forward_returns,
-            y_pred=predictions,
-            y_true=y,
-            no_of_classes=no_of_classes,
-            discretize=True,
-        )
-    else:
-        stats = None
 
-    return TrainingOutcomeWithoutTransformations(
+    return BaseTrainingOutcome(model_id, predictions, probabilities, model_over_time)
-        model_id, predictions, probabilities, stats, model_over_time
-    )

training/types.py

@@ -12,25 +12,22 @@ TransformationsOverTime = list[pd.Series]
 
 
 @dataclass
-class TrainingOutcomeWithoutTransformations:
+class BaseTrainingOutcome:
     model_id: str
     predictions: PredictionsSeries
     probabilities: ProbabilitiesDataFrame
-    stats: Optional[Stats]
     model_over_time: ModelOverTime
 
 
 @dataclass
-class TrainingOutcome(TrainingOutcomeWithoutTransformations):
+class TrainingOutcome(BaseTrainingOutcome):
     transformations: TransformationsOverTime
+    stats: Optional[Stats]
 
 
 @dataclass
-class BetSizingWithMetaOutcome:
+class BetSizingWithMetaOutcome(TrainingOutcome):
-    model_id: str
-    meta_training: TrainingOutcome
     weights: WeightsSeries
-    stats: Optional[Stats]
 
 
 @dataclass

utils/evaluate.py

@@ -1,16 +1,15 @@
-from typing import Literal, Callable
+from typing import Callable
 from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
 from quantstats.stats import skew, sortino
 from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio
 from utils.helpers import get_first_valid_return_index
 import pandas as pd
-import numpy as np
 from data_loader.types import ForwardReturnSeries, ySeries
 from training.types import Stats, WeightsSeries
 
 
 def backtest(
-    returns: pd.Series, signal: pd.Series, transaction_cost=0.002
+    returns: pd.Series, signal: pd.Series, transaction_cost: float
 ) -> pd.Series:
     delta_pos = signal.diff(1).abs().fillna(0.0)
     costs = transaction_cost * delta_pos
@@ -21,19 +20,18 @@ def __preprocess(
     forward_returns: ForwardReturnSeries,
     y_pred: pd.Series,
     y_true: pd.Series,
-    no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
+    discretize_func: Callable,
-    discretize: bool,
+    transaction_costs: float,
 ) -> pd.DataFrame:
     y_pred.name = "y_pred"
     forward_returns.name = "forward_returns"
     df = pd.concat([y_pred, forward_returns], axis=1).dropna()
 
-    discretize_func = get_discretize_function(no_of_classes)
     # make sure that we evaluate binary/three-way predictions even if the model is a regression
-    df["sign_pred"] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
+    df["sign_pred"] = df.y_pred.apply(discretize_func)
     df["sign_true"] = y_true
 
-    df["result"] = backtest(df.forward_returns, df.sign_pred)
+    df["result"] = backtest(df.forward_returns, df.y_pred, transaction_costs)
 
     return df
 
@@ -42,8 +40,9 @@ def evaluate_predictions(
     forward_returns: ForwardReturnSeries,
     y_pred: WeightsSeries,
     y_true: ySeries,
-    no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
+    discretize_func: Callable,
-    discretize: bool = False,
+    labels: list[int],
+    transaction_costs: float,
 ) -> Stats:
     # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
     evaluate_from = max(
@@ -54,7 +53,9 @@ def evaluate_predictions(
     forward_returns = pd.Series(forward_returns[evaluate_from:])
     y_pred = pd.Series(y_pred[evaluate_from:])
 
-    df = __preprocess(forward_returns, y_pred, y_true, no_of_classes, discretize)
+    df = __preprocess(
+        forward_returns, y_pred, y_true, discretize_func, transaction_costs
+    )
 
     scorecard = dict()
 
@@ -73,60 +74,27 @@ def evaluate_predictions(
     scorecard["sortino"] = sortino(df.result)
     scorecard["skew"] = skew(df.result)
 
-    labels = [1, -1] if no_of_classes == "two" else [1, -1, 0]
+    avg_type = "weighted" if len(labels) == 2 else "macro"
-    avg_type = "weighted" if no_of_classes == "two" else "macro"
 
-    if discretize == True:
+    scorecard["accuracy"] = accuracy_score(df.sign_true, df.sign_pred) * 100
-        scorecard["accuracy"] = accuracy_score(df.sign_true, df.sign_pred) * 100
+    scorecard["recall"] = recall_score(
-        scorecard["recall"] = recall_score(
+        df.sign_true, df.sign_pred, labels=labels, average=avg_type
-            df.sign_true, df.sign_pred, labels=labels, average=avg_type
+    )
-        )
+    scorecard["precision"] = precision_score(
-        scorecard["precision"] = precision_score(
+        df.sign_true, df.sign_pred, labels=labels, average=avg_type
-            df.sign_true, df.sign_pred, labels=labels, average=avg_type
+    )
-        )
+    scorecard["f1_score"] = f1_score(
-        scorecard["f1_score"] = f1_score(
+        df.sign_true, df.sign_pred, labels=labels, average=avg_type
-            df.sign_true, df.sign_pred, labels=labels, average=avg_type
+    )
-        )
     scorecard["edge"] = df.result.mean()
     scorecard["noise"] = df.y_pred.diff().abs().mean()
     scorecard["edge_to_noise"] = scorecard["edge"] / (scorecard["noise"] + 0.00001)
 
-    if discretize == True:
+    for index, row in df.sign_true.value_counts().iteritems():
-        for index, row in df.sign_true.value_counts().iteritems():
+        scorecard["sign_true_ratio_" + str(index)] = row / len(df.sign_true)
-            scorecard["sign_true_ratio_" + str(index)] = row / len(df.sign_true)
 
-        for index, row in df.sign_pred.value_counts().iteritems():
+    for index, row in df.sign_pred.value_counts().iteritems():
-            scorecard["sign_pred_ratio_" + str(index)] = row / len(df.sign_pred)
+        scorecard["sign_pred_ratio_" + str(index)] = row / len(df.sign_pred)
 
     scorecard = {k: round(float(v), 3) for k, v in scorecard.items()}
     return scorecard
-
-
-def __discretize_binary(x):
-    return 1 if x > 0 else -1
-
-
-def __discretize_threeway(x):
-    return 0 if x == 0 else 1 if x > 0 else -1
-
-
-def discretize_threeway_threshold(threshold: float) -> Callable:
-    def discretize(current_value):
-        lower_threshold = -threshold
-        upper_threshold = threshold
-        if np.isnan(current_value):
-            return np.nan
-        elif current_value <= lower_threshold:
-            return -1
-        elif current_value > lower_threshold and current_value < upper_threshold:
-            return 0
-        else:
-            return 1
-
-    return discretize
-
-
-def get_discretize_function(
-    no_of_classes: Literal["two", "three-balanced", "three-imbalanced"]
-) -> Callable:
-    return __discretize_binary if no_of_classes == "two" else __discretize_threeway