diff --git a/labeling/labellers/fixed_time_three_class_balanced.py b/labeling/labellers/fixed_time_three_class_balanced.py index 2be4a62..bece946 100644 --- a/labeling/labellers/fixed_time_three_class_balanced.py +++ b/labeling/labellers/fixed_time_three_class_balanced.py @@ -2,6 +2,7 @@ from data_loader.types import ReturnSeries from ..types import EventLabeller, EventsDataFrame import pandas as pd from .utils import create_forward_returns +from typing import Callable class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller): @@ -52,3 +53,9 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller): } ) return events + + def get_labels(self) -> list[int]: + return [-1, 0, 1] + + def get_discretize_function(self) -> Callable: + raise NotImplementedError diff --git a/labeling/labellers/fixed_time_three_class_imbalanced.py b/labeling/labellers/fixed_time_three_class_imbalanced.py index 22c4b76..b2278e6 100644 --- a/labeling/labellers/fixed_time_three_class_imbalanced.py +++ b/labeling/labellers/fixed_time_three_class_imbalanced.py @@ -1,6 +1,7 @@ from ..types import EventLabeller, EventsDataFrame, ReturnSeries import pandas as pd from .utils import create_forward_returns +from typing import Callable class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller): @@ -51,3 +52,9 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller): } ) return events + + def get_labels(self) -> list[int]: + return [-1, 0, 1] + + def get_discretize_function(self) -> Callable: + raise NotImplementedError diff --git a/labeling/labellers/fixed_time_two_class.py b/labeling/labellers/fixed_time_two_class.py index 25b2574..ca9cb17 100644 --- a/labeling/labellers/fixed_time_two_class.py +++ b/labeling/labellers/fixed_time_two_class.py @@ -1,6 +1,8 @@ from ..types import EventLabeller, EventsDataFrame, ReturnSeries import pandas as pd from .utils import create_forward_returns +from typing import Callable +from .utils import discretize_binary class FixedTimeHorionTwoClassEventLabeller(EventLabeller): @@ -32,3 +34,9 @@ class FixedTimeHorionTwoClassEventLabeller(EventLabeller): } ) return events + + def get_labels(self) -> list[int]: + return [-1, 1] + + def get_discretize_function(self) -> Callable: + return discretize_binary diff --git a/labeling/labellers/utils.py b/labeling/labellers/utils.py index ec436d6..e7e6754 100644 --- a/labeling/labellers/utils.py +++ b/labeling/labellers/utils.py @@ -1,6 +1,8 @@ import pandas as pd from data_loader.types import ForwardReturnSeries from labeling.types import EventsDataFrame +from typing import Callable +import numpy as np def create_forward_returns(series: pd.Series, period: int) -> ForwardReturnSeries: @@ -22,3 +24,31 @@ def purge_overlapping_events(events: EventsDataFrame) -> EventsDataFrame: last_event_end = row["end"] events.drop(indicies_to_remove, inplace=True) return events + + +def discretize_binary(x): + return 1 if x > 0 else -1 + + +def discretize_binary_zero_one(x): + return 1 if x > 0 else 0 + + +def discretize_threeway(x): + return 0 if x == 0 else 1 if x > 0 else -1 + + +def discretize_threeway_threshold(threshold: float) -> Callable: + def discretize(current_value): + lower_threshold = -threshold + upper_threshold = threshold + if np.isnan(current_value): + return np.nan + elif current_value <= lower_threshold: + return -1 + elif current_value > lower_threshold and current_value < upper_threshold: + return 0 + else: + return 1 + + return discretize diff --git a/labeling/types.py b/labeling/types.py index a2092d0..b813448 100644 --- a/labeling/types.py +++ b/labeling/types.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod import pandas as pd import pandera as pa from pandera.typing import DataFrame, Series +from typing import Callable class EventFilter(ABC): @@ -27,3 +28,9 @@ class EventLabeller(ABC): self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries ) -> EventsDataFrame: raise NotImplementedError + + def get_labels(self) -> list[int]: + raise NotImplementedError + + def get_discretize_function(self) -> Callable: + raise NotImplementedError diff --git a/run_inference.py b/run_inference.py index 252104d..09fd164 100644 --- a/run_inference.py +++ b/run_inference.py @@ -75,10 +75,9 @@ def __inference(config: Config, pipeline_outcome: PipelineOutcome): model=config.meta_model, transformations=config.transformations, config=config, - model_suffix="meta", from_index=inference_from, - transformations_over_time=pipeline_outcome.bet_sizing.meta_training.transformations, - preloaded_models=pipeline_outcome.bet_sizing.meta_training.model_over_time, + transformations_over_time=pipeline_outcome.bet_sizing.transformations, + preloaded_models=pipeline_outcome.bet_sizing.model_over_time, ) return PipelineOutcome(directional_training_outcome, bet_sizing_outcome) diff --git a/run_pipeline.py b/run_pipeline.py index 753ca66..170dbab 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -100,7 +100,6 @@ def run_training(config: Config) -> PipelineOutcome: config.meta_model, config.transformations, config, - "meta", None, None, None, diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index f4cfde7..16df438 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -4,6 +4,7 @@ from training.walk_forward import walk_forward_train, walk_forward_inference from models.base import Model from utils.evaluate import evaluate_predictions from sklearn.base import BaseEstimator, ClassifierMixin +from labeling.labellers.utils import discretize_binary no_of_rows = 100 @@ -97,8 +98,9 @@ def test_evaluation(): forward_returns=fake_forward_returns, y_pred=processed_predictions_to_match_returns, y_true=y, - no_of_classes="two", - discretize=True, + discretize_func=discretize_binary, + labels=[1, -1], + transaction_costs=0.002, ) assert result["accuracy"] == 100.0 diff --git a/training/bet_sizing.py b/training/bet_sizing.py index fdd58ac..50f46b4 100644 --- a/training/bet_sizing.py +++ b/training/bet_sizing.py @@ -1,20 +1,23 @@ from data_loader.types import ForwardReturnSeries, XDataFrame, ySeries -from utils.evaluate import discretize_threeway_threshold, evaluate_predictions +from utils.evaluate import evaluate_predictions from utils.helpers import equal_except_nan from .train_model import train_model import pandas as pd from models.base import Model -from models.model_map import default_feature_selector_classification from typing import Optional from config.types import Config from .types import ( BetSizingWithMetaOutcome, ModelOverTime, - TrainingOutcome, TransformationsOverTime, ) from training.walk_forward import walk_forward_process_transformations from transformations.base import Transformation +from labeling.labellers.utils import ( + discretize_binary_zero_one, + discretize_threeway_threshold, +) +import pprint def bet_sizing_with_meta_model( @@ -25,7 +28,6 @@ def bet_sizing_with_meta_model( model: Model, transformations: list[Transformation], config: Config, - model_suffix: str, from_index: Optional[pd.Timestamp], transformations_over_time: Optional[TransformationsOverTime] = None, preloaded_models: Optional[ModelOverTime] = None, @@ -63,36 +65,44 @@ def bet_sizing_with_meta_model( sliding_window_size=config.sliding_window_size, retrain_every=config.retrain_every, from_index=from_index, - no_of_classes="two", level="meta", - output_stats=config.mode == "training", transformations_over_time=transformations_over_time, model_over_time=preloaded_models, ) - meta_outcome = TrainingOutcome( - **vars(meta_outcome), transformations=transformations_over_time - ) meta_predictions = meta_outcome.predictions bet_size = meta_outcome.probabilities.iloc[:, 1] avg_predictions_with_sizing = input_predictions * meta_predictions * bet_size if config.mode == "training": + pp = pprint.PrettyPrinter(depth=2) + meta_stats = evaluate_predictions( + forward_returns=forward_returns, + y_pred=meta_outcome.predictions, + y_true=meta_y, + discretize_func=discretize_binary_zero_one, + labels=[0, 1], + transaction_costs=config.transaction_costs, + ) + pp.pprint(meta_stats) stats = evaluate_predictions( forward_returns=forward_returns, y_pred=avg_predictions_with_sizing, y_true=y, - no_of_classes="three-balanced", - discretize=False, + discretize_func=config.labeling.get_discretize_function(), + labels=config.labeling.get_labels(), + transaction_costs=config.transaction_costs, ) - print(stats) + pp.pprint(stats) else: stats = None - model_id = "model_" + config.target_asset[1] + "_" + model_suffix + model_id = "model_" + config.target_asset[1] + "_meta" + outcome_dict = vars(meta_outcome) + outcome_dict["model_id"] = model_id return BetSizingWithMetaOutcome( - model_id, - meta_outcome, - avg_predictions_with_sizing, - stats, + **outcome_dict, + transformations=transformations_over_time, + weights=avg_predictions_with_sizing, + stats=stats, ) diff --git a/training/directional_training.py b/training/directional_training.py index f865a26..bd4b93e 100644 --- a/training/directional_training.py +++ b/training/directional_training.py @@ -1,6 +1,7 @@ import pandas as pd from transformations.base import Transformation +from utils.evaluate import evaluate_predictions from .types import TrainingOutcome from training.train_model import train_model @@ -9,6 +10,7 @@ from training.walk_forward import walk_forward_process_transformations from typing import Optional from config.types import Config from models.base import Model +import pprint def train_directional_model( @@ -45,17 +47,30 @@ def train_directional_model( sliding_window_size=config.sliding_window_size, retrain_every=config.retrain_every, from_index=from_index, - no_of_classes=config.no_of_classes, level="primary", - output_stats=config.mode == "training", transformations_over_time=transformations_over_time, model_over_time=preloaded_training_step.model_over_time if preloaded_training_step else None, ) - if config.mode == "training": - print(training_outcome.stats) + + stats = ( + evaluate_predictions( + forward_returns=forward_returns, + y_pred=training_outcome.predictions, + y_true=y, + discretize_func=config.labeling.get_discretize_function(), + labels=config.labeling.get_labels(), + transaction_costs=config.transaction_costs, + ) + if config.mode == "training" + else None + ) + + if stats is not None: + pp = pprint.PrettyPrinter(depth=2) + pp.pprint(stats) return TrainingOutcome( - **vars(training_outcome), transformations=transformations_over_time + **vars(training_outcome), transformations=transformations_over_time, stats=stats ) diff --git a/training/train_model.py b/training/train_model.py index 5beaea6..098926c 100644 --- a/training/train_model.py +++ b/training/train_model.py @@ -5,12 +5,11 @@ from training.walk_forward import ( walk_forward_inference, walk_forward_inference_batched, ) -from utils.evaluate import evaluate_predictions from models.base import Model from .types import ( ModelOverTime, TransformationsOverTime, - TrainingOutcomeWithoutTransformations, + BaseTrainingOutcome, ) @@ -23,12 +22,17 @@ def train_model( sliding_window_size: int, retrain_every: int, from_index: Optional[pd.Timestamp], - no_of_classes: Literal["two", "three-balanced", "three-imbalanced"], level: str, - output_stats: bool, transformations_over_time: TransformationsOverTime, model_over_time: Optional[ModelOverTime], -) -> TrainingOutcomeWithoutTransformations: +) -> BaseTrainingOutcome: + + levelname = ("_" + level) if level == "meta" else "" + model_id = ( + "model_" + model.name + "_" + ticker_to_predict + levelname + if model_over_time is None + else model_over_time.name + ) if model_over_time is None: print("Train model") @@ -44,12 +48,6 @@ def train_model( transformations_over_time=transformations_over_time, ) - levelname = ("_" + level) if level == "meta" else "" - if model_over_time is None: - model_id = "model_" + model.name + "_" + ticker_to_predict + levelname - else: - model_id = model_over_time.name - inference_function = ( walk_forward_inference if from_index is not None @@ -67,17 +65,5 @@ def train_model( ) assert len(predictions) == len(y) - if output_stats: - stats = evaluate_predictions( - forward_returns=forward_returns, - y_pred=predictions, - y_true=y, - no_of_classes=no_of_classes, - discretize=True, - ) - else: - stats = None - return TrainingOutcomeWithoutTransformations( - model_id, predictions, probabilities, stats, model_over_time - ) + return BaseTrainingOutcome(model_id, predictions, probabilities, model_over_time) diff --git a/training/types.py b/training/types.py index e4bbe45..fb6dd35 100644 --- a/training/types.py +++ b/training/types.py @@ -12,25 +12,22 @@ TransformationsOverTime = list[pd.Series] @dataclass -class TrainingOutcomeWithoutTransformations: +class BaseTrainingOutcome: model_id: str predictions: PredictionsSeries probabilities: ProbabilitiesDataFrame - stats: Optional[Stats] model_over_time: ModelOverTime @dataclass -class TrainingOutcome(TrainingOutcomeWithoutTransformations): +class TrainingOutcome(BaseTrainingOutcome): transformations: TransformationsOverTime + stats: Optional[Stats] @dataclass -class BetSizingWithMetaOutcome: - model_id: str - meta_training: TrainingOutcome +class BetSizingWithMetaOutcome(TrainingOutcome): weights: WeightsSeries - stats: Optional[Stats] @dataclass diff --git a/utils/evaluate.py b/utils/evaluate.py index 22ec5a6..77ddf1e 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -1,16 +1,15 @@ -from typing import Literal, Callable +from typing import Callable from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score from quantstats.stats import skew, sortino from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio from utils.helpers import get_first_valid_return_index import pandas as pd -import numpy as np from data_loader.types import ForwardReturnSeries, ySeries from training.types import Stats, WeightsSeries def backtest( - returns: pd.Series, signal: pd.Series, transaction_cost=0.002 + returns: pd.Series, signal: pd.Series, transaction_cost: float ) -> pd.Series: delta_pos = signal.diff(1).abs().fillna(0.0) costs = transaction_cost * delta_pos @@ -21,19 +20,18 @@ def __preprocess( forward_returns: ForwardReturnSeries, y_pred: pd.Series, y_true: pd.Series, - no_of_classes: Literal["two", "three-balanced", "three-imbalanced"], - discretize: bool, + discretize_func: Callable, + transaction_costs: float, ) -> pd.DataFrame: y_pred.name = "y_pred" forward_returns.name = "forward_returns" df = pd.concat([y_pred, forward_returns], axis=1).dropna() - discretize_func = get_discretize_function(no_of_classes) # make sure that we evaluate binary/three-way predictions even if the model is a regression - df["sign_pred"] = df.y_pred.apply(discretize_func) if discretize else df.y_pred + df["sign_pred"] = df.y_pred.apply(discretize_func) df["sign_true"] = y_true - df["result"] = backtest(df.forward_returns, df.sign_pred) + df["result"] = backtest(df.forward_returns, df.y_pred, transaction_costs) return df @@ -42,8 +40,9 @@ def evaluate_predictions( forward_returns: ForwardReturnSeries, y_pred: WeightsSeries, y_true: ySeries, - no_of_classes: Literal["two", "three-balanced", "three-imbalanced"], - discretize: bool = False, + discretize_func: Callable, + labels: list[int], + transaction_costs: float, ) -> Stats: # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size) evaluate_from = max( @@ -54,7 +53,9 @@ def evaluate_predictions( forward_returns = pd.Series(forward_returns[evaluate_from:]) y_pred = pd.Series(y_pred[evaluate_from:]) - df = __preprocess(forward_returns, y_pred, y_true, no_of_classes, discretize) + df = __preprocess( + forward_returns, y_pred, y_true, discretize_func, transaction_costs + ) scorecard = dict() @@ -73,60 +74,27 @@ def evaluate_predictions( scorecard["sortino"] = sortino(df.result) scorecard["skew"] = skew(df.result) - labels = [1, -1] if no_of_classes == "two" else [1, -1, 0] - avg_type = "weighted" if no_of_classes == "two" else "macro" + avg_type = "weighted" if len(labels) == 2 else "macro" - if discretize == True: - scorecard["accuracy"] = accuracy_score(df.sign_true, df.sign_pred) * 100 - scorecard["recall"] = recall_score( - df.sign_true, df.sign_pred, labels=labels, average=avg_type - ) - scorecard["precision"] = precision_score( - df.sign_true, df.sign_pred, labels=labels, average=avg_type - ) - scorecard["f1_score"] = f1_score( - df.sign_true, df.sign_pred, labels=labels, average=avg_type - ) + scorecard["accuracy"] = accuracy_score(df.sign_true, df.sign_pred) * 100 + scorecard["recall"] = recall_score( + df.sign_true, df.sign_pred, labels=labels, average=avg_type + ) + scorecard["precision"] = precision_score( + df.sign_true, df.sign_pred, labels=labels, average=avg_type + ) + scorecard["f1_score"] = f1_score( + df.sign_true, df.sign_pred, labels=labels, average=avg_type + ) scorecard["edge"] = df.result.mean() scorecard["noise"] = df.y_pred.diff().abs().mean() scorecard["edge_to_noise"] = scorecard["edge"] / (scorecard["noise"] + 0.00001) - if discretize == True: - for index, row in df.sign_true.value_counts().iteritems(): - scorecard["sign_true_ratio_" + str(index)] = row / len(df.sign_true) + for index, row in df.sign_true.value_counts().iteritems(): + scorecard["sign_true_ratio_" + str(index)] = row / len(df.sign_true) - for index, row in df.sign_pred.value_counts().iteritems(): - scorecard["sign_pred_ratio_" + str(index)] = row / len(df.sign_pred) + for index, row in df.sign_pred.value_counts().iteritems(): + scorecard["sign_pred_ratio_" + str(index)] = row / len(df.sign_pred) scorecard = {k: round(float(v), 3) for k, v in scorecard.items()} return scorecard - - -def __discretize_binary(x): - return 1 if x > 0 else -1 - - -def __discretize_threeway(x): - return 0 if x == 0 else 1 if x > 0 else -1 - - -def discretize_threeway_threshold(threshold: float) -> Callable: - def discretize(current_value): - lower_threshold = -threshold - upper_threshold = threshold - if np.isnan(current_value): - return np.nan - elif current_value <= lower_threshold: - return -1 - elif current_value > lower_threshold and current_value < upper_threshold: - return 0 - else: - return 1 - - return discretize - - -def get_discretize_function( - no_of_classes: Literal["two", "three-balanced", "three-imbalanced"] -) -> Callable: - return __discretize_binary if no_of_classes == "two" else __discretize_threeway