diff --git a/config/preprocess.py b/config/preprocess.py index 5f1ce63..d327c61 100644 --- a/config/preprocess.py +++ b/config/preprocess.py @@ -15,6 +15,7 @@ def preprocess_config(raw_config: RawConfig) -> Config: config_dict = __preprocess_event_labeller_config(config_dict) config_dict['no_of_classes'] = 'two' + config_dict['mode'] = 'training' config = Config(**config_dict) validate_config(config) return config diff --git a/config/types.py b/config/types.py index 77c12c7..b6dbed3 100644 --- a/config/types.py +++ b/config/types.py @@ -57,6 +57,8 @@ class Config(BaseModel): labeling: EventLabeller no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'] + mode: Literal['training', 'inference'] + directional_models: list[Model] meta_models: list[Model] diff --git a/run_inference.py b/run_inference.py index 4851153..f934ef2 100644 --- a/run_inference.py +++ b/run_inference.py @@ -19,6 +19,7 @@ def run_inference(preload_models:bool, fallback_raw_config: RawConfig): else: pipeline_outcome, config = run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, raw_config=fallback_raw_config) + config.mode = 'inference' __inference(config, pipeline_outcome) @@ -39,9 +40,9 @@ def __inference(config: Config, pipeline_outcome: PipelineOutcome): events, X, y, forward_returns = label_data(config.event_filter, config.labeling, X, returns, forward_returns) - inference_from: pd.Timestamp = X.index[len(X.index) - 2] + inference_from: pd.Timestamp = X.index[len(X.index) - 1] - # 2. Filter for significant events when we want to trade, and label data + # 2. Filter for significant events when we want to trade, and label data events, X, y, forward_returns = label_data(config.event_filter, config.labeling, X, returns, forward_returns) # 3. Train directional models @@ -51,7 +52,7 @@ def __inference(config: Config, pipeline_outcome: PipelineOutcome): bet_sizing_outcomes = [bet_sizing_with_meta_models(X, training_outcome.predictions, y, forward_returns, config.meta_models, config, 'meta', from_index = inference_from, transformations_over_time = preloaded_outcome.meta_transformations, preloaded_models = [b.model_over_time for b in preloaded_outcome.meta_training]) for training_outcome, preloaded_outcome in zip(directional_training_outcome.training, pipeline_outcome.bet_sizing)] # 4. Ensemble weights - ensemble_outcome = ensemble_weights([o.weights for o in bet_sizing_outcomes], forward_returns, y, config.no_of_classes) + ensemble_outcome = ensemble_weights([o.weights for o in bet_sizing_outcomes], forward_returns, y, config.no_of_classes, config.mode == 'training') # 5. (Optional) Additional bet sizing on top of the ensembled weights ensemble_bet_sizing_outcome = bet_sizing_with_meta_models(X, ensemble_outcome.weights, y, forward_returns, config.meta_models, config, 'ensemble', from_index = inference_from, transformations_over_time = pipeline_outcome.secondary_bet_sizing.meta_transformations, preloaded_models= [b.model_over_time for b in pipeline_outcome.secondary_bet_sizing.meta_training]) if len(config.meta_models) > 0 else None diff --git a/run_pipeline.py b/run_pipeline.py index 1aa947a..4d7f247 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -69,7 +69,7 @@ def __run_training(config: Config) -> PipelineOutcome: bet_sizing_outcomes = [bet_sizing_with_meta_models(X, outcome.predictions, y, forward_returns, config.meta_models, config, 'meta', None, None, None) for outcome in directional_training_outcome.training] # 4. Ensemble weights - ensemble_outcome = ensemble_weights([o.weights for o in bet_sizing_outcomes], forward_returns, y, config.no_of_classes) + ensemble_outcome = ensemble_weights([o.weights for o in bet_sizing_outcomes], forward_returns, y, config.no_of_classes, config.mode == 'training') # 5. (Optional) Additional bet sizing on top of the ensembled weights ensemble_bet_sizing_outcome = bet_sizing_with_meta_models(X, ensemble_outcome.weights, y, forward_returns, config.meta_models, config, 'ensemble', None, None, None) if len(config.meta_models) > 0 else None diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 67734ee..9c48439 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -100,7 +100,6 @@ def test_evaluation(): y_pred=processed_predictions_to_match_returns, y_true=y, no_of_classes='two', - print_results = False, discretize=True ) diff --git a/training/bet_sizing.py b/training/bet_sizing.py index d911ae2..ea389a3 100644 --- a/training/bet_sizing.py +++ b/training/bet_sizing.py @@ -62,27 +62,31 @@ def bet_sizing_with_meta_models( from_index = from_index, no_of_classes = 'two', level = 'meta', - print_results = False, + output_stats = config.mode == 'training', transformations_over_time = transformations_over_time, models_over_time = preloaded_models, ) # Ensemble predictions if necessary if len(models) > 1: - # meta_predictions = pd.concat([outcome.predictions for outcome in meta_outcomes]).mean(axis = 1) + meta_predictions = pd.concat([outcome.predictions for outcome in meta_outcomes], axis = 1).mean(axis = 1).apply(discretize_threeway_threshold(0.5)) bet_size = pd.concat([outcome.probabilities[outcome.probabilities.columns[1::2]] for outcome in meta_outcomes], axis = 1).mean(axis = 1) else: + meta_predictions = meta_outcomes[0].predictions bet_size = meta_outcomes[0].probabilities.iloc[:,1] - avg_predictions_with_sizing = input_predictions * bet_size + avg_predictions_with_sizing = input_predictions * meta_predictions * bet_size - stats = evaluate_predictions( - forward_returns = forward_returns, - y_pred = avg_predictions_with_sizing, - y_true = y, - no_of_classes = 'two', - print_results = True, - discretize=False - ) + if config.mode == 'training': + stats = evaluate_predictions( + forward_returns = forward_returns, + y_pred = avg_predictions_with_sizing, + y_true = y, + no_of_classes = 'three-balanced', + discretize=False + ) + print(stats) + else: + stats = None model_id = "model_" + config.target_asset[1] + "_" + model_suffix return BetSizingWithMetaOutcome(model_id, meta_outcomes, transformations_over_time, avg_predictions_with_sizing, stats) diff --git a/training/directional_training.py b/training/directional_training.py index 42df5bd..bfb44b8 100644 --- a/training/directional_training.py +++ b/training/directional_training.py @@ -1,6 +1,6 @@ import pandas as pd -from .types import DirectionalTrainingOutcome +from .types import DirectionalTrainingOutcome, TrainingOutcome from training.train_model import train_model from training.walk_forward import walk_forward_process_transformations @@ -42,7 +42,12 @@ def train_directional_models( else: transformations_over_time = preloaded_training_step.transformations - training_outcomes = [train_model( + def print_stats(outcome: TrainingOutcome) -> TrainingOutcome: + if config.mode == 'training': + print(outcome.stats) + return outcome + + training_outcomes = [print_stats(train_model( ticker_to_predict = config.target_asset[1], X = X, y = y, @@ -54,9 +59,9 @@ def train_directional_models( from_index = from_index, no_of_classes = config.no_of_classes, level = 'primary', - print_results= True, + output_stats= config.mode == 'training', transformations_over_time = transformations_over_time, model_over_time = preloaded_training_step.training[index].model_over_time if preloaded_training_step else None - ) for index, model in enumerate(models)] + )) for index, model in enumerate(models)] return DirectionalTrainingOutcome(training_outcomes, transformations_over_time) diff --git a/training/ensemble.py b/training/ensemble.py index 7eaada6..02d1c67 100644 --- a/training/ensemble.py +++ b/training/ensemble.py @@ -9,14 +9,18 @@ def ensemble_weights( forward_returns: ForwardReturnSeries, y: ySeries, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], + output_stats: bool ) -> EnsembleOutcome: weights = pd.concat(input_weights, axis=1).mean(axis=1) - stats = evaluate_predictions( - forward_returns = forward_returns, - y_pred = weights, - y_true = y, - no_of_classes = no_of_classes, - print_results = False, - discretize = True, - ) + if output_stats: + stats = evaluate_predictions( + forward_returns = forward_returns, + y_pred = weights, + y_true = y, + no_of_classes = no_of_classes, + discretize = True, + ) + print(stats) + else: + stats = None return EnsembleOutcome(weights, stats) diff --git a/training/train_model.py b/training/train_model.py index e74b7ee..c900cbd 100644 --- a/training/train_model.py +++ b/training/train_model.py @@ -17,11 +17,11 @@ def train_models( from_index: Optional[pd.Timestamp], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], level: str, - print_results: bool, + output_stats: bool, transformations_over_time: TransformationsOverTime, models_over_time: Optional[list[ModelOverTime]] ) -> list[TrainingOutcome]: - return [train_model(ticker_to_predict, X, y, forward_returns, model, expanding_window, sliding_window_size, retrain_every, from_index, no_of_classes, level, print_results, transformations_over_time, models_over_time[index] if models_over_time else None) for index, model in enumerate(models)] + return [train_model(ticker_to_predict, X, y, forward_returns, model, expanding_window, sliding_window_size, retrain_every, from_index, no_of_classes, level, output_stats, transformations_over_time, models_over_time[index] if models_over_time else None) for index, model in enumerate(models)] def train_model( @@ -36,7 +36,7 @@ def train_model( from_index: Optional[pd.Timestamp], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], level: str, - print_results: bool, + output_stats: bool, transformations_over_time: TransformationsOverTime, model_over_time: Optional[ModelOverTime] ) -> TrainingOutcome: @@ -73,13 +73,15 @@ def train_model( ) assert len(predictions) == len(y) - stats = evaluate_predictions( - forward_returns = forward_returns, - y_pred = predictions, - y_true = y, - no_of_classes=no_of_classes, - print_results = print_results, - discretize=True - ) + if output_stats: + stats = evaluate_predictions( + forward_returns = forward_returns, + y_pred = predictions, + y_true = y, + no_of_classes=no_of_classes, + discretize=True + ) + else: + stats = None return TrainingOutcome(model_id, predictions, probabilities, stats, model_over_time) \ No newline at end of file diff --git a/training/types.py b/training/types.py index cd1f791..2e44970 100644 --- a/training/types.py +++ b/training/types.py @@ -16,13 +16,13 @@ class TrainingOutcome: model_id: str predictions: PredictionsSeries probabilities: ProbabilitiesDataFrame - stats: Stats + stats: Optional[Stats] model_over_time: ModelOverTime @dataclass class EnsembleOutcome: weights: WeightsSeries - stats: Stats + stats: Optional[Stats] @dataclass class BetSizingWithMetaOutcome: @@ -30,7 +30,7 @@ class BetSizingWithMetaOutcome: meta_training: list[TrainingOutcome] meta_transformations: TransformationsOverTime weights: WeightsSeries - stats: Stats + stats: Optional[Stats] @dataclass class DirectionalTrainingOutcome: @@ -47,5 +47,5 @@ class PipelineOutcome: def get_output_weights(self) -> WeightsSeries: return self.secondary_bet_sizing.weights if self.secondary_bet_sizing else self.ensemble.weights - def get_output_stats(self) -> Stats: + def get_output_stats(self) -> Optional[Stats]: return self.secondary_bet_sizing.stats if self.secondary_bet_sizing else self.ensemble.stats \ No newline at end of file diff --git a/utils/evaluate.py b/utils/evaluate.py index fdc3e56..665fd99 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -33,7 +33,6 @@ def evaluate_predictions( y_pred: WeightsSeries, y_true: ySeries, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], - print_results: bool, discretize: bool = False, ) -> Stats: # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size) @@ -78,8 +77,6 @@ def evaluate_predictions( scorecard['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred) scorecard = {k: round(float(v), 3) for k, v in scorecard.items()} - if print_results: - print(scorecard) return scorecard