diff --git a/config/preprocess.py b/config/preprocess.py index 75bd355..96f4bd0 100644 --- a/config/preprocess.py +++ b/config/preprocess.py @@ -1,4 +1,3 @@ -from sklearn.model_selection import TimeSeriesSplit from .types import Config, RawConfig from utils.helpers import flatten from feature_extractors.feature_extractor_presets import ( @@ -114,9 +113,12 @@ def __preprocess_data_collections_config(data_dict: dict) -> dict: return data_dict -def __preprocess_event_filter_config(data_dict: dict) -> dict: - data_dict["event_filter"] = eventfilters_map[data_dict["event_filter"]] - return data_dict +def __preprocess_event_filter_config(config_dict: dict) -> dict: + config_dict["event_filter"] = eventfilters_map[config_dict["event_filter"]]( + config_dict["event_filter_multiplier"] + ) + config_dict.pop("event_filter_multiplier") + return config_dict def __preprocess_event_labeller_config(config_dict: dict) -> dict: diff --git a/config/presets.py b/config/presets.py index 608aad8..c1a9c4d 100644 --- a/config/presets.py +++ b/config/presets.py @@ -1,4 +1,4 @@ -from .types import RawConfig, Config +from .types import RawConfig def get_default_config() -> RawConfig: @@ -22,14 +22,15 @@ def get_default_config() -> RawConfig: assets=["fivemin_crypto"], target_asset="BTCUSDT", other_assets=[], - exogenous_data=["daily_glassnode"], + exogenous_data=[], load_non_target_asset=True, own_features=["level_2"], other_features=["z_score"], - exogenous_features=["z_score"], + exogenous_features=[], directional_models=classification_models, meta_models=meta_models, event_filter="cusum_vol", + event_filter_multiplier=3.5, remove_overlapping_events=False, labeling="two_class", forecasting_horizon=10, diff --git a/config/sweep_ensemble.yaml b/config/sweep_ensemble.yaml deleted file mode 100644 index eca8127..0000000 --- a/config/sweep_ensemble.yaml +++ /dev/null @@ -1,44 +0,0 @@ -program: run_sweep.py -method: grid -project: price-forecasting -name: Meta labelling -metric: - goal: maximize - name: sharpe -parameters: - assets: - value: ['daily_crypto'] - other_assets: - value: ['daily_etf'] - exogenous_data: - value: ['daily_glassnode'] - initial_window_size: - value: 380 - distribution: categorical - n_features_to_select: - values: [40, 50, 60] - distribution: categorical - dimensionality_reduction_ratio: - value: 0.5 - retrain_every: - value: 20 - scaler: - value: 'minmax' - no_of_classes: - value: 'two' - load_non_target_asset: - value: True - directional_models: - distribution: categorical - values: - - ["LDA", "LogisticRegression_two_class", "KNN", "SVC", "CART", "NB", "AB", "RFC", "XGB_two_class", "LGBM", "StaticMom"] - - ["LogisticRegression_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"] - - ["LogisticRegression_two_class", "LDA", "LGBM", "RFC", "XGB_two_class"] - meta_models: - value: ["LGBM", "LogisticRegression_two_class"] - own_features: - value: ['date_days', 'level_2', 'lags_up_to_5'] - other_features: - value: ['level_2', 'lags_up_to_5'] - exogenous_features: - value: ['z_score'] diff --git a/config/types.py b/config/types.py index 8338c29..e52d7e3 100644 --- a/config/types.py +++ b/config/types.py @@ -26,6 +26,7 @@ class RawConfig(BaseModel): other_features: list[str] exogenous_features: list[str] event_filter: Literal["none", "cusum_vol", "cusum_fixed"] + event_filter_multiplier: float remove_overlapping_events: bool labeling: Literal["two_class", "three_class_balanced", "three_class_imbalanced"] forecasting_horizon: int diff --git a/environment.yml b/environment.yml index 51a36ec..43de08e 100644 --- a/environment.yml +++ b/environment.yml @@ -25,6 +25,7 @@ dependencies: - pytest - wandb - pandas-ta + - optuna - pip - pip: - black diff --git a/labeling/event_filters/cusum.py b/labeling/event_filters/cusum.py index 59e3f69..eb6a000 100644 --- a/labeling/event_filters/cusum.py +++ b/labeling/event_filters/cusum.py @@ -7,7 +7,7 @@ from numba.typed import List class CUSUMVolatilityEventFilter(EventFilter): - def __init__(self, vol_period: int, multiplier: float): + def __init__(self, multiplier: float, vol_period=100): self.vol_period = vol_period self.multiplier = multiplier @@ -23,13 +23,14 @@ class CUSUMVolatilityEventFilter(EventFilter): class CUSUMFixedEventFilter(EventFilter): - def __init__(self, threshold: float): - self.threshold = threshold + def __init__(self, threshold_multiplier: float): + self.threshold_multiplier = threshold_multiplier def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex: diffed_returns = returns.diff() - int_indicies = _process_fixed( - List(diffed_returns.to_list()), abs(returns.mean()) * self.threshold + int_indicies = _process( + List(diffed_returns.to_list()), + abs(returns.mean()) * self.threshold_multiplier, ) return pd.DatetimeIndex([returns.index[i] for i in int_indicies]) diff --git a/labeling/event_filters/nofilter.py b/labeling/event_filters/nofilter.py index 4ac7afd..619fcbd 100644 --- a/labeling/event_filters/nofilter.py +++ b/labeling/event_filters/nofilter.py @@ -4,5 +4,8 @@ import pandas as pd class NoEventFilter(EventFilter): + def __init__(self, ignored_threshold_multiplier: float): + pass + def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex: return returns.index diff --git a/labeling/eventfilters_map.py b/labeling/eventfilters_map.py index 0004fca..461d9df 100644 --- a/labeling/eventfilters_map.py +++ b/labeling/eventfilters_map.py @@ -2,7 +2,7 @@ from .event_filters.nofilter import NoEventFilter from .event_filters.cusum import CUSUMVolatilityEventFilter, CUSUMFixedEventFilter eventfilters_map = dict( - none=NoEventFilter(), - cusum_vol=CUSUMVolatilityEventFilter(vol_period=100, multiplier=3.5), - cusum_fixed=CUSUMFixedEventFilter(threshold=20), + none=NoEventFilter, + cusum_vol=CUSUMVolatilityEventFilter, + cusum_fixed=CUSUMFixedEventFilter, ) diff --git a/labeling/labellers/fixed_time_three_class_balanced.py b/labeling/labellers/fixed_time_three_class_balanced.py index 07ae563..b9af057 100644 --- a/labeling/labellers/fixed_time_three_class_balanced.py +++ b/labeling/labellers/fixed_time_three_class_balanced.py @@ -1,7 +1,7 @@ from data_loader.types import ReturnSeries from ..types import EventLabeller, EventsDataFrame import pandas as pd -from .utils import create_forward_returns +from .utils import create_forward_returns, discretize_threeway_threshold from typing import Callable @@ -58,4 +58,4 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller): return [-1, 0, 1] def get_discretize_function(self) -> Callable: - raise NotImplementedError + return discretize_threeway_threshold(0.02) diff --git a/labeling/labellers/fixed_time_three_class_imbalanced.py b/labeling/labellers/fixed_time_three_class_imbalanced.py index d9aa107..42a203c 100644 --- a/labeling/labellers/fixed_time_three_class_imbalanced.py +++ b/labeling/labellers/fixed_time_three_class_imbalanced.py @@ -57,4 +57,4 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller): return [-1, 0, 1] def get_discretize_function(self) -> Callable: - raise NotImplementedError + return discretize_threeway_threshold(0.02) diff --git a/reporting/reporting.py b/reporting/reporting.py index 00bbb83..0e63db7 100644 --- a/reporting/reporting.py +++ b/reporting/reporting.py @@ -11,7 +11,6 @@ def report_results( output_weights: WeightsSeries, config: Config, wandb, - sweep: bool, ): # Only send the results of the final model to wandb @@ -44,7 +43,3 @@ def report_results( "Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", output_stats["prob_sharpe"], ) - - if sweep: - if wandb.run is not None: - wandb.finish() diff --git a/reporting/wandb.py b/reporting/wandb.py index 8465461..99bb686 100644 --- a/reporting/wandb.py +++ b/reporting/wandb.py @@ -1,13 +1,9 @@ -import pandas as pd from config.types import RawConfig from typing import Optional -from utils.helpers import weighted_average from training.types import Stats -def launch_wandb( - project_name: str, default_config: RawConfig, sweep: bool = False -) -> Optional[object]: +def launch_wandb(project_name: str, default_config: RawConfig) -> Optional[object]: from wandb_setup import get_wandb wandb = get_wandb() @@ -15,10 +11,6 @@ def launch_wandb( raise Exception( "Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)" ) - - elif sweep: - wandb.init(project=project_name, config=vars(default_config)) - return wandb else: wandb.init(project=project_name, config=vars(default_config), reinit=True) return wandb diff --git a/run_evaluate_robustness.py b/run_evaluate_robustness.py index cbf6d88..6ebe79f 100644 --- a/run_evaluate_robustness.py +++ b/run_evaluate_robustness.py @@ -8,7 +8,6 @@ for index in range(6): _, _, _, _, results_1, predictions_1, _ = run_pipeline( project_name="price-prediction", with_wandb=False, - sweep=False, config=get_default_ensemble_config(), ) all_results.append(results_1) diff --git a/run_hpo.py b/run_hpo.py new file mode 100644 index 0000000..7ab8a91 --- /dev/null +++ b/run_hpo.py @@ -0,0 +1,39 @@ +from config.types import RawConfig +from run_pipeline import run_pipeline +from config.presets import get_default_config +import optuna +from utils.helpers import powerset + +config = get_default_config() +config.save_models = False + + +def train(trial): + search_space = { + "n_features_to_select": trial.suggest_int("n_features_to_select", 10, 200), + "forecasting_horizon": trial.suggest_int("forecasting_horizon", 3, 300), + "own_features": trial.suggest_categorical( + "own_features", powerset(["z_score", "level_2", "level_1"]) + ), + "other_features": trial.suggest_categorical( + "other_features", powerset(["z_score", "level_2", "level_1"]) + ), + "labeling": trial.suggest_categorical( + "labeling", ["two_class", "three_class_balanced", "three_class_imbalanced"] + ), + "scaler": trial.suggest_categorical( + "scaler", + ["normalize", "minmax", "standardize", "robust", "box-cox", "quantile"], + ), + } + trial_config = RawConfig(**vars(config) | search_space) + outcome, _ = run_pipeline( + project_name="test", with_wandb=False, raw_config=trial_config + ) + return 1 - outcome.get_output_stats()["sharpe"] + + +study = optuna.create_study() +study.optimize(train, n_trials=40) +print(study.best_params) +study.trials_dataframe().to_csv("output/trials.csv") diff --git a/run_inference.py b/run_inference.py index 09fd164..c683cfe 100644 --- a/run_inference.py +++ b/run_inference.py @@ -20,7 +20,6 @@ def run_inference(preload_models: bool, fallback_raw_config: RawConfig): pipeline_outcome, config = run_pipeline( project_name="price-prediction", with_wandb=False, - sweep=False, raw_config=fallback_raw_config, ) diff --git a/run_model_dev.py b/run_model_dev.py index ea73ad1..b963bc8 100644 --- a/run_model_dev.py +++ b/run_model_dev.py @@ -5,6 +5,5 @@ from config.presets import get_dev_config run_pipeline( project_name="price-prediction", with_wandb=False, - sweep=False, raw_config=get_dev_config(), ) diff --git a/run_multi_asset_pipeline.py b/run_multi_asset_pipeline.py index 41c27ca..6c7488f 100644 --- a/run_multi_asset_pipeline.py +++ b/run_multi_asset_pipeline.py @@ -6,13 +6,13 @@ from config.presets import get_default_config def run_multi_asset_pipeline( - project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig + project_name: str, with_wandb: bool, raw_config: RawConfig ): collection = data_collections["fivemin_crypto"] for asset in collection: - print(f"# Predicting asset: {asset[1]}\n") - raw_config.target_asset = asset[1] - wandb, config = setup_config(project_name, with_wandb, sweep, raw_config) + print(f"# Predicting asset: {asset.file_name}\n") + raw_config.target_asset = asset.file_name + wandb, config = setup_config(project_name, with_wandb, raw_config) pipeline_outcome = run_training(config) report_results( pipeline_outcome.directional_training.training.stats, @@ -20,7 +20,6 @@ def run_multi_asset_pipeline( pipeline_outcome.get_output_weights(), config, wandb, - sweep, ) @@ -28,6 +27,5 @@ if __name__ == "__main__": run_multi_asset_pipeline( project_name="price-prediction", with_wandb=False, - sweep=False, raw_config=get_default_config(), ) diff --git a/run_pipeline.py b/run_pipeline.py index b7db9c2..fa997e2 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -19,31 +19,28 @@ from training.types import PipelineOutcome def run_pipeline( - project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig + project_name: str, with_wandb: bool, raw_config: RawConfig ) -> tuple[PipelineOutcome, Config]: - wandb, config = setup_config(project_name, with_wandb, sweep, raw_config) - pipeline_outcome = run_training(config) + wandb, config = setup_config(project_name, with_wandb, raw_config) + outcome = run_training(config) report_results( - pipeline_outcome.directional_training.stats, - pipeline_outcome.get_output_stats(), - pipeline_outcome.get_output_weights(), + outcome.directional_training.stats, + outcome.get_output_stats(), + outcome.get_output_weights(), config, wandb, - sweep, ) if config.save_models: - save_models(pipeline_outcome, config) - return pipeline_outcome, config + save_models(outcome, config) + return outcome, config def setup_config( - project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig + project_name: str, with_wandb: bool, raw_config: RawConfig ) -> tuple[Optional[object], Config]: wandb = None if with_wandb: - wandb = launch_wandb( - project_name=project_name, default_config=raw_config, sweep=sweep - ) + wandb = launch_wandb(project_name=project_name, default_config=raw_config) raw_config = override_config_with_wandb_values(wandb, raw_config) config = preprocess_config(raw_config) @@ -108,6 +105,5 @@ if __name__ == "__main__": run_pipeline( project_name="price-prediction", with_wandb=False, - sweep=False, raw_config=get_default_config(), ) diff --git a/run_sweep.py b/run_sweep.py deleted file mode 100644 index 7cd1b56..0000000 --- a/run_sweep.py +++ /dev/null @@ -1,9 +0,0 @@ -from run_pipeline import run_pipeline -from config.presets import get_default_ensemble_config - -run_pipeline( - project_name="price-prediction", - with_wandb=True, - sweep=True, - raw_config=get_default_ensemble_config(), -) diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 16df438..d0fd9b0 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -84,6 +84,7 @@ def test_evaluation(): expanding_window=False, window_size=window_length, retrain_every=retrain_every, + class_labels=[0, 1], from_index=None, ) diff --git a/tests/test_walk_forward.py b/tests/test_walk_forward.py index b7baf32..b64d01d 100644 --- a/tests/test_walk_forward.py +++ b/tests/test_walk_forward.py @@ -80,6 +80,7 @@ def test_walk_forward_train_test(): expanding_window=False, window_size=window_length, retrain_every=retrain_every, + class_labels=[0, 1], from_index=None, ) diff --git a/training/bet_sizing.py b/training/bet_sizing.py index c36df2c..dfcb35e 100644 --- a/training/bet_sizing.py +++ b/training/bet_sizing.py @@ -66,6 +66,7 @@ def bet_sizing_with_meta_model( retrain_every=config.retrain_every, from_index=from_index, level="meta", + class_labels=[0, 1], transformations_over_time=transformations_over_time, model_over_time=preloaded_models, ) diff --git a/training/directional_training.py b/training/directional_training.py index 612a22c..59ebb72 100644 --- a/training/directional_training.py +++ b/training/directional_training.py @@ -48,6 +48,7 @@ def train_directional_model( retrain_every=config.retrain_every, from_index=from_index, level="primary", + class_labels=config.labeling.get_labels(), transformations_over_time=transformations_over_time, model_over_time=preloaded_training_step.model_over_time if preloaded_training_step diff --git a/training/train_model.py b/training/train_model.py index 0dc9669..ee72dde 100644 --- a/training/train_model.py +++ b/training/train_model.py @@ -23,6 +23,7 @@ def train_model( retrain_every: int, from_index: Optional[pd.Timestamp], level: str, + class_labels: list[int], transformations_over_time: TransformationsOverTime, model_over_time: Optional[ModelOverTime], ) -> BaseTrainingOutcome: @@ -61,6 +62,7 @@ def train_model( expanding_window=True, window_size=initial_window_size, retrain_every=retrain_every, + class_labels=class_labels, from_index=from_index, ) diff --git a/training/walk_forward/inference.py b/training/walk_forward/inference.py index 3af1883..189c967 100644 --- a/training/walk_forward/inference.py +++ b/training/walk_forward/inference.py @@ -20,10 +20,13 @@ def walk_forward_inference( expanding_window: bool, window_size: int, retrain_every: int, + class_labels: list[int], from_index: Optional[pd.Timestamp], ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: predictions = pd.Series(index=X.index, dtype="object").rename(model_name) - probabilities = pd.DataFrame(index=X.index) + probabilities = pd.DataFrame( + index=X.index, columns=[str(label) for label in class_labels] + ) inference_from = ( max( diff --git a/training/walk_forward/inference_batched.py b/training/walk_forward/inference_batched.py index 9631809..3c9e46c 100644 --- a/training/walk_forward/inference_batched.py +++ b/training/walk_forward/inference_batched.py @@ -20,10 +20,13 @@ def walk_forward_inference_batched( expanding_window: bool, window_size: int, retrain_every: int, + class_labels: list[int], from_index: Optional[pd.Timestamp], ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: predictions = pd.Series(index=X.index, dtype="object").rename(model_name) - probabilities = pd.DataFrame(index=X.index, columns=["0", "1"]) + probabilities = pd.DataFrame( + index=X.index, columns=[str(label) for label in class_labels] + ) inference_from = ( max( diff --git a/training/walk_forward/inference_parallel.py b/training/walk_forward/inference_parallel.py index eeaa0dd..b6cb264 100644 --- a/training/walk_forward/inference_parallel.py +++ b/training/walk_forward/inference_parallel.py @@ -22,10 +22,13 @@ def walk_forward_inference( expanding_window: bool, window_size: int, retrain_every: int, + class_labels: list[int], from_index: Optional[pd.Timestamp], ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: predictions = pd.Series(index=X.index, dtype="object").rename(model_name) - probabilities = pd.DataFrame(index=X.index) + probabilities = pd.DataFrame( + index=X.index, columns=[str(label) for label in class_labels] + ) inference_from = ( max( diff --git a/transformations/pca.py b/transformations/pca.py index 5a56882..b5601a0 100644 --- a/transformations/pca.py +++ b/transformations/pca.py @@ -19,7 +19,8 @@ class PCATransformation(Transformation): n_components=min( int(len(X.columns) * self.ratio_components_to_keep), self.initial_window_size, - ) + ), + # whiten=True, ) self.pca.fit(X, y) diff --git a/transformations/retrieve.py b/transformations/retrieve.py index 7dd7ea1..2156276 100644 --- a/transformations/retrieve.py +++ b/transformations/retrieve.py @@ -3,9 +3,18 @@ from .pca import PCATransformation from .sklearn import SKLearnTransformation from typing import Literal, Optional from models.model_map import default_feature_selector_classification -from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler, RobustScaler +from sklearn.preprocessing import ( + MinMaxScaler, + Normalizer, + StandardScaler, + RobustScaler, + PowerTransformer, + QuantileTransformer, +) -ScalerTypes = Literal["normalize", "minmax", "standardize", "robust"] +ScalerTypes = Literal[ + "normalize", "minmax", "standardize", "robust", "box-cox", "quantile" +] def get_rfe(n_feature_to_select: int) -> Optional[RFETransformation]: @@ -42,5 +51,13 @@ def get_scaler(type: ScalerTypes) -> SKLearnTransformation: return SKLearnTransformation( RobustScaler(with_centering=False, quantile_range=(0.10, 0.90)) ) + elif type == "box-cox": + return SKLearnTransformation( + PowerTransformer(method="box-cox", standardize=True) + ) + elif type == "quantile": + return SKLearnTransformation( + QuantileTransformer(n_quantiles=100, output_distribution="normal") + ) else: raise Exception("Scaler type not supported") diff --git a/utils/helpers.py b/utils/helpers.py index d17a159..26b6b56 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -3,7 +3,7 @@ import numpy as np import os import string import random -from itertools import dropwhile +from itertools import chain, combinations, dropwhile def get_files_from_dir(path: str) -> list[str]: @@ -78,3 +78,8 @@ def drop_until_first_valid_index( get_first_valid_return_index(series), ) return df.iloc[first_valid_index:], series.iloc[first_valid_index:] + + +def powerset(input: list) -> list[list]: + p_set = list(chain.from_iterable(combinations(input, r) for r in range(len(input) + 1))) # type: ignore + return [list(item) for item in p_set if len(item) > 0]