feat(HPO): added run_hpo script (#237)

* feat(HPO): added `run_hpo` script

* fix(Linter): ran

* feat(HPO): removed any reference to sweep (superseeded by optuna)

* fix(HPO): optimize for sharpe

* fix(Config): removed glassnode data, save trials from hpo

* feat(Labelling): added three-balanced method works again

* fix(BetSizing): set the correct class labels

* fix(HPO): powerset should return what's expected, added two new normalization methods

* fix(Linter): ran

* fix(DataLoader): sort the dataframe when fetching data

* fix(Config): only take z-score of other assets

config/preprocess.py

@@ -1,4 +1,3 @@
-from sklearn.model_selection import TimeSeriesSplit
 from .types import Config, RawConfig
 from utils.helpers import flatten
 from feature_extractors.feature_extractor_presets import (
@@ -114,9 +113,12 @@ def __preprocess_data_collections_config(data_dict: dict) -> dict:
     return data_dict
 
 
-def __preprocess_event_filter_config(data_dict: dict) -> dict:
-    data_dict["event_filter"] = eventfilters_map[data_dict["event_filter"]]
-    return data_dict
+def __preprocess_event_filter_config(config_dict: dict) -> dict:
+    config_dict["event_filter"] = eventfilters_map[config_dict["event_filter"]](
+        config_dict["event_filter_multiplier"]
+    )
+    config_dict.pop("event_filter_multiplier")
+    return config_dict
 
 
 def __preprocess_event_labeller_config(config_dict: dict) -> dict:

config/presets.py

@@ -1,4 +1,4 @@
-from .types import RawConfig, Config
+from .types import RawConfig
 
 
 def get_default_config() -> RawConfig:
@@ -22,14 +22,15 @@ def get_default_config() -> RawConfig:
         assets=["fivemin_crypto"],
         target_asset="BTCUSDT",
         other_assets=[],
-        exogenous_data=["daily_glassnode"],
+        exogenous_data=[],
         load_non_target_asset=True,
         own_features=["level_2"],
         other_features=["z_score"],
-        exogenous_features=["z_score"],
+        exogenous_features=[],
         directional_models=classification_models,
         meta_models=meta_models,
         event_filter="cusum_vol",
+        event_filter_multiplier=3.5,
         remove_overlapping_events=False,
         labeling="two_class",
         forecasting_horizon=10,

config/sweep_ensemble.yaml

@@ -1,44 +0,0 @@
-program: run_sweep.py
-method: grid
-project: price-forecasting
-name: Meta labelling
-metric:
-  goal: maximize
-  name: sharpe
-parameters:
-    assets:
-      value: ['daily_crypto']
-    other_assets:
-      value: ['daily_etf']
-    exogenous_data:
-      value: ['daily_glassnode']
-    initial_window_size: 
-      value: 380
-      distribution: categorical
-    n_features_to_select:
-      values: [40, 50, 60]
-      distribution: categorical
-    dimensionality_reduction_ratio:
-      value: 0.5
-    retrain_every: 
-      value: 20
-    scaler:
-      value: 'minmax'
-    no_of_classes:
-      value: 'two'
-    load_non_target_asset: 
-      value: True
-    directional_models: 
-      distribution: categorical
-      values:
-      - ["LDA", "LogisticRegression_two_class", "KNN", "SVC", "CART", "NB", "AB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
-      - ["LogisticRegression_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
-      - ["LogisticRegression_two_class", "LDA", "LGBM", "RFC", "XGB_two_class"]
-    meta_models: 
-      value: ["LGBM", "LogisticRegression_two_class"]
-    own_features:
-      value: ['date_days', 'level_2', 'lags_up_to_5']
-    other_features:
-      value: ['level_2', 'lags_up_to_5']
-    exogenous_features:
-      value: ['z_score']

config/types.py

@@ -26,6 +26,7 @@ class RawConfig(BaseModel):
     other_features: list[str]
     exogenous_features: list[str]
     event_filter: Literal["none", "cusum_vol", "cusum_fixed"]
+    event_filter_multiplier: float
     remove_overlapping_events: bool
     labeling: Literal["two_class", "three_class_balanced", "three_class_imbalanced"]
     forecasting_horizon: int

environment.yml

@@ -25,6 +25,7 @@ dependencies:
   - pytest
   - wandb
   - pandas-ta
+  - optuna
   - pip
   - pip:
     - black

labeling/event_filters/cusum.py

@@ -7,7 +7,7 @@ from numba.typed import List
 
 
 class CUSUMVolatilityEventFilter(EventFilter):
-    def __init__(self, vol_period: int, multiplier: float):
+    def __init__(self, multiplier: float, vol_period=100):
         self.vol_period = vol_period
         self.multiplier = multiplier
 
@@ -23,13 +23,14 @@ class CUSUMVolatilityEventFilter(EventFilter):
 
 
 class CUSUMFixedEventFilter(EventFilter):
-    def __init__(self, threshold: float):
-        self.threshold = threshold
+    def __init__(self, threshold_multiplier: float):
+        self.threshold_multiplier = threshold_multiplier
 
     def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex:
         diffed_returns = returns.diff()
-        int_indicies = _process_fixed(
-            List(diffed_returns.to_list()), abs(returns.mean()) * self.threshold
+        int_indicies = _process(
+            List(diffed_returns.to_list()),
+            abs(returns.mean()) * self.threshold_multiplier,
         )
 
         return pd.DatetimeIndex([returns.index[i] for i in int_indicies])

labeling/event_filters/nofilter.py

@@ -4,5 +4,8 @@ import pandas as pd
 
 
 class NoEventFilter(EventFilter):
+    def __init__(self, ignored_threshold_multiplier: float):
+        pass
+
     def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex:
         return returns.index

labeling/eventfilters_map.py

@@ -2,7 +2,7 @@ from .event_filters.nofilter import NoEventFilter
 from .event_filters.cusum import CUSUMVolatilityEventFilter, CUSUMFixedEventFilter
 
 eventfilters_map = dict(
-    none=NoEventFilter(),
-    cusum_vol=CUSUMVolatilityEventFilter(vol_period=100, multiplier=3.5),
-    cusum_fixed=CUSUMFixedEventFilter(threshold=20),
+    none=NoEventFilter,
+    cusum_vol=CUSUMVolatilityEventFilter,
+    cusum_fixed=CUSUMFixedEventFilter,
 )

labeling/labellers/fixed_time_three_class_balanced.py

@@ -1,7 +1,7 @@
 from data_loader.types import ReturnSeries
 from ..types import EventLabeller, EventsDataFrame
 import pandas as pd
-from .utils import create_forward_returns
+from .utils import create_forward_returns, discretize_threeway_threshold
 from typing import Callable
 
 
@@ -58,4 +58,4 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
         return [-1, 0, 1]
 
     def get_discretize_function(self) -> Callable:
-        raise NotImplementedError
+        return discretize_threeway_threshold(0.02)

labeling/labellers/fixed_time_three_class_imbalanced.py

@@ -57,4 +57,4 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
         return [-1, 0, 1]
 
     def get_discretize_function(self) -> Callable:
-        raise NotImplementedError
+        return discretize_threeway_threshold(0.02)

reporting/reporting.py

@@ -11,7 +11,6 @@ def report_results(
     output_weights: WeightsSeries,
     config: Config,
     wandb,
-    sweep: bool,
 ):
 
     # Only send the results of the final model to wandb
@@ -44,7 +43,3 @@ def report_results(
         "Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ",
         output_stats["prob_sharpe"],
     )
-
-    if sweep:
-        if wandb.run is not None:
-            wandb.finish()

reporting/wandb.py

@@ -1,13 +1,9 @@
-import pandas as pd
 from config.types import RawConfig
 from typing import Optional
-from utils.helpers import weighted_average
 from training.types import Stats
 
 
-def launch_wandb(
-    project_name: str, default_config: RawConfig, sweep: bool = False
-) -> Optional[object]:
+def launch_wandb(project_name: str, default_config: RawConfig) -> Optional[object]:
     from wandb_setup import get_wandb
 
     wandb = get_wandb()
@@ -15,10 +11,6 @@ def launch_wandb(
         raise Exception(
             "Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)"
         )
-
-    elif sweep:
-        wandb.init(project=project_name, config=vars(default_config))
-        return wandb
     else:
         wandb.init(project=project_name, config=vars(default_config), reinit=True)
         return wandb

run_evaluate_robustness.py

@@ -8,7 +8,6 @@ for index in range(6):
     _, _, _, _, results_1, predictions_1, _ = run_pipeline(
         project_name="price-prediction",
         with_wandb=False,
-        sweep=False,
         config=get_default_ensemble_config(),
     )
     all_results.append(results_1)

run_hpo.py

@@ -0,0 +1,39 @@
+from config.types import RawConfig
+from run_pipeline import run_pipeline
+from config.presets import get_default_config
+import optuna
+from utils.helpers import powerset
+
+config = get_default_config()
+config.save_models = False
+
+
+def train(trial):
+    search_space = {
+        "n_features_to_select": trial.suggest_int("n_features_to_select", 10, 200),
+        "forecasting_horizon": trial.suggest_int("forecasting_horizon", 3, 300),
+        "own_features": trial.suggest_categorical(
+            "own_features", powerset(["z_score", "level_2", "level_1"])
+        ),
+        "other_features": trial.suggest_categorical(
+            "other_features", powerset(["z_score", "level_2", "level_1"])
+        ),
+        "labeling": trial.suggest_categorical(
+            "labeling", ["two_class", "three_class_balanced", "three_class_imbalanced"]
+        ),
+        "scaler": trial.suggest_categorical(
+            "scaler",
+            ["normalize", "minmax", "standardize", "robust", "box-cox", "quantile"],
+        ),
+    }
+    trial_config = RawConfig(**vars(config) | search_space)
+    outcome, _ = run_pipeline(
+        project_name="test", with_wandb=False, raw_config=trial_config
+    )
+    return 1 - outcome.get_output_stats()["sharpe"]
+
+
+study = optuna.create_study()
+study.optimize(train, n_trials=40)
+print(study.best_params)
+study.trials_dataframe().to_csv("output/trials.csv")

run_inference.py

@@ -20,7 +20,6 @@ def run_inference(preload_models: bool, fallback_raw_config: RawConfig):
         pipeline_outcome, config = run_pipeline(
             project_name="price-prediction",
             with_wandb=False,
-            sweep=False,
             raw_config=fallback_raw_config,
         )
 

run_model_dev.py

@@ -5,6 +5,5 @@ from config.presets import get_dev_config
 run_pipeline(
     project_name="price-prediction",
     with_wandb=False,
-    sweep=False,
     raw_config=get_dev_config(),
 )

run_multi_asset_pipeline.py

@@ -6,13 +6,13 @@ from config.presets import get_default_config
 
 
 def run_multi_asset_pipeline(
-    project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig
+    project_name: str, with_wandb: bool, raw_config: RawConfig
 ):
     collection = data_collections["fivemin_crypto"]
     for asset in collection:
-        print(f"# Predicting asset: {asset[1]}\n")
-        raw_config.target_asset = asset[1]
-        wandb, config = setup_config(project_name, with_wandb, sweep, raw_config)
+        print(f"# Predicting asset: {asset.file_name}\n")
+        raw_config.target_asset = asset.file_name
+        wandb, config = setup_config(project_name, with_wandb, raw_config)
         pipeline_outcome = run_training(config)
         report_results(
             pipeline_outcome.directional_training.training.stats,
@@ -20,7 +20,6 @@ def run_multi_asset_pipeline(
             pipeline_outcome.get_output_weights(),
             config,
             wandb,
-            sweep,
         )
 
 
@@ -28,6 +27,5 @@ if __name__ == "__main__":
     run_multi_asset_pipeline(
         project_name="price-prediction",
         with_wandb=False,
-        sweep=False,
         raw_config=get_default_config(),
     )

run_pipeline.py

@@ -19,31 +19,28 @@ from training.types import PipelineOutcome
 
 
 def run_pipeline(
-    project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig
+    project_name: str, with_wandb: bool, raw_config: RawConfig
 ) -> tuple[PipelineOutcome, Config]:
-    wandb, config = setup_config(project_name, with_wandb, sweep, raw_config)
-    pipeline_outcome = run_training(config)
+    wandb, config = setup_config(project_name, with_wandb, raw_config)
+    outcome = run_training(config)
     report_results(
-        pipeline_outcome.directional_training.stats,
-        pipeline_outcome.get_output_stats(),
-        pipeline_outcome.get_output_weights(),
+        outcome.directional_training.stats,
+        outcome.get_output_stats(),
+        outcome.get_output_weights(),
         config,
         wandb,
-        sweep,
     )
     if config.save_models:
-        save_models(pipeline_outcome, config)
-    return pipeline_outcome, config
+        save_models(outcome, config)
+    return outcome, config
 
 
 def setup_config(
-    project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig
+    project_name: str, with_wandb: bool, raw_config: RawConfig
 ) -> tuple[Optional[object], Config]:
     wandb = None
     if with_wandb:
-        wandb = launch_wandb(
-            project_name=project_name, default_config=raw_config, sweep=sweep
-        )
+        wandb = launch_wandb(project_name=project_name, default_config=raw_config)
         raw_config = override_config_with_wandb_values(wandb, raw_config)
     config = preprocess_config(raw_config)
 
@@ -108,6 +105,5 @@ if __name__ == "__main__":
     run_pipeline(
         project_name="price-prediction",
         with_wandb=False,
-        sweep=False,
         raw_config=get_default_config(),
     )

run_sweep.py

@@ -1,9 +0,0 @@
-from run_pipeline import run_pipeline
-from config.presets import get_default_ensemble_config
-
-run_pipeline(
-    project_name="price-prediction",
-    with_wandb=True,
-    sweep=True,
-    raw_config=get_default_ensemble_config(),
-)

tests/test_evaluation.py

@@ -84,6 +84,7 @@ def test_evaluation():
         expanding_window=False,
         window_size=window_length,
         retrain_every=retrain_every,
+        class_labels=[0, 1],
         from_index=None,
     )
 

tests/test_walk_forward.py

@@ -80,6 +80,7 @@ def test_walk_forward_train_test():
         expanding_window=False,
         window_size=window_length,
         retrain_every=retrain_every,
+        class_labels=[0, 1],
         from_index=None,
     )
 

training/bet_sizing.py

@@ -66,6 +66,7 @@ def bet_sizing_with_meta_model(
         retrain_every=config.retrain_every,
         from_index=from_index,
         level="meta",
+        class_labels=[0, 1],
         transformations_over_time=transformations_over_time,
         model_over_time=preloaded_models,
     )

training/directional_training.py

@@ -48,6 +48,7 @@ def train_directional_model(
         retrain_every=config.retrain_every,
         from_index=from_index,
         level="primary",
+        class_labels=config.labeling.get_labels(),
         transformations_over_time=transformations_over_time,
         model_over_time=preloaded_training_step.model_over_time
         if preloaded_training_step

training/train_model.py

@@ -23,6 +23,7 @@ def train_model(
     retrain_every: int,
     from_index: Optional[pd.Timestamp],
     level: str,
+    class_labels: list[int],
     transformations_over_time: TransformationsOverTime,
     model_over_time: Optional[ModelOverTime],
 ) -> BaseTrainingOutcome:
@@ -61,6 +62,7 @@ def train_model(
         expanding_window=True,
         window_size=initial_window_size,
         retrain_every=retrain_every,
+        class_labels=class_labels,
         from_index=from_index,
     )
 

training/walk_forward/inference.py

@@ -20,10 +20,13 @@ def walk_forward_inference(
     expanding_window: bool,
     window_size: int,
     retrain_every: int,
+    class_labels: list[int],
     from_index: Optional[pd.Timestamp],
 ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
     predictions = pd.Series(index=X.index, dtype="object").rename(model_name)
-    probabilities = pd.DataFrame(index=X.index)
+    probabilities = pd.DataFrame(
+        index=X.index, columns=[str(label) for label in class_labels]
+    )
 
     inference_from = (
         max(

training/walk_forward/inference_batched.py

@@ -20,10 +20,13 @@ def walk_forward_inference_batched(
     expanding_window: bool,
     window_size: int,
     retrain_every: int,
+    class_labels: list[int],
     from_index: Optional[pd.Timestamp],
 ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
     predictions = pd.Series(index=X.index, dtype="object").rename(model_name)
-    probabilities = pd.DataFrame(index=X.index, columns=["0", "1"])
+    probabilities = pd.DataFrame(
+        index=X.index, columns=[str(label) for label in class_labels]
+    )
 
     inference_from = (
         max(

training/walk_forward/inference_parallel.py

@@ -22,10 +22,13 @@ def walk_forward_inference(
     expanding_window: bool,
     window_size: int,
     retrain_every: int,
+    class_labels: list[int],
     from_index: Optional[pd.Timestamp],
 ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
     predictions = pd.Series(index=X.index, dtype="object").rename(model_name)
-    probabilities = pd.DataFrame(index=X.index)
+    probabilities = pd.DataFrame(
+        index=X.index, columns=[str(label) for label in class_labels]
+    )
 
     inference_from = (
         max(

transformations/pca.py

@@ -19,7 +19,8 @@ class PCATransformation(Transformation):
             n_components=min(
                 int(len(X.columns) * self.ratio_components_to_keep),
                 self.initial_window_size,
-            )
+            ),
+            # whiten=True,
         )
         self.pca.fit(X, y)
 

transformations/retrieve.py

@@ -3,9 +3,18 @@ from .pca import PCATransformation
 from .sklearn import SKLearnTransformation
 from typing import Literal, Optional
 from models.model_map import default_feature_selector_classification
-from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler, RobustScaler
+from sklearn.preprocessing import (
+    MinMaxScaler,
+    Normalizer,
+    StandardScaler,
+    RobustScaler,
+    PowerTransformer,
+    QuantileTransformer,
+)
 
-ScalerTypes = Literal["normalize", "minmax", "standardize", "robust"]
+ScalerTypes = Literal[
+    "normalize", "minmax", "standardize", "robust", "box-cox", "quantile"
+]
 
 
 def get_rfe(n_feature_to_select: int) -> Optional[RFETransformation]:
@@ -42,5 +51,13 @@ def get_scaler(type: ScalerTypes) -> SKLearnTransformation:
         return SKLearnTransformation(
             RobustScaler(with_centering=False, quantile_range=(0.10, 0.90))
         )
+    elif type == "box-cox":
+        return SKLearnTransformation(
+            PowerTransformer(method="box-cox", standardize=True)
+        )
+    elif type == "quantile":
+        return SKLearnTransformation(
+            QuantileTransformer(n_quantiles=100, output_distribution="normal")
+        )
     else:
         raise Exception("Scaler type not supported")

utils/helpers.py

@@ -3,7 +3,7 @@ import numpy as np
 import os
 import string
 import random
-from itertools import dropwhile
+from itertools import chain, combinations, dropwhile
 
 
 def get_files_from_dir(path: str) -> list[str]:
@@ -78,3 +78,8 @@ def drop_until_first_valid_index(
         get_first_valid_return_index(series),
     )
     return df.iloc[first_valid_index:], series.iloc[first_valid_index:]
+
+
+def powerset(input: list) -> list[list]:
+    p_set = list(chain.from_iterable(combinations(input, r) for r in range(len(input) + 1)))  # type: ignore
+    return [list(item) for item in p_set if len(item) > 0]