feat(Project): use SKLearn models directly, removed custom ensembling, use 5 minute data, batch inference, numba cusum filter (#192)

* feat(Project): use 5 minute data, running training in parallel, sped up cusum filter by 10x with numba

* fix(WalkForward): inference mini-batch parallelization

* fix(WalkForward): don't use the parallel version of any of the functions

* feat(CI): download the data required

* fix(Project): 5min_crypto folder added

* fix(Evaluate): make sure we have numerical stability in returns

* feat(Models): use SKLearn models directly to enable composability

* feat(Inference): batched inference now working, added forecasting_horizon

* fix(Inference): works again

* fix(Inference)

* chore(Models): remove unused Ensemble model

* fix(Labeller): don't just forward shift returns, also take the sum of the data happened until then

* Update test.yml

config/preprocess.py

@@ -5,6 +5,8 @@ from models.model_map import get_model
 from data_loader.collections import data_collections
 from labeling.eventfilters_map import eventfilters_map
 from labeling.labellers_map import labellers_map
+from models.sklearn import SKLearnModel
+from sklearn.ensemble import VotingClassifier
 
 def preprocess_config(raw_config: RawConfig) -> Config:
     config_dict = vars(raw_config)
@@ -17,7 +19,6 @@ def preprocess_config(raw_config: RawConfig) -> Config:
     config_dict['no_of_classes'] = 'two'
     config_dict['mode'] = 'training'
     config = Config(**config_dict)
-    validate_config(config)
     return config
 
 def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
@@ -28,10 +29,14 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
         data_dict[key] = flatten([feature_extractor_presets[preset_name] for preset_name in preset_names])
     return data_dict
 
-def __preprocess_model_config(model_config:dict) -> dict:
-    model_config['directional_models'] = [get_model(model_name) for model_name in model_config['directional_models']]
+def __preprocess_model_config(model_config: dict) -> dict:
+    directional_models = [get_model(model_name) for model_name in model_config['directional_models']]
+    model_config.pop('directional_models')
+    model_config['directional_model'] = SKLearnModel(VotingClassifier([(m.name, m)for m in directional_models], voting ='soft'))
     if len(model_config['meta_models']) > 0:
-        model_config['meta_models'] = [get_model(model_name) for model_name in  model_config['meta_models']]
+        meta_models = [get_model(model_name) for model_name in model_config['meta_models']]
+        model_config['meta_model'] = SKLearnModel(VotingClassifier([(m.name, m)for m in meta_models], voting ='soft'))
+    model_config.pop('meta_models')
 
     return model_config
 
@@ -49,16 +54,9 @@ def __preprocess_event_filter_config(data_dict: dict) -> dict:
     data_dict['event_filter'] = eventfilters_map[data_dict['event_filter']]
     return data_dict
 
-def __preprocess_event_labeller_config(data_dict: dict) -> dict:
-    data_dict['labeling'] = labellers_map[data_dict['labeling']]
-    return data_dict
+def __preprocess_event_labeller_config(config_dict: dict) -> dict:
+    config_dict['labeling'] = labellers_map[config_dict['labeling']](config_dict['forecasting_horizon'])
+    return config_dict
     
 
-def validate_config(config: Config):
-    # We need to make sure there's only one output from the pipeline
-    # If meta model is there, we need more than one directional models to train
-    if len(config.meta_models) > 1: assert len(config.directional_models) > 0
-    # If there's no level-2 model, we need to have only one level-1 model
-    if len(config.meta_models) == 0: assert len(config.directional_models) == 1
-
 

config/presets.py

@@ -2,7 +2,6 @@ from .types import RawConfig, Config
 
 def get_dev_config() -> RawConfig:
   
-    regression_models = ["Lasso"]
     classification_models = ["LogisticRegression_two_class"]
 
     return RawConfig(
@@ -29,13 +28,13 @@ def get_dev_config() -> RawConfig:
         meta_models = [],
 
         event_filter = 'none',
-        labeling = 'two_class'
+        labeling = 'two_class',
+        forecasting_horizon = 100,
     )
 
 
 def get_default_ensemble_config() -> RawConfig:
   
-    regression_models = ["Lasso", "KNN", "RFR"]
     classification_models = ["LogisticRegression_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
     meta_models = ['LogisticRegression_two_class', 'LGBM']
 
@@ -63,42 +62,43 @@ def get_default_ensemble_config() -> RawConfig:
         meta_models = meta_models,
 
         event_filter = 'cusum_vol',
-        labeling = 'two_class'
+        labeling = 'two_class',
+        forecasting_horizon = 100,
     )
 
 
 
 def get_lightweight_ensemble_config() -> RawConfig:
   
-    regression_models = ["Lasso", "KNN"]
-    classification_models = ['LogisticRegression_two_class', 'SVC']
+    classification_models = ['LogisticRegression_two_class', 'LGBM']
     meta_models = ['LogisticRegression_two_class', 'LGBM']
 
     return RawConfig(
         directional_models_meta = True,
         dimensionality_reduction = True,
         n_features_to_select = 30,
-        expanding_window_base = False,
+        expanding_window_base = True,
         expanding_window_meta = True,
-        sliding_window_size_base = 380,
-        sliding_window_size_meta = 240,
-        retrain_every = 40,
+        sliding_window_size_base = 3800,
+        sliding_window_size_meta = 2400,
+        retrain_every = 1000,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
 
-        assets = ['daily_crypto_lightweight'],
-        target_asset = 'BCH_USD',
-        other_assets = ['daily_etf'],
-        exogenous_data = ['daily_glassnode'],
-        load_non_target_asset= True,
-        own_features = ['level_2' ],
-        other_features = ['level_2'],
-        exogenous_features = ['z_score'],
+        assets = ['fivemin_crypto'],
+        target_asset = 'BTC_USD',
+        other_assets = [],
+        exogenous_data = [],
+        load_non_target_asset= False,
+        own_features = ['level_1'],
+        other_features = [],
+        exogenous_features = [],
 
         directional_models = classification_models,
         meta_models = meta_models,
 
-        event_filter = 'none',
-        labeling = 'two_class'
+        event_filter = 'cusum_fixed',
+        labeling = 'two_class',
+        forecasting_horizon = 50,
     )
 
 

config/types.py

@@ -1,11 +1,12 @@
-from pydantic import BaseModel
+from pydantic import BaseModel, validator
 from typing import Literal, Optional
 from labeling.types import EventFilter
 from models.base import Model
 from data_loader.types import DataCollection, DataSource
 from feature_extractors.types import FeatureExtractor, ScalerTypes
 from labeling.types import EventFilter, EventLabeller
-
+from sklearn.base import BaseEstimator
+from dataclasses import dataclass
 
 # RawConfig is needed to ensure we can declare config presets here with static typing, we then convert it to Config
 class RawConfig(BaseModel):
@@ -29,12 +30,14 @@ class RawConfig(BaseModel):
     exogenous_features: list[str]
     event_filter: Literal['none', 'cusum_vol', 'cusum_fixed']
     labeling: Literal['two_class', 'three_class_balanced', 'three_class_imbalanced']
+    forecasting_horizon: int
 
     directional_models: list[str]
     meta_models: list[str]
 
 
-class Config(BaseModel):
+@dataclass
+class Config:
     directional_models_meta: bool
     dimensionality_reduction: bool
     n_features_to_select: int
@@ -55,14 +58,29 @@ class Config(BaseModel):
     exogenous_features: list[tuple[str, FeatureExtractor, list[int]]]
     event_filter: EventFilter
     labeling: EventLabeller
+    forecasting_horizon: int
     no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
 
     mode: Literal['training', 'inference']
 
-    directional_models: list[Model]
-    meta_models: list[Model]
+    directional_model: Model
+    meta_model: Model
 
-    class Config:
-        arbitrary_types_allowed = True
+    @validator('directional_model', 'meta_model')
+    def check_model(cls, v):
+        assert isinstance(v, BaseEstimator)
+        return v
+    
+    @validator('event_filter')
+    def check_event_filter(cls, v):
+        assert isinstance(v, EventFilter)
+        return v
+
+    @validator('labeling')
+    def check_labeling(cls, v):
+        assert isinstance(v, EventLabeller)
+        return v
+    # class Config:
+    #     arbitrary_types_allowed = True