gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-16 12:28:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Events): added EventFilter, EventLabeller (#186)

Browse Source
This commit is contained in:
Mark Aron Szulyovszky
2022-01-26 23:22:43 +01:00
committed by GitHub
parent 1042c82333
commit 42a1bc59cb
65 changed files with 759 additions and 276571 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/hashing.py
+1 -4
View File
@@ -1,5 +1,5 @@
from utils.types import DataCollection
from data_loader.types import DataCollection
def hash_data_config(data_config: dict) -> str:
@@ -12,10 +12,7 @@ def hash_data_config(data_config: dict) -> str:
hash_data_collection(data_config['exogenous_data']),
data_config['target_asset'][0] + data_config['target_asset'][1],
data_config['load_non_target_asset'],
data_config['log_returns'],
data_config['forecasting_horizon'],
hash_feature_extractors(data_config['own_features']),
hash_feature_extractors(data_config['other_features']),
hash_feature_extractors(data_config['exogenous_features']),
data_config['no_of_classes'],
]))
config/preprocess.py
+18 -8
View File
@@ -1,16 +1,20 @@
from config.config import Config, RawConfig
from .types import Config, RawConfig
from utils.helpers import flatten
from feature_extractors.feature_extractor_presets import presets as feature_extractor_presets
from models.model_map import get_model_map
from models.model_map import get_model
from data_loader.collections import data_collections
from labeling.eventfilters_map import eventfilters_map
from labeling.labellers_map import labellers_map
def preprocess_config(raw_config: RawConfig) -> Config:
config_dict = vars(raw_config)
config_dict = __preprocess_model_config(config_dict)
config_dict = __preprocess_feature_extractors_config(config_dict)
config_dict = __preprocess_data_collections_config(config_dict)
config_dict = __preprocess_event_filter_config(config_dict)
config_dict = __preprocess_event_labeller_config(config_dict)
config_dict['no_of_classes'] = 'two'
config = Config(**config_dict)
validate_config(config)
return config
@@ -24,17 +28,15 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
return data_dict
def __preprocess_model_config(model_config:dict) -> dict:
model_map = get_model_map(model_config)
model_config['primary_models'] = [(model_name, model_map['primary_models'][model_name]) for model_name in model_config['primary_models']]
model_config['primary_models'] = [(model_name, get_model(model_name)) for model_name in model_config['primary_models']]
if len(model_config['meta_labeling_models']) > 0:
model_config['meta_labeling_models'] = [(model_name, model_map['primary_models'][model_name]) for model_name in model_config['meta_labeling_models']]
model_config['meta_labeling_models'] = [(model_name, get_model(model_name)) for model_name in model_config['meta_labeling_models']]
if model_config['ensemble_model'] is not None:
model_config['ensemble_model'] = (model_config['ensemble_model'], model_map['ensemble_models'][model_config['ensemble_model']])
model_config['ensemble_model'] = (model_config['ensemble_model'], get_model(model_config['ensemble_model']))
return model_config
def __preprocess_data_collections_config(data_dict: dict) -> dict:
data_dict = data_dict.copy()
keys = ['assets', 'other_assets', 'exogenous_data']
for key in keys:
preset_names = data_dict[key]
@@ -44,6 +46,14 @@ def __preprocess_data_collections_config(data_dict: dict) -> dict:
data_dict['target_asset'] = target_asset
return data_dict
def __preprocess_event_filter_config(data_dict: dict) -> dict:
data_dict['event_filter'] = eventfilters_map[data_dict['event_filter']]
return data_dict
def __preprocess_event_labeller_config(data_dict: dict) -> dict:
data_dict['labeling'] = labellers_map[data_dict['labeling']]
return data_dict
def validate_config(config: Config):
# We need to make sure there's only one output from the pipeline
config/config.py → config/presets.py
+13 -77
View File
@@ -1,68 +1,4 @@
from pydantic import BaseModel
from typing import Literal, Optional
from models.base import Model
from utils.types import DataCollection, DataSource, FeatureExtractor
# RawConfig is needed to ensure we can declare config presets here with static typing, we then convert it to Config
class RawConfig(BaseModel):
primary_models_meta_labeling: bool
dimensionality_reduction: bool
n_features_to_select: int
expanding_window_base: bool
expanding_window_meta_labeling: bool
sliding_window_size_base: int
sliding_window_size_meta_labeling: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
assets: list[str]
target_asset: str
other_assets: list[str]
exogenous_data: list[str]
load_non_target_asset: bool
log_returns: bool
forecasting_horizon: int
own_features: list[str]
other_features: list[str]
exogenous_features: list[str]
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
primary_models: list[str]
meta_labeling_models: list[str]
ensemble_model: Optional[str]
class Config(BaseModel):
primary_models_meta_labeling: bool
dimensionality_reduction: bool
n_features_to_select: int
expanding_window_base: bool
expanding_window_meta_labeling: bool
sliding_window_size_base: int
sliding_window_size_meta_labeling: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
assets: DataCollection
target_asset: DataSource
other_assets: DataCollection
exogenous_data: DataCollection
load_non_target_asset: bool
log_returns: bool
forecasting_horizon: int
own_features: list[tuple[str, FeatureExtractor, list[int]]]
other_features: list[tuple[str, FeatureExtractor, list[int]]]
exogenous_features: list[tuple[str, FeatureExtractor, list[int]]]
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
primary_models: list[tuple[str, Model]]
meta_labeling_models: list[tuple[str, Model]]
ensemble_model: Optional[tuple[str, Model]]
class Config:
arbitrary_types_allowed = True
from .types import RawConfig, Config
def get_dev_config() -> RawConfig:
@@ -85,16 +21,16 @@ def get_dev_config() -> RawConfig:
other_assets = [],
exogenous_data = [],
load_non_target_asset= True,
log_returns= True,
forecasting_horizon = 1,
own_features = ['level_2', 'date_days'],
other_features = ['single_mom'],
exogenous_features = ['z_score'],
no_of_classes= 'two',
primary_models = classification_models,
meta_labeling_models = [],
ensemble_model = None
ensemble_model = None,
event_filter = 'none',
labeling = 'two_class'
)
@@ -121,16 +57,16 @@ def get_default_ensemble_config() -> RawConfig:
other_assets = ['daily_etf'],
exogenous_data = ['daily_glassnode'],
load_non_target_asset= True,
log_returns= True,
forecasting_horizon = 1,
own_features = ['level_2', 'date_days', 'lags_up_to_5'],
other_features = ['level_2', 'lags_up_to_5'],
exogenous_features = ['z_score'],
no_of_classes= 'two',
primary_models = classification_models,
meta_labeling_models = meta_labeling_models,
ensemble_model = ensemble_model
ensemble_model = ensemble_model,
event_filter = 'cusum_vol',
labeling = 'two_class'
)
@@ -158,16 +94,16 @@ def get_lightweight_ensemble_config() -> RawConfig:
other_assets = ['daily_etf'],
exogenous_data = ['daily_glassnode'],
load_non_target_asset= True,
log_returns= True,
forecasting_horizon = 1,
own_features = ['level_2' ],
other_features = ['level_2'],
exogenous_features = ['z_score'],
no_of_classes= 'two',
primary_models = classification_models,
meta_labeling_models = meta_labeling_models,
ensemble_model = ensemble_model
ensemble_model = ensemble_model,
event_filter = 'none',
labeling = 'two_class'
)
config/sweep_ensemble.yaml
-4
View File
@@ -34,12 +34,8 @@ parameters:
value: 'minmax'
no_of_classes:
value: 'two'
forecasting_horizon:
value: 1
load_non_target_asset:
value: True
log_returns:
value: True
primary_models:
distribution: categorical
values:
config/sweep_meta_labeling.yaml
-4
View File
@@ -39,13 +39,9 @@ parameters:
no_of_classes:
values: ['two', 'three-balanced', 'three-imbalanced']
distribution: categorical
forecasting_horizon:
value: 1
load_non_target_asset:
values: [True, False]
distribution: categorical
log_returns:
value: True
primary_models:
value: ["LogisticRegression_two_class", "LDA", "KNN", "CART", "NB", "AB", "RFC", "StaticMom"]
meta_labeling_models:
config/sweep_primary_models.yaml
-4
View File
@@ -34,12 +34,8 @@ parameters:
value: 'minmax'
no_of_classes:
value: 'two'
forecasting_horizon:
value: 1
load_non_target_asset:
value: True
log_returns:
value: True
primary_models:
values: [['LogisticRegression_two_class'], ['SVC'], ['LDA'], ['KNN'], ['CART'], ['MNB'], ['NB'], ['AB'], ['RFC'], ['XGB_two_class'], ['LGBM']]
distribution: categorical
config/types.py
+68
View File
@@ -0,0 +1,68 @@
from pydantic import BaseModel
from typing import Literal, Optional
from labeling.types import EventFilter
from models.base import Model
from data_loader.types import DataCollection, DataSource
from feature_extractors.types import FeatureExtractor, ScalerTypes
from labeling.types import EventFilter, EventLabeller
# RawConfig is needed to ensure we can declare config presets here with static typing, we then convert it to Config
class RawConfig(BaseModel):
primary_models_meta_labeling: bool
dimensionality_reduction: bool
n_features_to_select: int
expanding_window_base: bool
expanding_window_meta_labeling: bool
sliding_window_size_base: int
sliding_window_size_meta_labeling: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
assets: list[str]
target_asset: str
other_assets: list[str]
exogenous_data: list[str]
load_non_target_asset: bool
own_features: list[str]
other_features: list[str]
exogenous_features: list[str]
event_filter: Literal['none', 'cusum_vol', 'cusum_fixed']
labeling: Literal['two_class', 'three_class_balanced', 'three_class_imbalanced']
primary_models: list[str]
meta_labeling_models: list[str]
ensemble_model: Optional[str]
class Config(BaseModel):
primary_models_meta_labeling: bool
dimensionality_reduction: bool
n_features_to_select: int
expanding_window_base: bool
expanding_window_meta_labeling: bool
sliding_window_size_base: int
sliding_window_size_meta_labeling: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
assets: DataCollection
target_asset: DataSource
other_assets: DataCollection
exogenous_data: DataCollection
load_non_target_asset: bool
own_features: list[tuple[str, FeatureExtractor, list[int]]]
other_features: list[tuple[str, FeatureExtractor, list[int]]]
exogenous_features: list[tuple[str, FeatureExtractor, list[int]]]
event_filter: EventFilter
labeling: EventLabeller
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
primary_models: list[tuple[str, Model]]
meta_labeling_models: list[tuple[str, Model]]
ensemble_model: Optional[tuple[str, Model]]
class Config:
arbitrary_types_allowed = True