gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-07-27 18:57:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor(Training): new outcome types, representative pipeline steps, bet-sizing (#187)

Browse Source 
* refactor(Training): added InferenceResult & TrainedModel types

* refactor(Pipeline): introduced TrainingOutcome, BetSizingWithMetaOutcome, etc.

* fix(Pipeline): getting it to compile

* refactor(WalkForward): separate preprocessing step

* feat(Pipeline): separate out transformations processing step

* refactor(Pipeline): use the Directional model terminology, put bet_sizing into pipeline instead of hiding it in a step

* refactor(WalkForward): moved functions to separate folder

* fix(WalkForward): use sparse array to store models, process transformations in parallel (lot faster)

* fix(Tests): and evaluation

* fix(Tests): for realz

* fix(Inference): preloading everything now, renamed primary models to directional models

* fix(BetSizing): was running transformations on the wrong data, oops

* fix(BetSizing): concatenated on the wrong axis accidentally

* fix(Reporting): able to use the new Stats type

* fix(BetSizing): renamed int column names

* fix(Portfolio): name the column properly

* fix(Reporting): rename the correct Series, lol

* fix(Inference): walk_forwad_inference() can deal with models not being aligned with the starting index

* fix(WalkForward): accidentally using the wrong index

* fix(WalkForward): use the correct indicies to fetch last model/transformations

* fix(CI): changed the name of the results
This commit is contained in:
Mark Aron Szulyovszky
2022-01-29 06:41:40 +01:00
committed by GitHub
parent 42a1bc59cb
commit 3eb3ea94e3
42 changed files with 772 additions and 736 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/preprocess.py
+6 -8
View File
@@ -28,11 +28,9 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
return data_dict
def __preprocess_model_config(model_config:dict) -> dict:
model_config['primary_models'] = [(model_name, get_model(model_name)) for model_name in model_config['primary_models']]
if len(model_config['meta_labeling_models']) > 0:
model_config['meta_labeling_models'] = [(model_name, get_model(model_name)) for model_name in model_config['meta_labeling_models']]
if model_config['ensemble_model'] is not None:
model_config['ensemble_model'] = (model_config['ensemble_model'], get_model(model_config['ensemble_model']))
model_config['directional_models'] = [get_model(model_name) for model_name in model_config['directional_models']]
if len(model_config['meta_models']) > 0:
model_config['meta_models'] = [get_model(model_name) for model_name in model_config['meta_models']]
return model_config
@@ -57,9 +55,9 @@ def __preprocess_event_labeller_config(data_dict: dict) -> dict:
def validate_config(config: Config):
# We need to make sure there's only one output from the pipeline
# If level-2 model is there, we need more than one level-1 models to train
if len(config.meta_labeling_models) > 1: assert len(config.primary_models) > 0
# If meta model is there, we need more than one directional models to train
if len(config.meta_models) > 1: assert len(config.directional_models) > 0
# If there's no level-2 model, we need to have only one level-1 model
if len(config.meta_labeling_models) == 0: assert len(config.primary_models) == 1
if len(config.meta_models) == 0: assert len(config.directional_models) == 1
config/presets.py
+17 -22
View File
@@ -6,13 +6,13 @@ def get_dev_config() -> RawConfig:
classification_models = ["LogisticRegression_two_class"]
return RawConfig(
primary_models_meta_labeling = False,
directional_models_meta = False,
dimensionality_reduction = False,
n_features_to_select = 30,
expanding_window_base = False,
expanding_window_meta_labeling = False,
expanding_window_meta = False,
sliding_window_size_base = 380,
sliding_window_size_meta_labeling = 1,
sliding_window_size_meta = 1,
retrain_every = 20,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
@@ -25,9 +25,8 @@ def get_dev_config() -> RawConfig:
other_features = ['single_mom'],
exogenous_features = ['z_score'],
primary_models = classification_models,
meta_labeling_models = [],
ensemble_model = None,
directional_models = classification_models,
meta_models = [],
event_filter = 'none',
labeling = 'two_class'
@@ -38,17 +37,16 @@ def get_default_ensemble_config() -> RawConfig:
regression_models = ["Lasso", "KNN", "RFR"]
classification_models = ["LogisticRegression_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
meta_labeling_models = ['LogisticRegression_two_class', 'LGBM']
ensemble_model = 'Average'
meta_models = ['LogisticRegression_two_class', 'LGBM']
return RawConfig(
primary_models_meta_labeling = True,
directional_models_meta = True,
dimensionality_reduction = False,
n_features_to_select = 30,
expanding_window_base = False,
expanding_window_meta_labeling = True,
expanding_window_meta = True,
sliding_window_size_base = 380,
sliding_window_size_meta_labeling = 240,
sliding_window_size_meta = 240,
retrain_every = 10,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
@@ -61,9 +59,8 @@ def get_default_ensemble_config() -> RawConfig:
other_features = ['level_2', 'lags_up_to_5'],
exogenous_features = ['z_score'],
primary_models = classification_models,
meta_labeling_models = meta_labeling_models,
ensemble_model = ensemble_model,
directional_models = classification_models,
meta_models = meta_models,
event_filter = 'cusum_vol',
labeling = 'two_class'
@@ -75,17 +72,16 @@ def get_lightweight_ensemble_config() -> RawConfig:
regression_models = ["Lasso", "KNN"]
classification_models = ['LogisticRegression_two_class', 'SVC']
meta_labeling_models = ['LogisticRegression_two_class', 'LGBM']
ensemble_model = 'Average'
meta_models = ['LogisticRegression_two_class', 'LGBM']
return RawConfig(
primary_models_meta_labeling = True,
directional_models_meta = True,
dimensionality_reduction = True,
n_features_to_select = 30,
expanding_window_base = False,
expanding_window_meta_labeling = True,
expanding_window_meta = True,
sliding_window_size_base = 380,
sliding_window_size_meta_labeling = 240,
sliding_window_size_meta = 240,
retrain_every = 40,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
@@ -98,9 +94,8 @@ def get_lightweight_ensemble_config() -> RawConfig:
other_features = ['level_2'],
exogenous_features = ['z_score'],
primary_models = classification_models,
meta_labeling_models = meta_labeling_models,
ensemble_model = ensemble_model,
directional_models = classification_models,
meta_models = meta_models,
event_filter = 'none',
labeling = 'two_class'
config/sweep_ensemble.yaml
+5 -5
View File
@@ -6,7 +6,7 @@ metric:
goal: maximize
name: sharpe
parameters:
primary_models_meta_labeling:
directional_models_meta:
value: True
assets:
value: ['daily_crypto']
@@ -16,11 +16,11 @@ parameters:
value: ['daily_glassnode']
expanding_window_base:
value: True
expanding_window_meta_labeling:
expanding_window_meta:
value: True
sliding_window_size_base:
value: 380
sliding_window_size_meta_labeling:
sliding_window_size_meta:
values: [250, 300, 380]
distribution: categorical
n_features_to_select:
@@ -36,13 +36,13 @@ parameters:
value: 'two'
load_non_target_asset:
value: True
primary_models:
directional_models:
distribution: categorical
values:
- ["LDA", "LogisticRegression_two_class", "KNN", "SVC", "CART", "NB", "AB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
- ["LogisticRegression_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
- ["LogisticRegression_two_class", "LDA", "LGBM", "RFC", "XGB_two_class"]
meta_labeling_models:
meta_models:
value: ["LGBM", "LogisticRegression_two_class"]
own_features:
value: ['date_days', 'level_2', 'lags_up_to_5']
config/sweep_meta_labeling.yaml
+5 -5
View File
@@ -6,7 +6,7 @@ metric:
goal: maximize
name: sharpe
parameters:
primary_models_meta_labeling:
directional_models_meta:
value: True
assets:
value: ['daily_crypto']
@@ -17,7 +17,7 @@ parameters:
expanding_window_base:
values: [True, False]
distribution: categorical
expanding_window_meta_labeling:
expanding_window_meta:
values: [True, False]
distribution: categorical
n_features_to_select:
@@ -28,7 +28,7 @@ parameters:
sliding_window_size_base:
values: [180, 280, 380]
distribution: categorical
sliding_window_size_meta_labeling:
sliding_window_size_meta:
values: [180, 280, 380]
distribution: categorical
retrain_every:
@@ -42,9 +42,9 @@ parameters:
load_non_target_asset:
values: [True, False]
distribution: categorical
primary_models:
directional_models:
value: ["LogisticRegression_two_class", "LDA", "KNN", "CART", "NB", "AB", "RFC", "StaticMom"]
meta_labeling_models:
meta_models:
values: ["LogisticRegression_two_class", "LDA", "KNN", "CART", "NB", "AB", "RFC"]
distribution: categorical
own_features:
config/sweep_primary_models.yaml
+5 -5
View File
@@ -6,7 +6,7 @@ metric:
goal: maximize
name: sharpe
parameters:
primary_models_meta_labeling:
directional_models_meta:
value: True
assets:
value: ['daily_crypto']
@@ -17,7 +17,7 @@ parameters:
expanding_window_base:
values: [True, False]
distribution: categorical
expanding_window_meta_labeling:
expanding_window_meta:
value: False
n_features_to_select:
value: 50
@@ -25,7 +25,7 @@ parameters:
value: True
sliding_window_size_base:
value: 380
sliding_window_size_meta_labeling:
sliding_window_size_meta:
value: 380
retrain_every:
values: [10, 20, 30]
@@ -36,10 +36,10 @@ parameters:
value: 'two'
load_non_target_asset:
value: True
primary_models:
directional_models:
values: [['LogisticRegression_two_class'], ['SVC'], ['LDA'], ['KNN'], ['CART'], ['MNB'], ['NB'], ['AB'], ['RFC'], ['XGB_two_class'], ['LGBM']]
distribution: categorical
meta_labeling_models:
meta_models:
value: ["LGBM", "LogisticRegression_two_class"]
own_features:
value: ['date_days', 'level_2', 'lags_up_to_5']
config/types.py
+10 -12
View File
@@ -9,13 +9,13 @@ from labeling.types import EventFilter, EventLabeller
# RawConfig is needed to ensure we can declare config presets here with static typing, we then convert it to Config
class RawConfig(BaseModel):
primary_models_meta_labeling: bool
directional_models_meta: bool
dimensionality_reduction: bool
n_features_to_select: int
expanding_window_base: bool
expanding_window_meta_labeling: bool
expanding_window_meta: bool
sliding_window_size_base: int
sliding_window_size_meta_labeling: int
sliding_window_size_meta: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
@@ -30,19 +30,18 @@ class RawConfig(BaseModel):
event_filter: Literal['none', 'cusum_vol', 'cusum_fixed']
labeling: Literal['two_class', 'three_class_balanced', 'three_class_imbalanced']
primary_models: list[str]
meta_labeling_models: list[str]
ensemble_model: Optional[str]
directional_models: list[str]
meta_models: list[str]
class Config(BaseModel):
primary_models_meta_labeling: bool
directional_models_meta: bool
dimensionality_reduction: bool
n_features_to_select: int
expanding_window_base: bool
expanding_window_meta_labeling: bool
expanding_window_meta: bool
sliding_window_size_base: int
sliding_window_size_meta_labeling: int
sliding_window_size_meta: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
@@ -58,9 +57,8 @@ class Config(BaseModel):
labeling: EventLabeller
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
primary_models: list[tuple[str, Model]]
meta_labeling_models: list[tuple[str, Model]]
ensemble_model: Optional[tuple[str, Model]]
directional_models: list[Model]
meta_models: list[Model]
class Config:
arbitrary_types_allowed = True