gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-15 03:48:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(Linter): reformatted code with black (#211)

Browse Source 
* chore(Linter): reformatted code with black

* Create black.yaml
This commit is contained in:
Mark Aron Szulyovszky
2022-02-17 19:22:17 +01:00
committed by GitHub
parent f3fee4a4e1
commit 8dd2d88740
101 changed files with 2595 additions and 2319 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/hashing.py
+24 -15
View File
@@ -1,18 +1,27 @@
from data_loader.types import DataCollection
def hash_data_config(data_config: dict) -> str:
def hash_data_collection(data_collection: DataCollection) -> str: return ''.join([a[0] + a[1] for a in data_collection])
def hash_feature_extractors(feature_extractos) -> str: return ''.join([f[0] for f in feature_extractos])
def to_str(x): return ''.join([str(i) for i in x])
return '_'.join(to_str([
hash_data_collection(data_config['assets']),
hash_data_collection(data_config['other_assets']),
hash_data_collection(data_config['exogenous_data']),
data_config['target_asset'][0] + data_config['target_asset'][1],
data_config['load_non_target_asset'],
hash_feature_extractors(data_config['own_features']),
hash_feature_extractors(data_config['other_features']),
hash_feature_extractors(data_config['exogenous_features']),
]))
def hash_data_collection(data_collection: DataCollection) -> str:
return "".join([a[0] + a[1] for a in data_collection])
def hash_feature_extractors(feature_extractos) -> str:
return "".join([f[0] for f in feature_extractos])
def to_str(x):
return "".join([str(i) for i in x])
return "_".join(
to_str(
[
hash_data_collection(data_config["assets"]),
hash_data_collection(data_config["other_assets"]),
hash_data_collection(data_config["exogenous_data"]),
data_config["target_asset"][0] + data_config["target_asset"][1],
data_config["load_non_target_asset"],
hash_feature_extractors(data_config["own_features"]),
hash_feature_extractors(data_config["other_features"]),
hash_feature_extractors(data_config["exogenous_features"]),
]
)
)
config/preprocess.py
+51 -22
View File
@@ -1,6 +1,8 @@
from .types import Config, RawConfig
from utils.helpers import flatten
from feature_extractors.feature_extractor_presets import presets as feature_extractor_presets
from feature_extractors.feature_extractor_presets import (
presets as feature_extractor_presets,
)
from models.model_map import get_model
from data_loader.collections import data_collections
from labeling.eventfilters_map import eventfilters_map
@@ -8,6 +10,7 @@ from labeling.labellers_map import labellers_map
from models.sklearn import SKLearnModel
from sklearn.ensemble import VotingClassifier
def preprocess_config(raw_config: RawConfig) -> Config:
config_dict = vars(raw_config)
config_dict = __preprocess_model_config(config_dict)
@@ -16,47 +19,73 @@ def preprocess_config(raw_config: RawConfig) -> Config:
config_dict = __preprocess_event_filter_config(config_dict)
config_dict = __preprocess_event_labeller_config(config_dict)
config_dict['no_of_classes'] = 'two'
config_dict['mode'] = 'training'
config_dict["no_of_classes"] = "two"
config_dict["mode"] = "training"
config = Config(**config_dict)
return config
def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
data_dict = data_dict.copy()
keys = ['own_features', 'other_features', 'exogenous_features']
keys = ["own_features", "other_features", "exogenous_features"]
for key in keys:
preset_names = data_dict[key]
data_dict[key] = flatten([feature_extractor_presets[preset_name] for preset_name in preset_names])
data_dict[key] = flatten(
[feature_extractor_presets[preset_name] for preset_name in preset_names]
)
return data_dict
def __preprocess_model_config(model_config: dict) -> dict:
directional_models = [get_model(model_name) for model_name in model_config['directional_models']]
model_config.pop('directional_models')
model_config['directional_model'] = SKLearnModel(VotingClassifier([(m.name, m)for m in directional_models], voting ='soft'))
if len(model_config['meta_models']) > 0:
meta_models = [get_model(model_name) for model_name in model_config['meta_models']]
model_config['meta_model'] = SKLearnModel(VotingClassifier([(m.name, m)for m in meta_models], voting ='soft'))
model_config.pop('meta_models')
directional_models = [
get_model(model_name) for model_name in model_config["directional_models"]
]
model_config.pop("directional_models")
model_config["directional_model"] = SKLearnModel(
VotingClassifier([(m.name, m) for m in directional_models], voting="soft")
)
if len(model_config["meta_models"]) > 0:
meta_models = [
get_model(model_name) for model_name in model_config["meta_models"]
]
model_config["meta_model"] = SKLearnModel(
VotingClassifier([(m.name, m) for m in meta_models], voting="soft")
)
model_config.pop("meta_models")
return model_config
def __preprocess_data_collections_config(data_dict: dict) -> dict:
keys = ['assets', 'other_assets', 'exogenous_data']
keys = ["assets", "other_assets", "exogenous_data"]
for key in keys:
preset_names = data_dict[key]
data_dict[key] = flatten([data_collections[preset_name] for preset_name in preset_names])
target_asset = next(iter([asset for asset in data_dict['assets'] if asset[1] == data_dict['target_asset']]), None)
if target_asset is None: raise Exception('Target asset wasnt found in assets')
data_dict['target_asset'] = target_asset
data_dict[key] = flatten(
[data_collections[preset_name] for preset_name in preset_names]
)
target_asset = next(
iter(
[
asset
for asset in data_dict["assets"]
if asset[1] == data_dict["target_asset"]
]
),
None,
)
if target_asset is None:
raise Exception("Target asset wasnt found in assets")
data_dict["target_asset"] = target_asset
return data_dict
def __preprocess_event_filter_config(data_dict: dict) -> dict:
data_dict['event_filter'] = eventfilters_map[data_dict['event_filter']]
data_dict["event_filter"] = eventfilters_map[data_dict["event_filter"]]
return data_dict
def __preprocess_event_labeller_config(config_dict: dict) -> dict:
config_dict['labeling'] = labellers_map[config_dict['labeling']](config_dict['forecasting_horizon'])
config_dict["labeling"] = labellers_map[config_dict["labeling"]](
config_dict["forecasting_horizon"]
)
return config_dict
config/presets.py
+82 -85
View File
@@ -1,104 +1,101 @@
from .types import RawConfig, Config
def get_dev_config() -> RawConfig:
classification_models = ["LogisticRegression_two_class"]
return RawConfig(
directional_models_meta = False,
dimensionality_reduction = False,
n_features_to_select = 30,
expanding_window_base = False,
expanding_window_meta = False,
sliding_window_size_base = 380,
sliding_window_size_meta = 1,
retrain_every = 20,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
assets = ['daily_only_btc'],
target_asset = 'BTC_USD',
other_assets = [],
exogenous_data = [],
load_non_target_asset= True,
own_features = ['level_2', 'date_days'],
other_features = ['single_mom'],
exogenous_features = ['z_score'],
directional_models = classification_models,
meta_models = [],
event_filter = 'none',
labeling = 'two_class',
forecasting_horizon = 100,
directional_models_meta=False,
dimensionality_reduction=False,
n_features_to_select=30,
expanding_window_base=False,
expanding_window_meta=False,
sliding_window_size_base=380,
sliding_window_size_meta=1,
retrain_every=20,
scaler="minmax", # 'normalize' 'minmax' 'standardize'
assets=["daily_only_btc"],
target_asset="BTC_USD",
other_assets=[],
exogenous_data=[],
load_non_target_asset=True,
own_features=["level_2", "date_days"],
other_features=["single_mom"],
exogenous_features=["z_score"],
directional_models=classification_models,
meta_models=[],
event_filter="none",
labeling="two_class",
forecasting_horizon=100,
)
def get_default_ensemble_config() -> RawConfig:
classification_models = ["LogisticRegression_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
meta_models = ['LogisticRegression_two_class', 'LGBM']
classification_models = [
"LogisticRegression_two_class",
"LDA",
"NB",
"RFC",
"XGB_two_class",
"LGBM",
"StaticMom",
]
meta_models = ["LogisticRegression_two_class", "LGBM"]
return RawConfig(
directional_models_meta = True,
dimensionality_reduction = False,
n_features_to_select = 30,
expanding_window_base = False,
expanding_window_meta = True,
sliding_window_size_base = 380,
sliding_window_size_meta = 240,
retrain_every = 10,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
assets = ['daily_crypto'],
target_asset = 'BTC_USD',
other_assets = ['daily_etf'],
exogenous_data = ['daily_glassnode'],
load_non_target_asset= True,
own_features = ['level_2', 'date_days', 'lags_up_to_5'],
other_features = ['level_2', 'lags_up_to_5'],
exogenous_features = ['z_score'],
directional_models = classification_models,
meta_models = meta_models,
event_filter = 'cusum_vol',
labeling = 'two_class',
forecasting_horizon = 100,
directional_models_meta=True,
dimensionality_reduction=False,
n_features_to_select=30,
expanding_window_base=False,
expanding_window_meta=True,
sliding_window_size_base=380,
sliding_window_size_meta=240,
retrain_every=10,
scaler="minmax", # 'normalize' 'minmax' 'standardize'
assets=["daily_crypto"],
target_asset="BTC_USD",
other_assets=["daily_etf"],
exogenous_data=["daily_glassnode"],
load_non_target_asset=True,
own_features=["level_2", "date_days", "lags_up_to_5"],
other_features=["level_2", "lags_up_to_5"],
exogenous_features=["z_score"],
directional_models=classification_models,
meta_models=meta_models,
event_filter="cusum_vol",
labeling="two_class",
forecasting_horizon=100,
)
def get_lightweight_ensemble_config() -> RawConfig:
classification_models = ['LogisticRegression_two_class', 'LGBM']
meta_models = ['LogisticRegression_two_class', 'LGBM']
classification_models = ["LogisticRegression_two_class", "LGBM"]
meta_models = ["LogisticRegression_two_class", "LGBM"]
return RawConfig(
directional_models_meta = True,
dimensionality_reduction = True,
n_features_to_select = 30,
expanding_window_base = True,
expanding_window_meta = True,
sliding_window_size_base = 3800,
sliding_window_size_meta = 2400,
retrain_every = 1000,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
assets = ['fivemin_crypto'],
target_asset = 'BTC_USD',
other_assets = [],
exogenous_data = [],
load_non_target_asset= False,
own_features = ['level_1'],
other_features = [],
exogenous_features = [],
directional_models = classification_models,
meta_models = meta_models,
event_filter = 'cusum_fixed',
labeling = 'two_class',
forecasting_horizon = 50,
directional_models_meta=True,
dimensionality_reduction=True,
n_features_to_select=30,
expanding_window_base=True,
expanding_window_meta=True,
sliding_window_size_base=3800,
sliding_window_size_meta=2400,
retrain_every=1000,
scaler="minmax", # 'normalize' 'minmax' 'standardize'
assets=["fivemin_crypto"],
target_asset="BTC_USD",
other_assets=[],
exogenous_data=[],
load_non_target_asset=False,
own_features=["level_1"],
other_features=[],
exogenous_features=[],
directional_models=classification_models,
meta_models=meta_models,
event_filter="cusum_fixed",
labeling="two_class",
forecasting_horizon=50,
)
config/types.py
+11 -12
View File
@@ -18,7 +18,7 @@ class RawConfig(BaseModel):
sliding_window_size_base: int
sliding_window_size_meta: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
scaler: Literal["normalize", "minmax", "standardize"]
assets: list[str]
target_asset: str
@@ -28,8 +28,8 @@ class RawConfig(BaseModel):
own_features: list[str]
other_features: list[str]
exogenous_features: list[str]
event_filter: Literal['none', 'cusum_vol', 'cusum_fixed']
labeling: Literal['two_class', 'three_class_balanced', 'three_class_imbalanced']
event_filter: Literal["none", "cusum_vol", "cusum_fixed"]
labeling: Literal["two_class", "three_class_balanced", "three_class_imbalanced"]
forecasting_horizon: int
directional_models: list[str]
@@ -46,7 +46,7 @@ class Config:
sliding_window_size_base: int
sliding_window_size_meta: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
scaler: Literal["normalize", "minmax", "standardize"]
assets: DataCollection
target_asset: DataSource
@@ -59,28 +59,27 @@ class Config:
event_filter: EventFilter
labeling: EventLabeller
forecasting_horizon: int
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"]
mode: Literal['training', 'inference']
mode: Literal["training", "inference"]
directional_model: Model
meta_model: Model
@validator('directional_model', 'meta_model')
@validator("directional_model", "meta_model")
def check_model(cls, v):
assert isinstance(v, BaseEstimator)
return v
@validator('event_filter')
@validator("event_filter")
def check_event_filter(cls, v):
assert isinstance(v, EventFilter)
return v
@validator('labeling')
@validator("labeling")
def check_labeling(cls, v):
assert isinstance(v, EventLabeller)
return v
# class Config:
# arbitrary_types_allowed = True