gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-06 07:37:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refractor(Main Pipeline): Refractored the two main steps and the data processing. (#156)

Browse Source 
* refr: Took out main primary and secondary loops and data processing.

* feat: Tidied the code up.

* feat: Saving models and results now works in a type safe way.

* fix: There was error in the saving function.

* chore: Took out some remaining comments.

* fix: Fixed the previous data checking process.

* feat: Fixed model selection method. I will continue the inference after we merged.

Co-authored-by: Daniel Szemerey <szemereydaniel@gmail.com>
This commit is contained in:
Daniel Szemerey
2022-01-12 23:10:18 +01:00
committed by GitHub
parent c611481eb6
commit 3084f5e271
9 changed files with 277 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
training/inference.py
+22 -17
View File
@@ -4,46 +4,51 @@ from data_loader.load_data import load_data
from typing import Optional, Union
import warnings
from utils.encapsulation import Asset, Single_Model, Training_Step
def run_inference_pipeline(data_config:dict, training_config:dict, all_models_all_assets:dict):
def run_inference_pipeline(data_config:dict, training_config:dict, all_models_all_assets:list[Asset]):
data_params = data_config.copy()
data_params['target_asset'] = data_params['assets'][0]
X, y, _ = load_data(**data_params)
input_features = __select_data(X, training_config)
primary_models, secondary_models = __select_models(data_params, all_models_all_assets)
primary_step, secondary_step = __select_models(data_params, all_models_all_assets)
result = __inference(input_features, primary_models, secondary_models)
result = __inference(input_features, primary_step, secondary_step)
return result
def __inference(data:pd.DataFrame, primary_models:Union[dict,None], secondary_models:Union[dict,None]) -> pd.DataFrame:
assert primary_models is not None, "No primary models found. Cancelling Inference."
def __inference(data:pd.DataFrame, primary_step:Union[Training_Step,None], secondary_step:Union[Training_Step,None]) -> pd.DataFrame:
assert primary_step is not None, "No primary models found. Cancelling Inference."
data = __primary_models(data, primary_models)
data = __primary_models(data, primary_step)
if secondary_models is not None:
if secondary_step is not None:
warnings.warn("Secondary models are not specified.")
data = __secondary_models(data, secondary_models)
data = __secondary_models(data, secondary_step)
return data
def __select_models( data_params:dict, all_models_all_assets:dict)-> tuple[Optional[dict],Optional[dict]]:
def __select_models( data_params:dict, all_models_all_assets:list[Asset])-> tuple[Union[Training_Step,None], Union[Training_Step, None]]:
target_asset_name = data_params['target_asset'][1]
primary_models, secondary_models = None, None
primary_step, secondary_step, = None, None
target_asset_models = next((x for x in all_models_all_assets if x.name == target_asset_name), None)
if 'primary_models' in all_models_all_assets[target_asset_name]:
primary_models = all_models_all_assets[target_asset_name]['primary_models']
if target_asset_models is not None:
if len(target_asset_models.primary.base)>0:
primary_step = target_asset_models.primary
else: warnings.warn("No primary models found for {}.".format(target_asset_name))
if len(target_asset_models.secondary.base)>0:
secondary_step = target_asset_models.secondary
else: warnings.warn("No secondary models found for {}.".format(target_asset_name))
else:
assert("No primary models found for asset: " + target_asset_name)
assert("No models found for asset: " + target_asset_name)
if 'secondary_model' in all_models_all_assets[target_asset_name]:
secondary_models = all_models_all_assets[target_asset_name]['secondary_model']
return primary_models, secondary_models
return primary_step, secondary_step
def __select_data(X:pd.DataFrame, training_config:dict)-> pd.DataFrame: