mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-28 11:17:47 +00:00
Daniel Szemerey
55f083638f
* feat: Basic scaffolding up for inference process after training. * feat: Saving and loading models works. Inference works nearly. * feat: Added inference pipeline. * feat: Saving model now accoring to date and time; loading models now selects from latest file. Fixed the creation of dictionary of models. * feat: Added lightweight asset config, but full pipeline. * feat: Added new naming for dictionary. * fix: Fixed dictionary naming convention. * fix: Fixed naming again, now the model structure is good * fix: Changed the output path and the return values from run_pipeline. * feat: Added function to make sure folder exists for output models. Co-authored-by: Daniel Szemerey <szemereydaniel@gmail.com>
38 lines
1.3 KiB
Python
38 lines
1.3 KiB
Python
import pickle
|
|
import datetime
|
|
from typing import Optional, Union
|
|
import os
|
|
import warnings
|
|
|
|
|
|
|
|
def save_models(all_models_for_all_assets: dict, data_config:dict, training_config:dict) -> None:
|
|
all_models_for_all_assets['training_config'] = training_config
|
|
all_models_for_all_assets['data_config'] = data_config
|
|
|
|
date_string = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")
|
|
|
|
if not os.path.exists('output/models'):
|
|
warnings.warn("No folder exists, creating one.")
|
|
os.makedirs('output/models')
|
|
|
|
pickle.dump( all_models_for_all_assets, open( "output/models/{}.p".format(date_string), "wb" ) )
|
|
|
|
|
|
def load_models(file_name:Union[str, None]) -> tuple[dict, dict, dict]:
|
|
|
|
if file_name is None:
|
|
warnings.warn("No file name provided, will load latest models and configurations.")
|
|
files_in_directory:list = os.listdir('output/models')
|
|
|
|
assert len(files_in_directory) > 0, "No models found in output/models."
|
|
file_name = sorted(files_in_directory)[-1]
|
|
|
|
all_models_for_all_assets = pickle.load( open( "output/models/{}".format(file_name), "rb" ) )
|
|
|
|
data_config = all_models_for_all_assets['data_config']
|
|
training_config = all_models_for_all_assets['training_config']
|
|
|
|
return all_models_for_all_assets, data_config, training_config
|
|
|