mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-28 03:08:01 +00:00
Daniel Szemerey
516c8bcc87
* fix, feat: Fixed inference processing data. Add transformation attribute. * feat: Added transformations step, refractored the loop to make more sense (divided the train and inference loop). * feat: Truncated models over time and transformations over time. Fixed some typing aswell. * fix: Fixed a number of out of array problems. * feat: Inference now works! * fix(Steps): runtime error not checking for None * fix(Steps): preloaded transformers are not optional anymore, sped up training by temporary increasing the retrain_every * fix(CI): disable ray memory monitoring * refactor(Inference): removed truncate_models and replaced it with filling X with NaN until inference should start * feat(Inference): added index_from parameter * fix(Tests): walk_forward test * refactor(Pipeline): only predict one asset * refactor(Inference): removed select_models step, inference code moved to run_inference.py so it matches convention (similar to run_pipeline.py) * fix(Evaluation): adjust transaction costs * fix(Config): adjusted retrain_every Co-authored-by: Daniel Szemerey <szemereydaniel@gmail.com> Co-authored-by: Mark Aron Szulyovszky <mark.szulyovszky@gmail.com>
44 lines
1.6 KiB
Python
44 lines
1.6 KiB
Python
import pickle
|
|
import datetime
|
|
from typing import Optional, Union
|
|
import os
|
|
import warnings
|
|
from reporting.types import Reporting
|
|
|
|
|
|
|
|
def save_models(all_models: Reporting.Asset, data_config:dict, training_config:dict, model_config:dict) -> None:
|
|
dict_for_pickle = dict()
|
|
dict_for_pickle['training_config'] = training_config
|
|
dict_for_pickle['data_config'] = data_config
|
|
dict_for_pickle['model_config'] = model_config
|
|
dict_for_pickle['all_models'] = all_models
|
|
|
|
date_string = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")
|
|
|
|
if not os.path.exists('output/models'):
|
|
warnings.warn("No folder exists, creating one.")
|
|
os.makedirs('output/models')
|
|
|
|
pickle.dump( dict_for_pickle, open( "output/models/{}.p".format(date_string), "wb" ) )
|
|
|
|
|
|
def load_models(file_name:Union[str, None]) -> tuple[Reporting.Asset, dict, dict, dict]:
|
|
|
|
if file_name is None:
|
|
warnings.warn("No file name provided, will load latest models and configurations.")
|
|
files_in_directory:list = os.listdir('output/models')
|
|
|
|
assert len(files_in_directory) > 0, "No models found in output/models."
|
|
file_name = sorted(files_in_directory)[-1]
|
|
|
|
packacked_dict = pickle.load( open( "output/models/{}".format(file_name), "rb" ) )
|
|
|
|
data_config = packacked_dict.pop("data_config", None)
|
|
training_config = packacked_dict.pop("training_config", None)
|
|
model_config = packacked_dict.pop("model_config", None)
|
|
all_models = packacked_dict.pop("all_models", None)
|
|
|
|
return all_models, data_config, training_config, model_config
|
|
|