diff --git a/default_config.py b/config.py similarity index 61% rename from default_config.py rename to config.py index 18f7a2c..c714ca4 100644 --- a/default_config.py +++ b/config.py @@ -35,4 +35,22 @@ def get_default_config() -> tuple[dict, dict, dict]: level_2_models = regression_ensemble_models if data_config['method'] == 'regression' else classification_ensemble_models ) - return model_config, training_config, data_config \ No newline at end of file + return model_config, training_config, data_config + + +def validate_config(model_config:dict, training_config:dict, data_config:dict): + # We need to make sure there's only one output from the pipeline + # We're not prepared for more than 1 level-2 models at the moment + assert len(model_config["level_2_models"]) <= 1 + # If level-2 model is there, we need more than one level-1 models to train + if len(model_config["level_2_models"]) == 1: assert len(model_config["level_1_models"]) > 0 + # If there's no level-2 model, we need to have only one level-1 model + if len(model_config["level_2_models"]) == 0: assert len(model_config["level_1_models"]) == 1 + +def get_model_name(model_config:dict) -> str: + if len(model_config["level_2_models"]) == 1: + return model_config["level_2_models"][0][0] + elif len(model_config["level_1_models"]) == 1: + return model_config["level_1_models"][0][0] + else: + raise Exception("No model name found") \ No newline at end of file diff --git a/models/model_map.py b/models/model_map.py index 4f39d5a..df81f65 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -46,8 +46,8 @@ model_map = { ) } -model_names_classification = model_map["classification_models"].keys() -model_names_regression = model_map["regression_models"].keys() +model_names_classification = list(model_map["classification_models"].keys()) +model_names_regression = list(model_map["regression_models"].keys()) def map_model_name_to_function(model_config:dict, method:str) -> dict: diff --git a/reporting/wandb.py b/reporting/wandb.py new file mode 100644 index 0000000..4a0abf8 --- /dev/null +++ b/reporting/wandb.py @@ -0,0 +1,43 @@ +import pandas as pd +from typing import Optional + +def launch_wandb(project_name:str, default_config:dict, sweep:bool=False): + from wandb_setup import get_wandb + wandb = get_wandb() + + if wandb is None: + return None + elif sweep: + wandb.init(project=project_name, config = default_config) + return wandb + else: + wandb.init(project=project_name, config = default_config, reinit=True) + return wandb + + +def seperate_configs(wandb: Optional[object], model_config:dict, training_config:dict, data_config:dict) -> tuple[dict,dict,dict]: + config: dict = wandb.config + + if type(wandb) is not type(None): + for k in training_config: training_config[k] = config[k] + for k in model_config: model_config[k] = config[k] + # for k in data_config: data_config[k] = config[k] + + return model_config, training_config, data_config + + +def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_name: str, model_name: str): + if wandb is None: return + + run = wandb.init(project=project_name, config={"model_type": model_name}, reinit=True) + wandb.run.name = model_name+ "-" + wandb.run.id + wandb.run.save() + + mean_results = results.mean(axis = 1) + for key, value in mean_results.iteritems(): + run.log({"model_type": model_name, key: value }) + + run.finish() + + + \ No newline at end of file diff --git a/run_pipeline.py b/run_pipeline.py index 159f2da..3c1234e 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -1,9 +1,9 @@ from utils.load_data import load_data import pandas as pd from training.training import run_single_asset_trainig -from utils.launch_wandb import launch_wandb, seperate_configs +from reporting.wandb import launch_wandb, send_report_to_wandb, seperate_configs from models.model_map import map_model_name_to_function -from default_config import get_default_config +from config import get_default_config, validate_config, get_model_name def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): model_config, training_config, data_config = get_default_config() @@ -17,8 +17,10 @@ def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): pipeline(project_name, wandb, sweep, model_config, training_config, data_config) + def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict ): results = pd.DataFrame() + validate_config(model_config, training_config, data_config) for asset in data_config['all_assets']: print('--------\nPredicting: ', asset) @@ -72,7 +74,9 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co results = pd.concat([results, ensemble_result], axis=1) all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1) - + + # 4. Save & report results + send_report_to_wandb(results, wandb, project_name, get_model_name(model_config)) results.to_csv('results.csv') level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]] diff --git a/sweep.yaml b/sweep.yaml index 167bee6..0487469 100644 --- a/sweep.yaml +++ b/sweep.yaml @@ -1,5 +1,5 @@ program: run_sweep.py -method: grid +method: bayes project: price-forecasting name: Finding best hyperparameters for price prediction # early_terminate: @@ -12,10 +12,10 @@ parameters: path : value: 'data/' sliding_window_size: - values: [50, 90, 130, 160, 180, 280, 380, 500] + values: [90, 130, 160, 180, 280, 380] distribution: categorical retrain_every: - values: [7, 14, 30, 60, 100] + values: [14, 30, 60, 100] distribution: categorical scaler: values: ['minmax', 'normalize', 'minmax', 'standardize', 'none'] @@ -24,8 +24,7 @@ parameters: values: [True, False] distribution: categorical method: - values: ['classification', 'regression'] - distribution: categorical + value: 'classification' forecasting_horizon: values: [1,2,3,4,5,6,7,8,9,10] distribution: categorical @@ -42,20 +41,9 @@ parameters: index_column: value: 'int' level_1_models: - value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF"] - distribution: constant + values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"]] + distribution: categorical level_2_models: - value: ['Ensemble_Average'] + value: [] distribution: constant - - - - - - - - - - - diff --git a/training/training.py b/training/training.py index e299dc1..bbd3348 100644 --- a/training/training.py +++ b/training/training.py @@ -61,21 +61,5 @@ def run_single_asset_trainig( # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary predictions["model_" + column_name] = preds - if wandb_active and not sweep: - run = wandb.init(project=project_name, config={"model_type": model_name, "ticker": ticker_to_predict}, reinit=True) - wandb.run.name = ticker_to_predict + "-" + model_name+ "-" + wandb.run.id - wandb.run.save() - - for rownum,(indx,val) in enumerate(result.iteritems()): - run.log({"model_type": model_name, indx:val }) - - run.finish() - - if wandb_active and sweep: - mean_results = results.mean() - - wandb.log({"model_type": 'avarage_model', 'results':results }) - for rownum,(indx,val) in enumerate(mean_results.iteritems()): - wandb.log({"model_type": 'avarage_model', indx:val }) return results, predictions \ No newline at end of file diff --git a/utils/get_prices.py b/utils/get_prices.py index 12ccdc2..a63d2aa 100644 --- a/utils/get_prices.py +++ b/utils/get_prices.py @@ -2,9 +2,6 @@ import requests import pandas as pd -AV_API_KEY = 'UY5VGSWBE88SHGI6' -CC_API_KEY = 'bfb8b5f54b21354608020a6654b370617b2fcabd2c8c2ce04ab881682a1d9dc9' - # %% def get_crypto_price_crypto_compare(symbol: str, exchange: str, days: int) -> pd.DataFrame: api_url = f'https://min-api.cryptocompare.com/data/v2/histoday?fsym={symbol}&tsym={exchange}&limit={days}&api_key={CC_API_KEY}' diff --git a/utils/launch_wandb.py b/utils/launch_wandb.py deleted file mode 100644 index d2c802c..0000000 --- a/utils/launch_wandb.py +++ /dev/null @@ -1,27 +0,0 @@ - - -def launch_wandb(project_name:str, default_config:dict, sweep:bool=False): - from wandb_setup import get_wandb - wandb = get_wandb() - - if type(wandb) == type(None): - return None - elif sweep: - wandb.init(project=project_name, config = default_config) - return wandb - else: - wandb.init(project=project_name, config = default_config, reinit=True) - return wandb - - -def seperate_configs(wandb, model_config:dict, training_config:dict, data_config:dict) -> tuple[dict,dict,dict]: - config:dict = wandb.config - - if type(wandb) is not type(None): - for k in training_config: training_config[k] = config[k] - for k in model_config: model_config[k] = config[k] - # for k in data_config: data_config[k] = config[k] - - return model_config, training_config, data_config - -