refactor(Reporting): only report the last model's results, moved wandb-related functions to reporting (#69)

* refactor(Reporting): only report the last model's results, moved wandb-related functions to `reporting`

* fix(Reporting): use .mean() on axis 1 to retain the metrics, fixed get_model_name()

* fix(Config): sweep file syntax

* fix(Config): changed hyperparameter search method to "bayes"

* chore(Sweep): adjusted sweep config based on the results we saw (removed Momentum as well)

* fix(Sweep): only use classification method for now, we're not yet prepared for regression

config.py

@@ -35,4 +35,22 @@ def get_default_config() -> tuple[dict, dict, dict]:
         level_2_models = regression_ensemble_models if data_config['method'] == 'regression' else classification_ensemble_models
     )
     
-    return model_config, training_config, data_config
+    return model_config, training_config, data_config
+
+
+def validate_config(model_config:dict, training_config:dict, data_config:dict):
+    # We need to make sure there's only one output from the pipeline
+    # We're not prepared for more than 1 level-2 models at the moment
+    assert len(model_config["level_2_models"]) <= 1
+    # If level-2 model is there, we need more than one level-1 models to train
+    if len(model_config["level_2_models"]) == 1: assert len(model_config["level_1_models"]) > 0
+    # If there's no level-2 model, we need to have only one level-1 model
+    if len(model_config["level_2_models"]) == 0: assert len(model_config["level_1_models"]) == 1
+
+def get_model_name(model_config:dict) -> str:
+    if len(model_config["level_2_models"]) == 1:
+        return model_config["level_2_models"][0][0]
+    elif len(model_config["level_1_models"]) == 1:
+        return model_config["level_1_models"][0][0]
+    else:
+        raise Exception("No model name found")

models/model_map.py

@@ -46,8 +46,8 @@ model_map = {
     )
 }
 
-model_names_classification = model_map["classification_models"].keys()
-model_names_regression = model_map["regression_models"].keys()
+model_names_classification = list(model_map["classification_models"].keys())
+model_names_regression = list(model_map["regression_models"].keys())
 
 
 def map_model_name_to_function(model_config:dict, method:str) -> dict:

reporting/wandb.py

@@ -0,0 +1,43 @@
+import pandas as pd
+from typing import Optional
+
+def launch_wandb(project_name:str, default_config:dict, sweep:bool=False):
+    from wandb_setup import get_wandb
+    wandb = get_wandb()   
+    
+    if wandb is None: 
+        return None
+    elif sweep:
+        wandb.init(project=project_name,  config = default_config)             
+        return wandb
+    else:
+        wandb.init(project=project_name, config = default_config, reinit=True)
+        return wandb
+    
+    
+def seperate_configs(wandb: Optional[object], model_config:dict, training_config:dict, data_config:dict) -> tuple[dict,dict,dict]:
+    config: dict = wandb.config
+
+    if type(wandb) is not type(None):
+        for k in training_config: training_config[k] = config[k]
+        for k in model_config: model_config[k] = config[k]
+        # for k in data_config: data_config[k] = config[k]
+
+    return model_config, training_config, data_config
+
+
+def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_name: str, model_name: str):
+    if wandb is None: return
+
+    run = wandb.init(project=project_name, config={"model_type": model_name}, reinit=True)
+    wandb.run.name = model_name+ "-" + wandb.run.id
+    wandb.run.save()
+
+    mean_results = results.mean(axis = 1)
+    for key, value in mean_results.iteritems():
+        run.log({"model_type": model_name, key: value })
+
+    run.finish()
+
+
+        

run_pipeline.py

@@ -1,9 +1,9 @@
 from utils.load_data import load_data
 import pandas as pd
 from training.training import run_single_asset_trainig
-from utils.launch_wandb import launch_wandb, seperate_configs
+from reporting.wandb import launch_wandb, send_report_to_wandb, seperate_configs
 from models.model_map import map_model_name_to_function
-from default_config import get_default_config
+from config import get_default_config, validate_config, get_model_name
 
 def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
     model_config, training_config, data_config = get_default_config()
@@ -17,8 +17,10 @@ def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
     pipeline(project_name, wandb, sweep, model_config, training_config, data_config)  
     
 
+
 def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_config:dict, data_config:dict ):
     results = pd.DataFrame()
+    validate_config(model_config, training_config, data_config)
 
     for asset in data_config['all_assets']:
         print('--------\nPredicting: ', asset)
@@ -72,7 +74,9 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
 
             results = pd.concat([results, ensemble_result], axis=1)
             all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
-
+        
+    # 4. Save & report results
+    send_report_to_wandb(results, wandb, project_name, get_model_name(model_config))
     results.to_csv('results.csv')
 
     level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]

sweep.yaml

@@ -1,5 +1,5 @@
 program: run_sweep.py
-method: grid
+method: bayes
 project: price-forecasting
 name: Finding best hyperparameters for price prediction
 # early_terminate:
@@ -12,10 +12,10 @@ parameters:
     path : 
       value: 'data/'
     sliding_window_size: 
-      values: [50, 90, 130, 160, 180, 280, 380, 500]
+      values: [90, 130, 160, 180, 280, 380]
       distribution: categorical
     retrain_every: 
-      values: [7, 14, 30, 60, 100]
+      values: [14, 30, 60, 100]
       distribution: categorical
     scaler:
       values: ['minmax', 'normalize', 'minmax', 'standardize', 'none']
@@ -24,8 +24,7 @@ parameters:
       values: [True, False]
       distribution: categorical
     method: 
-      values: ['classification', 'regression']
-      distribution: categorical
+      value: 'classification'
     forecasting_horizon: 
       values: [1,2,3,4,5,6,7,8,9,10]
       distribution: categorical
@@ -42,20 +41,9 @@ parameters:
     index_column: 
       value: 'int'
     level_1_models: 
-      value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF"]
-      distribution: constant
+      values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"]]
+      distribution: categorical
     level_2_models: 
-      value: ['Ensemble_Average']
+      value: []
       distribution: constant
 
-
-
-
-
-
-
-
-
-
-
-

training/training.py

@@ -61,21 +61,5 @@ def run_single_asset_trainig(
         # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary
         predictions["model_" + column_name] = preds
         
-        if wandb_active and not sweep:
-                run = wandb.init(project=project_name, config={"model_type": model_name, "ticker": ticker_to_predict}, reinit=True)
-                wandb.run.name = ticker_to_predict + "-" + model_name+ "-" + wandb.run.id
-                wandb.run.save()
-            
-                for rownum,(indx,val) in enumerate(result.iteritems()):
-                    run.log({"model_type": model_name, indx:val })
-                    
-                run.finish()
-    
-    if wandb_active and sweep:
-        mean_results = results.mean()
-         
-        wandb.log({"model_type": 'avarage_model', 'results':results })
-        for rownum,(indx,val) in enumerate(mean_results.iteritems()):
-            wandb.log({"model_type": 'avarage_model', indx:val })
 
     return results, predictions

utils/get_prices.py

@@ -2,9 +2,6 @@
 import requests
 import pandas as pd
 
-AV_API_KEY = 'UY5VGSWBE88SHGI6'
-CC_API_KEY = 'bfb8b5f54b21354608020a6654b370617b2fcabd2c8c2ce04ab881682a1d9dc9'
-
 # %%
 def get_crypto_price_crypto_compare(symbol: str, exchange: str, days: int) -> pd.DataFrame:
     api_url = f'https://min-api.cryptocompare.com/data/v2/histoday?fsym={symbol}&tsym={exchange}&limit={days}&api_key={CC_API_KEY}'

utils/launch_wandb.py

@@ -1,27 +0,0 @@
-
-
-def launch_wandb(project_name:str, default_config:dict, sweep:bool=False):
-    from wandb_setup import get_wandb
-    wandb = get_wandb()   
-    
-    if type(wandb) == type(None): 
-        return None
-    elif sweep:
-        wandb.init(project=project_name,  config = default_config)             
-        return wandb
-    else:
-        wandb.init(project=project_name, config = default_config, reinit=True)
-        return wandb
-    
-    
-def seperate_configs(wandb, model_config:dict, training_config:dict, data_config:dict) -> tuple[dict,dict,dict]:
-    config:dict = wandb.config
-
-    if type(wandb) is not type(None):
-        for k in training_config: training_config[k] = config[k]
-        for k in model_config: model_config[k] = config[k]
-        # for k in data_config: data_config[k] = config[k]
-
-    return model_config, training_config, data_config
-
-