feat: Added Weight and Biases single run logging. (#58)

* feat: initial wandb configured. Sweep parameters aren't configured yet.

* feat: Wandb logs now results.

* feat: gitignore.

* fix: Took out print()

* feat: Changed default value of wandb to False.

* feat: Added wandb to turn of automatically if there is no environment variable to start it (when we push it). Added environment configuration aswell.

* feat: Each assets model is seperated into a run that tracks the results.

* fix: Nonetype error, truncated assets.

* fix: Fixed the logging to wandb.

run_pipeline.py

@@ -17,17 +17,17 @@ from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTree
 import feature_extractors.feature_extractor_presets as feature_extractor_presets
 from training.pipeline import run_single_asset_trainig_pipeline
 
-
+from typing import Tuple
-WANDB=True
 
 
-# Parameters
+def get_config()->Tuple[dict, dict, dict]:
-model_config = dict(
+    # Parameters
+    model_config = dict(
         regression_models = [
             # ('Lasso', Lasso(alpha=0.1, max_iter=1000)),
             ('Ridge', Ridge(alpha=0.1)),
             ('BayesianRidge', BayesianRidge()),
-        ('KNN', KNeighborsRegressor(n_neighbors=25)),
+            # ('KNN', KNeighborsRegressor(n_neighbors=25)),
             # ('AB', AdaBoostRegressor(random_state=1)),
             # ('LR', LinearRegression(n_jobs=-1)),
             # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
@@ -38,63 +38,75 @@ model_config = dict(
 
         classification_models = [
             ('LR', LogisticRegression(n_jobs=-1)),
-        ('LDA', LinearDiscriminantAnalysis()),
+            # ('LDA', LinearDiscriminantAnalysis()),
-        ('KNN', KNeighborsClassifier()),
+            # ('KNN', KNeighborsClassifier()),
-        ('CART', DecisionTreeClassifier()),
+            # ('CART', DecisionTreeClassifier()),
-        ('NB', GaussianNB()),
+            # ('NB', GaussianNB()),
             # ('AB', AdaBoostClassifier()),
             # ('RF', RandomForestClassifier(n_jobs=-1))
         ],
         classification_ensemble_model = [('Ensemble - CART', DecisionTreeClassifier())]
-)
+    )
 
-training_config = dict(
+    training_config = dict(
-    path = 'data/',
+        # path = 'data/',
         sliding_window_size = 150,
         retrain_every = 20,
         scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
         include_original_data_in_ensemble = True,
-    method = 'classification',
+        # method = 'regression',
-    forecasting_horizon = 1)
+        # forecasting_horizon = 1
+        )
 
-feature_extractors = feature_extractor_presets.date + feature_extractor_presets.level1
+    data_config = dict(
-data_config = dict(
+        path='data/',
-    path=training_config['path'],
+        all_assets = get_crypto_assets('data/'),
-    all_assets = get_crypto_assets(training_config['path']),
         load_other_assets= False,
         log_returns= True,
-    forecasting_horizon = training_config['forecasting_horizon'],
+        forecasting_horizon = 1,
-    own_features= feature_extractors,
+        own_features= feature_extractor_presets.date + feature_extractor_presets.level1,
         other_features= [],
         index_column= 'int',
-    method= training_config['method'],
+        method= 'regression',
-)
+    )
     
+    return model_config, training_config, data_config
 
-if WANDB:
+def launch_wandb(config, sweep=False):
         from wandb_setup import get_wandb
         wandb = get_wandb()   
         
         if type(wandb) == type(None): 
-        WANDB = False
+            return None
+        elif sweep:
+            wandb.init(project="price-forecasting",  config = config)             
+            return wandb
         else:
-        ''' 3. Initialize Weights and Biases with default values, then grab the config file (necessary for sweep) '''
+            wandb.init(project="price-forecasting", config=config, reinit=True)
-        wandb.init(project="price-forecasting",  
+            return wandb
-                config={"data_config":data_config, "training_config":training_config, "model_config": model_config}) # default config
-
-        training_config = wandb.config['training_config']
-        # vvv this doesnt work, wandb casts the functions to strings vvv
-        # data_config = wandb.config['data_config'] 
-        # model_config = wandb.config['model_config'] 
     
 
+def run_pipeline(with_wandb, sweep):
+    model_config, training_config, data_config = get_config()
+    
+    wandb = None
+    if with_wandb: 
+        wandb = launch_wandb(dict(**model_config, **training_config, **data_config), sweep)
+        
+        if type(wandb) is not type(None):
+            for k in training_config: training_config[k] = wandb.config[k]
+        # for k in model_config: model_config[k] = wandb.config[k]
+        # for k in data_config: data_config[k] = wandb.config[k]
+            
+    pipeline(model_config, training_config, data_config, wandb)  
     
 
 # Run pipeline
 
-results = pd.DataFrame()
+def pipeline(model_config:dict, training_config:dict, data_config:dict, wandb):
+    results = pd.DataFrame()
 
-for asset in data_config['all_assets']:
+    for asset in data_config['all_assets']:
         print('--------\nPredicting: ', asset)
         all_predictions = pd.DataFrame()
 
@@ -110,11 +122,12 @@ for asset in data_config['all_assets']:
             X = X,
             y = y,
             target_returns = target_returns,
-        models = model_config['regression_models'] if training_config['method'] == 'regression' else model_config['classification_models'],
+            models = model_config['regression_models'] if data_config['method'] == 'regression' else model_config['classification_models'],
-        method = training_config['method'],
+            method = data_config['method'],
             sliding_window_size = training_config['sliding_window_size'],
             retrain_every =  training_config['retrain_every'],
-        scaler =  training_config['scaler']
+            scaler =  training_config['scaler'],
+            wandb = wandb
         )
         results = pd.concat([results, current_result], axis=1)
         all_predictions = pd.concat([all_predictions, current_predictions], axis=1)
@@ -129,28 +142,27 @@ for asset in data_config['all_assets']:
             X = ensemble_X,
             y = y,
             target_returns = target_returns,
-        models = model_config['regression_ensemble_model'] if training_config['method'] == 'regression' else model_config['classification_ensemble_model'],
+            models = model_config['regression_ensemble_model'] if data_config['method'] == 'regression' else model_config['classification_ensemble_model'],
-        method = training_config['method'],
+            method = data_config['method'],
             sliding_window_size = training_config['sliding_window_size'],
             retrain_every = training_config['retrain_every'],
-        scaler = training_config['scaler']
+            scaler = training_config['scaler'],
+            wandb = wandb
         )
 
         results = pd.concat([results, ensemble_result], axis=1)
         all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
 
-if WANDB: 
+
-    combined_metrics = results.mean(axis=1)
+    results.to_csv('results.csv')
-    wandb.log({'results': results})
+
+    level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]
+    ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]]
+
+    print("Mean Sharpe ratio for Level-1 models: ", level1_columns.loc['sharpe'].mean())
+    print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", ensemble_columns.loc['sharpe'].mean())
 
     
-    if wandb.run is not None:
-        wandb.finish()
     
-results.to_csv('results.csv')
+if __name__ == '__main__':
-
+    run_pipeline(False, False)
-level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]
-ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]]
-
-print("Mean Sharpe ratio for Level-1 models: ", level1_columns.loc['sharpe'].mean())
-print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", ensemble_columns.loc['sharpe'].mean())

training/pipeline.py

@@ -25,6 +25,7 @@ def run_single_asset_trainig_pipeline(
                     sliding_window_size: int,
                     retrain_every: int,
                     scaler: Literal['normalize', 'minmax', 'standardize', 'none'],
+                    wandb
                     ) -> tuple[pd.DataFrame, pd.DataFrame]:
 
 
@@ -33,8 +34,9 @@ def run_single_asset_trainig_pipeline(
     results = pd.DataFrame()
     predictions = pd.DataFrame()
     
-    for model_name, model in models:
+    wandb_active = type(wandb) is not type(None)
 
+    for model_name, model in models:
         model_over_time, preds = walk_forward_train_test(
             model_name=model_name,
             model = model,
@@ -56,4 +58,14 @@ def run_single_asset_trainig_pipeline(
         results[column_name] = result
         predictions[column_name] = preds
         
+        if wandb_active:
+            run = wandb.init(project="price-forecasting", config={"model_type": model_name, "ticker": ticker_to_predict}, reinit=True)
+            wandb.run.name = ticker_to_predict + "-" + model_name+ "-" + wandb.run.id
+            wandb.run.save()
+            
+            for rownum,(indx,val) in enumerate(result.iteritems()):
+                run.log({"model_type": model_name, indx:val })
+                
+            run.finish()
+
     return results, predictions