feat: Added Weight and Biases single run logging. (#58)

* feat: initial wandb configured. Sweep parameters aren't configured yet.

* feat: Wandb logs now results.

* feat: gitignore.

* fix: Took out print()

* feat: Changed default value of wandb to False.

* feat: Added wandb to turn of automatically if there is no environment variable to start it (when we push it). Added environment configuration aswell.

* feat: Each assets model is seperated into a run that tracks the results.

* fix: Nonetype error, truncated assets.

* fix: Fixed the logging to wandb.

run_pipeline.py

@@ -17,140 +17,152 @@ from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTree
 import feature_extractors.feature_extractor_presets as feature_extractor_presets
 from training.pipeline import run_single_asset_trainig_pipeline
 
-
+from typing import Tuple
-WANDB=True
 
 
-# Parameters
+def get_config()->Tuple[dict, dict, dict]:
-model_config = dict(
+    # Parameters
-    regression_models = [
+    model_config = dict(
-        # ('Lasso', Lasso(alpha=0.1, max_iter=1000)),
+        regression_models = [
-        ('Ridge', Ridge(alpha=0.1)),
+            # ('Lasso', Lasso(alpha=0.1, max_iter=1000)),
-        ('BayesianRidge', BayesianRidge()),
+            ('Ridge', Ridge(alpha=0.1)),
-        ('KNN', KNeighborsRegressor(n_neighbors=25)),
+            ('BayesianRidge', BayesianRidge()),
-        # ('AB', AdaBoostRegressor(random_state=1)),
+            # ('KNN', KNeighborsRegressor(n_neighbors=25)),
-        # ('LR', LinearRegression(n_jobs=-1)),
+            # ('AB', AdaBoostRegressor(random_state=1)),
-        # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
+            # ('LR', LinearRegression(n_jobs=-1)),
-        # ('RF', RandomForestRegressor(n_jobs=-1)),
+            # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
-        # ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1))
+            # ('RF', RandomForestRegressor(n_jobs=-1)),
-    ],
+            # ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1))
-    regression_ensemble_model = [('Ensemble - Ridge', Ridge(alpha=0.1))],
+        ],
+        regression_ensemble_model = [('Ensemble - Ridge', Ridge(alpha=0.1))],
 
-    classification_models = [
+        classification_models = [
-        ('LR', LogisticRegression(n_jobs=-1)),
+            ('LR', LogisticRegression(n_jobs=-1)),
-        ('LDA', LinearDiscriminantAnalysis()),
+            # ('LDA', LinearDiscriminantAnalysis()),
-        ('KNN', KNeighborsClassifier()),
+            # ('KNN', KNeighborsClassifier()),
-        ('CART', DecisionTreeClassifier()),
+            # ('CART', DecisionTreeClassifier()),
-        ('NB', GaussianNB()),
+            # ('NB', GaussianNB()),
-        # ('AB', AdaBoostClassifier()),
+            # ('AB', AdaBoostClassifier()),
-        # ('RF', RandomForestClassifier(n_jobs=-1))
+            # ('RF', RandomForestClassifier(n_jobs=-1))
-    ],
+        ],
-    classification_ensemble_model = [('Ensemble - CART', DecisionTreeClassifier())]
+        classification_ensemble_model = [('Ensemble - CART', DecisionTreeClassifier())]
-)
+    )
 
-training_config = dict(
+    training_config = dict(
-    path = 'data/',
+        # path = 'data/',
-    sliding_window_size = 150,
+        sliding_window_size = 150,
-    retrain_every = 20,
+        retrain_every = 20,
-    scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
+        scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
-    include_original_data_in_ensemble = True,
+        include_original_data_in_ensemble = True,
-    method = 'classification',
+        # method = 'regression',
-    forecasting_horizon = 1)
+        # forecasting_horizon = 1
+        )
 
-feature_extractors = feature_extractor_presets.date + feature_extractor_presets.level1
+    data_config = dict(
-data_config = dict(
+        path='data/',
-    path=training_config['path'],
+        all_assets = get_crypto_assets('data/'),
-    all_assets = get_crypto_assets(training_config['path']),
+        load_other_assets= False,
-    load_other_assets= False,
+        log_returns= True,
-    log_returns= True,
+        forecasting_horizon = 1,
-    forecasting_horizon = training_config['forecasting_horizon'],
+        own_features= feature_extractor_presets.date + feature_extractor_presets.level1,
-    own_features= feature_extractors,
+        other_features= [],
-    other_features= [],
+        index_column= 'int',
-    index_column= 'int',
+        method= 'regression',
-    method= training_config['method'],
+    )
-)
+    
+    return model_config, training_config, data_config
+
+def launch_wandb(config, sweep=False):
+        from wandb_setup import get_wandb
+        wandb = get_wandb()   
+        
+        if type(wandb) == type(None): 
+            return None
+        elif sweep:
+            wandb.init(project="price-forecasting",  config = config)             
+            return wandb
+        else:
+            wandb.init(project="price-forecasting", config=config, reinit=True)
+            return wandb
     
 
-if WANDB:
+def run_pipeline(with_wandb, sweep):
-    from wandb_setup import get_wandb
+    model_config, training_config, data_config = get_config()
-    wandb = get_wandb()   
     
-    if type(wandb) == type(None):
+    wandb = None
-        WANDB = False
+    if with_wandb: 
-    else:
+        wandb = launch_wandb(dict(**model_config, **training_config, **data_config), sweep)
-        ''' 3. Initialize Weights and Biases with default values, then grab the config file (necessary for sweep) '''
-        wandb.init(project="price-forecasting",  
-                config={"data_config":data_config, "training_config":training_config, "model_config": model_config}) # default config
-
-        training_config = wandb.config['training_config']
-        # vvv this doesnt work, wandb casts the functions to strings vvv
-        # data_config = wandb.config['data_config'] 
-        # model_config = wandb.config['model_config'] 
         
+        if type(wandb) is not type(None):
+            for k in training_config: training_config[k] = wandb.config[k]
+        # for k in model_config: model_config[k] = wandb.config[k]
+        # for k in data_config: data_config[k] = wandb.config[k]
             
+    pipeline(model_config, training_config, data_config, wandb)  
     
 
 # Run pipeline
 
-results = pd.DataFrame()
+def pipeline(model_config:dict, training_config:dict, data_config:dict, wandb):
+    results = pd.DataFrame()
 
-for asset in data_config['all_assets']:
+    for asset in data_config['all_assets']:
-    print('--------\nPredicting: ', asset)
+        print('--------\nPredicting: ', asset)
-    all_predictions = pd.DataFrame()
+        all_predictions = pd.DataFrame()
 
-    # 1. Load data
+        # 1. Load data
-    data_params = data_config.copy()
+        data_params = data_config.copy()
-    data_params['target_asset'] = asset
+        data_params['target_asset'] = asset
 
-    X, y, target_returns = load_data(**data_params)
+        X, y, target_returns = load_data(**data_params)
 
-    # 2. Train Level-1 models
+        # 2. Train Level-1 models
-    current_result, current_predictions = run_single_asset_trainig_pipeline(
+        current_result, current_predictions = run_single_asset_trainig_pipeline(
-        ticker_to_predict = asset,
+            ticker_to_predict = asset,
-        X = X,
+            X = X,
-        y = y,
+            y = y,
-        target_returns = target_returns,
+            target_returns = target_returns,
-        models = model_config['regression_models'] if training_config['method'] == 'regression' else model_config['classification_models'],
+            models = model_config['regression_models'] if data_config['method'] == 'regression' else model_config['classification_models'],
-        method = training_config['method'],
+            method = data_config['method'],
-        sliding_window_size = training_config['sliding_window_size'],
+            sliding_window_size = training_config['sliding_window_size'],
-        retrain_every =  training_config['retrain_every'],
+            retrain_every =  training_config['retrain_every'],
-        scaler =  training_config['scaler']
+            scaler =  training_config['scaler'],
-    )
+            wandb = wandb
-    results = pd.concat([results, current_result], axis=1)
+        )
-    all_predictions = pd.concat([all_predictions, current_predictions], axis=1)
+        results = pd.concat([results, current_result], axis=1)
+        all_predictions = pd.concat([all_predictions, current_predictions], axis=1)
 
-    # 3. Train Level-2 (Ensemble) model
+        # 3. Train Level-2 (Ensemble) model
-    ensemble_X = all_predictions
+        ensemble_X = all_predictions
-    if training_config['include_original_data_in_ensemble']:
+        if training_config['include_original_data_in_ensemble']:
-        ensemble_X = pd.concat([ensemble_X, X], axis=1)
+            ensemble_X = pd.concat([ensemble_X, X], axis=1)
 
-    ensemble_result, ensemble_preds = run_single_asset_trainig_pipeline(
+        ensemble_result, ensemble_preds = run_single_asset_trainig_pipeline(
-        ticker_to_predict = asset,
+            ticker_to_predict = asset,
-        X = ensemble_X,
+            X = ensemble_X,
-        y = y,
+            y = y,
-        target_returns = target_returns,
+            target_returns = target_returns,
-        models = model_config['regression_ensemble_model'] if training_config['method'] == 'regression' else model_config['classification_ensemble_model'],
+            models = model_config['regression_ensemble_model'] if data_config['method'] == 'regression' else model_config['classification_ensemble_model'],
-        method = training_config['method'],
+            method = data_config['method'],
-        sliding_window_size = training_config['sliding_window_size'],
+            sliding_window_size = training_config['sliding_window_size'],
-        retrain_every = training_config['retrain_every'],
+            retrain_every = training_config['retrain_every'],
-        scaler = training_config['scaler']
+            scaler = training_config['scaler'],
-    )
+            wandb = wandb
+        )
 
-    results = pd.concat([results, ensemble_result], axis=1)
+        results = pd.concat([results, ensemble_result], axis=1)
-    all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
+        all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
-
-if WANDB: 
-    combined_metrics = results.mean(axis=1)
-    wandb.log({'results': results})
 
 
-    if wandb.run is not None:
+    results.to_csv('results.csv')
-        wandb.finish()
 
-results.to_csv('results.csv')
+    level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]
+    ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]]
 
-level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]
+    print("Mean Sharpe ratio for Level-1 models: ", level1_columns.loc['sharpe'].mean())
-ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]]
+    print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", ensemble_columns.loc['sharpe'].mean())
 
-print("Mean Sharpe ratio for Level-1 models: ", level1_columns.loc['sharpe'].mean())
+    
-print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", ensemble_columns.loc['sharpe'].mean())
+    
+if __name__ == '__main__':
+    run_pipeline(False, False)

training/pipeline.py

@@ -25,6 +25,7 @@ def run_single_asset_trainig_pipeline(
                     sliding_window_size: int,
                     retrain_every: int,
                     scaler: Literal['normalize', 'minmax', 'standardize', 'none'],
+                    wandb
                     ) -> tuple[pd.DataFrame, pd.DataFrame]:
 
 
@@ -33,8 +34,9 @@ def run_single_asset_trainig_pipeline(
     results = pd.DataFrame()
     predictions = pd.DataFrame()
     
-    for model_name, model in models:
+    wandb_active = type(wandb) is not type(None)
 
+    for model_name, model in models:
         model_over_time, preds = walk_forward_train_test(
             model_name=model_name,
             model = model,
@@ -56,4 +58,14 @@ def run_single_asset_trainig_pipeline(
         results[column_name] = result
         predictions[column_name] = preds
         
+        if wandb_active:
+            run = wandb.init(project="price-forecasting", config={"model_type": model_name, "ticker": ticker_to_predict}, reinit=True)
+            wandb.run.name = ticker_to_predict + "-" + model_name+ "-" + wandb.run.id
+            wandb.run.save()
+            
+            for rownum,(indx,val) in enumerate(result.iteritems()):
+                run.log({"model_type": model_name, indx:val })
+                
+            run.finish()
+
     return results, predictions