feature: Added Weights and Biases configuration to the repo. (#48)

* feat: initial wandb configured. Sweep parameters aren't configured yet.

* feat: Wandb logs now results.

* feat: gitignore.

* fix: Took out print()

* feat: Changed default value of wandb to False.

* feat: Added wandb to turn of automatically if there is no environment variable to start it (when we push it). Added environment configuration aswell.

* fix(Dependencies): the package name seems to be python-dotenv

Co-authored-by: Mark Aron Szulyovszky <mark.szulyovszky@gmail.com>

.gitignore

@@ -129,4 +129,5 @@ dmypy.json
 .pyre/
 lightning/lightning_logs/
 
-results.csv
+results.csv
+wandb/

environment.yml

@@ -17,4 +17,6 @@ dependencies:
   - quantstats
   - pytorch-lightning
   - pytest
+  - wandb
+  - python-dotenv
 prefix: /usr/local/anaconda3/envs/quant

run_pipeline.py

@@ -18,62 +18,88 @@ import feature_extractors.feature_extractor_presets as feature_extractor_presets
 from training.pipeline import run_single_asset_trainig_pipeline
 
 
+WANDB=True
+
+
 # Parameters
-regression_models = [
-    # ('Lasso', Lasso(alpha=0.1, max_iter=1000)),
-    ('Ridge', Ridge(alpha=0.1)),
-    ('BayesianRidge', BayesianRidge()),
-    ('KNN', KNeighborsRegressor(n_neighbors=25)),
-    # ('AB', AdaBoostRegressor(random_state=1)),
-    # ('LR', LinearRegression(n_jobs=-1)),
-    # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
-    # ('RF', RandomForestRegressor(n_jobs=-1)),
-    # ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1))
-]
-regression_ensemble_model = [('Ensemble - Ridge', Ridge(alpha=0.1))]
+model_config = dict(
+    regression_models = [
+        # ('Lasso', Lasso(alpha=0.1, max_iter=1000)),
+        ('Ridge', Ridge(alpha=0.1)),
+        ('BayesianRidge', BayesianRidge()),
+        ('KNN', KNeighborsRegressor(n_neighbors=25)),
+        # ('AB', AdaBoostRegressor(random_state=1)),
+        # ('LR', LinearRegression(n_jobs=-1)),
+        # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
+        # ('RF', RandomForestRegressor(n_jobs=-1)),
+        # ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1))
+    ],
+    regression_ensemble_model = [('Ensemble - Ridge', Ridge(alpha=0.1))],
 
-classification_models = [
-    ('LR', LogisticRegression(n_jobs=-1)),
-    ('LDA', LinearDiscriminantAnalysis()),
-    ('KNN', KNeighborsClassifier()),
-    ('CART', DecisionTreeClassifier()),
-    ('NB', GaussianNB()),
-    # ('AB', AdaBoostClassifier()),
-    # ('RF', RandomForestClassifier(n_jobs=-1))
-]
-classification_ensemble_model = [('Ensemble - CART', DecisionTreeClassifier())]
+    classification_models = [
+        ('LR', LogisticRegression(n_jobs=-1)),
+        ('LDA', LinearDiscriminantAnalysis()),
+        ('KNN', KNeighborsClassifier()),
+        ('CART', DecisionTreeClassifier()),
+        ('NB', GaussianNB()),
+        # ('AB', AdaBoostClassifier()),
+        # ('RF', RandomForestClassifier(n_jobs=-1))
+    ],
+    classification_ensemble_model = [('Ensemble - CART', DecisionTreeClassifier())]
+)
 
+training_config = dict(
+    path = 'data/',
+    sliding_window_size = 150,
+    retrain_every = 20,
+    scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
+    include_original_data_in_ensemble = True,
+    method = 'classification',
+    forecasting_horizon = 1)
 
-path = 'data/'
-all_assets = get_crypto_assets(path)
-
-sliding_window_size = 150
-retrain_every = 20
-scaler = 'minmax' # 'normalize' 'minmax' 'standardize' 'none'
-include_original_data_in_ensemble = True
-method = 'classification'
-forecasting_horizon = 1
-data_parameters = dict(path=path,
+feature_extractors = feature_extractor_presets.date + feature_extractor_presets.level1
+data_config = dict(
+    path=training_config['path'],
+    all_assets = get_crypto_assets(training_config['path']),
     load_other_assets= False,
     log_returns= True,
-    forecasting_horizon = forecasting_horizon,
-    own_features= feature_extractor_presets.date + feature_extractor_presets.level1,
+    forecasting_horizon = training_config['forecasting_horizon'],
+    own_features= feature_extractors,
     other_features= [],
     index_column= 'int',
-    method= method,
+    method= training_config['method'],
 )
 
 
+if WANDB:
+    from wandb_setup import get_wandb
+    wandb = get_wandb()   
+    
+    if type(wandb) == type(None):
+        WANDB = False
+    else:
+        ''' 3. Initialize Weights and Biases with default values, then grab the config file (necessary for sweep) '''
+        wandb.init(project="price-forecasting",  
+                config={"data_config":data_config, "training_config":training_config, "model_config": model_config}) # default config
+
+        training_config = wandb.config['training_config']
+        # vvv this doesnt work, wandb casts the functions to strings vvv
+        # data_config = wandb.config['data_config'] 
+        # model_config = wandb.config['model_config'] 
+
+        
+
+
 # Run pipeline
 
 results = pd.DataFrame()
 
-for asset in all_assets:
+for asset in data_config['all_assets']:
     print('--------\nPredicting: ', asset)
     all_predictions = pd.DataFrame()
 
     # 1. Load data
-    data_params = data_parameters.copy()
+    data_params = data_config.copy()
     data_params['target_asset'] = asset
 
     X, y, target_returns = load_data(**data_params)
@@ -84,18 +110,18 @@ for asset in all_assets:
         X = X,
         y = y,
         target_returns = target_returns,
-        models = regression_models if method == 'regression' else classification_models,
-        method = method,
-        sliding_window_size = sliding_window_size,
-        retrain_every = retrain_every,
-        scaler = scaler
+        models = model_config['regression_models'] if training_config['method'] == 'regression' else model_config['classification_models'],
+        method = training_config['method'],
+        sliding_window_size = training_config['sliding_window_size'],
+        retrain_every =  training_config['retrain_every'],
+        scaler =  training_config['scaler']
     )
     results = pd.concat([results, current_result], axis=1)
     all_predictions = pd.concat([all_predictions, current_predictions], axis=1)
 
     # 3. Train Level-2 (Ensemble) model
     ensemble_X = all_predictions
-    if include_original_data_in_ensemble:
+    if training_config['include_original_data_in_ensemble']:
         ensemble_X = pd.concat([ensemble_X, X], axis=1)
 
     ensemble_result, ensemble_preds = run_single_asset_trainig_pipeline(
@@ -103,16 +129,25 @@ for asset in all_assets:
         X = ensemble_X,
         y = y,
         target_returns = target_returns,
-        models = regression_ensemble_model if method == 'regression' else classification_ensemble_model,
-        method = method,
-        sliding_window_size = sliding_window_size,
-        retrain_every = retrain_every,
-        scaler = scaler
+        models = model_config['regression_ensemble_model'] if training_config['method'] == 'regression' else model_config['classification_ensemble_model'],
+        method = training_config['method'],
+        sliding_window_size = training_config['sliding_window_size'],
+        retrain_every = training_config['retrain_every'],
+        scaler = training_config['scaler']
     )
 
     results = pd.concat([results, ensemble_result], axis=1)
     all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
 
+if WANDB: 
+    combined_metrics = results.mean(axis=1)
+    wandb.log({'results': results})
+    wandb.log({'combined': combined_metrics})
+    
+    
+    if wandb.run is not None:
+        wandb.finish()
+
 results.to_csv('results.csv')
 
 level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]

sweep.yaml

@@ -0,0 +1,47 @@
+program: rnn_sweep.py
+method: bayes
+project: integer-sequence
+name: Finding best hyperparameters for price prediction
+early_terminate:
+  type: hyperband
+  min_iter: 2000
+metric:
+  goal: maximize
+  name: accuracy_test
+parameters:
+    path : 'data/'
+    sliding_window_size: 
+      values:
+        - 90
+        - 150
+        - 365
+        - 730
+      distribution: categorical
+    retrain_every: 
+      values:
+        - 7
+        - 14
+        - 30
+        - 60
+      distribution: categorical
+    scaler: 'minmax' # 'normalize' 'minmax' 'standardize' 'none'
+    include_original_data_in_ensemble: True
+    method: 
+      values:
+        - 'classification'
+        - 'regression'
+      distribution: categorical
+    forecasting_horizon: 
+      value: 1
+      distribution: constant
+
+
+
+
+
+
+
+
+
+
+

utils/load_data.py

@@ -27,6 +27,7 @@ def load_data(path: str,
             index_column: Literal['date', 'int'],
             method: Literal['regression', 'classification'],
             narrow_format: bool = False,
+            all_assets:list=[]
         ) -> tuple[pd.DataFrame, pd.Series, pd.Series]:
     """
     Loads asset data from the specified path.
@@ -35,6 +36,7 @@ def load_data(path: str,
         - Series `y` with the target asset returns shifted by 1 day OR if it's a classification problem, the target class)
         - Series `forward_returns` with the target asset returns shifted by 1 day
     """
+
     
     files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
     files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(target_asset))]

wandb_setup.py

@@ -0,0 +1,17 @@
+import wandb
+import os
+
+
+def get_wandb():
+    from dotenv import load_dotenv
+    load_dotenv()
+    
+    ''' 0. Login to Weights and Biases '''
+    wsb_token = os.environ.get('WANDB_API_KEY')
+    if wsb_token: 
+        wandb.login(key=wsb_token)
+        return wandb
+    else: return None #wandb.login()
+    
+    # return wandb
+