diff --git a/.gitignore b/.gitignore index 241e131..e266286 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,5 @@ dmypy.json .pyre/ lightning/lightning_logs/ -results.csv \ No newline at end of file +results.csv +wandb/ diff --git a/environment.yml b/environment.yml index 691d211..b7345c0 100644 --- a/environment.yml +++ b/environment.yml @@ -17,4 +17,6 @@ dependencies: - quantstats - pytorch-lightning - pytest + - wandb + - python-dotenv prefix: /usr/local/anaconda3/envs/quant diff --git a/run_pipeline.py b/run_pipeline.py index 512f833..661b96b 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -18,62 +18,88 @@ import feature_extractors.feature_extractor_presets as feature_extractor_presets from training.pipeline import run_single_asset_trainig_pipeline +WANDB=True + + # Parameters -regression_models = [ - # ('Lasso', Lasso(alpha=0.1, max_iter=1000)), - ('Ridge', Ridge(alpha=0.1)), - ('BayesianRidge', BayesianRidge()), - ('KNN', KNeighborsRegressor(n_neighbors=25)), - # ('AB', AdaBoostRegressor(random_state=1)), - # ('LR', LinearRegression(n_jobs=-1)), - # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)), - # ('RF', RandomForestRegressor(n_jobs=-1)), - # ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1)) -] -regression_ensemble_model = [('Ensemble - Ridge', Ridge(alpha=0.1))] +model_config = dict( + regression_models = [ + # ('Lasso', Lasso(alpha=0.1, max_iter=1000)), + ('Ridge', Ridge(alpha=0.1)), + ('BayesianRidge', BayesianRidge()), + ('KNN', KNeighborsRegressor(n_neighbors=25)), + # ('AB', AdaBoostRegressor(random_state=1)), + # ('LR', LinearRegression(n_jobs=-1)), + # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)), + # ('RF', RandomForestRegressor(n_jobs=-1)), + # ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1)) + ], + regression_ensemble_model = [('Ensemble - Ridge', Ridge(alpha=0.1))], -classification_models = [ - ('LR', LogisticRegression(n_jobs=-1)), - ('LDA', LinearDiscriminantAnalysis()), - ('KNN', KNeighborsClassifier()), - ('CART', DecisionTreeClassifier()), - ('NB', GaussianNB()), - # ('AB', AdaBoostClassifier()), - # ('RF', RandomForestClassifier(n_jobs=-1)) -] -classification_ensemble_model = [('Ensemble - CART', DecisionTreeClassifier())] + classification_models = [ + ('LR', LogisticRegression(n_jobs=-1)), + ('LDA', LinearDiscriminantAnalysis()), + ('KNN', KNeighborsClassifier()), + ('CART', DecisionTreeClassifier()), + ('NB', GaussianNB()), + # ('AB', AdaBoostClassifier()), + # ('RF', RandomForestClassifier(n_jobs=-1)) + ], + classification_ensemble_model = [('Ensemble - CART', DecisionTreeClassifier())] +) +training_config = dict( + path = 'data/', + sliding_window_size = 150, + retrain_every = 20, + scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' + include_original_data_in_ensemble = True, + method = 'classification', + forecasting_horizon = 1) -path = 'data/' -all_assets = get_crypto_assets(path) - -sliding_window_size = 150 -retrain_every = 20 -scaler = 'minmax' # 'normalize' 'minmax' 'standardize' 'none' -include_original_data_in_ensemble = True -method = 'classification' -forecasting_horizon = 1 -data_parameters = dict(path=path, +feature_extractors = feature_extractor_presets.date + feature_extractor_presets.level1 +data_config = dict( + path=training_config['path'], + all_assets = get_crypto_assets(training_config['path']), load_other_assets= False, log_returns= True, - forecasting_horizon = forecasting_horizon, - own_features= feature_extractor_presets.date + feature_extractor_presets.level1, + forecasting_horizon = training_config['forecasting_horizon'], + own_features= feature_extractors, other_features= [], index_column= 'int', - method= method, + method= training_config['method'], ) +if WANDB: + from wandb_setup import get_wandb + wandb = get_wandb() + + if type(wandb) == type(None): + WANDB = False + else: + ''' 3. Initialize Weights and Biases with default values, then grab the config file (necessary for sweep) ''' + wandb.init(project="price-forecasting", + config={"data_config":data_config, "training_config":training_config, "model_config": model_config}) # default config + + training_config = wandb.config['training_config'] + # vvv this doesnt work, wandb casts the functions to strings vvv + # data_config = wandb.config['data_config'] + # model_config = wandb.config['model_config'] + + + + # Run pipeline results = pd.DataFrame() -for asset in all_assets: +for asset in data_config['all_assets']: print('--------\nPredicting: ', asset) all_predictions = pd.DataFrame() # 1. Load data - data_params = data_parameters.copy() + data_params = data_config.copy() data_params['target_asset'] = asset X, y, target_returns = load_data(**data_params) @@ -84,18 +110,18 @@ for asset in all_assets: X = X, y = y, target_returns = target_returns, - models = regression_models if method == 'regression' else classification_models, - method = method, - sliding_window_size = sliding_window_size, - retrain_every = retrain_every, - scaler = scaler + models = model_config['regression_models'] if training_config['method'] == 'regression' else model_config['classification_models'], + method = training_config['method'], + sliding_window_size = training_config['sliding_window_size'], + retrain_every = training_config['retrain_every'], + scaler = training_config['scaler'] ) results = pd.concat([results, current_result], axis=1) all_predictions = pd.concat([all_predictions, current_predictions], axis=1) # 3. Train Level-2 (Ensemble) model ensemble_X = all_predictions - if include_original_data_in_ensemble: + if training_config['include_original_data_in_ensemble']: ensemble_X = pd.concat([ensemble_X, X], axis=1) ensemble_result, ensemble_preds = run_single_asset_trainig_pipeline( @@ -103,16 +129,25 @@ for asset in all_assets: X = ensemble_X, y = y, target_returns = target_returns, - models = regression_ensemble_model if method == 'regression' else classification_ensemble_model, - method = method, - sliding_window_size = sliding_window_size, - retrain_every = retrain_every, - scaler = scaler + models = model_config['regression_ensemble_model'] if training_config['method'] == 'regression' else model_config['classification_ensemble_model'], + method = training_config['method'], + sliding_window_size = training_config['sliding_window_size'], + retrain_every = training_config['retrain_every'], + scaler = training_config['scaler'] ) results = pd.concat([results, ensemble_result], axis=1) all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1) +if WANDB: + combined_metrics = results.mean(axis=1) + wandb.log({'results': results}) + wandb.log({'combined': combined_metrics}) + + + if wandb.run is not None: + wandb.finish() + results.to_csv('results.csv') level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]] diff --git a/sweep.yaml b/sweep.yaml new file mode 100644 index 0000000..3492327 --- /dev/null +++ b/sweep.yaml @@ -0,0 +1,47 @@ +program: rnn_sweep.py +method: bayes +project: integer-sequence +name: Finding best hyperparameters for price prediction +early_terminate: + type: hyperband + min_iter: 2000 +metric: + goal: maximize + name: accuracy_test +parameters: + path : 'data/' + sliding_window_size: + values: + - 90 + - 150 + - 365 + - 730 + distribution: categorical + retrain_every: + values: + - 7 + - 14 + - 30 + - 60 + distribution: categorical + scaler: 'minmax' # 'normalize' 'minmax' 'standardize' 'none' + include_original_data_in_ensemble: True + method: + values: + - 'classification' + - 'regression' + distribution: categorical + forecasting_horizon: + value: 1 + distribution: constant + + + + + + + + + + + diff --git a/utils/load_data.py b/utils/load_data.py index be0e8f6..445112d 100644 --- a/utils/load_data.py +++ b/utils/load_data.py @@ -27,6 +27,7 @@ def load_data(path: str, index_column: Literal['date', 'int'], method: Literal['regression', 'classification'], narrow_format: bool = False, + all_assets:list=[] ) -> tuple[pd.DataFrame, pd.Series, pd.Series]: """ Loads asset data from the specified path. @@ -35,6 +36,7 @@ def load_data(path: str, - Series `y` with the target asset returns shifted by 1 day OR if it's a classification problem, the target class) - Series `forward_returns` with the target asset returns shifted by 1 day """ + files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')] files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(target_asset))] diff --git a/wandb_setup.py b/wandb_setup.py new file mode 100644 index 0000000..b5857cd --- /dev/null +++ b/wandb_setup.py @@ -0,0 +1,17 @@ +import wandb +import os + + +def get_wandb(): + from dotenv import load_dotenv + load_dotenv() + + ''' 0. Login to Weights and Biases ''' + wsb_token = os.environ.get('WANDB_API_KEY') + if wsb_token: + wandb.login(key=wsb_token) + return wandb + else: return None #wandb.login() + + # return wandb +