diff --git a/config/config.py b/config/config.py index 2a1c473..dfe66ff 100644 --- a/config/config.py +++ b/config/config.py @@ -1,15 +1,14 @@ -def get_default_level_1_daily_config() -> tuple[dict, dict, dict]: +def get_dev_config() -> tuple[dict, dict, dict]: training_config = dict( - meta_labeling_lvl_1 = False, + primary_models_meta_labeling = False, dimensionality_reduction = True, n_features_to_select = 30, - dynamic_feature_selection = True, - expanding_window_level1 = False, - expanding_window_level2 = False, - sliding_window_size_level1 = 380, - sliding_window_size_level2 = 1, + expanding_window_primary = False, + expanding_window_meta_labeling = False, + sliding_window_size_primary = 380, + sliding_window_size_meta_labeling = 1, retrain_every = 20, scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' ) @@ -26,76 +25,33 @@ def get_default_level_1_daily_config() -> tuple[dict, dict, dict]: exogenous_features = ['standard_scaling'], index_column= 'int', method= 'classification', - no_of_classes= 'three-balanced', + no_of_classes= 'two', narrow_format = False, ) regression_models = ["Lasso"] - classification_models = ["KNN"] + classification_models = ["LR_two_class"] model_config = dict( - level_1_models = regression_models if data_config['method'] == 'regression' else classification_models, - level_2_model = None + primary_models = regression_models if data_config['method'] == 'regression' else classification_models, + meta_labeling_models = [], + ensemble_model = None ) return model_config, training_config, data_config -def get_default_level_2_hourly_config() -> tuple[dict, dict, dict]: + +def get_default_ensemble_config() -> tuple[dict, dict, dict]: training_config = dict( - meta_labeling_lvl_1 = True, + primary_models_meta_labeling = True, dimensionality_reduction = True, n_features_to_select = 30, - dynamic_feature_selection = True, - expanding_window_level1 = True, - expanding_window_level2 = False, - sliding_window_size_level1 = 2480, - sliding_window_size_level2 = 1, - retrain_every = 100, - scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' - ) - - data_config = dict( - assets = ['hourly_crypto'], - other_assets = [], - exogenous_data = [], - load_non_target_asset= True, - log_returns= True, - forecasting_horizon = 1, - own_features = ['level_2', 'date_days', 'lags_up_to_5'], - other_features = ['level_2'], - exogenous_features = ['standard_scaling'], - index_column= 'int', - method= 'classification', - no_of_classes= 'three-balanced', - narrow_format = False, - ) - - regression_models = ["Lasso", "KNN", "RF"] - regression_ensemble_model = 'KNN' - classification_models = ["LDA", "KNN", "CART", "RF", "StaticMom"] - classification_ensemble_model = 'Ensemble_Average' - - model_config = dict( - level_1_models = regression_models if data_config['method'] == 'regression' else classification_models, - level_2_model = regression_ensemble_model if data_config['method'] == 'regression' else classification_ensemble_model - ) - - return model_config, training_config, data_config - - -def get_default_level_2_daily_config() -> tuple[dict, dict, dict]: - - training_config = dict( - meta_labeling_lvl_1 = True, - dimensionality_reduction = True, - n_features_to_select = 30, - dynamic_feature_selection = True, - expanding_window_level1 = False, - expanding_window_level2 = True, - sliding_window_size_level1 = 380, - sliding_window_size_level2 = 240, + expanding_window_primary = False, + expanding_window_meta_labeling = True, + sliding_window_size_primary = 380, + sliding_window_size_meta_labeling = 240, retrain_every = 20, scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' ) @@ -117,13 +73,14 @@ def get_default_level_2_daily_config() -> tuple[dict, dict, dict]: ) regression_models = ["Lasso", "KNN", "RF"] - regression_ensemble_model = 'KNN' - classification_models = ['SVC', 'LDA', 'KNN', 'CART', 'NB', 'AB', 'RF', 'XGB_two_class', 'LGBM', 'StaticMom'] - classification_ensemble_model = 'LDA' + classification_models = ['LR_two_class', 'SVC', 'KNN', 'CART', 'NB', 'AB', 'RF', 'XGB_two_class', 'LGBM', 'StaticMom'] + meta_labeling_models = ['LR_two_class', 'LGBM'] + ensemble_model = 'Average' model_config = dict( - level_1_models = regression_models if data_config['method'] == 'regression' else classification_models, - level_2_model = regression_ensemble_model if data_config['method'] == 'regression' else classification_ensemble_model + primary_models = regression_models if data_config['method'] == 'regression' else classification_models, + meta_labeling_models = meta_labeling_models, + ensemble_model = ensemble_model ) return model_config, training_config, data_config diff --git a/config/preprocess.py b/config/preprocess.py index bac1ab9..c97fcb2 100644 --- a/config/preprocess.py +++ b/config/preprocess.py @@ -21,9 +21,11 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict: return data_dict def __preprocess_model_config(model_config:dict, method:str) -> dict: - model_config['level_1_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in model_config['level_1_models']] - if model_config['level_2_model'] is not None: - model_config['level_2_model'] = (model_config['level_2_model'], model_map[method + '_models'][model_config['level_2_model']]) + model_config['primary_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in model_config['primary_models']] + if len(model_config['meta_labeling_models']) > 0: + model_config['meta_labeling_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in model_config['meta_labeling_models']] + if model_config['ensemble_model'] is not None: + model_config['ensemble_model'] = (model_config['ensemble_model'], model_map['ensemble_models'][model_config['ensemble_model']]) return model_config @@ -39,15 +41,15 @@ def __preprocess_data_collections_config(data_dict: dict) -> dict: def validate_config(model_config:dict, training_config:dict, data_config:dict): # We need to make sure there's only one output from the pipeline # If level-2 model is there, we need more than one level-1 models to train - if model_config["level_2_model"] is not None: assert len(model_config["level_1_models"]) > 0 + if len(model_config["meta_labeling_models"]) > 1: assert len(model_config["primary_models"]) > 0 # If there's no level-2 model, we need to have only one level-1 model - if model_config["level_2_model"] is None: assert len(model_config["level_1_models"]) == 1 + if len(model_config["meta_labeling_models"]) == 0: assert len(model_config["primary_models"]) == 1 def get_model_name(model_config:dict) -> str: - if model_config["level_2_model"] is not None: - return model_config["level_2_model"][0] - elif len(model_config["level_1_models"]) == 1: - return model_config["level_1_models"][0][0] + if len(model_config["meta_labeling_models"]) > 0: + return model_config["meta_labeling_models"][0][0] + elif len(model_config["primary_models"]) == 1: + return model_config["primary_models"][0][0] else: raise Exception("No model name found") diff --git a/exploration.ipynb b/exploration.ipynb index a2a7739..e98f412 100644 --- a/exploration.ipynb +++ b/exploration.ipynb @@ -25,13 +25,13 @@ "source": [ "import pandas as pd\n", "import pandas_ta as ta\n", - "from config.config import get_default_level_2_daily_config\n", + "from config.config import get_default_ensemble_config\n", "from config.preprocess import preprocess_config\n", "from data_loader.load_data import load_data\n", "import seaborn as sns\n", "import numpy as np\n", "\n", - "model_config, training_config, data_config = get_default_level_2_daily_config()\n", + "model_config, training_config, data_config = get_default_ensemble_config()\n", "model_config, training_config, data_config = preprocess_config(model_config, training_config, data_config)\n", "\n", "data_config['target_asset'] = data_config['assets'][0]\n", diff --git a/feature_selection/feature_selection.py b/feature_selection/feature_selection.py index fe3ee36..623045c 100644 --- a/feature_selection/feature_selection.py +++ b/feature_selection/feature_selection.py @@ -5,20 +5,20 @@ from models.base import Model from models.sklearn import SKLearnModel from utils.scaler import get_scaler from utils.types import ScalerTypes -from utils.hashing import hash_df, hash_series -from diskcache import Cache -cache = Cache(".cachedir/feature_selection") +# from utils.hashing import hash_df, hash_series +# from diskcache import Cache +# cache = Cache(".cachedir/feature_selection") -def select_features(**kwargs) -> pd.DataFrame: - hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling'] + str(kwargs['dynamic_feature_selection']) - if hashed in cache: - return cache.get(hashed) - else: - return_value = __select_features(**kwargs) - cache[hashed] = return_value - return return_value +# def select_features(**kwargs) -> pd.DataFrame: +# hashed = kwargs['data_config_hash'] + kwargs['model'].get_name() + str(kwargs['n_features_to_select']) + kwargs['backup_model'].get_name() + kwargs['scaling'] +# if hashed in cache: +# return cache.get(hashed) +# else: +# return_value = __select_features(**kwargs) +# cache[hashed] = return_value +# return return_value -def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel, scaling: ScalerTypes, dynamic_feature_selection: bool, data_config_hash: str) -> pd.DataFrame: +def select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel, scaling: ScalerTypes) -> pd.DataFrame: ''' Select features using RFECV, returns a pd.DataFrame (X) with only the selected features.''' if model.model_type != 'ml': return X @@ -34,7 +34,7 @@ def __select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to feat_selector_model = backup_model.model # selector = RFECV(feat_selector_model, cv = cv, step=5, min_features_to_select=min_features_to_select) - step = 0.05 if dynamic_feature_selection else 5 + step = 0.05 selector = RFE(feat_selector_model, n_features_to_select= n_features_to_select, step=step) selector = selector.fit(X_scaled, y) print("Kept %d features out of %d" % (selector.n_features_, X_scaled.shape[1])) diff --git a/models/lightgbm.py b/models/lightgbm.py deleted file mode 100644 index 5ab6330..0000000 --- a/models/lightgbm.py +++ /dev/null @@ -1,2 +0,0 @@ -import lightgbm as lgb - diff --git a/models/model_map.py b/models/model_map.py index 59cbfd4..b810f32 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -1,5 +1,6 @@ from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, Ridge -from sklearnex.linear_model import LogisticRegression +from sklearn.linear_model import LogisticRegression +from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX from sklearn.tree import DecisionTreeClassifier from sklearnex.neighbors import KNeighborsRegressor, KNeighborsClassifier from sklearn.discriminant_analysis import LinearDiscriminantAnalysis @@ -44,7 +45,8 @@ model_map = { ) ), "classification_models": dict( - LR= SKLearnModel(LogisticRegression(C=10, random_state=1, max_iter=1000, n_jobs=-1)), + LR_two_class= SKLearnModel(LogisticRegression(C=10, random_state=1, solver='liblinear', max_iter=1000)), + LR_three_class= SKLearnModel(LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1)), LDA= SKLearnModel(LinearDiscriminantAnalysis()), KNN= SKLearnModel(KNeighborsClassifier()), CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)), @@ -55,9 +57,11 @@ model_map = { XGB_two_class= XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')), LGBM = SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)), StaticMom= StaticMomentumModel(allow_short=True), - Ensemble_Average= StaticAverageModel(), # ExpSmoothing = SKLearnModel(ExponentialSmoothing(trend='add', seasonal='add', seasonal_periods=30)), ), + "ensemble_models": dict( + Average= StaticAverageModel(), + ) } model_names_classification = list(model_map["classification_models"].keys()) diff --git a/reporting/reporting.py b/reporting/reporting.py index daad5ab..593beee 100644 --- a/reporting/reporting.py +++ b/reporting/reporting.py @@ -5,36 +5,36 @@ from utils.helpers import weighted_average def report_results(results:pd.DataFrame, all_predictions:pd.DataFrame, model_config:dict, wandb, sweep: bool, project_name:str): - level1_columns = results[[column for column in results.columns if 'lvl1' in column]] - level2_columns = results[[column for column in results.columns if 'lvl2' in column]] + primary_results = results[[column for column in results.columns if 'primary' in column]] + ensemble_results = results[[column for column in results.columns if 'ensemble' in column]] # Only send the results of the final model to wandb - results_to_send = level2_columns if level2_columns.shape[1] > 0 else level1_columns + results_to_send = ensemble_results if ensemble_results.shape[1] > 0 else primary_results send_report_to_wandb(results_to_send, wandb, project_name, get_model_name(model_config)) results.to_csv('output/results.csv') - level1_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl1' in column]] - level2_predictions = all_predictions[[column for column in all_predictions.columns if 'lvl2' in column]] - predictions_to_save = level2_predictions if level2_predictions.shape[1] > 0 else level1_predictions + primary_weights = all_predictions[[column for column in all_predictions.columns if 'primary' in column]] + ensemble_weights = all_predictions[[column for column in all_predictions.columns if 'ensemble' in column]] + predictions_to_save = ensemble_weights if ensemble_weights.shape[1] > 0 else primary_weights predictions_to_save.to_csv('output/predictions.csv') print("\n--------\n") all_avg_results = weighted_average(results, 'no_of_samples') - lvl1_avg_results = weighted_average(level1_columns, 'no_of_samples') - lvl2_avg_results = weighted_average(level2_columns, 'no_of_samples') + primary_avg_results = weighted_average(primary_results, 'no_of_samples') + ensemble_avg_results = weighted_average(ensemble_results, 'no_of_samples') print("Benchmark buy-and-hold sharpe: ", round(all_avg_results.loc['benchmark_sharpe'], 3)) - print("Level-1: Number of samples evaluated: ", level1_columns.loc['no_of_samples'].sum()) - print("Mean Sharpe ratio for Level-1 models: ", round(lvl1_avg_results.loc['sharpe'], 3)) - print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(lvl1_avg_results.loc['prob_sharpe'].mean(), 3)) + print("Level-1: Number of samples evaluated: ", primary_results.loc['no_of_samples'].sum()) + print("Mean Sharpe ratio for Level-1 models: ", round(primary_avg_results.loc['sharpe'], 3)) + print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(primary_avg_results.loc['prob_sharpe'].mean(), 3)) - if model_config['level_2_model'] is not None: - print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum()) - print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(lvl2_avg_results.loc['sharpe'].mean(), 3)) - print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(lvl2_avg_results.loc['prob_sharpe'].mean(), 3)) + if len(model_config['meta_labeling_models']) > 0: + print("Level-2 (Ensemble): Number of samples evaluated: ", ensemble_results.loc['no_of_samples'].sum()) + print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_avg_results.loc['sharpe'].mean(), 3)) + print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_avg_results.loc['prob_sharpe'].mean(), 3)) - lvl2_avg_results.to_csv('output/results_level2.csv') + ensemble_avg_results.to_csv('output/results_level2.csv') if sweep: if wandb.run is not None: diff --git a/run_feature_selection.py b/run_feature_selection.py index 21e2a8b..2f30c80 100644 --- a/run_feature_selection.py +++ b/run_feature_selection.py @@ -1,12 +1,12 @@ # #%% # import pandas as pd # import pandas_ta as ta -# from config.config import get_default_level_2_daily_config +# from config.config import get_default_ensemble_config # from config.preprocess import preprocess_config # from data_loader.load_data import load_data # # %% -# model_config, training_config, data_config = get_default_level_2_daily_config() +# model_config, training_config, data_config = get_default_ensemble_config() # model_config, training_config, data_config = preprocess_config(model_config, training_config, data_config) # data_config['target_asset'] = data_config['assets'][0] diff --git a/run_model_dev.py b/run_model_dev.py index 3b2b971..5a7ba6e 100644 --- a/run_model_dev.py +++ b/run_model_dev.py @@ -1,5 +1,5 @@ from run_pipeline import run_pipeline -from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config +from config.config import get_dev_config -run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_level_1_daily_config) +run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_dev_config) diff --git a/run_pipeline.py b/run_pipeline.py index adfefc0..55c5008 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -1,16 +1,15 @@ from config.hashing import hash_data_config from data_loader.load_data import load_data import pandas as pd -from training.training import run_single_asset_trainig -from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_with_wandb +from training.primary_model import train_primary_model +from reporting.wandb import launch_wandb, register_config_with_wandb from models.model_map import default_feature_selector_regression, default_feature_selector_classification from utils.helpers import get_first_valid_return_index -from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config +from config.config import get_default_ensemble_config from config.preprocess import validate_config, preprocess_config from feature_selection.feature_selection import select_features from feature_selection.dim_reduction import reduce_dimensionality -from training.meta_labeling import run_meta_labeling_training -from training.averaged import average_and_evaluate_predictions +from training.meta_labeling import train_meta_labeling_model from reporting.reporting import report_results from typing import Callable, Optional import ray @@ -52,7 +51,7 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): original_X = X.copy() first_valid_index = get_first_valid_return_index(X.iloc[:,0]) samples_to_train = len(y) - first_valid_index - if samples_to_train < training_config['sliding_window_size_level1'] * 3: + if samples_to_train < training_config['sliding_window_size_primary'] * 3: print("Not enough samples to train") continue @@ -67,23 +66,24 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): print("Feature Selection started") # TODO: this needs to be done per model! backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification - X = select_features(X = X, y = y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], dynamic_feature_selection = training_config['dynamic_feature_selection'], data_config_hash = hash_data_config(data_params)) + X = select_features(X = X, y = y, model = model_config['primary_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler']) - # 3. Train Level-1 models - current_result, current_predictions, current_probabilities, all_models_for_single_asset = run_single_asset_trainig( + # 3. Train Primary models + current_result, current_predictions, current_probabilities, all_models_for_single_asset = train_primary_model( ticker_to_predict = asset[1], original_X = original_X, X = X, y = y, target_returns = target_returns, - models = model_config['level_1_models'], + models = model_config['primary_models'], method = data_config['method'], - expanding_window = training_config['expanding_window_level1'], - sliding_window_size = training_config['sliding_window_size_level1'], + expanding_window = training_config['expanding_window_primary'], + sliding_window_size = training_config['sliding_window_size_primary'], retrain_every = training_config['retrain_every'], scaler = training_config['scaler'], no_of_classes = data_config['no_of_classes'], - level = 1 + level = 'primary', + print_results= True ) all_models_for_all_assets[asset[1]] = dict( @@ -91,25 +91,24 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): models=all_models_for_single_asset ) - # 4. Train a Meta-Labeling model for each Level-1 model and replace its predictions with the meta-labeling predictions - if training_config['meta_labeling_lvl_1'] == True: + # 4. Train a Meta-Labeling model for each Primary model and replace their predictions with the meta-labeling predictions + if training_config['primary_models_meta_labeling'] == True: for model_name in current_result.columns: - lvl1_model_predictions = current_predictions[model_name] - prev_sharpe = current_result[model_name]['sharpe'] - lvl1_meta_result, lvl1_meta_preds, lvl1_meta_probabilities, meta_labeling_models = run_meta_labeling_training( + primary_model_predictions = current_predictions[model_name] + primary_meta_result, primary_meta_preds, primary_meta_probabilities, meta_labeling_models = train_meta_labeling_model( target_asset=asset[1], X_pca = X_pca, - input_predictions= lvl1_model_predictions, + input_predictions= primary_model_predictions, y = y, target_returns = target_returns, + models = model_config['meta_labeling_models'], data_config= data_config, model_config= model_config, - training_config= training_config + training_config= training_config, + model_suffix = 'meta' ) - new_sharpe = lvl1_meta_result['sharpe'] - print("Improvement in sharpe for the meta model: ", ((new_sharpe / prev_sharpe) - 1) * 100, "%") - current_result[model_name] = lvl1_meta_result - current_predictions[model_name] = lvl1_meta_preds + current_result[model_name] = primary_meta_result + current_predictions[model_name] = primary_meta_preds all_models_for_all_assets[asset[1]][model_name] = meta_labeling_models @@ -118,26 +117,46 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): all_predictions = pd.concat([all_predictions, current_predictions], axis=1).fillna(0.) all_probabilities = pd.concat([all_probabilities, current_probabilities], axis=1).fillna(0.) - if model_config['level_2_model'] is not None: + # 5. Ensemble primary model predictions (If Ensemble model is present) + if model_config['ensemble_model'] is not None: - # 3. Average the Level-1 model predictions - averaged_predictions, averaged_results = average_and_evaluate_predictions(current_predictions, y, target_returns, data_config) - - # 3. Train a Meta-labeling model on the averaged level-1 model predictions - meta_result, avg_predictions_with_sizing, meta_probabilities, meta_labeling_models = run_meta_labeling_training( - target_asset=asset[1], - X_pca = X_pca, - input_predictions= averaged_predictions, + ensemble_result, ensemble_predictions, _, _ = train_primary_model( + ticker_to_predict = asset[1], + original_X = current_predictions, + X = current_predictions, y = y, target_returns = target_returns, - data_config= data_config, - model_config= model_config, - training_config= training_config + models = [model_config['ensemble_model']], + method = data_config['method'], + expanding_window = False, + sliding_window_size = 1, + retrain_every = training_config['retrain_every'], + scaler = training_config['scaler'], + no_of_classes = data_config['no_of_classes'], + level = 'ensemble', + print_results= True, ) + ensemble_result, ensemble_predictions = ensemble_result.iloc[:,0], ensemble_predictions.iloc[:,0] - results = pd.concat([results, meta_result], axis=1) - all_predictions = pd.concat([all_predictions, avg_predictions_with_sizing], axis=1) - all_probabilities = pd.concat([all_probabilities, meta_probabilities], axis=1).fillna(0.) + if len(model_config['meta_labeling_models']) > 0: + + # 3. Train a Meta-labeling model on the averaged level-1 model predictions + ensemble_meta_result, ensemble_meta_predictions, ensemble_meta_probabilities, ensemble_meta_labeling_models = train_meta_labeling_model( + target_asset=asset[1], + X_pca = X_pca, + input_predictions= ensemble_predictions, + y = y, + target_returns = target_returns, + models = model_config['meta_labeling_models'], + data_config= data_config, + model_config= model_config, + training_config= training_config, + model_suffix = 'ensemble' + ) + + results = pd.concat([results, ensemble_meta_result], axis=1) + all_predictions = pd.concat([all_predictions, ensemble_meta_predictions], axis=1) + all_probabilities = pd.concat([all_probabilities, ensemble_meta_probabilities], axis=1).fillna(0.) return results, all_predictions, all_probabilities @@ -145,4 +164,4 @@ def __run_training(model_config:dict, training_config:dict, data_config:dict): if __name__ == '__main__': - run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_level_2_daily_config) \ No newline at end of file + run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, get_config=get_default_ensemble_config) \ No newline at end of file diff --git a/run_sweep.py b/run_sweep.py index 703c2d1..9e0ea9a 100644 --- a/run_sweep.py +++ b/run_sweep.py @@ -1,4 +1,4 @@ from run_pipeline import run_pipeline -from config.config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config +from config.config import get_default_ensemble_config -run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True, get_config= get_default_level_2_daily_config) \ No newline at end of file +run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True, get_config= get_default_ensemble_config) \ No newline at end of file diff --git a/sweep_dynamic_features.yaml b/sweep_ensemble.yaml similarity index 68% rename from sweep_dynamic_features.yaml rename to sweep_ensemble.yaml index 5e990eb..7b50ef0 100644 --- a/sweep_dynamic_features.yaml +++ b/sweep_ensemble.yaml @@ -6,10 +6,7 @@ metric: goal: maximize name: sharpe parameters: - dynamic_feature_selection: - values: [True, False] - distribution: 'categorical' - meta_labeling_lvl_1: + primary_models_meta_labeling: value: True assets: value: ['daily_crypto'] @@ -17,18 +14,16 @@ parameters: value: ['daily_etf'] exogenous_data: value: ['daily_glassnode'] - expanding_window_level1: - values: [False, True] - distribution: 'categorical' - expanding_window_level2: + expanding_window_primary: value: True - sliding_window_size_level1: + expanding_window_meta_labeling: + value: True + sliding_window_size_primary: value: 380 - sliding_window_size_level2: + sliding_window_size_meta_labeling: value: 380 n_features_to_select: - values: [30, 50, 70, 80] - distribution: 'categorical' + value: 50 dimensionality_reduction: value: True retrain_every: @@ -47,10 +42,10 @@ parameters: value: True index_column: value: 'int' - level_1_models: - value: ["LDA", "KNN", "SVC", "CART", "NB", "AB", "RF", "XGB_two_class", "StaticMom"] - level_2_model: - values: ["LDA", "XGB_two_class"] + primary_models: + value: ["LDA", "KNN", "SVC", "CART", "NB", "AB", "RF", "XGB_two_class", "LGBM", "StaticMom"] + meta_labeling_models: + values: [["LDA"], ["XGB_two_class"], ["LR_two_class"], ["LGBM"], ["LGBM", "LR_two_class"], ["XGB_two_class", "LDA"], ["XGB_two_class", "LR_two_class"]] distribution: categorical own_features: value: ['date_days', 'level_2', 'lags_up_to_5'] diff --git a/sweep_level_2.yaml b/sweep_meta_labeling.yaml similarity index 78% rename from sweep_level_2.yaml rename to sweep_meta_labeling.yaml index 813d79e..42d03f5 100644 --- a/sweep_level_2.yaml +++ b/sweep_meta_labeling.yaml @@ -6,7 +6,7 @@ metric: goal: maximize name: sharpe parameters: - meta_labeling_lvl_1: + primary_models_meta_labeling: value: True assets: value: ['daily_crypto'] @@ -14,10 +14,10 @@ parameters: value: ['daily_etf'] exogenous_data: value: ['daily_glassnode'] - expanding_window_level1: + expanding_window_primary: values: [True, False] distribution: categorical - expanding_window_level2: + expanding_window_meta_labeling: values: [True, False] distribution: categorical n_features_to_select: @@ -25,10 +25,10 @@ parameters: distribution: categorical dimensionality_reduction: value: True - sliding_window_size_level1: + sliding_window_size_primary: values: [180, 280, 380] distribution: categorical - sliding_window_size_level2: + sliding_window_size_meta_labeling: values: [180, 280, 380] distribution: categorical retrain_every: @@ -50,10 +50,10 @@ parameters: value: True index_column: value: 'int' - level_1_models: - value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"] - level_2_model: - values: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "Ensemble_Average"] + primary_models: + value: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"] + meta_labeling_models: + values: ["LR_two_class", "LDA", "KNN", "CART", "NB", "AB", "RF"] distribution: categorical own_features: values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']] diff --git a/sweep_level_1.yaml b/sweep_primary_models.yaml similarity index 80% rename from sweep_level_1.yaml rename to sweep_primary_models.yaml index 929b4b3..30dbe7a 100644 --- a/sweep_level_1.yaml +++ b/sweep_primary_models.yaml @@ -6,7 +6,7 @@ metric: goal: maximize name: sharpe parameters: - meta_labeling_lvl_1: + primary_models_meta_labeling: value: True assets: value: ['daily_crypto'] @@ -14,20 +14,20 @@ parameters: value: ['daily_etf'] exogenous_data: value: ['daily_glassnode'] - expanding_window_level1: + expanding_window_primary: values: [True, False] distribution: categorical - expanding_window_level2: + expanding_window_meta_labeling: value: False n_features_to_select: values: [10, 20, 30] distribution: categorical dimensionality_reduction: value: True - sliding_window_size_level1: + sliding_window_size_primary: values: [180, 280, 380, 480, 580] distribution: categorical - sliding_window_size_level2: + sliding_window_size_meta_labeling: value: 1 retrain_every: values: [10, 20, 30] @@ -48,11 +48,11 @@ parameters: value: True index_column: value: 'int' - level_1_models: - values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"]] + primary_models: + values: [["LR_two_class"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"]] distribution: categorical - level_2_model: - value: None + meta_labeling_models: + value: [] own_features: values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']] distribution: categorical diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index deb9baa..83df433 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -95,6 +95,7 @@ def test_evaluation(): y_true=y, method='classification', no_of_classes='two', + print_results = False, discretize=True ) diff --git a/training/averaged.py b/training/averaged.py deleted file mode 100644 index 81b5420..0000000 --- a/training/averaged.py +++ /dev/null @@ -1,25 +0,0 @@ - -import pandas as pd -from utils.evaluate import evaluate_predictions - -def average_and_evaluate_predictions(predictions: pd.DataFrame, y: pd.Series, target_returns: pd.Series, data_config: dict) -> tuple[pd.Series, pd.DataFrame]: - averaged_predictions = predictions.mean(axis = 1) - non_discretized_result = evaluate_predictions( - model_name = 'Averaged - Non-discrete', - target_returns = target_returns, - y_pred = averaged_predictions, - y_true = y, - method = 'classification', - no_of_classes = data_config['no_of_classes'], - discretize=False - ) - discretized_result = evaluate_predictions( - model_name = 'Averaged - Discrete', - target_returns = target_returns, - y_pred = averaged_predictions, - y_true = y, - method = 'classification', - no_of_classes = data_config['no_of_classes'], - discretize=True - ) - return averaged_predictions, pd.concat([non_discretized_result, discretized_result], axis = 1) \ No newline at end of file diff --git a/training/meta_labeling.py b/training/meta_labeling.py index 90f1107..f9ec879 100644 --- a/training/meta_labeling.py +++ b/training/meta_labeling.py @@ -1,20 +1,23 @@ from utils.evaluate import discretize_threeway_threshold, evaluate_predictions from utils.helpers import random_string, equal_except_nan, drop_until_first_valid_index -from training.training import run_single_asset_trainig +from training.primary_model import train_primary_model from feature_selection.feature_selection import select_features import pandas as pd from models.model_map import default_feature_selector_regression, default_feature_selector_classification +from models.base import Model -def run_meta_labeling_training( +def train_meta_labeling_model( target_asset: str, X_pca: pd.DataFrame, input_predictions: pd.Series, y: pd.Series, target_returns: pd.Series, + models: list[tuple[str, Model]], data_config: dict, model_config: dict, - training_config: dict + training_config: dict, + model_suffix: str ) -> tuple[pd.Series, pd.Series, pd.DataFrame, dict]: discretize = discretize_threeway_threshold(0.33) @@ -24,29 +27,34 @@ def run_meta_labeling_training( print("Feature Selection started") backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X_pca, meta_y) - feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'], dynamic_feature_selection = training_config['dynamic_feature_selection'], data_config_hash = random_string(10)) + feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = models[0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler']) meta_selected_features_X = X_pca[feature_selection_output.columns] meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1) - _, meta_preds, meta_probabilities, all_models_single_asset = run_single_asset_trainig( + _, meta_preds, meta_probabilities, all_models_single_asset = train_primary_model( ticker_to_predict = "prediction_correct", original_X = meta_X, X = meta_X, y = meta_y, target_returns = target_returns, - models = [model_config['level_2_model']], - method = data_config['method'], - expanding_window = training_config['expanding_window_level2'], - sliding_window_size = training_config['sliding_window_size_level2'], + models = models, + method = 'classification', + expanding_window = training_config['expanding_window_meta_labeling'], + sliding_window_size = training_config['sliding_window_size_meta_labeling'], retrain_every = training_config['retrain_every'], scaler = training_config['scaler'], no_of_classes = 'two', - level = 2 + level = 'meta_labeling', + print_results = False ) - bet_size = meta_probabilities.iloc[:,1] + if len(models) > 1: + meta_preds = meta_preds.mean(axis = 1) + bet_size = meta_probabilities[meta_probabilities.columns[1::2]].mean(axis = 1) + else: + bet_size = meta_probabilities.iloc[:,1] avg_predictions_with_sizing = input_predictions * bet_size - avg_predictions_with_sizing.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True) + avg_predictions_with_sizing.rename("model_" + target_asset + "_" + model_suffix, inplace=True) meta_result = evaluate_predictions( model_name = "Meta", @@ -54,9 +62,10 @@ def run_meta_labeling_training( y_pred = avg_predictions_with_sizing, y_true = y, method = 'classification', - no_of_classes = data_config['no_of_classes'], + no_of_classes = 'two', + print_results = True, discretize=False ) - meta_result.rename("model_" + target_asset + "_meta_lvl" + str(2), inplace=True) + meta_result.rename("model_" + target_asset + "_" + model_suffix, inplace=True) return meta_result, avg_predictions_with_sizing, meta_probabilities, all_models_single_asset diff --git a/training/training.py b/training/primary_model.py similarity index 93% rename from training/training.py rename to training/primary_model.py index bc2b271..d19a8ac 100644 --- a/training/training.py +++ b/training/primary_model.py @@ -6,7 +6,7 @@ from models.base import Model from utils.scaler import get_scaler from utils.types import ScalerTypes -def run_single_asset_trainig( +def train_primary_model( ticker_to_predict: str, original_X: pd.DataFrame, X: pd.DataFrame, @@ -19,10 +19,10 @@ def run_single_asset_trainig( retrain_every: int, scaler: ScalerTypes, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], - level: int + level: str, + print_results: bool ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]: - scaler = get_scaler(scaler) results = pd.DataFrame() @@ -51,14 +51,15 @@ def run_single_asset_trainig( y_true = y, method = method, no_of_classes=no_of_classes, + print_results = print_results, discretize=True ) - column_name = "model_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level) + column_name = "model_" + ticker_to_predict + "_" + model_name + "_" + level results[column_name] = result all_models_single_asset[model_name] = model_over_time # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary predictions[column_name] = preds - probs_column_name = "probs_" + ticker_to_predict + "_" + model_name + "_lvl" + str(level) + probs_column_name = "probs_" + ticker_to_predict + "_" + model_name + "_" + level probs.columns = [probs_column_name + "_" + c for c in probs.columns] probabilities = pd.concat([probabilities, probs], axis=1) diff --git a/utils/evaluate.py b/utils/evaluate.py index a0d6a04..d7bbc51 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -37,6 +37,7 @@ def evaluate_predictions( y_true: pd.Series, method: Literal['classification', 'regression'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], + print_results: bool, discretize: bool = False, ) -> pd.Series: # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size) @@ -100,8 +101,9 @@ def evaluate_predictions( # scorecard.loc['edge_to_mae'] = 0. scorecard = scorecard.round(3) - print("Model name: ", model_name) - print(scorecard) + if print_results: + print("Model name: ", model_name) + print(scorecard) return scorecard diff --git a/utils/helpers.py b/utils/helpers.py index 5285958..2833c0d 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -21,7 +21,7 @@ def flatten(list_of_lists: list) -> list: return [item for sublist in list_of_lists for item in sublist] def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.Series: - if df.shape[0] == 0: + if df.shape[1] == 0: return df mean_df = df.iloc[:,0] weights = df.loc[weights_source]