diff --git a/config.py b/config.py index c63750d..d9e16b4 100644 --- a/config.py +++ b/config.py @@ -7,8 +7,8 @@ def get_default_config() -> tuple[dict, dict, dict]: training_config = dict( expanding_window = False, - sliding_window_size = 200, - retrain_every = 100, + sliding_window_size = 220, + retrain_every = 20, scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' include_original_data_in_ensemble = True, ) @@ -26,12 +26,10 @@ def get_default_config() -> tuple[dict, dict, dict]: no_of_classes= 'two' ) - # regression_models = ["Lasso", "Ridge", "BayesianRidge", "KNN", "AB", "LR", "MLP", "RF", "SVR"] regression_models = ["Lasso", "KNN", "RF"] - regression_ensemble_models = ['Ensemble_Average'] - classification_models = ["LR", "LDA", "KNN", "CART", "RF"] - # classification_models = model_names_classification - classification_ensemble_models = ['Ensemble_Average'] + regression_ensemble_models = ['KNN'] + classification_models = ["LR", "LDA", "KNN", "CART", "RF", "StaticMom"] + classification_ensemble_models = ['LR'] model_config = dict( level_1_models = regression_models if data_config['method'] == 'regression' else classification_models, diff --git a/models/model_map.py b/models/model_map.py index ff18aa0..9226146 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -35,15 +35,8 @@ model_map = { AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)), RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)), StaticMom= StaticMomentumModel(allow_short=True), - ), - "classification_ensemble_models": dict( - Ensemble_CART = SKLearnModel(DecisionTreeClassifier()), Ensemble_Average = StaticAverageModel(), ), - "regression_ensemble_models": dict( - Ensemble_Ridge = SKLearnModel(Ridge(alpha=0.1)), - Ensemble_Average = StaticAverageModel(), - ) } model_names_classification = list(model_map["classification_models"].keys()) @@ -52,7 +45,7 @@ model_names_regression = list(model_map["regression_models"].keys()) def map_model_name_to_function(model_config:dict, method:str) -> dict: for level in ['level_1_models', 'level_2_models']: - model_category = method + '_models' if level=='level_1_models' else method + '_ensemble_models' + model_category = method + '_models' model_config[level] = [(model_name, model_map[model_category][model_name]) for model_name in model_config[level]] return model_config \ No newline at end of file diff --git a/run_pipeline.py b/run_pipeline.py index a9d3434..c23778d 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -5,6 +5,7 @@ from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_ from models.model_map import map_model_name_to_function from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config from config import get_default_config, validate_config, get_model_name +from utils.helpers import get_first_valid_return_index def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): model_config, training_config, data_config = get_default_config() @@ -33,6 +34,11 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co data_params['target_asset'] = asset X, y, target_returns = load_data(**data_params) + first_valid_index = get_first_valid_return_index(X.iloc[:,0]) + samples_to_train = len(y) - first_valid_index + if samples_to_train < training_config['sliding_window_size'] * 2.6: + print("Not enough samples to train") + continue # 2. Train Level-1 models current_result, current_predictions = run_single_asset_trainig( @@ -46,14 +52,14 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co sliding_window_size = training_config['sliding_window_size'], retrain_every = training_config['retrain_every'], scaler = training_config['scaler'], - no_of_classes = data_config['no_of_classes'] + no_of_classes = data_config['no_of_classes'], + level = 1 ) results = pd.concat([results, current_result], axis=1) all_predictions = pd.concat([all_predictions, current_predictions], axis=1) if len(model_config['level_2_models']) > 0: # 3. Train Level-2 (Ensemble) model - ensemble_X = all_predictions if training_config['include_original_data_in_ensemble']: ensemble_X = pd.concat([ensemble_X, X], axis=1) @@ -69,7 +75,8 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co sliding_window_size = training_config['sliding_window_size'], retrain_every = training_config['retrain_every'], scaler = training_config['scaler'], - no_of_classes = data_config['no_of_classes'] + no_of_classes = data_config['no_of_classes'], + level = 2 ) results = pd.concat([results, ensemble_result], axis=1) diff --git a/sweep.yaml b/sweep_level_1.yaml similarity index 77% rename from sweep.yaml rename to sweep_level_1.yaml index e861a4f..cef55a8 100644 --- a/sweep.yaml +++ b/sweep_level_1.yaml @@ -1,10 +1,7 @@ program: run_sweep.py method: bayes project: price-forecasting -name: Finding best hyperparameters for price prediction -# early_terminate: -# type: hyperband -# min_iter: 2000 +name: Level-1 models metric: goal: maximize name: sharpe @@ -15,14 +12,13 @@ parameters: values: [True, False] distribution: categorical sliding_window_size: - values: [180, 280, 380] + values: [180, 280, 380, 480, 580] distribution: categorical retrain_every: values: [10, 20, 30] distribution: categorical scaler: - values: ['minmax', 'none'] - distribution: categorical + value: 'minmax' include_original_data_in_ensemble: value: False method: @@ -45,7 +41,8 @@ parameters: level_2_models: value: [] own_features: - values: [['only_mom', 'date_days'], [], ['level_1', 'date_days'], ['level_1', 'date_days', 'level_2']] + values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']] distribution: categorical other_features: - value: [] \ No newline at end of file + values: [[], ['level_1'], ['level_2']] + distribution: categorical diff --git a/sweep_level_2.yaml b/sweep_level_2.yaml new file mode 100644 index 0000000..489bc36 --- /dev/null +++ b/sweep_level_2.yaml @@ -0,0 +1,49 @@ +program: run_sweep.py +method: bayes +project: price-forecasting +name: Level-2 models +metric: + goal: maximize + name: sharpe +parameters: + path : + value: 'data/' + expanding_window: + values: [True, False] + distribution: categorical + sliding_window_size: + values: [180, 280, 380] + distribution: categorical + retrain_every: + values: [10, 20, 30] + distribution: categorical + scaler: + value: 'minmax' + include_original_data_in_ensemble: + values: [True, False] + distribution: categorical + method: + value: 'classification' + no_of_classes: + values: ['two', 'three-balanced', 'three-imbalanced'] + distribution: categorical + forecasting_horizon: + value: 1 + load_other_assets: + values: [True, False] + distribution: categorical + log_returns: + value: True + index_column: + value: 'int' + level_1_models: + value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"] + level_2_models: + values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"], ["Ensemble_Average"]] + distribution: categorical + own_features: + values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']] + distribution: categorical + other_features: + values: [[], ['level_1'], ['level_2']] + distribution: categorical \ No newline at end of file diff --git a/training/training.py b/training/training.py index 9b8d985..4afe7f9 100644 --- a/training/training.py +++ b/training/training.py @@ -26,7 +26,8 @@ def run_single_asset_trainig( sliding_window_size: int, retrain_every: int, scaler: Literal['normalize', 'minmax', 'standardize', 'none'], - no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'] + no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], + level: int ) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -56,7 +57,7 @@ def run_single_asset_trainig( method = method, no_of_classes=no_of_classes ) - column_name = ticker_to_predict + "_" + model_name + column_name = ticker_to_predict + "_" + model_name + "_" + str(level) results[column_name] = result # column names for model outputs should be different, so we can differentiate between original data and model predictions later, where necessary predictions["model_" + column_name] = preds diff --git a/utils/evaluate.py b/utils/evaluate.py index 3b70a09..7444174 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -1,10 +1,9 @@ from typing import Literal -from sklearn.metrics import mean_absolute_error, accuracy_score, r2_score, f1_score, precision_score, recall_score +from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score from quantstats.stats import skew, sortino -from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio, average_holding_period +from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio from utils.helpers import get_first_valid_return_index import pandas as pd -import numpy as np def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.00) -> pd.Series: delta_pos = signal.diff(1).abs().fillna(0.)