From cc70d3f90798e51e7fa1de222d7ee79df259e99f Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Mon, 27 Dec 2021 21:59:22 +0100 Subject: [PATCH] feat(Selection): added toggleable feature selection step into the pipeline (#83) * feat(Selection): added prototype feature selection python script * feat(Utils): added some helpers for the future from Advances in Financial ML book * feat(Selection): added RFECV * feat(Selection): added configurable feature selection step into pipeline * feat(Config): added level_1 & level_2 default config, PCA before feature selection process starts * feat(Selection): added backup feature selector models if current one can't output feature importance, removed unnecessary array for level-2 models * fix(Training): deal with zero first value coming out of static models * feat(Sweep): added feature selection sweep * fix(Sweep): config problem * fix(Sweep): config * chore(Utils): removed unnecessary purged k-fold crossval class * feat(Config): added dimensionality_reduction as a separate flag * fix(Sweep): config updated * fix(Sweep): sweep name * chore(Config): updated level_2 config to the best performing configuation --- config.py | 74 ++++-- environment.yml | 1 + .../feature_extractor_presets.py | 1 + feature_extractors/utils/fracdiff.py | 240 ++++++++++++++++++ feature_selection/dim_reduction.py | 8 + feature_selection/feature_selection.py | 23 ++ models/average.py | 3 +- models/base.py | 6 +- models/model_map.py | 12 +- models/momentum.py | 3 +- models/naive.py | 3 +- reporting/wandb.py | 17 +- run_pipeline.py | 44 +++- sweep_level_1.yaml | 4 +- sweep_level_2.yaml | 4 +- sweep_selection.yaml | 47 ++++ training/training.py | 3 +- utils/helpers.py | 2 + utils/rolling.py | 6 - 19 files changed, 442 insertions(+), 59 deletions(-) create mode 100644 feature_extractors/utils/fracdiff.py create mode 100644 feature_selection/dim_reduction.py create mode 100644 feature_selection/feature_selection.py create mode 100644 sweep_selection.yaml delete mode 100644 utils/rolling.py diff --git a/config.py b/config.py index d9e16b4..b9dd9eb 100644 --- a/config.py +++ b/config.py @@ -3,37 +3,77 @@ from utils.load_data import get_crypto_assets from feature_extractors.feature_extractor_presets import presets from models.model_map import model_names_classification, model_names_regression -def get_default_config() -> tuple[dict, dict, dict]: + +def get_default_level_1_config() -> tuple[dict, dict, dict]: training_config = dict( + dimensionality_reduction = False, + feature_selection = True, expanding_window = False, - sliding_window_size = 220, + sliding_window_size = 380, retrain_every = 20, scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' - include_original_data_in_ensemble = True, + include_original_data_in_ensemble = False, ) data_config = dict( path='data/', all_assets = get_crypto_assets('data/'), - load_other_assets= False, + load_other_assets= True, log_returns= True, forecasting_horizon = 1, - own_features = ['level_1', 'date_days'], - other_features = [], + own_features = ['level_2', 'date_days'], + other_features = ['single_mom'], index_column= 'int', method= 'classification', - no_of_classes= 'two' + no_of_classes= 'three-balanced' ) - regression_models = ["Lasso", "KNN", "RF"] - regression_ensemble_models = ['KNN'] - classification_models = ["LR", "LDA", "KNN", "CART", "RF", "StaticMom"] - classification_ensemble_models = ['LR'] + regression_models = ["Lasso"] + classification_models = ["KNN"] model_config = dict( level_1_models = regression_models if data_config['method'] == 'regression' else classification_models, - level_2_models = regression_ensemble_models if data_config['method'] == 'regression' else classification_ensemble_models + level_2_model = None + ) + + return model_config, training_config, data_config + + + +def get_default_level_2_config() -> tuple[dict, dict, dict]: + + training_config = dict( + dimensionality_reduction = True, + feature_selection = True, + expanding_window = True, + sliding_window_size = 380, + retrain_every = 20, + scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' + include_original_data_in_ensemble = False, + ) + + data_config = dict( + path='data/', + all_assets = get_crypto_assets('data/'), + load_other_assets= True, + log_returns= True, + forecasting_horizon = 1, + own_features = ['level_2', 'date_days', 'lags_up_to_5'], + other_features = ['level_2', 'lags_up_to_5'], + index_column= 'int', + method= 'classification', + no_of_classes= 'three-balanced' + ) + + regression_models = ["Lasso", "KNN", "RF"] + regression_ensemble_model = 'KNN' + classification_models = ["LR", "LDA", "KNN", "CART", "RF", "StaticMom"] + classification_ensemble_model = 'Ensemble_Average' + + model_config = dict( + level_1_models = regression_models if data_config['method'] == 'regression' else classification_models, + level_2_model = regression_ensemble_model if data_config['method'] == 'regression' else classification_ensemble_model ) return model_config, training_config, data_config @@ -41,16 +81,14 @@ def get_default_config() -> tuple[dict, dict, dict]: def validate_config(model_config:dict, training_config:dict, data_config:dict): # We need to make sure there's only one output from the pipeline - # We're not prepared for more than 1 level-2 models at the moment - assert len(model_config["level_2_models"]) <= 1 # If level-2 model is there, we need more than one level-1 models to train - if len(model_config["level_2_models"]) == 1: assert len(model_config["level_1_models"]) > 0 + if model_config["level_2_model"] is not None: assert len(model_config["level_1_models"]) > 0 # If there's no level-2 model, we need to have only one level-1 model - if len(model_config["level_2_models"]) == 0: assert len(model_config["level_1_models"]) == 1 + if model_config["level_2_model"] is None: assert len(model_config["level_1_models"]) == 1 def get_model_name(model_config:dict) -> str: - if len(model_config["level_2_models"]) == 1: - return model_config["level_2_models"][0][0] + if model_config["level_2_model"] is not None: + return model_config["level_2_model"][0] elif len(model_config["level_1_models"]) == 1: return model_config["level_1_models"][0][0] else: diff --git a/environment.yml b/environment.yml index 1fa97bc..f751771 100644 --- a/environment.yml +++ b/environment.yml @@ -20,4 +20,5 @@ dependencies: - pytest - wandb - python-dotenv + - tscv prefix: /usr/local/anaconda3/envs/quant diff --git a/feature_extractors/feature_extractor_presets.py b/feature_extractors/feature_extractor_presets.py index cfd0d7b..c23c9a5 100644 --- a/feature_extractors/feature_extractor_presets.py +++ b/feature_extractors/feature_extractor_presets.py @@ -30,6 +30,7 @@ presets = __presets | dict( ) def preprocess_feature_extractors_config(data_dict: dict) -> dict: + data_dict = data_dict.copy() keys = ['own_features', 'other_features'] for key in keys: preset_names = data_dict[key] diff --git a/feature_extractors/utils/fracdiff.py b/feature_extractors/utils/fracdiff.py new file mode 100644 index 0000000..a9303cb --- /dev/null +++ b/feature_extractors/utils/fracdiff.py @@ -0,0 +1,240 @@ +""" +Fractional differentiation is a technique to make a time series stationary but also +retain as much memory as possible. This is done by differencing by a positive real +number. Fractionally differenced series can be used as a feature in machine learning +process. +""" + +import numpy as np +import pandas as pd + + +class FractionalDifferentiation: + """ + FractionalDifferentiation class encapsulates the functions that can + be used to compute fractionally differentiated series. + """ + + @staticmethod + def get_weights(diff_amt, size): + """ + Advances in Financial Machine Learning, Chapter 5, section 5.4.2, page 79. + + The helper function generates weights that are used to compute fractionally + differentiated series. It computes the weights that get used in the computation + of fractionally differentiated series. This generates a non-terminating series + that approaches zero asymptotically. The side effect of this function is that + it leads to negative drift "caused by an expanding window's added weights" + (see page 83 AFML) + + When diff_amt is real (non-integer) positive number then it preserves memory. + + The book does not discuss what should be expected if d is a negative real + number. Conceptually (from set theory) negative d leads to set of negative + number of elements. And that translates into a set whose elements can be + selected more than once or as many times as one chooses (multisets with + unbounded multiplicity) - see http://faculty.uml.edu/jpropp/msri-up12.pdf. + + :param diff_amt: (float) Differencing amount + :param size: (int) Length of the series + :return: (np.ndarray) Weight vector + """ + + # The algorithm below executes the iterative estimation (section 5.4.2, page 78) + weights = [1.] # create an empty list and initialize the first element with 1. + for k in range(1, size): + weights_ = -weights[-1] * (diff_amt - k + 1) / k # compute the next weight + weights.append(weights_) + + # Now, reverse the list, convert into a numpy column vector + weights = np.array(weights[::-1]).reshape(-1, 1) + return weights + + @staticmethod + def frac_diff(series, diff_amt, thresh=0.01): + """ + Advances in Financial Machine Learning, Chapter 5, section 5.5, page 82. + + References: + https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086 + https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf + https://en.wikipedia.org/wiki/Fractional_calculus + + The steps are as follows: + - Compute weights (this is a one-time exercise) + - Iteratively apply the weights to the price series and generate output points + + This is the expanding window variant of the fracDiff algorithm + Note 1: For thresh-1, nothing is skipped + Note 2: diff_amt can be any positive fractional, not necessarility bounded [0, 1] + + :param series: (pd.Series) A time series that needs to be differenced + :param diff_amt: (float) Differencing amount + :param thresh: (float) Threshold or epsilon + :return: (pd.DataFrame) Differenced series + """ + + # 1. Compute weights for the longest series + weights = get_weights(diff_amt, series.shape[0]) + + # 2. Determine initial calculations to be skipped based on weight-loss threshold + weights_ = np.cumsum(abs(weights)) + weights_ /= weights_[-1] + skip = weights_[weights_ > thresh].shape[0] + + # 3. Apply weights to values + output_df = {} + for name in series.columns: + series_f = series[[name]].fillna(method='ffill').dropna() + output_df_ = pd.Series(index=series.index, dtype='float64') + + for iloc in range(skip, series_f.shape[0]): + loc = series_f.index[iloc] + + # At this point all entries are non-NAs so no need for the following check + # if np.isfinite(series.loc[loc, name]): + output_df_[loc] = np.dot(weights[-(iloc + 1):, :].T, series_f.loc[:loc])[0, 0] + + output_df[name] = output_df_.copy(deep=True) + output_df = pd.concat(output_df, axis=1) + return output_df + + @staticmethod + def get_weights_ffd(diff_amt, thresh, lim): + """ + Advances in Financial Machine Learning, Chapter 5, section 5.4.2, page 83. + + The helper function generates weights that are used to compute fractionally + differentiate dseries. It computes the weights that get used in the computation + of fractionally differentiated series. The series is of fixed width and same + weights (generated by this function) can be used when creating fractional + differentiated series. + This makes the process more efficient. But the side-effect is that the + fractionally differentiated series is skewed and has excess kurtosis. In + other words, it is not Gaussian any more. + + The discussion of positive and negative d is similar to that in get_weights + (see the function get_weights) + + :param diff_amt: (float) Differencing amount + :param thresh: (float) Threshold for minimum weight + :param lim: (int) Maximum length of the weight vector + :return: (np.ndarray) Weight vector + """ + + weights = [1.] + k = 1 + + # The algorithm below executes the iterativetive estimation (section 5.4.2, page 78) + # The output weights array is of the indicated length (specified by lim) + ctr = 0 + while True: + # compute the next weight + weights_ = -weights[-1] * (diff_amt - k + 1) / k + + if abs(weights_) < thresh: + break + + weights.append(weights_) + k += 1 + ctr += 1 + if ctr == lim - 1: # if we have reached the size limit, exit the loop + break + + # Now, reverse the list, convert into a numpy column vector + weights = np.array(weights[::-1]).reshape(-1, 1) + return weights + + @staticmethod + def frac_diff_ffd(series, diff_amt, thresh=1e-5): + """ + Advances in Financial Machine Learning, Chapter 5, section 5.5, page 83. + + References: + + * https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086 + * https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf + * https://en.wikipedia.org/wiki/Fractional_calculus + + The steps are as follows: + + - Compute weights (this is a one-time exercise) + - Iteratively apply the weights to the price series and generate output points + + Constant width window (new solution) + Note 1: thresh determines the cut-off weight for the window + Note 2: diff_amt can be any positive fractional, not necessarity bounded [0, 1]. + + :param series: (pd.Series) A time series that needs to be differenced + :param diff_amt: (float) Differencing amount + :param thresh: (float) Threshold for minimum weight + :return: (pd.DataFrame) A data frame of differenced series + """ + + # 1) Compute weights for the longest series + weights = get_weights_ffd(diff_amt, thresh, series.shape[0]) + width = len(weights) - 1 + + # 2) Apply weights to values + # 2.1) Start by creating a dictionary to hold all the fractionally differenced series + output_df = {} + + # 2.2) compute fractionally differenced series for each stock + for name in series.columns: + series_f = series[[name]].fillna(method='ffill').dropna() + temp_df_ = pd.Series(index=series.index, dtype='float64') + for iloc1 in range(width, series_f.shape[0]): + loc0 = series_f.index[iloc1 - width] + loc1 = series.index[iloc1] + + # At this point all entries are non-NAs, hence no need for the following check + # if np.isfinite(series.loc[loc1, name]): + temp_df_[loc1] = np.dot(weights.T, series_f.loc[loc0:loc1])[0, 0] + + output_df[name] = temp_df_.copy(deep=True) + + # transform the dictionary into a data frame + output_df = pd.concat(output_df, axis=1) + return output_df + + +def get_weights(diff_amt, size): + """ This is a pass-through function """ + return FractionalDifferentiation.get_weights(diff_amt, size) + + +def frac_diff(series, diff_amt, thresh=0.01): + """ This is a pass-through function """ + return FractionalDifferentiation.frac_diff(series, diff_amt, thresh) + + +def get_weights_ffd(diff_amt, thresh, lim): + """ This is a pass-through function """ + return FractionalDifferentiation.get_weights_ffd(diff_amt, thresh, lim) + + +def frac_diff_ffd(series, diff_amt, thresh=1e-5): + """ + Advances in Financial Machine Learning, Chapter 5, section 5.5, page 83. + + References: + + * https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086 + * https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf + * https://en.wikipedia.org/wiki/Fractional_calculus + + The steps are as follows: + + - Compute weights (this is a one-time exercise) + - Iteratively apply the weights to the price series and generate output points + + Constant width window (new solution) + Note 1: thresh determines the cut-off weight for the window + Note 2: diff_amt can be any positive fractional, not necessarity bounded [0, 1]. + + :param series: (pd.Series) A time series that needs to be differenced + :param diff_amt: (float) Differencing amount + :param thresh: (float) Threshold for minimum weight + :return: (pd.DataFrame) A data frame of differenced series + """ + return FractionalDifferentiation.frac_diff_ffd(series, diff_amt, thresh) \ No newline at end of file diff --git a/feature_selection/dim_reduction.py b/feature_selection/dim_reduction.py new file mode 100644 index 0000000..a112036 --- /dev/null +++ b/feature_selection/dim_reduction.py @@ -0,0 +1,8 @@ +import pandas as pd +from sklearn.decomposition import PCA + +def reduce_dimensionality(X: pd.DataFrame, no_of_compoments: int) -> pd.DataFrame: + pca = PCA(n_components= no_of_compoments) + result = pd.DataFrame(pca.fit_transform(X), index= X.index) + result.columns = ['PCA_' + str(i) for i in range(1, no_of_compoments+1)] + return result diff --git a/feature_selection/feature_selection.py b/feature_selection/feature_selection.py new file mode 100644 index 0000000..57d7052 --- /dev/null +++ b/feature_selection/feature_selection.py @@ -0,0 +1,23 @@ +from sklearn.feature_selection import RFE +from sklearn.model_selection import TimeSeriesSplit +import pandas as pd +from models.base import Model, SKLearnModel +from sklearn.decomposition import PCA + +def select_features(X: pd.DataFrame, y: pd.Series, model: Model, min_features_to_select: int, backup_model: SKLearnModel) -> pd.DataFrame: + ''' Select features using RFECV, returns a pd.DataFrame (X) with only the selected features.''' + if model.model_type != 'ml': return X + + # 2. Recursive feature selection + cv = TimeSeriesSplit(n_splits=5) + + feat_selector_model = model.model + if hasattr(feat_selector_model, 'feature_importances_') == False and hasattr(feat_selector_model, 'coef_') == False: + feat_selector_model = backup_model.model + +# selector = RFECV(feat_selector_model, cv = cv, step=5, min_features_to_select=min_features_to_select) + selector = RFE(feat_selector_model, n_features_to_select=10) + selector = selector.fit(X, y) + print("Kept %d features out of %d" % (selector.n_features_, X.shape[1])) + + return pd.DataFrame(X[X.columns[selector.support_]], index= X.index) diff --git a/models/average.py b/models/average.py index 0b490d5..ce962da 100644 --- a/models/average.py +++ b/models/average.py @@ -6,9 +6,10 @@ class StaticAverageModel(Model): Model that averages . ''' - # data_format = 'dataframe' data_scaling = 'unscaled' only_column = 'model_' + feature_selection = 'off' + model_type = 'static' def fit(self, X, y, prev_model): # This is a static model, it can' learn anything diff --git a/models/base.py b/models/base.py index 1be5bda..31f8244 100644 --- a/models/base.py +++ b/models/base.py @@ -5,10 +5,11 @@ from abc import ABC, abstractmethod, abstractproperty class Model(ABC): - # data_format: Literal['dataframe', 'numpy'] data_scaling: Literal["scaled", "unscaled"] + feature_selection: Literal["on", "off"] # data_format: Literal["wide", "narrow"] only_column: Optional[str] + model_type: Literal['ml', 'static'] @abstractmethod def fit(self, X, y, prev_model): @@ -25,9 +26,10 @@ class Model(ABC): class SKLearnModel(Model): - # data_format = 'numpy' data_scaling = 'scaled' only_column = None + feature_selection = 'on' + model_type = 'ml' def __init__(self, model): self.model = model diff --git a/models/model_map.py b/models/model_map.py index 9226146..ba68555 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -22,12 +22,12 @@ model_map = { KNN = SKLearnModel(KNeighborsRegressor(n_neighbors=25)), AB = SKLearnModel(AdaBoostRegressor(random_state=1)), MLP = SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)), - RF = SKLearnModel(RandomForestRegressor(n_jobs=-1, random_state=1)), + RF = SKLearnModel(RandomForestRegressor(n_jobs=-1, max_depth=20, random_state=1)), SVR = SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1)), StaticNaive = StaticNaiveModel(), ), "classification_models": dict( - LR= SKLearnModel(LogisticRegression(C=10, random_state=1, max_iter=1000)), + LR= SKLearnModel(LogisticRegression(C=10, random_state=1, max_iter=1000, n_jobs=-1)), LDA= SKLearnModel(LinearDiscriminantAnalysis()), KNN= SKLearnModel(KNeighborsClassifier()), CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)), @@ -42,10 +42,12 @@ model_map = { model_names_classification = list(model_map["classification_models"].keys()) model_names_regression = list(model_map["regression_models"].keys()) +default_feature_selector_regression = model_map['regression_models']['RF'] +default_feature_selector_classification = model_map['classification_models']['RF'] def map_model_name_to_function(model_config:dict, method:str) -> dict: - for level in ['level_1_models', 'level_2_models']: - model_category = method + '_models' - model_config[level] = [(model_name, model_map[model_category][model_name]) for model_name in model_config[level]] + model_config['level_1_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in model_config['level_1_models']] + if model_config['level_2_model'] is not None: + model_config['level_2_model'] = (model_config['level_2_model'], model_map[method + '_models'][model_config['level_2_model']]) return model_config \ No newline at end of file diff --git a/models/momentum.py b/models/momentum.py index e838fb6..cdd10bb 100644 --- a/models/momentum.py +++ b/models/momentum.py @@ -6,9 +6,10 @@ class StaticMomentumModel(Model): Model that uses only one feature: momentum. It's positive if momentum is greater than 0, otherwise it's negative. ''' - # data_format = 'dataframe' data_scaling = 'unscaled' only_column = 'mom' + feature_selection = 'off' + model_type = 'static' def __init__(self, allow_short: bool) -> None: super().__init__() diff --git a/models/naive.py b/models/naive.py index 7822a7f..b134675 100644 --- a/models/naive.py +++ b/models/naive.py @@ -6,9 +6,10 @@ class StaticNaiveModel(Model): Model that carries the last observation (from returns) to the next one, naively. ''' - # data_format = 'dataframe' data_scaling = 'unscaled' only_column = None + feature_selection = 'off' + model_type = 'static' def fit(self, X, y, prev_model): # This is a static model, it can' learn anything diff --git a/reporting/wandb.py b/reporting/wandb.py index 0002ff0..4973e05 100644 --- a/reporting/wandb.py +++ b/reporting/wandb.py @@ -17,15 +17,16 @@ def launch_wandb(project_name:str, default_config:dict, sweep:bool=False): def register_config_with_wandb(wandb: Optional[object], model_config:dict, training_config:dict, data_config:dict): - config: dict = wandb.config + if wandb is None: return - if type(wandb) is not type(None): - for k in training_config: - training_config[k] = config[k] - for k in model_config: - model_config[k] = config[k] - for k in data_config: - data_config[k] = config[k] + config: dict = wandb.config + + for k in training_config: + training_config[k] = config[k] + for k in model_config: + model_config[k] = config[k] + for k in data_config: + data_config[k] = config[k] def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object], project_name: str, model_name: str): diff --git a/run_pipeline.py b/run_pipeline.py index e97abb6..8fe136d 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -2,13 +2,15 @@ from utils.load_data import load_data import pandas as pd from training.training import run_single_asset_trainig from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_with_wandb -from models.model_map import map_model_name_to_function +from models.model_map import map_model_name_to_function, default_feature_selector_regression, default_feature_selector_classification from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config -from config import get_default_config, validate_config, get_model_name from utils.helpers import get_first_valid_return_index, weighted_average +from config import get_default_level_1_config, get_default_level_2_config, validate_config, get_model_name +from feature_selection.feature_selection import select_features +from feature_selection.dim_reduction import reduce_dimensionality def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool): - model_config, training_config, data_config = get_default_config() + model_config, training_config, data_config = get_default_level_2_config() wandb = None if with_wandb: @@ -34,15 +36,30 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co data_params['target_asset'] = asset X, y, target_returns = load_data(**data_params) + original_X = X.copy() first_valid_index = get_first_valid_return_index(X.iloc[:,0]) samples_to_train = len(y) - first_valid_index if samples_to_train < training_config['sliding_window_size'] * 3: print("Not enough samples to train") continue - # 2. Train Level-1 models + # 2a. Dimensionality Reduction (optional) + if training_config['dimensionality_reduction']: + X = reduce_dimensionality(X, int(len(X.columns) / 2)) + + # 2b. Feature Selection (optional) + if training_config['feature_selection']: + print("Feature Selection started") + # TODO: this needs to be done per model! + backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification + X = select_features(X, y, model_config['level_1_models'][0][1], min_features_to_select = 10, backup_model = backup_model) + print("Feature Selection ended") + + + # 3. Train Level-1 models current_result, current_predictions = run_single_asset_trainig( ticker_to_predict = asset, + original_X = original_X, X = X, y = y, target_returns = target_returns, @@ -56,20 +73,22 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co level = 1 ) results = pd.concat([results, current_result], axis=1) - all_predictions = pd.concat([all_predictions, current_predictions], axis=1) + # With static models, because of the lag in the indicator, the first prediction is NA, so we fill it with zero. + all_predictions = pd.concat([all_predictions, current_predictions], axis=1).fillna(0.) - if len(model_config['level_2_models']) > 0: - # 3. Train Level-2 (Ensemble) model + # 3. Train Level-2 (Ensemble) model (Optional) + if model_config['level_2_model'] is not None: ensemble_X = all_predictions if training_config['include_original_data_in_ensemble']: ensemble_X = pd.concat([ensemble_X, X], axis=1) ensemble_result, ensemble_preds = run_single_asset_trainig( ticker_to_predict = asset, + original_X = ensemble_X, X = ensemble_X, y = y, target_returns = target_returns, - models = model_config['level_2_models'], + models = [model_config['level_2_model']], method = data_config['method'], expanding_window = training_config['expanding_window'], sliding_window_size = training_config['sliding_window_size'], @@ -82,7 +101,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co results = pd.concat([results, ensemble_result], axis=1) all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1) - # 4. Save & report results + results.to_csv('results.csv') level1_columns = results[[column for column in results.columns if 'lvl1' in column]] @@ -99,9 +118,10 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co print("Mean Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['sharpe'], 3)) print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(weighted_average(level1_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3)) - print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum()) - print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['sharpe'].mean(), 3)) - print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3)) + if model_config['level_2_model'] is not None: + print("Level-2 (Ensemble): Number of samples evaluated: ", level2_columns.loc['no_of_samples'].sum()) + print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['sharpe'].mean(), 3)) + print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(weighted_average(level2_columns, 'no_of_samples').loc['prob_sharpe'].mean(), 3)) if sweep: if wandb.run is not None: diff --git a/sweep_level_1.yaml b/sweep_level_1.yaml index cef55a8..a767353 100644 --- a/sweep_level_1.yaml +++ b/sweep_level_1.yaml @@ -38,8 +38,8 @@ parameters: level_1_models: values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"]] distribution: categorical - level_2_models: - value: [] + level_2_model: + value: None own_features: values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']] distribution: categorical diff --git a/sweep_level_2.yaml b/sweep_level_2.yaml index c164ae5..bef58bc 100644 --- a/sweep_level_2.yaml +++ b/sweep_level_2.yaml @@ -38,8 +38,8 @@ parameters: value: 'int' level_1_models: value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"] - level_2_models: - values: [["LR"], ["LDA"], ["KNN"], ["CART"], ["NB"], ["AB"], ["RF"], ["Ensemble_Average"]] + level_2_model: + values: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "Ensemble_Average"] distribution: categorical own_features: values: [['single_mom', 'date_days'], [], ['level_1', 'date_days'], ['date_days', 'level_2']] diff --git a/sweep_selection.yaml b/sweep_selection.yaml new file mode 100644 index 0000000..92676c6 --- /dev/null +++ b/sweep_selection.yaml @@ -0,0 +1,47 @@ +program: run_sweep.py +method: bayes +project: price-forecasting +name: Feature selection +metric: + goal: maximize + name: sharpe +parameters: + path : + value: 'data/' + expanding_window: + value: True + sliding_window_size: + value: 380 + feature_selection: + values: [True, False] + distribution: categorical + dimensionality_reduction: + values: [True, False] + distribution: categorical + retrain_every: + value: 20 + scaler: + value: 'minmax' + include_original_data_in_ensemble: + value: False + method: + value: 'classification' + no_of_classes: + values: ['two', 'three-balanced', 'three-imbalanced'] + distribution: categorical + forecasting_horizon: + value: 1 + load_other_assets: + value: True + log_returns: + value: True + index_column: + value: 'int' + level_1_models: + value: ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"] + level_2_model: + value: "Ensemble_Average" + own_features: + value: ['date_days', 'level_2', 'lags_up_to_5'] + other_features: + value: ['level_2', 'lags_up_to_5'] diff --git a/training/training.py b/training/training.py index f95e5f4..87bed18 100644 --- a/training/training.py +++ b/training/training.py @@ -17,6 +17,7 @@ def __get_scaler(type: Literal['normalize', 'minmax', 'standardize', 'none']): def run_single_asset_trainig( ticker_to_predict: str, + original_X: pd.DataFrame, X: pd.DataFrame, y: pd.Series, target_returns: pd.Series, @@ -40,7 +41,7 @@ def run_single_asset_trainig( model_over_time, preds = walk_forward_train_test( model_name=model_name, model = model, - X = X, + X = X if model.feature_selection == 'on' else original_X, y = y, target_returns = target_returns, expanding_window = expanding_window, diff --git a/utils/helpers.py b/utils/helpers.py index 09761ea..eeed3e3 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -15,6 +15,8 @@ def flatten(list_of_lists: list) -> list: return [item for sublist in list_of_lists for item in sublist] def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.DataFrame: + if df.shape[0] == 0: + return df mean_df = df.iloc[:,0] weights = df.loc[weights_source] diff --git a/utils/rolling.py b/utils/rolling.py deleted file mode 100644 index e080174..0000000 --- a/utils/rolling.py +++ /dev/null @@ -1,6 +0,0 @@ -import numpy as np - -def rolling_window(a, window): - shape = a.shape[:-1] + (a.shape[-1] - window + 1, window) - strides = a.strides + (a.strides[-1],) - return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides) \ No newline at end of file