From 6ae8acf70e005c35261f465d9f8a21407795568d Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Wed, 22 Dec 2021 16:59:03 +0100 Subject: [PATCH] feat(Models): added `debug_future_lookahead`, sped up LogisticRegression & DecisionTreeClassifier (#74) * feat(Models): added `debug_future_lookahead`, sped up LogisticRegression & DecisionTreeClassifier * feat(Training): added ability to train on expanding_window * feat(Models): tuned some hyperparameters, added expanding_window to sweep config, fixed tests * feat(Models): tune parameters of ensemble models * fix(Config): use window size that works with ensembling --- config.py | 5 +++-- environment.yml | 1 + feature_extractors/feature_extractor_presets.py | 4 +++- feature_extractors/feature_extractors.py | 4 ++++ models/model_map.py | 12 ++++++------ run_pipeline.py | 2 ++ sweep.yaml | 3 +++ tests/test_evaluation.py | 1 + tests/test_walk_forward.py | 1 + training/training.py | 2 ++ training/walk_forward.py | 7 ++++++- 11 files changed, 32 insertions(+), 10 deletions(-) diff --git a/config.py b/config.py index c714ca4..de7b902 100644 --- a/config.py +++ b/config.py @@ -5,8 +5,9 @@ from models.model_map import model_names_classification, model_names_regression def get_default_config() -> tuple[dict, dict, dict]: training_config = dict( - sliding_window_size = 150, - retrain_every = 20, + expanding_window = True, + sliding_window_size = 200, + retrain_every = 100, scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' include_original_data_in_ensemble = True, ) diff --git a/environment.yml b/environment.yml index 402bbd0..1fa97bc 100644 --- a/environment.yml +++ b/environment.yml @@ -8,6 +8,7 @@ dependencies: - seaborn - scikit-learn-intelex=2021.4.0 - ipython + - ipykernel - scipy - scikit-learn - numba diff --git a/feature_extractors/feature_extractor_presets.py b/feature_extractors/feature_extractor_presets.py index 55056a0..536ba76 100644 --- a/feature_extractors/feature_extractor_presets.py +++ b/feature_extractors/feature_extractor_presets.py @@ -1,4 +1,6 @@ -from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_vol, feature_day_of_month, feature_day_of_week, feature_month +from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead + +debug_future_lookahead = [('debug_future', feature_debug_future_lookahead, [1])] lags = [('lag', feature_lag, [1,2,3,4,5,6,7,8,9])] diff --git a/feature_extractors/feature_extractors.py b/feature_extractors/feature_extractors.py index be64c45..6add0fd 100644 --- a/feature_extractors/feature_extractors.py +++ b/feature_extractors/feature_extractors.py @@ -11,7 +11,11 @@ def __get_close_low_high(df: pd.DataFrame) -> tuple[pd.Series, pd.Series, pd.Ser ## Feature extractors +def feature_debug_future_lookahead(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series: + return df['returns'].shift(-period) + def feature_lag(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series: + assert period > 0 return df['returns'].shift(period) def feature_day_of_week(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.DataFrame: diff --git a/models/model_map.py b/models/model_map.py index df81f65..eb333be 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -15,25 +15,25 @@ from models.naive import StaticNaiveModel model_map = { "regression_models": dict( - Lasso = SKLearnModel(Lasso(alpha=0.1, max_iter=1000)), + LR = SKLearnModel(LinearRegression(n_jobs=-1)), + Lasso = SKLearnModel(Lasso(alpha=100, random_state=1)), Ridge = SKLearnModel(Ridge(alpha=0.1)), BayesianRidge = SKLearnModel(BayesianRidge()), KNN = SKLearnModel(KNeighborsRegressor(n_neighbors=25)), AB = SKLearnModel(AdaBoostRegressor(random_state=1)), - LR = SKLearnModel(LinearRegression(n_jobs=-1)), MLP = SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)), RF = SKLearnModel(RandomForestRegressor(n_jobs=-1)), SVR = SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1)), StaticNaive = StaticNaiveModel(), ), "classification_models": dict( - LR= SKLearnModel(LogisticRegression(n_jobs=-1)), + LR= SKLearnModel(LogisticRegression(solver='liblinear', C=10, max_iter=1000)), LDA= SKLearnModel(LinearDiscriminantAnalysis()), KNN= SKLearnModel(KNeighborsClassifier()), - CART= SKLearnModel(DecisionTreeClassifier()), + CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)), NB= SKLearnModel(GaussianNB()), - AB= SKLearnModel(AdaBoostClassifier()), - RF= SKLearnModel(RandomForestClassifier(n_jobs=-1)), + AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)), + RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20)), StaticMom= StaticMomentumModel(allow_short=True), ), "classification_ensemble_models": dict( diff --git a/run_pipeline.py b/run_pipeline.py index 3c1234e..4811ae3 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -40,6 +40,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co target_returns = target_returns, models = model_config['level_1_models'], method = data_config['method'], + expanding_window = training_config['expanding_window'], sliding_window_size = training_config['sliding_window_size'], retrain_every = training_config['retrain_every'], scaler = training_config['scaler'], @@ -64,6 +65,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co target_returns = target_returns, models = model_config['level_2_models'], method = data_config['method'], + expanding_window = training_config['expanding_window'], sliding_window_size = training_config['sliding_window_size'], retrain_every = training_config['retrain_every'], scaler = training_config['scaler'], diff --git a/sweep.yaml b/sweep.yaml index 0487469..387a81c 100644 --- a/sweep.yaml +++ b/sweep.yaml @@ -11,6 +11,9 @@ metric: parameters: path : value: 'data/' + expanding_window: + values: [True, False] + distribution: categorical sliding_window_size: values: [90, 130, 160, 180, 280, 380] distribution: categorical diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 53ef86e..8b8fd5b 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -68,6 +68,7 @@ def test_evaluation(): X=X, y=y, target_returns=y, + expanding_window=False, window_size=window_length, retrain_every=10, scaler=scaler diff --git a/tests/test_walk_forward.py b/tests/test_walk_forward.py index 1286b0c..ce763f4 100644 --- a/tests/test_walk_forward.py +++ b/tests/test_walk_forward.py @@ -65,6 +65,7 @@ def test_walk_forward_train_test(): X=X, y=y, target_returns=y, + expanding_window=False, window_size=window_length, retrain_every=10, scaler=scaler) diff --git a/training/training.py b/training/training.py index bbd3348..db3a5c9 100644 --- a/training/training.py +++ b/training/training.py @@ -22,6 +22,7 @@ def run_single_asset_trainig( target_returns: pd.Series, models: list[tuple[str, Model]], method: Literal['regression', 'classification'], + expanding_window: bool, sliding_window_size: int, retrain_every: int, scaler: Literal['normalize', 'minmax', 'standardize', 'none'], @@ -45,6 +46,7 @@ def run_single_asset_trainig( X = X, y = y, target_returns = target_returns, + expanding_window = expanding_window, window_size = sliding_window_size, retrain_every = retrain_every, scaler = scaler diff --git a/training/walk_forward.py b/training/walk_forward.py index c829677..37eadcd 100644 --- a/training/walk_forward.py +++ b/training/walk_forward.py @@ -11,6 +11,7 @@ def walk_forward_train_test( X: pd.DataFrame, y: pd.Series, target_returns: pd.Series, + expanding_window: bool, window_size: int, retrain_every: int, scaler, @@ -35,7 +36,11 @@ def walk_forward_train_test( for index in range(train_from, train_till): if iterations_before_retrain <= 0 or pd.isna(models[index-1]): - train_window_start = index - window_size - 1 + if expanding_window: + train_window_start = first_nonzero_return + else: + train_window_start = index - window_size - 1 + train_window_end = index - 1 if is_scaling_on: