diff --git a/config/config.py b/config/config.py index 911ba3c..fd303b8 100644 --- a/config/config.py +++ b/config/config.py @@ -25,7 +25,6 @@ def get_dev_config() -> tuple[dict, dict, dict]: other_features = ['single_mom'], exogenous_features = ['z_score'], index_column= 'int', - method= 'classification', no_of_classes= 'two', narrow_format = False, ) @@ -34,7 +33,7 @@ def get_dev_config() -> tuple[dict, dict, dict]: classification_models = ["LogisticRegression_two_class"] model_config = dict( - primary_models = regression_models if data_config['method'] == 'regression' else classification_models, + primary_models = classification_models, meta_labeling_models = [], ensemble_model = None ) @@ -69,7 +68,6 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]: other_features = ['level_2', 'lags_up_to_5'], exogenous_features = ['z_score'], index_column= 'int', - method= 'classification', no_of_classes= 'two', narrow_format = False, ) @@ -80,7 +78,7 @@ def get_default_ensemble_config() -> tuple[dict, dict, dict]: ensemble_model = 'Average' model_config = dict( - primary_models = regression_models if data_config['method'] == 'regression' else classification_models, + primary_models = classification_models, meta_labeling_models = meta_labeling_models, ensemble_model = ensemble_model ) @@ -115,7 +113,6 @@ def get_lightweight_ensemble_config() -> tuple[dict, dict, dict]: other_features = ['level_2'], exogenous_features = ['z_score'], index_column= 'int', - method= 'classification', no_of_classes= 'two', narrow_format = False, ) @@ -126,7 +123,7 @@ def get_lightweight_ensemble_config() -> tuple[dict, dict, dict]: ensemble_model = 'Average' model_config = dict( - primary_models = regression_models if data_config['method'] == 'regression' else classification_models, + primary_models = classification_models, meta_labeling_models = meta_labeling_models, ensemble_model = ensemble_model ) diff --git a/config/hashing.py b/config/hashing.py index 594c14d..9842ad8 100644 --- a/config/hashing.py +++ b/config/hashing.py @@ -18,7 +18,6 @@ def hash_data_config(data_config: dict) -> str: hash_feature_extractors(data_config['other_features']), hash_feature_extractors(data_config['exogenous_features']), data_config['index_column'], - data_config['method'], data_config['no_of_classes'], data_config['narrow_format'] ])) diff --git a/config/preprocess.py b/config/preprocess.py index b1797e1..185f301 100644 --- a/config/preprocess.py +++ b/config/preprocess.py @@ -5,7 +5,7 @@ from models.model_map import get_model_map from data_loader.collections import data_collections def preprocess_config(model_config:dict, training_config:dict, data_config:dict) -> tuple[dict, dict, dict]: - model_config = __preprocess_model_config(model_config, data_config['method']) + model_config = __preprocess_model_config(model_config) data_config = __preprocess_feature_extractors_config(data_config) data_config = __preprocess_data_collections_config(data_config) @@ -20,7 +20,7 @@ def __preprocess_feature_extractors_config(data_dict: dict) -> dict: data_dict[key] = flatten([feature_extractor_presets[preset_name] for preset_name in preset_names]) return data_dict -def __preprocess_model_config(model_config:dict, method:str) -> dict: +def __preprocess_model_config(model_config:dict) -> dict: model_map = get_model_map(model_config) model_config['primary_models'] = [(model_name, model_map['primary_models'][model_name]) for model_name in model_config['primary_models']] if len(model_config['meta_labeling_models']) > 0: diff --git a/sweep_ensemble.yaml b/config/sweep_ensemble.yaml similarity index 97% rename from sweep_ensemble.yaml rename to config/sweep_ensemble.yaml index d1cf254..d328bc9 100644 --- a/sweep_ensemble.yaml +++ b/config/sweep_ensemble.yaml @@ -32,8 +32,6 @@ parameters: value: 20 scaler: value: 'minmax' - method: - value: 'classification' no_of_classes: value: 'two' forecasting_horizon: diff --git a/sweep_meta_labeling.yaml b/config/sweep_meta_labeling.yaml similarity index 97% rename from sweep_meta_labeling.yaml rename to config/sweep_meta_labeling.yaml index 5d1b0a5..2e5117f 100644 --- a/sweep_meta_labeling.yaml +++ b/config/sweep_meta_labeling.yaml @@ -36,8 +36,6 @@ parameters: distribution: categorical scaler: value: 'minmax' - method: - value: 'classification' no_of_classes: values: ['two', 'three-balanced', 'three-imbalanced'] distribution: categorical diff --git a/sweep_primary_models.yaml b/config/sweep_primary_models.yaml similarity index 97% rename from sweep_primary_models.yaml rename to config/sweep_primary_models.yaml index c34b30e..bb733ce 100644 --- a/sweep_primary_models.yaml +++ b/config/sweep_primary_models.yaml @@ -32,8 +32,6 @@ parameters: distribution: categorical scaler: value: 'minmax' - method: - value: 'classification' no_of_classes: value: 'two' forecasting_horizon: diff --git a/data_loader/load_data.py b/data_loader/load_data.py index 03595de..451a24a 100644 --- a/data_loader/load_data.py +++ b/data_loader/load_data.py @@ -30,7 +30,6 @@ def __load_data(assets: DataCollection, other_features: list[tuple[str, FeatureExtractor, list[int]]], exogenous_features: list[tuple[str, FeatureExtractor, list[int]]], index_column: Literal['date', 'int'], - method: Literal['regression', 'classification'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], narrow_format: bool = False ) -> tuple[pd.DataFrame, pd.Series, pd.Series]: @@ -105,10 +104,7 @@ def __load_data(assets: DataCollection, target_col = 'target' returns_col = target_asset[1] + '_returns' forward_returns = __create_target_cum_forward_returns(df_target_asset_only_returns, returns_col, forecasting_horizon) - if method == 'regression': - dfs[target_col] = forward_returns - elif method == 'classification': - dfs[target_col] = __create_target_classes(dfs, returns_col, forecasting_horizon, no_of_classes) + dfs[target_col] = __create_target_classes(dfs, returns_col, forecasting_horizon, no_of_classes) # we need to drop the last row, because we forward-shift the target (see what happens if you call .shift[-1] on a pd.Series) dfs = dfs.iloc[:-forecasting_horizon] forward_returns = forward_returns.iloc[:-forecasting_horizon] diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 93ca444..17c2137 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -102,7 +102,6 @@ def test_evaluation(): target_returns=fake_target_returns, y_pred=processed_predictions_to_match_returns, y_true=y, - method='classification', no_of_classes='two', print_results = False, discretize=True diff --git a/training/meta_labeling.py b/training/meta_labeling.py index 40f5d74..dea6125 100644 --- a/training/meta_labeling.py +++ b/training/meta_labeling.py @@ -2,7 +2,6 @@ from utils.evaluate import discretize_threeway_threshold, evaluate_predictions from utils.helpers import equal_except_nan from training.primary_model import train_primary_model import pandas as pd -from models.model_map import default_feature_selector_classification, default_feature_selector_regression from models.base import Model from reporting.types import Reporting from typing import Union, Optional @@ -35,7 +34,6 @@ def train_meta_labeling_model( y = meta_y, target_returns = target_returns, models = models, - method = 'classification', expanding_window = training_config['expanding_window_meta_labeling'], sliding_window_size = training_config['sliding_window_size_meta_labeling'], retrain_every = training_config['retrain_every'], @@ -59,7 +57,6 @@ def train_meta_labeling_model( target_returns = target_returns, y_pred = avg_predictions_with_sizing, y_true = y, - method = 'classification', no_of_classes = 'two', print_results = True, discretize=False diff --git a/training/primary_model.py b/training/primary_model.py index b8f61f2..9998cd8 100644 --- a/training/primary_model.py +++ b/training/primary_model.py @@ -15,7 +15,6 @@ def train_primary_model( y: pd.Series, target_returns: pd.Series, models: list[tuple[str, Model]], - method: Literal['regression', 'classification'], expanding_window: bool, sliding_window_size: int, retrain_every: int, @@ -80,7 +79,6 @@ def train_primary_model( target_returns = target_returns, y_pred = preds, y_true = y, - method = method, no_of_classes=no_of_classes, print_results = print_results, discretize=True diff --git a/training/training_steps.py b/training/training_steps.py index 58c46c6..7cb6ea8 100644 --- a/training/training_steps.py +++ b/training/training_steps.py @@ -28,7 +28,6 @@ def primary_step( y = y, target_returns = target_returns, models = model_config['primary_models'], - method = data_config['method'], expanding_window = training_config['expanding_window_primary'], sliding_window_size = training_config['sliding_window_size_primary'], retrain_every = training_config['retrain_every'], @@ -94,9 +93,7 @@ def secondary_step( X = current_predictions, y = y, target_returns = target_returns, - models = [model_config['ensemble_model']], - method = data_config['method'], - expanding_window = False, + models = [model_config['ensemble_model']], expanding_window = False, sliding_window_size = 1, retrain_every = training_config['retrain_every'], from_index = from_index, diff --git a/utils/evaluate.py b/utils/evaluate.py index 40adfee..0e78a79 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -12,19 +12,15 @@ def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.002) -> return (signal * returns) - costs -def __preprocess(target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, method: Literal['classification', 'regression'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame: +def __preprocess(target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame: y_pred.name = 'y_pred' target_returns.name = 'target_returns' df = pd.concat([y_pred, target_returns],axis=1).dropna() discretize_func = get_discretize_function(no_of_classes) # make sure that we evaluate binary/three-way predictions even if the model is a regression - if method == 'regression': - df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred - df['sign_true'] = df.target_returns.apply(discretize_func) - else: - df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred - df['sign_true'] = y_true + df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred + df['sign_true'] = y_true df['result'] = backtest(df.target_returns, df.sign_pred) @@ -35,7 +31,6 @@ def evaluate_predictions( target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, - method: Literal['classification', 'regression'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], print_results: bool, discretize: bool = False, @@ -45,14 +40,9 @@ def evaluate_predictions( evaluate_from = first_nonzero_return + 1 target_returns = pd.Series(target_returns[evaluate_from:]) - if method == 'regression': - # if there are lots of zeros in the ground truth returns, probably something is wrong, but we can tolerate a couple of days of missing data. - is_zero = target_returns[target_returns == 0] - assert len(is_zero) < 15 - y_pred = pd.Series(y_pred[evaluate_from:]) - df = __preprocess(target_returns, y_pred, y_true, method, no_of_classes, discretize) + df = __preprocess(target_returns, y_pred, y_true, no_of_classes, discretize) scorecard = pd.Series()