From 95573eb9ddd9842185a6040153caad0cd4132053 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Thu, 23 Dec 2021 13:24:56 +0100 Subject: [PATCH] feat(Data): add option to predict 3 classes (#79) * feat(Data): add option to predict 3 classes * feat(Evaluation): added ability to evaluate 3 class predictions * chore(Config): set sensible config for regression models * feat(Data): added option to use balanced or imbalanced three-class data * feat(Evaluate): correctly track "no_of_samples" now that we have three classes * chore(Sweep): remove probably not useful scaler values from sweep --- config.py | 9 +-- models/model_map.py | 6 +- run_pipeline.py | 13 ++-- sweep.yaml | 14 ++--- tests/test_evaluation.py | 4 +- training/training.py | 10 ++-- utils/evaluate.py | 49 +++++++-------- utils/load_data.py | 125 +++++++++++++++++++++++++-------------- 8 files changed, 134 insertions(+), 96 deletions(-) diff --git a/config.py b/config.py index 928fdb1..c63750d 100644 --- a/config.py +++ b/config.py @@ -6,7 +6,7 @@ from models.model_map import model_names_classification, model_names_regression def get_default_config() -> tuple[dict, dict, dict]: training_config = dict( - expanding_window = True, + expanding_window = False, sliding_window_size = 200, retrain_every = 100, scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none' @@ -23,13 +23,14 @@ def get_default_config() -> tuple[dict, dict, dict]: other_features = [], index_column= 'int', method= 'classification', + no_of_classes= 'two' ) # regression_models = ["Lasso", "Ridge", "BayesianRidge", "KNN", "AB", "LR", "MLP", "RF", "SVR"] - regression_models = model_names_regression + regression_models = ["Lasso", "KNN", "RF"] regression_ensemble_models = ['Ensemble_Average'] - # classification_models = ["LR", "LDA", "KNN", "CART", "NB", "AB", "RF", "StaticMom"] - classification_models = model_names_classification + classification_models = ["LR", "LDA", "KNN", "CART", "RF"] + # classification_models = model_names_classification classification_ensemble_models = ['Ensemble_Average'] model_config = dict( diff --git a/models/model_map.py b/models/model_map.py index eb333be..ff18aa0 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -22,18 +22,18 @@ model_map = { KNN = SKLearnModel(KNeighborsRegressor(n_neighbors=25)), AB = SKLearnModel(AdaBoostRegressor(random_state=1)), MLP = SKLearnModel(MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)), - RF = SKLearnModel(RandomForestRegressor(n_jobs=-1)), + RF = SKLearnModel(RandomForestRegressor(n_jobs=-1, random_state=1)), SVR = SKLearnModel(SVR(kernel='rbf', C=1e3, gamma=0.1)), StaticNaive = StaticNaiveModel(), ), "classification_models": dict( - LR= SKLearnModel(LogisticRegression(solver='liblinear', C=10, max_iter=1000)), + LR= SKLearnModel(LogisticRegression(C=10, random_state=1, max_iter=1000)), LDA= SKLearnModel(LinearDiscriminantAnalysis()), KNN= SKLearnModel(KNeighborsClassifier()), CART= SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)), NB= SKLearnModel(GaussianNB()), AB= SKLearnModel(AdaBoostClassifier(n_estimators=15)), - RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20)), + RF= SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)), StaticMom= StaticMomentumModel(allow_short=True), ), "classification_ensemble_models": dict( diff --git a/run_pipeline.py b/run_pipeline.py index baa578c..f1127dd 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -46,9 +46,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co sliding_window_size = training_config['sliding_window_size'], retrain_every = training_config['retrain_every'], scaler = training_config['scaler'], - wandb = wandb, - project_name=project_name, - sweep=sweep + no_of_classes = data_config['no_of_classes'] ) results = pd.concat([results, current_result], axis=1) all_predictions = pd.concat([all_predictions, current_predictions], axis=1) @@ -71,9 +69,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co sliding_window_size = training_config['sliding_window_size'], retrain_every = training_config['retrain_every'], scaler = training_config['scaler'], - wandb = wandb, - project_name=project_name, - sweep=sweep + no_of_classes = data_config['no_of_classes'] ) results = pd.concat([results, ensemble_result], axis=1) @@ -86,8 +82,9 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]] ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]] - print("Mean Sharpe ratio for Level-1 models: ", level1_columns.loc['sharpe'].mean()) - print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", ensemble_columns.loc['sharpe'].mean()) + print("Mean no of samples: ", results.loc['no_of_samples'].mean()) + print("Mean Sharpe ratio for Level-1 models: ", round(level1_columns.loc['sharpe'].mean(), 3)) + print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_columns.loc['sharpe'].mean(), 3)) if sweep: if wandb.run is not None: diff --git a/sweep.yaml b/sweep.yaml index 1f2a8ae..a9f9edc 100644 --- a/sweep.yaml +++ b/sweep.yaml @@ -21,22 +21,23 @@ parameters: values: [10, 20, 30] distribution: categorical scaler: - values: ['minmax', 'normalize', 'minmax', 'standardize', 'none'] + values: ['minmax', 'none'] distribution: categorical include_original_data_in_ensemble: - values: [True, False] - distribution: categorical + values: False method: value: 'classification' + no_of_classes: + values: ['two', 'three-balanced', 'three-imbalanced'] + distribution: categorical forecasting_horizon: - values: [1,2,3,4,5,6,7,8,9,10] + values: 1 distribution: categorical load_other_assets: values: [True, False] distribution: categorical log_returns: - values: [True, False] - distribution: categorical + value: True index_column: value: 'int' level_1_models: @@ -44,7 +45,6 @@ parameters: distribution: categorical level_2_models: value: [] - distribution: constant own_features: values: [['only_mom', 'date_days'], [], ['level_1', 'date_days'], ['level_1', 'date_days', 'level_2']] distribution: categorical diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 58be78f..57c9a0f 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -85,7 +85,9 @@ def test_evaluation(): model_name='test', target_returns=fake_target_returns, y_pred=processed_predictions_to_match_returns, - method='classification' + y_true=y, + method='classification', + no_of_classes='two' ) assert result['accuracy'] == 100.0 diff --git a/training/training.py b/training/training.py index db3a5c9..9b8d985 100644 --- a/training/training.py +++ b/training/training.py @@ -26,10 +26,8 @@ def run_single_asset_trainig( sliding_window_size: int, retrain_every: int, scaler: Literal['normalize', 'minmax', 'standardize', 'none'], - wandb, - project_name:str, - sweep:bool - ) -> tuple[pd.DataFrame, pd.DataFrame]: + no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'] + ) -> tuple[pd.DataFrame, pd.DataFrame]: scaler = __get_scaler(scaler) @@ -37,8 +35,6 @@ def run_single_asset_trainig( results = pd.DataFrame() predictions = pd.DataFrame() - wandb_active = type(wandb) is not type(None) - for model_name, model in models: model_over_time, preds = walk_forward_train_test( model_name=model_name, @@ -56,7 +52,9 @@ def run_single_asset_trainig( model_name = model_name, target_returns = target_returns, y_pred = preds, + y_true = y, method = method, + no_of_classes=no_of_classes ) column_name = ticker_to_predict + "_" + model_name results[column_name] = result diff --git a/utils/evaluate.py b/utils/evaluate.py index 7d00cb4..c1967e2 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -11,23 +11,22 @@ def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.00) -> return (signal * returns) - costs -def __preprocess(target_returns: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']) -> pd.DataFrame: +def __preprocess(target_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, method: Literal['classification', 'regression'], no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']) -> pd.DataFrame: y_pred.name = 'y_pred' target_returns.name = 'target_returns' df = pd.concat([y_pred, target_returns],axis=1).dropna() - df['sign_pred'] = df.y_pred.apply(lambda x: 1 if x>0 else -1) - def sign_true(x): - if x > 0: - return 1 - else: - return -1 - df['sign_true'] = df.target_returns.apply(sign_true) - df['is_correct'] = 0 - df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct - df['is_incorrect'] = 0 - df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong - df['is_predicted'] = df.is_correct + df.is_incorrect + def categorize_binary(x): return 1 if x > 0 else -1 + def categorize_threeway(x): return 0 if x == 0 else 1 if x > 0 else -1 + categorize = categorize_binary if no_of_classes == 'two' else categorize_threeway + # make sure that we evaluate binary/three-way predictions even if the model is a regression + if method == 'regression': + df['sign_pred'] = df.y_pred.apply(categorize) + df['sign_true'] = df.target_returns.apply(categorize) + else: + df['sign_pred'] = df.y_pred.apply(categorize) + df['sign_true'] = y_true + df['result'] = backtest(df.target_returns, df.sign_pred) return df @@ -36,7 +35,9 @@ def evaluate_predictions( model_name: str, target_returns: pd.Series, y_pred: pd.Series, - method: Literal['classification', 'regression'] + y_true: pd.Series, + method: Literal['classification', 'regression'], + no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], ) -> pd.Series: # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size) first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(y_pred)) @@ -50,7 +51,7 @@ def evaluate_predictions( y_pred = pd.Series(y_pred[evaluate_from:]) - df = __preprocess(target_returns, y_pred, method) + df = __preprocess(target_returns, y_pred, y_true, method, no_of_classes) scorecard = pd.Series() if method == 'regression': @@ -62,15 +63,20 @@ def evaluate_predictions( sign_true = df.sign_true.astype(int) sign_pred = df.sign_pred.astype(int) - scorecard.loc['no_of_samples'] = len(target_returns) - evaluate_from + def count_non_zero(series: pd.Series) -> int: + return len(series[series != 0]) + scorecard.loc['no_of_samples'] = count_non_zero(y_pred[evaluate_from:]) scorecard.loc['sharpe'] = sharpe(df.result) scorecard.loc['sortino'] = sortino(df.result) scorecard.loc['skew'] = skew(df.result) + labels = [1, -1] if no_of_classes == 'two' else [1, -1, 0] + avg_type = 'weighted' if no_of_classes == 'two' else 'macro' + scorecard.loc['accuracy'] = accuracy_score(sign_true, sign_pred) * 100 - scorecard.loc['recall'] = recall_score(sign_true, sign_pred, labels = [1, -1]) - scorecard.loc['precision'] = precision_score(sign_true, sign_pred, labels = [1, -1]) - scorecard.loc['f1_score'] = f1_score(sign_true, sign_pred, labels = [1, -1]) + scorecard.loc['recall'] = recall_score(sign_true, sign_pred, labels = labels, average=avg_type) + scorecard.loc['precision'] = precision_score(sign_true, sign_pred, labels = labels, average=avg_type) + scorecard.loc['f1_score'] = f1_score(sign_true, sign_pred, labels = labels, average=avg_type) scorecard.loc['edge'] = df.result.mean() scorecard.loc['noise'] = df.y_pred.diff().abs().mean() scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise'] @@ -82,11 +88,6 @@ def evaluate_predictions( scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(sign_pred) - - # scorecard.loc['ratio_of_classes_y'] = ' / '.join([str(index) + " " + str(round(row / len(sign_true), 2)) for index, row in sign_true.value_counts().iteritems()]) - # scorecard.loc['ratio_of_classes_pred'] = ' / '.join([str(round(row / len(sign_pred), 2)) for index, row in sign_pred.value_counts().iteritems()]) - - if method == 'regression': scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE'] elif method == 'classification': diff --git a/utils/load_data.py b/utils/load_data.py index 445112d..c0d734a 100644 --- a/utils/load_data.py +++ b/utils/load_data.py @@ -2,11 +2,8 @@ import pandas as pd import os import numpy as np -from pandas.core.frame import DataFrame -from utils.technical_indicators import ROC, RSI, STOK, STOD from utils.typing import FeatureExtractor -from typing import Callable, Literal -from sklearn.preprocessing import OneHotEncoder +from typing import Literal #%% @@ -26,6 +23,7 @@ def load_data(path: str, other_features: list[tuple[str, FeatureExtractor, list[int]]], index_column: Literal['date', 'int'], method: Literal['regression', 'classification'], + no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], narrow_format: bool = False, all_assets:list=[] ) -> tuple[pd.DataFrame, pd.Series, pd.Series]: @@ -68,7 +66,7 @@ def load_data(path: str, if method == 'regression': dfs[target_col] = forward_returns elif method == 'classification': - dfs[target_col] = __create_target_classes(dfs, returns_col, forecasting_horizon, 'two') + dfs[target_col] = __create_target_classes(dfs, returns_col, forecasting_horizon, no_of_classes) # we need to drop the last row, because we forward-shift the target (see what happens if you call .shift[-1] on a pd.Series) dfs = dfs.iloc[:-forecasting_horizon] forward_returns = forward_returns.iloc[:-forecasting_horizon] @@ -78,29 +76,6 @@ def load_data(path: str, return X, y, forward_returns -# %% - -def load_crypto_only_returns(path: str, index_column: Literal['date', 'int'], returns: Literal['price', 'returns']) -> pd.DataFrame: - files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')] - dfs = [__load_df( - path=os.path.join(path,f), - prefix=f.split('.')[0], - returns=returns, - feature_extractors=[], - narrow_format=False, - ) for f in files] - dfs = pd.concat(dfs, axis=1) - dfs = dfs.applymap(lambda x: np.nan if x == 0 else x) - dfs.index = pd.DatetimeIndex(dfs.index) - dfs.columns = [column.split('_')[0] for column in dfs.columns] - if index_column == 'int': - dfs.reset_index(drop=True, inplace=True) - - return dfs - -def load_crypto_assets_availability(path: str, index_column: Literal['date', 'int']) -> pd.DataFrame: - return load_crypto_only_returns(path, index_column, 'returns').applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1) - def __load_df(path: str, prefix: str, @@ -156,24 +131,88 @@ def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, pe def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.Series: assert period > 0 - def get_class_binary(x): + def get_class_binary(x: float) -> int: return -1 if x <= 0.0 else 1 - def get_class_threeway(x): - bins = pd.qcut(df[source_column], 4, duplicates='raise', retbins=True)[1] - lower_threshold = bins[1] - upper_threshold = bins[3] - if x <= lower_threshold: - return -1 - elif x > lower_threshold and x < upper_threshold: - return 0 - else: - return 1 + def get_class_threeway_balanced(series: pd.Series) -> pd.Series: + + def get_bins_threeway(x): + bins = pd.qcut(df[source_column], 3, retbins=True, duplicates = 'drop')[1] + + if len(bins) != 4: + # if we don't have enough data for the quantiles, we'll need to add hard-coded values + lower_bound = bins[0] + upper_bound = bins[-1] + bins = [lower_bound] + [-0.02, 0.02] + [upper_bound] + return bins + bins = get_bins_threeway(series) + + def map_class_threeway(current_value): + lower_threshold = bins[1] + upper_threshold = bins[2] + if current_value <= lower_threshold: + return -1 + elif current_value > lower_threshold and current_value < upper_threshold: + return 0 + else: + return 1 + return series.map(map_class_threeway) + + def get_class_threeway_imbalanced(series: pd.Series) -> pd.Series: + + def get_bins_threeway(x): + bins = pd.qcut(df[source_column], 4, retbins=True, duplicates = 'drop')[1] + + if len(bins) != 5: + # if we don't have enough data for the quantiles, we'll need to add hard-coded values + lower_bound = bins[0] + upper_bound = bins[-1] + bins = [lower_bound] + [-0.02, 0.0, 0.02] + [upper_bound] + return bins + bins = get_bins_threeway(series) + + def map_class_threeway(current_value): + lower_threshold = bins[1] + upper_threshold = bins[3] + if current_value <= lower_threshold: + return -1 + elif current_value > lower_threshold and current_value < upper_threshold: + return 0 + else: + return 1 + return series.map(map_class_threeway) target_column = df[source_column].shift(-period) - get_class_function = get_class_binary - if no_of_classes == "three": - get_class_function = get_class_threeway + if no_of_classes == "three-balanced": + return get_class_threeway_balanced(target_column) + elif no_of_classes == "three-imbalanced": + return get_class_threeway_imbalanced(target_column) + else: + return target_column.map(get_class_binary) + + + + +# These are needed for the portfolio feature, maybe we can do this in a more elegant way +# def load_crypto_only_returns(path: str, index_column: Literal['date', 'int'], returns: Literal['price', 'returns']) -> pd.DataFrame: +# files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')] +# dfs = [__load_df( +# path=os.path.join(path,f), +# prefix=f.split('.')[0], +# returns=returns, +# feature_extractors=[], +# narrow_format=False, +# ) for f in files] +# dfs = pd.concat(dfs, axis=1) +# dfs = dfs.applymap(lambda x: np.nan if x == 0 else x) +# dfs.index = pd.DatetimeIndex(dfs.index) +# dfs.columns = [column.split('_')[0] for column in dfs.columns] +# if index_column == 'int': +# dfs.reset_index(drop=True, inplace=True) + +# return dfs + +# def load_crypto_assets_availability(path: str, index_column: Literal['date', 'int']) -> pd.DataFrame: +# return load_crypto_only_returns(path, index_column, 'returns').applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1) - return target_column.map(get_class_function) \ No newline at end of file