From cc7061b456581bc90ae3f8453ab89808acfd49c2 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Fri, 17 Dec 2021 14:32:17 +0100 Subject: [PATCH] feat(Core): ensemble models, correct forward returns calculation, scaling, only train from when asset returns are available, major bug fixed in walk_forward_train_test (#35) * fix(Core): correct forward returns calculation, classifiers are now working again, only train from when asset returns are available * feat(Utils): added get_first_valid_return_index() * feat(Ensemble): return models from `run_whole_pipeline` * feat(Ensemble): added ensemble step, fixed walk_forward_train_test predictions index confusion, * chore(Pipeline): remove unnecessary extra ensemble results dataframe * refactor(Core): removed unnecessary ensemble_train_predict, moved run_single_asset_trainig_pipeline to a separate file * feat(Training): added scaling on expanding window (the past) to walk_forward_train_test(), now printing out mean sharpe ratio * feat(CI): added environment.yml file * chore(Environment): update env.yml * feat(CI): added testing workflow * fix(CI): renamed enviroment.yml * fix(Tests): added missing new parameter to walk_forward_train_test() --- .github/workflows/test.yml | 42 ++++++++++ README.md | 4 +- environment.yml | 20 +++++ load_data.py | 82 ++++++++++++------ run_pipeline.py | 166 +++++++++++++++++++------------------ tests/test_walk_forward.py | 27 ++++-- training/pipeline.py | 60 ++++++++++++++ training/walk_forward.py | 63 ++++++++++++++ utils/evaluate.py | 52 +++++++----- utils/get_prices.py | 10 --- utils/helpers.py | 6 ++ utils/walk_forward.py | 48 ----------- 12 files changed, 385 insertions(+), 195 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 environment.yml create mode 100644 training/pipeline.py create mode 100644 training/walk_forward.py create mode 100644 utils/helpers.py delete mode 100644 utils/walk_forward.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..23dc951 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,42 @@ +name: run-test +on: [push] +jobs: + run: + runs-on: [ubuntu-latest] + # optionally use a convenient Ubuntu LTS + CUDA + DVC + CML image + # container: docker://dvcorg/cml:0-dvc2-base1-gpu + # container: docker://ghcr.io/iterative/cml:0-dvc2-base1-gpu + steps: + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: false + activate-environment: quant + environment-file: environment.yml + python-version: 3.9 + - uses: iterative/setup-cml@v1 + - name: Run pytest + shell: bash -l {0} +# env: +# WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} + run: | + pytest --junit-xml pytest.xml + - name: Upload Unit Test Results + if: always() + uses: actions/upload-artifact@v2 + with: + name: Unit Test Results (Python ${{ matrix.python-version }}) + path: pytest.xml + - name: Publish Unit Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + if: always() + with: + files: pytest.xml + +# - name: Write CML report +# env: +# REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# run: | +# # Post reports as comments in GitHub PRs +# # cat results.txt >> report.md +# # cml-send-comment report.md diff --git a/README.md b/README.md index 3fe1661..19133d8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# Price prediction models +# Financial time series prediction models ## Installation -Make sure you use keras `2.6.0`, otherwise you won't be able to train models. \ No newline at end of file +Use the conda environment file attached!:) \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..691d211 --- /dev/null +++ b/environment.yml @@ -0,0 +1,20 @@ +name: quant +channels: + - johnsnowlabs + - conda-forge + - defaults +dependencies: + - python=3.9 + - seaborn + - scikit-learn-intelex + - ipython + - scipy + - scikit-learn + - numba + - pytorch + - matplotlib + - numpy + - quantstats + - pytorch-lightning + - pytest +prefix: /usr/local/anaconda3/envs/quant diff --git a/load_data.py b/load_data.py index 6bc9c98..6a10a4e 100644 --- a/load_data.py +++ b/load_data.py @@ -10,7 +10,7 @@ from sklearn.preprocessing import OneHotEncoder #%% def get_crypto_assets(path: str) -> list[str]: - return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' in f and not f.startswith('.')]) + return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')]) def get_etf_assets(path: str) -> list[str]: return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' not in f and not f.startswith('.')]) @@ -29,7 +29,14 @@ def load_data(path: str, index_column: Literal['date', 'int'], method: Literal['regression', 'classification'], narrow_format: bool = False, - ) -> tuple[pd.DataFrame, pd.Series]: + ) -> tuple[pd.DataFrame, pd.Series, pd.Series]: + """ + Loads asset data from the specified path. + Returns: + - DataFrame `X` with all the training data + - Series `y` with the target asset returns shifted by 1 day OR if it's a classification problem, the target class) + - Series `forward_returns` with the target asset returns shifted by 1 day + """ files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')] files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(target_asset))] @@ -37,7 +44,7 @@ def load_data(path: str, dfs = [__load_df( path=os.path.join(path,f), prefix=f.split('.')[0], - log_returns=log_returns, + returns='log_returns' if log_returns else 'returns', technical_features=own_technical_features if is_target_asset(target_asset, f) else other_technical_features, lags= target_asset_lags if is_target_asset(target_asset, f) else other_asset_lags, narrow_format=narrow_format, @@ -63,28 +70,60 @@ def load_data(path: str, ## Create target target_col = 'target' returns_col = target_asset + '_returns' + prediction_horizon = 1 + forward_returns = __create_target_cum_forward_returns(dfs, returns_col, prediction_horizon) if method == 'regression': - dfs = __create_target_cum_forward_returns(dfs, returns_col, 1) + dfs[target_col] = forward_returns elif method == 'classification': - dfs = __create_target_classes(dfs, returns_col, 1, 'two') - + dfs[target_col] = __create_target_classes(dfs, returns_col, prediction_horizon, 'two') + # we need to drop the last row, because we forward-shift the target (see what happens if you call .shift[-1] on a pd.Series) + dfs = dfs.iloc[:-prediction_horizon] + forward_returns = forward_returns.iloc[:-prediction_horizon] + X = dfs.drop(columns=[target_col]) y = dfs[target_col] - return X, y + return X, y, forward_returns -def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Literal['none', 'level1', 'level2'], lags: list[int], narrow_format: bool = False) -> pd.DataFrame: +# %% + +def load_crypto_only_returns(path: str, index_column: Literal['date', 'int'], returns: Literal['price', 'returns']) -> pd.DataFrame: + files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')] + dfs = [__load_df( + path=os.path.join(path,f), + prefix=f.split('.')[0], + returns=returns, + technical_features='none', + lags=[], + narrow_format=False, + ) for f in files] + dfs = pd.concat(dfs, axis=1) + dfs = dfs.applymap(lambda x: np.nan if x == 0 else x) + dfs.index = pd.DatetimeIndex(dfs.index) + dfs.columns = [column.split('_')[0] for column in dfs.columns] + if index_column == 'int': + dfs.reset_index(drop=True, inplace=True) + + return dfs + +def load_crypto_assets_availability(path: str, index_column: Literal['date', 'int']) -> pd.DataFrame: + return load_crypto_only_returns(path, index_column, 'returns').applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1) + + +def __load_df(path: str, prefix: str, returns: Literal['price', 'returns', 'log_returns'], technical_features: Literal['none', 'level1', 'level2'], lags: list[int], narrow_format: bool = False) -> pd.DataFrame: df = pd.read_csv(path, header=0, index_col=0).fillna(0) - if log_returns: + if returns == 'log_returns': df['returns'] = np.log(df['close']).diff(1) + elif returns == 'price': + df['returns'] = df['close'] else: df['returns'] = df['close'].pct_change() for lag in lags: df[f'lag_{lag}'] = df['returns'].shift(lag) - df = __augment_derived_features(df, log_returns=log_returns, technical_features=technical_features) + df = __augment_derived_features(df, log_returns=True if returns == 'log_returns' else False, technical_features=technical_features) df = df.replace([np.inf, -np.inf], 0.) df = df.drop(columns=['open', 'high', 'low', 'close']) @@ -100,7 +139,7 @@ def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Lit def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_features: Literal['none', 'level1', 'level2']) -> pd.DataFrame: if technical_features == 'level1' or technical_features == 'level2': - # volatility (10, 20, 30 days) + # volatility (10, 20, 30, 60 days) df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5) df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5) df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5) @@ -137,16 +176,15 @@ def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_fe return df -# %% # %% -def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame: - df['target'] = df[source_column].diff(period).shift(-period) - df = df.iloc[:-period] - return df +def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.Series: + assert period > 0 + return df[source_column].shift(-period) -def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.DataFrame: +def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.Series: + assert period > 0 def get_class_binary(x): return 0 if x <= 0.0 else 1 @@ -162,16 +200,10 @@ def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, n else: return 1 - if period > 0: - df['target'] = df[source_column].shift(-period) - else: - df['target'] = df[source_column] + target_column = df[source_column].shift(-period) get_class_function = get_class_binary if no_of_classes == "three": get_class_function = get_class_threeway - df['target'] = df['target'].map(get_class_function) - if period > 0: - df = df.iloc[:-period] - return df \ No newline at end of file + return target_column.map(get_class_function) \ No newline at end of file diff --git a/run_pipeline.py b/run_pipeline.py index 46fb241..b46c8f8 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -1,12 +1,10 @@ -from logging import log -from typing import Literal from sklearnex import patch_sklearn patch_sklearn() from load_data import get_crypto_assets, get_etf_assets, load_data -from utils.evaluate import evaluate_predictions import pandas as pd +import numpy as np from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression, Ridge from sklearn.tree import DecisionTreeClassifier from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier @@ -15,106 +13,110 @@ from sklearn.svm import SVR from sklearn.naive_bayes import GaussianNB from sklearn.neural_network import MLPRegressor, MLPClassifier from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier -from sklearn.preprocessing import MinMaxScaler -from utils.walk_forward import walk_forward_train_test +from training.pipeline import run_single_asset_trainig_pipeline + +# Parameters regression_models = [ - # ('LR', LinearRegression(n_jobs=-1)), - ('Lasso', Lasso(alpha=0.1, max_iter=10000)), + ('Lasso', Lasso(alpha=1.0, max_iter=10000)), ('Ridge', Ridge(alpha=1.0)), ('BayesianRidge', BayesianRidge()), ('KNN', KNeighborsRegressor(n_neighbors=15)), + # ('AB', AdaBoostRegressor()), + # ('LR', LinearRegression(n_jobs=-1)), # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)), - ('AB', AdaBoostRegressor()), # ('RF', RandomForestRegressor(n_jobs=-1)), # ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1)) ] +ensemble_model = [('Ensemble - Lasso', Lasso(alpha=1.0, max_iter=10000, positive=True))] + classification_models = [ ('LR', LogisticRegression(n_jobs=-1)), - ('LDA', LinearDiscriminantAnalysis()), - ('KNN', KNeighborsClassifier()), - ('CART', DecisionTreeClassifier()), - ('NB', GaussianNB()), - ('AB', AdaBoostClassifier()), - ('RF', RandomForestClassifier(n_jobs=-1)) + # ('LDA', LinearDiscriminantAnalysis()), + # ('KNN', KNeighborsClassifier()), + # ('CART', DecisionTreeClassifier()), + # ('NB', GaussianNB()), + # ('AB', AdaBoostClassifier()), + # ('RF', RandomForestClassifier(n_jobs=-1)) ] +path = 'data/' +all_assets = get_crypto_assets(path) -def run_whole_pipeline( - ticker_to_predict: str, - load_data_args: dict, - models, - method: Literal['regression', 'classification'], - sliding_window_size: int, - retrain_every: int, - scaling: bool, - ): - print('--------\nPredicting: ', ticker_to_predict) +sliding_window_size = 200 +retrain_every = 100 +scaler = 'none' # 'normalize' 'minmax' 'standardize' 'none' +include_original_data_in_ensemble = True +method = 'regression' +data_parameters = dict(path=path, + target_asset_lags= [1,2,3,4,5,6,8,10,15], + load_other_assets= True, + other_asset_lags= [], + log_returns= True, + add_date_features= True, + own_technical_features= 'level2', + other_technical_features= 'none', + exogenous_features= 'none', + index_column= 'int', + method= method, +) - X, y = load_data(**load_data_args) - if scaling: - # TODO: should move scaling to an expanding window compomenent, probably worth not turning it on for now - feature_scaler = MinMaxScaler(feature_range= (-1, 1)) - X = pd.DataFrame(feature_scaler.fit_transform(X), columns=X.columns, index=X.index) - # TODO: should scale y as well probably - - results = pd.DataFrame() - - for model_name, model in models: - - model_over_time, preds = walk_forward_train_test( - model_name=model_name, - model = model, - X = X, - y = y, - window_size = sliding_window_size, - retrain_every = retrain_every - ) - result = evaluate_predictions( - model_name = model_name, - y_true = y, - y_pred = preds, - sliding_window_size = sliding_window_size, - method = method, - ) - column_name = ticker_to_predict + "_" + model_name - results[column_name] = result - - return results +# Run pipeline results = pd.DataFrame() -all_assets = get_crypto_assets('data/') - - for asset in all_assets: - for method in ['regression']: - load_data_args = dict(path='data/', - target_asset= asset, - target_asset_lags= [1,2,3,4,5,6,8,10,15], - load_other_assets= False, - other_asset_lags= [], - log_returns= True, - add_date_features= True, - own_technical_features= 'level2', - other_technical_features= 'none', - exogenous_features= 'none', - index_column= 'int', - method= method, - ) + print('--------\nPredicting: ', asset) + all_predictions = pd.DataFrame() - current_result = run_whole_pipeline( - ticker_to_predict = asset, - load_data_args = load_data_args, - models = regression_models if method == 'regression' else classification_models, - method = method, - sliding_window_size = 120, - retrain_every = 50, - scaling = False - ) - results = pd.concat([results, current_result], axis=1) + # 1. Load data + data_params = data_parameters.copy() + data_params['target_asset'] = asset -results.to_csv('results.csv') \ No newline at end of file + X, y, target_returns = load_data(**data_params) + + # 2. Train Level-1 models + current_result, current_predictions = run_single_asset_trainig_pipeline( + ticker_to_predict = asset, + X = X, + y = y, + target_returns = target_returns, + models = regression_models if method == 'regression' else classification_models, + method = method, + sliding_window_size = sliding_window_size, + retrain_every = retrain_every, + scaler = scaler + ) + results = pd.concat([results, current_result], axis=1) + all_predictions = pd.concat([all_predictions, current_predictions], axis=1) + + # 3. Train Level-2 (Ensemble) model + ensemble_X = all_predictions + if include_original_data_in_ensemble: + ensemble_X = pd.concat([ensemble_X, X], axis=1) + + ensemble_result, ensemble_preds = run_single_asset_trainig_pipeline( + ticker_to_predict = asset, + X = ensemble_X, + y = target_returns, + target_returns = target_returns, + models = ensemble_model, + method = 'regression', + sliding_window_size = sliding_window_size, + retrain_every = retrain_every, + scaler = scaler + ) + + results = pd.concat([results, ensemble_result], axis=1) + all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1) + +results.to_csv('results.csv') + +level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]] +ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]] + +print("Mean Sharpe ratio for Level-1 models: ", level1_columns.loc['sharpe'].mean()) +print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", ensemble_columns.loc['sharpe'].mean()) \ No newline at end of file diff --git a/tests/test_walk_forward.py b/tests/test_walk_forward.py index be0212b..96a9d34 100644 --- a/tests/test_walk_forward.py +++ b/tests/test_walk_forward.py @@ -1,20 +1,21 @@ import pytest import numpy as np import pandas as pd -from utils.walk_forward import walk_forward_train_test +from training.walk_forward import walk_forward_train_test from sklearn.base import BaseEstimator -def __generate_test_data(): +no_of_rows = 100 + +def __generate_test_data(no_of_rows): no_columns = 6 - no_rows = 100 - X = [[row] * no_columns for row in range(no_rows)] + X = [[row] * no_columns for row in range(no_of_rows)] assert X[0][0] == 0 assert X[1][0] == 1 assert X[2][0] == 2 assert X[3][0] == 3 X = pd.DataFrame(X) - y = [row+1 for row in range(no_rows)] + y = [row+1 for row in range(no_of_rows)] assert y[0] == 1 assert y[1] == 2 assert y[2] == 3 @@ -24,7 +25,7 @@ def __generate_test_data(): def test_walk_forward_train_test(): - X, y = __generate_test_data() + X, y = __generate_test_data(no_of_rows) window_length = 10 class StubModel(BaseEstimator): @@ -38,4 +39,16 @@ def test_walk_forward_train_test(): return np.array([X[0][0] + 1]) model = StubModel() - walk_forward_train_test('test', model, X, y, window_length, 10) + scaler = None + models, predictions = walk_forward_train_test( + model_name='test', + model=model, + X=X, + y=y, + target_returns=y, + window_size=window_length, + retrain_every=10, + scaler=scaler) + for i in range(window_length, no_of_rows): + predictions[i] == y[i] + diff --git a/training/pipeline.py b/training/pipeline.py new file mode 100644 index 0000000..74d33b3 --- /dev/null +++ b/training/pipeline.py @@ -0,0 +1,60 @@ +import pandas as pd +from typing import Literal +from training.walk_forward import walk_forward_train_test +from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler +from utils.evaluate import evaluate_predictions +from utils.typing import SKLearnModel + +def __get_scaler(type: Literal['normalize', 'minmax', 'standardize', 'none']): + if type == 'normalize': + return Normalizer() + elif type == 'minmax': + return MinMaxScaler(feature_range= (-1, 1)) + elif type == 'standardize': + return StandardScaler() + else: + return None + +def run_single_asset_trainig_pipeline( + ticker_to_predict: str, + X: pd.DataFrame, + y: pd.Series, + target_returns: pd.Series, + models: list[tuple[str, SKLearnModel]], + method: Literal['regression', 'classification'], + sliding_window_size: int, + retrain_every: int, + scaler: Literal['normalize', 'minmax', 'standardize', 'none'], + ) -> tuple[pd.DataFrame, pd.DataFrame]: + + + scaler = __get_scaler(scaler) + + results = pd.DataFrame() + predictions = pd.DataFrame() + + for model_name, model in models: + + model_over_time, preds = walk_forward_train_test( + model_name=model_name, + model = model, + X = X, + y = y, + target_returns = target_returns, + window_size = sliding_window_size, + retrain_every = retrain_every, + scaler = scaler + ) + assert len(preds) == len(y) + result = evaluate_predictions( + model_name = model_name, + target_returns = target_returns, + y_pred = preds, + sliding_window_size = sliding_window_size, + method = method, + ) + column_name = ticker_to_predict + "_" + model_name + results[column_name] = result + predictions[column_name] = preds + + return results, predictions \ No newline at end of file diff --git a/training/walk_forward.py b/training/walk_forward.py new file mode 100644 index 0000000..877ec7b --- /dev/null +++ b/training/walk_forward.py @@ -0,0 +1,63 @@ +import pandas as pd +from sklearn.base import clone +from utils.typing import SKLearnModel +import numpy as np +from utils.helpers import get_first_valid_return_index + +def walk_forward_train_test( + model_name: str, + model: SKLearnModel, + X: pd.DataFrame, + y: pd.Series, + target_returns: pd.Series, + window_size: int, + retrain_every: int, + scaler, + ) -> tuple[pd.Series, pd.Series]: + assert len(X) == len(y) + predictions = pd.Series(index=y.index).rename(model_name) + models = pd.Series(index=y.index).rename(model_name) + + first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(X.iloc[:,0])) + train_from = first_nonzero_return + window_size + 1 + train_till = len(y) + + iterations_since_retrain = 0 + if scaler is not None: + scaler = clone(scaler) + + for index in range(train_from, train_till): + + iterations_since_retrain += 1 + + if iterations_since_retrain >= retrain_every or pd.isna(models[index-1]): + train_window_start = index - window_size - 1 + train_window_end = index - 1 + + if scaler is not None: + # First we need to fit on the expanding window data slice + # This is our only way to avoid lookahead bia + X_expanding_window = X[first_nonzero_return:train_window_end] + scaler.fit(X_expanding_window) + + X_slice = X[train_window_start:train_window_end] + y_slice = y[train_window_start:train_window_end] + + if scaler is not None: + X_slice = scaler.transform(X_slice) + else: + X_slice = X_slice.to_numpy() + + current_model = clone(model) + current_model.fit(X_slice, y_slice.to_numpy()) + iterations_since_retrain = 0 + else: + current_model = models[index-1] + + models[index] = current_model + + next_timestep = X.iloc[index].to_numpy().reshape(1, -1) + prediction = current_model.predict(next_timestep).item() + predictions[index] = prediction + + return models, predictions diff --git a/utils/evaluate.py b/utils/evaluate.py index a2b4b11..0f52caa 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -1,53 +1,63 @@ from typing import Literal from sklearn.metrics import mean_absolute_error, accuracy_score, r2_score, f1_score, precision_score, recall_score from quantstats.stats import expected_return, sharpe, skew, sortino +from utils.helpers import get_first_valid_return_index import pandas as pd import numpy as np -def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.02) -> pd.Series: +def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.01) -> pd.Series: delta_pos = signal.diff(1).abs().fillna(0.) costs = transaction_cost * delta_pos return (signal * returns) - costs -def __preprocess(y_true: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']): +def __preprocess(target_returns: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']) -> pd.DataFrame: y_pred.name = 'y_pred' - y_true.name = 'y_true' - df = pd.concat([y_pred, y_true],axis=1).dropna() + target_returns.name = 'target_returns' + df = pd.concat([y_pred, target_returns],axis=1).dropna() - if method == 'regression': - df['sign_pred'] = df.y_pred.apply(np.sign) - else: - df['sign_pred'] = df.y_pred.apply(lambda x: 1 if x>0 else -1) - df['sign_true'] = df.y_true.apply(np.sign) + df['sign_pred'] = df.y_pred.apply(lambda x: 1 if x>0 else -1) + def sign_true(x): + if x > 0: + return 1 + else: + return -1 + df['sign_true'] = df.target_returns.apply(sign_true) df['is_correct'] = 0 df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct df['is_incorrect'] = 0 df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong df['is_predicted'] = df.is_correct + df.is_incorrect - df['result'] = backtest(df.y_true, df.sign_pred) + df['result'] = backtest(df.target_returns, df.sign_pred) return df -def evaluate_predictions(model_name: str, y_true: pd.Series, y_pred: pd.Series, sliding_window_size: int, method: Literal['classification', 'regression']): +def evaluate_predictions( + model_name: str, + target_returns: pd.Series, + y_pred: pd.Series, + sliding_window_size: int, + method: Literal['classification', 'regression'] + ) -> pd.Series: # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size) - first_nonzero_return = np.where(y_true != 0)[0][0] + first_nonzero_return = get_first_valid_return_index(target_returns) evaluate_from = first_nonzero_return + sliding_window_size + 1 - y_true = pd.Series(y_true[evaluate_from:]) - # if there are lots of zeros in the ground truth returns, probably something is wrong, but we can tolerate a couple of days of missing data. - is_zero = y_true[y_true == 0] - assert len(is_zero) < 5 - # we can't deal with 0 returns, so we'll just remap the few examples to 1 - y_true = y_true.apply(lambda x: 1 if x == 0 else x) + target_returns = pd.Series(target_returns[evaluate_from:]) + if method == 'regression': + # if there are lots of zeros in the ground truth returns, probably something is wrong, but we can tolerate a couple of days of missing data. + is_zero = target_returns[target_returns == 0] + assert len(is_zero) < 15 + # we can't deal with 0 returns, so we'll just remap the few examples to 0.0001 + target_returns = target_returns.apply(lambda x: 0.0001 if x == 0 else x) y_pred = pd.Series(y_pred[evaluate_from:]) - df = __preprocess(y_true, y_pred, method) + df = __preprocess(target_returns, y_pred, method) scorecard = pd.Series() if method == 'regression': - scorecard.loc['RSQ'] = r2_score(df.y_true, df.y_pred) - scorecard.loc['MAE'] = mean_absolute_error(df.y_true, df.y_pred) + scorecard.loc['RSQ'] = r2_score(df.target_returns, df.y_pred) + scorecard.loc['MAE'] = mean_absolute_error(df.target_returns, df.y_pred) elif method == 'classification': scorecard.loc['RSQ'] = 0. scorecard.loc['MAE Matrix'] = 0. diff --git a/utils/get_prices.py b/utils/get_prices.py index 837b51c..12ccdc2 100644 --- a/utils/get_prices.py +++ b/utils/get_prices.py @@ -49,13 +49,3 @@ def get_stock_price_av(symbol: str, start_date: str = None) -> pd.DataFrame: df = df.rename_axis('time') return df - - -# %% -# btc = get_crypto_price_av(symbol = 'BTC', exchange = 'USD', start_date = '2018-01-01') -# btc - -# %% - -# spy = get_stock_price_av(symbol = 'SPY', start_date = '2018-01-01') -# spy \ No newline at end of file diff --git a/utils/helpers.py b/utils/helpers.py new file mode 100644 index 0000000..a49f716 --- /dev/null +++ b/utils/helpers.py @@ -0,0 +1,6 @@ +import pandas as pd +import numpy as np + +def get_first_valid_return_index(series: pd.Series) -> int: + first_nonzero_return = np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series))))[0][0] + return first_nonzero_return \ No newline at end of file diff --git a/utils/walk_forward.py b/utils/walk_forward.py deleted file mode 100644 index 740b6b8..0000000 --- a/utils/walk_forward.py +++ /dev/null @@ -1,48 +0,0 @@ -import pandas as pd -from sklearn.base import clone -from utils.typing import SKLearnModel -import numpy as np - -def walk_forward_train_test( - model_name: str, - model: SKLearnModel, - X: pd.DataFrame, - y: pd.Series, - window_size: int, - retrain_every: int - ) -> tuple[pd.Series, pd.Series]: - - predictions = pd.Series(index=y.index).rename(model_name) - models = pd.Series(index=y.index).rename(model_name) - - train_from = window_size - train_till = y.index[-1] - - iterations_since_retrain = 0 - - for i in range(train_from, train_till): - - iterations_since_retrain += 1 - window_start = i - window_size - window_end = i - X_slice = X[window_start:window_end] - y_slice = y[window_start:window_end] - - if iterations_since_retrain >= retrain_every or pd.isna(models[i-1]): - current_model = clone(model) - current_model.fit(X_slice.to_numpy(), y_slice.to_numpy()) - iterations_since_retrain = 0 - else: - current_model = models[i-1] - - models[window_end] = current_model - - next_timestep = X.iloc[window_end+1].to_numpy().reshape(1, -1) - prediction = current_model.predict(next_timestep).item() - if prediction == 0.: - # TODO: we shouldn't feed in zeros to the model, and skip training / predicting when everything is 0 - # print("Warning: model predicted 0., overriding it with 0.0001") - prediction = 0.0001 - predictions[window_end+1] = prediction - - return models, predictions