diff --git a/.gitignore b/.gitignore index a1eefbc..7aa671c 100644 --- a/.gitignore +++ b/.gitignore @@ -129,9 +129,7 @@ dmypy.json .pyre/ lightning/lightning_logs/ -output/results.csv -output/results_level2.csv -output/predictions.csv +output/** wandb/ .cachedir/** diff --git a/data_loader/load_data.py b/data_loader/load_data.py index 57fd47e..e56f2b4 100644 --- a/data_loader/load_data.py +++ b/data_loader/load_data.py @@ -224,25 +224,24 @@ def datasource_to_file(data_source: DataSource) -> str: return data_source[0] + '/' + data_source[1] + '.csv' -# These are needed for the portfolio feature, maybe we can do this in a more elegant way -# def load_crypto_only_returns(path: str, index_column: Literal['date', 'int'], returns: Literal['price', 'returns']) -> pd.DataFrame: -# files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')] -# dfs = [__load_df( -# path=os.path.join(path,f), -# prefix=f.split('.')[0], -# returns=returns, -# feature_extractors=[], -# narrow_format=False, -# ) for f in files] -# dfs = pd.concat(dfs, axis=1) -# dfs = dfs.applymap(lambda x: np.nan if x == 0 else x) -# dfs.index = pd.DatetimeIndex(dfs.index) -# dfs.columns = [column.split('_')[0] for column in dfs.columns] -# if index_column == 'int': -# dfs.reset_index(drop=True, inplace=True) -# return dfs +def load_only_returns(assets: DataCollection, index_column: Literal['date', 'int'], returns: Literal['price', 'returns']) -> pd.DataFrame: -# def load_crypto_assets_availability(path: str, index_column: Literal['date', 'int']) -> pd.DataFrame: -# return load_crypto_only_returns(path, index_column, 'returns').applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1) + assets_future = [__load_df.remote( + data_source=data_source, + prefix=data_source[1], + returns=returns, + feature_extractors=[], + narrow_format=False, + ) for data_source in assets] + target_asset_df = ray.get(assets_future) + dfs = [deduplicate_indexes(df) for df in target_asset_df] + dfs = pd.concat(dfs, axis=1) + # dfs = dfs.applymap(lambda x: np.nan if x == 0 else x) + dfs.index = pd.DatetimeIndex(dfs.index) + + if index_column == 'int': + dfs.reset_index(drop=True, inplace=True) + + return dfs diff --git a/environment.yml b/environment.yml index c210fc5..37c203e 100644 --- a/environment.yml +++ b/environment.yml @@ -3,6 +3,7 @@ channels: - johnsnowlabs - conda-forge - defaults + - ml4t dependencies: - python=3.9 - seaborn @@ -25,6 +26,8 @@ dependencies: - pip - pandas-ta - xgboost + - alphalens-reloaded + - pyfolio-reloaded - pip: - fracdiff - ray diff --git a/run_feature_selection.py b/run_feature_selection.py index 62b5f97..21e2a8b 100644 --- a/run_feature_selection.py +++ b/run_feature_selection.py @@ -1,39 +1,39 @@ -#%% -import pandas as pd -import pandas_ta as ta -from config.config import get_default_level_2_daily_config -from config.preprocess import preprocess_config -from data_loader.load_data import load_data +# #%% +# import pandas as pd +# import pandas_ta as ta +# from config.config import get_default_level_2_daily_config +# from config.preprocess import preprocess_config +# from data_loader.load_data import load_data -# %% -model_config, training_config, data_config = get_default_level_2_daily_config() -model_config, training_config, data_config = preprocess_config(model_config, training_config, data_config) +# # %% +# model_config, training_config, data_config = get_default_level_2_daily_config() +# model_config, training_config, data_config = preprocess_config(model_config, training_config, data_config) -data_config['target_asset'] = data_config['assets'][0] -X, y, target_returns = load_data(**data_config) -# %% -X.ta.donchian() +# data_config['target_asset'] = data_config['assets'][0] +# X, y, target_returns = load_data(**data_config) +# # %% +# X.ta.donchian() -# %% -X.ta.ema() -# %% -X.ta.adjusted = "ADA_USD_returns" +# # %% +# X.ta.ema() +# # %% +# X.ta.adjusted = "ADA_USD_returns" -# %% -X.ta.sma(length=10) +# # %% +# X.ta.sma(length=10) -# %% -X -# %% -X.ta.categories +# # %% +# X +# # %% +# X.ta.categories -# %% -ind_list = X.ta.indicators(as_list=True) +# # %% +# ind_list = X.ta.indicators(as_list=True) -# %% -ind_list -# %% -X.ta.ao('ADA_USD_returns', length=10) -# %% -ta.ao() \ No newline at end of file +# # %% +# ind_list +# # %% +# X.ta.ao('ADA_USD_returns', length=10) +# # %% +# ta.ao() \ No newline at end of file diff --git a/run_portfolio_reporting.py b/run_portfolio_reporting.py index 25879bf..7314dc7 100644 --- a/run_portfolio_reporting.py +++ b/run_portfolio_reporting.py @@ -3,8 +3,7 @@ import pandas as pd import numpy as np from data_loader.load_data import load_only_returns from data_loader.collections import data_collections -import vectorbt as vbt -from vectorbt.portfolio.enums import SizeType, CallSeqType + from utils.helpers import get_first_valid_return_index from alphalens.tears import (create_returns_tear_sheet, create_information_tear_sheet, @@ -13,6 +12,8 @@ from alphalens.tears import (create_returns_tear_sheet, create_full_tear_sheet, create_event_returns_tear_sheet, create_event_study_tear_sheet) +import alphalens +import pyfolio from alphalens.utils import get_clean_factor_and_forward_returns @@ -35,7 +36,7 @@ def fixed_weight(row: pd.Series, availability_row: pd.Series, allow_short: bool) def limit_weight(row: pd.Series) -> pd.Series: if row.sum() > 1: row = row / row.sum() - elif row.sum() < 1: + elif row.sum() < -1: row = row * (1. / abs(row.sum())) return row @@ -81,7 +82,7 @@ def create_naive_portfolio_weights(predictions: pd.DataFrame, availability: pd.D # row = row / row.sum() # row = only_top_bottom_2(row) # row = equal_weight(row, availability.iloc[index]) - row = limit_weight(row) + # row = limit_weight(row) weights.iloc[index] = row return weights @@ -96,51 +97,67 @@ close = load_only_returns(data_collections['daily_crypto'], 'date', 'price') close = close.iloc[first_index:-1] close.columns = [col.replace("_returns", "") for col in close.columns] close = close[predictions.columns] -close.reset_index(drop=True, inplace=True) -# returns = load_only_returns(data_collections['daily_crypto'], 'date', 'returns') -# returns = returns.iloc[first_index:-1] -# returns.columns = [col.replace("_returns", "") for col in returns.columns] -# returns = returns[predictions.columns] -# returns.reset_index(drop=True, inplace=True) - -# predictions = predictions.reindex(close.index, method='ffill') availability = close.applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1) weights = create_naive_portfolio_weights(predictions, availability, allow_short=True) -# weights_long = pd.melt(weights,id_vars=['index']) +weights.index = close.index + +weights_long = pd.melt(weights.reset_index(), id_vars=['time'], value_vars=weights.columns).set_index(['time', 'variable']) +close_long = pd.melt(close.reset_index(), id_vars=['time'], value_vars=close.columns).set_index(['time', 'variable']) +#%% +factor_data = get_clean_factor_and_forward_returns( + weights_long, + close, + # groupby=weights.columns.to_list(), + quantiles=4, + periods=(1, 2, 3, 4, 5, 6, 10), + filter_zscore=None) + +#%% +factor_data.head(10) + +#%% +create_full_tear_sheet(factor_data, long_short=True) + +from matplotlib.backends.backend_pdf import PdfPages + +mean_return_by_q_daily, std_err = alphalens.performance.mean_return_by_quantile(factor_data, by_date=True) +mean_return_by_q, std_err_by_q = alphalens.performance.mean_return_by_quantile(factor_data, by_group=False) +plot1 = alphalens.plotting.plot_quantile_returns_bar(mean_return_by_q) +plot2 = alphalens.plotting.plot_quantile_returns_violin(mean_return_by_q_daily) +plot3 = alphalens.plotting.plot_cumulative_returns_by_quantile(mean_return_by_q_daily, period='D') +full_tear = create_full_tear_sheet(factor_data, long_short=True) +avg_returns = create_event_returns_tear_sheet(factor_data, close, avgretplot=(1, 3, 5), long_short=True) -# factor_data = get_clean_factor_and_forward_returns( -# weights, -# close, -# groupby=factor_groups, -# quantiles=4, -# periods=(1, 3), -# filter_zscore=None) +with PdfPages('output/factors.pdf') as pdf: + pdf.savefig(plot1.figure) + pdf.savefig(plot2.figure) + pdf.savefig(plot3.figure) +# create_event_returns_tear_sheet(factor_data, close, avgretplot=(1, 3, 5), long_short=True) +#%% + +pf_returns, pf_positions, pf_benchmark = alphalens.performance.create_pyfolio_input(factor_data, + period='1D', + capital=100000, + long_short=True, + equal_weight=True, + quantiles=[1,4], + groups=None, + benchmark_period='1D') + +pyfolio.tears.create_full_tear_sheet(pf_returns, + positions=pf_positions, + benchmark_rets=pf_benchmark) # rebalance every n days # weights.iloc[np.arange(len(weights)) % 7 != 0] = np.nan -# portfolio = vbt.Portfolio.from_orders( -# close=close, -# size=weights, -# size_type=SizeType.TargetPercent, -# cash_sharing=True, -# call_seq=CallSeqType.Auto, -# group_by=True, -# freq='1D', -# raise_reject=True, -# fees=0.01, -# seed=1, -# init_cash=1e5, -# ) - -# print(portfolio.stats()) # from pypfopt import risk_models # from pypfopt import expected_returns @@ -153,4 +170,3 @@ weights = create_naive_portfolio_weights(predictions, availability, allow_short= # cleaned_weights = ef.clean_weights() # print(ef.portfolio_performance(verbose=True)) -# %%