From 8dd2d887400d51e4f44dcbcf253deb24f3578c45 Mon Sep 17 00:00:00 2001 From: Mark Aron Szulyovszky Date: Thu, 17 Feb 2022 19:22:17 +0100 Subject: [PATCH] chore(Linter): reformatted code with black (#211) * chore(Linter): reformatted code with black * Create black.yaml --- .github/workflows/black.yaml | 10 + .vscode/settings.json | 10 + archive/forecasting.py | 119 --------- archive/lightning/create_dataset.py | 64 ----- archive/lightning/define_model.py | 0 archive/lightning/models/built_in_models.py | 19 -- archive/lightning/models/custom_model.py | 98 -------- archive/lightning/options.py | 63 ----- archive/lightning/run_pipeline.py | 41 ---- archive/lightning/train_predict.py | 67 ------ archive/model_lightning.py | 60 ----- archive/run_lightning.py | 48 ---- config/hashing.py | 39 +-- config/preprocess.py | 73 ++++-- config/presets.py | 167 +++++++------ config/types.py | 23 +- data_loader/collections.py | 96 +++++--- data_loader/get_prices.py | 71 ++++-- data_loader/load.py | 156 +++++++----- data_loader/process.py | 15 +- data_loader/types.py | 2 +- data_loader/utils.py | 4 +- docs/conf.py | 23 +- environment.yml | 1 + .../feature_extractor_presets.py | 74 +++--- feature_extractors/feature_extractors.py | 65 +++-- .../fractional_differentiation.py | 7 +- feature_extractors/ta.py | 1 + feature_extractors/types.py | 2 +- feature_extractors/utils.py | 7 +- labeling/event_filters/cusum.py | 27 ++- labeling/event_filters/nofilter.py | 3 +- labeling/eventfilters_map.py | 8 +- .../fixed_time_three_class_balanced.py | 29 ++- .../fixed_time_three_class_imbalanced.py | 28 ++- labeling/labellers/fixed_time_two_class.py | 26 +- labeling/labellers/utils.py | 5 +- labeling/labellers_map.py | 16 +- labeling/process.py | 28 ++- labeling/types.py | 9 +- models/base.py | 4 +- models/model_map.py | 72 ++++-- models/momentum.py | 17 +- models/sklearn.py | 8 +- models/xgboost.py | 3 +- mycodemod/replace_functions.py | 65 +++-- reporting/reporting.py | 43 +++- reporting/saving.py | 40 +-- reporting/wandb.py | 40 +-- run_clear_cache.py | 3 +- run_evaluate_robustness.py | 13 +- run_fetch_data_5min.py | 40 +-- run_fetch_data_daily.py | 21 +- run_fetch_data_glassnode.py | 65 ++--- run_inference.py | 75 ++++-- run_model_dev.py | 7 +- run_pipeline.py | 90 +++++-- run_portfolio_reporting.py | 100 +++++--- run_sweep.py | 7 +- tests/test_evaluation.py | 41 ++-- tests/test_walk_forward.py | 30 ++- training/bet_sizing.py | 109 +++++---- training/directional_training.py | 78 +++--- training/train_model.py | 95 ++++---- training/types.py | 7 +- training/walk_forward/__init__.py | 2 +- training/walk_forward/inference.py | 84 +++++-- training/walk_forward/inference_batched.py | 91 +++++-- training/walk_forward/inference_parallel.py | 98 ++++++-- .../walk_forward/process_transformations.py | 63 +++-- .../process_transformations_parallel.py | 83 +++++-- training/walk_forward/train.py | 58 +++-- training/walk_forward/train_parallel.py | 82 +++++-- transformations/base.py | 8 +- transformations/pca.py | 24 +- transformations/rfe.py | 26 +- transformations/scaler.py | 13 +- transformations/sklearn.py | 13 +- utils/evaluate.py | 126 ++++++---- utils/glassnode/blockchain.py | 119 ++++----- utils/glassnode/client.py | 38 +-- utils/glassnode/derivatives.py | 129 +++++----- utils/glassnode/distribution.py | 77 +++--- utils/glassnode/endpoints.py | 14 +- utils/glassnode/entities.py | 121 +++++----- utils/glassnode/eth2.py | 53 ++-- utils/glassnode/fees.py | 95 ++++---- utils/glassnode/indicators.py | 82 +++---- utils/glassnode/market.py | 71 +++--- utils/glassnode/mining.py | 79 +++--- utils/glassnode/protocols.py | 31 +-- utils/glassnode/supply.py | 227 +++++++++--------- utils/glassnode/utils.py | 17 +- utils/hashing.py | 4 +- utils/helpers.py | 44 +++- utils/metrics.py | 190 ++++++++++----- utils/parallel.py | 4 +- utils/resample.py | 11 +- utils/sktime.py | 13 +- utils/technical_indicators.py | 34 ++- wandb_setup.py | 15 +- 101 files changed, 2596 insertions(+), 2320 deletions(-) create mode 100644 .github/workflows/black.yaml create mode 100644 .vscode/settings.json delete mode 100644 archive/forecasting.py delete mode 100644 archive/lightning/create_dataset.py delete mode 100644 archive/lightning/define_model.py delete mode 100644 archive/lightning/models/built_in_models.py delete mode 100644 archive/lightning/models/custom_model.py delete mode 100644 archive/lightning/options.py delete mode 100644 archive/lightning/run_pipeline.py delete mode 100644 archive/lightning/train_predict.py delete mode 100644 archive/model_lightning.py delete mode 100644 archive/run_lightning.py diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml new file mode 100644 index 0000000..609d2b9 --- /dev/null +++ b/.github/workflows/black.yaml @@ -0,0 +1,10 @@ +name: Lint + +on: push + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: psf/black@stable diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9f9b965 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "python.testing.pytestArgs": [ + "." + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.linting.enabled": true, + "python.formatting.provider": "black", + "editor.formatOnSave": true, +} \ No newline at end of file diff --git a/archive/forecasting.py b/archive/forecasting.py deleted file mode 100644 index 227905a..0000000 --- a/archive/forecasting.py +++ /dev/null @@ -1,119 +0,0 @@ -#%% -import pytorch_lightning as pl -from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor - -from pytorch_forecasting import TimeSeriesDataSet, TemporalFusionTransformer -from pytorch_forecasting.data import GroupNormalizer -from pytorch_forecasting.metrics import QuantileLoss - -import sys -sys.path.insert(0, '..') - -from data_loader import load_files - -print("success") - -#%% -# load data -data = load_files('../data/', add_features=True, log_returns=False, narrow_format=True) -# need to treat time as an independent column -data = data.reset_index().rename({'index':'time'}, axis = 'columns') -# we need a `time_idx` column for pytorch-forecasting, so we convert the time column to a time index by encoding the dates as consecutive days from the first date. -data['time_idx'] = (data['time']-data['time'].min()).astype('timedelta64[D]').astype(int)+1 -# volume needs some love before we can use it -data.drop(columns=['volume'], inplace=True) - -data['month'] = data['month'].astype(str) -data['day_month'] = data['day_month'].astype(str) -data['day_week'] = data['day_week'].astype(str) - - - -#%% -# define dataset -max_encoder_length = 36 # this is the look-back window, see https://github.com/jdb78/pytorch-forecasting/issues/448 -max_prediction_length = 6 -# training_cutoff = "YYYY-MM-DD" # day for cutoff - -# training_cutoff = data["time_idx"].max() - max_prediction_length - -#%% -data.head() -data.describe() - -#%% -training = TimeSeriesDataSet( - data, # data[lambda x: x.date < training_cutoff], - time_idx= 'time_idx', - target= 'returns', - # weight="weight", - group_ids=[ 'ticker' ], - min_encoder_length = max_encoder_length,##max_encoder_length//2, - max_encoder_length = max_encoder_length, - min_prediction_length = 1, - max_prediction_length = 1,#max_prediction_length, - static_categoricals=[ ], - static_reals=[ ], - time_varying_known_categoricals=[ 'day_month', 'day_week', 'month' ], - time_varying_known_reals=[ 'vol_10', 'vol_20', 'vol_30', 'vol_60', 'mom_10', 'mom_20', 'mom_30', 'mom_60', 'mom_90'], - time_varying_unknown_categoricals=[ ], - time_varying_unknown_reals=[ 'returns' ], - allow_missing_timesteps=True, - target_normalizer=GroupNormalizer( - groups=['ticker'], transformation="softplus") -) - -#%% -print(training.index.time) -print(training.index.time.max()) - -#%% -# create validation and training dataset - -validation = TimeSeriesDataSet.from_dataset(training, data, predict=True, stop_randomization=True) -batch_size = 128 -train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0) -val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=0) - -# define trainer with early stopping -early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min") -lr_logger = LearningRateMonitor() -trainer = pl.Trainer( - max_epochs=100, - gpus=0, - gradient_clip_val=0.1, - limit_train_batches=30, - callbacks=[lr_logger, early_stop_callback], -) - -# create the model -tft = TemporalFusionTransformer.from_dataset( - training, - learning_rate=0.03, - hidden_size=32, - attention_head_size=1, - dropout=0.1, - hidden_continuous_size=16, - output_size=7, - loss=QuantileLoss(), - log_interval=2, - reduce_on_plateau_patience=4 -) -print(f"Number of parameters in network: {tft.size()/1e3:.1f}k") - -# find optimal learning rate (set limit_train_batches to 1.0 and log_interval = -1) -res = trainer.tuner.lr_find( - tft, train_dataloader=train_dataloader, val_dataloaders=val_dataloader, early_stop_threshold=1000.0, max_lr=0.3, -) - -print(f"suggested learning rate: {res.suggestion()}") -fig = res.plot(show=True, suggest=True) -fig.show() - -# fit the model -trainer.fit( - tft, train_dataloader=train_dataloader, val_dataloaders=val_dataloader, -) - - -# %% diff --git a/archive/lightning/create_dataset.py b/archive/lightning/create_dataset.py deleted file mode 100644 index ba802c7..0000000 --- a/archive/lightning/create_dataset.py +++ /dev/null @@ -1,64 +0,0 @@ -import pytorch_lightning as pl -from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor - -from pytorch_forecasting import TimeSeriesDataSet, TemporalFusionTransformer -from pytorch_forecasting.data import GroupNormalizer -from pytorch_forecasting.metrics import QuantileLoss - -import sys -sys.path.insert(0, '..') - -from data_loader.load_data import load_files - -print("success") - -def load_format_data(data_dir): - print("Data Starting ===>", end=" ") - # load data - data = load_files(data_dir, add_features=True, log_returns=False, narrow_format=True) - # need to treat time as an independent column - data = data.reset_index().rename({'index':'time'}, axis = 'columns') - # we need a `time_idx` column for pytorch-forecasting, so we convert the time column to a time index by encoding the dates as consecutive days from the first date. - data['time_idx'] = (data['time']-data['time'].min()).astype('timedelta64[D]').astype(int)+1 - # volume needs some love before we can use it - data.drop(columns=['volume'], inplace=True) - - data['month'] = data['month'].astype(str) - data['day_month'] = data['day_month'].astype(str) - data['day_week'] = data['day_week'].astype(str) - - - print("<=== Data Loaded") - print(data.head(3)) - print(data.describe()) - print("") - - - - - return data - - -def create_dataloaders(data, kwargs): - print("DataLoader Starting ===>", end=" ") - # training_cutoff = "YYYY-MM-DD" # day for cutoff - # training_cutoff = data["time_idx"].max() - max_prediction_length - batch_size = kwargs['batch_size'] - del kwargs['batch_size'] - - training_dataset = TimeSeriesDataSet( - data, # data[lambda x: x.date < training_cutoff], - **kwargs - ) - - #%% - # create validation and training dataset - validation = TimeSeriesDataSet.from_dataset(training_dataset, data, predict=True, stop_randomization=True) - train_dataloader = training_dataset.to_dataloader(train=True, batch_size=batch_size, num_workers=0) - val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=0) - - - print("<=== DataLoader Created") - - - return training_dataset, train_dataloader, val_dataloader \ No newline at end of file diff --git a/archive/lightning/define_model.py b/archive/lightning/define_model.py deleted file mode 100644 index e69de29..0000000 diff --git a/archive/lightning/models/built_in_models.py b/archive/lightning/models/built_in_models.py deleted file mode 100644 index b2fe570..0000000 --- a/archive/lightning/models/built_in_models.py +++ /dev/null @@ -1,19 +0,0 @@ -from pytorch_forecasting import TemporalFusionTransformer -from pytorch_forecasting.metrics import QuantileLoss - -def create_TemporalFusionTransformer(training_dataset, model_options): - # create the model - tft = TemporalFusionTransformer.from_dataset( - training_dataset, - learning_rate=0.03, - hidden_size=32, - attention_head_size=1, - dropout=0.1, - hidden_continuous_size=16, - output_size=7, - loss=QuantileLoss(), - log_interval=2, - reduce_on_plateau_patience=4 - ) - print(f"Number of parameters in network: {tft.size()/1e3:.1f}k") - return tft \ No newline at end of file diff --git a/archive/lightning/models/custom_model.py b/archive/lightning/models/custom_model.py deleted file mode 100644 index 94c3372..0000000 --- a/archive/lightning/models/custom_model.py +++ /dev/null @@ -1,98 +0,0 @@ -#%% -import warnings -from typing import Dict - -warnings.filterwarnings("ignore") - -import torch -from torch import nn - -from pytorch_forecasting.models import BaseModel -from pytorch_forecasting import TimeSeriesDataSet - - -#%% -class FullyConnectedModule(nn.Module): - def __init__(self, input_size: int, output_size: int, hidden_size: int, n_hidden_layers: int): - super().__init__() - - # input layer - module_list = [nn.Linear(input_size, hidden_size), nn.ReLU()] - # hidden layers - for _ in range(n_hidden_layers): - module_list.extend([nn.Linear(hidden_size, hidden_size), nn.ReLU()]) - # output layer - module_list.append(nn.Linear(hidden_size, output_size)) - - self.sequential = nn.Sequential(*module_list) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - # x of shape: batch_size x n_timesteps_in - # output of shape batch_size x n_timesteps_out - return self.sequential(x) - - - -#%% -class FullyConnectedModel(BaseModel): - def __init__(self, input_size: int, output_size: int, hidden_size: int, n_hidden_layers: int, **kwargs): - # saves arguments in signature to `.hparams` attribute, mandatory call - do not skip this - self.save_hyperparameters() - # pass additional arguments to BaseModel.__init__, mandatory call - do not skip this - super().__init__(**kwargs) - self.network = FullyConnectedModule( - input_size=self.hparams.input_size, - output_size=self.hparams.output_size, - hidden_size=self.hparams.hidden_size, - n_hidden_layers=self.hparams.n_hidden_layers, - ) - - def forward(self, x: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - # x is a batch generated based on the TimeSeriesDataset - network_input = x["encoder_cont"].squeeze(-1) - prediction = self.network(network_input) - - # rescale predictions into target space - prediction = self.transform_output(prediction, target_scale=x["target_scale"]) - - # We need to return a dictionary that at least contains the prediction - # The parameter can be directly forwarded from the input. - # The conversion to a named tuple can be directly achieved with the `to_network_output` function. - return self.to_network_output(prediction=prediction) - - @classmethod - def from_dataset(cls, dataset: TimeSeriesDataSet, **kwargs): - new_kwargs = { - "output_size": dataset.max_prediction_length, - "input_size": dataset.max_encoder_length, - } - new_kwargs.update(kwargs) # use to pass real hyperparameters and override defaults set by dataset - # example for dataset validation - assert dataset.max_prediction_length == dataset.min_prediction_length, "Decoder only supports a fixed length" - assert dataset.min_encoder_length == dataset.max_encoder_length, "Encoder only supports a fixed length" - assert ( - len(dataset.time_varying_known_categoricals) == 0 - and len(dataset.time_varying_known_reals) == 0 - and len(dataset.time_varying_unknown_categoricals) == 0 - and len(dataset.static_categoricals) == 0 - and len(dataset.static_reals) == 0 - and len(dataset.time_varying_unknown_reals) == 1 - and dataset.time_varying_unknown_reals[0] == dataset.target - ), "Only covariate should be the target in 'time_varying_unknown_reals'" - - return super().from_dataset(dataset, **new_kwargs) - - def calculate_prediction_actual_by_variable(x, train): - pass - - - - - -# %% -def create_FullyConnectedModel(training_dataset, kwargs): - model = FullyConnectedModel.from_dataset(training_dataset, **kwargs) - model.summarize("full") # print model summary - model.hparams - - return model diff --git a/archive/lightning/options.py b/archive/lightning/options.py deleted file mode 100644 index bbf6c9a..0000000 --- a/archive/lightning/options.py +++ /dev/null @@ -1,63 +0,0 @@ -from pytorch_forecasting.data import GroupNormalizer -from pytorch_forecasting.models.temporal_fusion_transformer import TemporalFusionTransformer - -from lightning.models.custom_model import FullyConnectedModel - -training_options = dict( - max_epochs=100, - gpus=0, - gradient_clip_val=0.1, - limit_train_batches=30 -) - -model_options_fcn = dict( - hidden_size=64, - n_hidden_layers=2, -) - -model_options_tft = dict( - hidden_size=64, - n_hidden_layers=2, -) - -dataset_options_tft = dict( - time_idx= 'time_idx', - target= 'returns', - # weight="weight", - group_ids=[ 'ticker' ], - min_encoder_length = 36, # this is the look-back window, see https://github.com/jdb78/pytorch-forecasting/issues/448 - max_encoder_length = 36, - min_prediction_length = 1, - max_prediction_length = 1, - static_categoricals=[ ], - static_reals=[ ], - time_varying_known_categoricals=[ 'day_month', 'day_week', 'month' ], - time_varying_known_reals=[ 'vol_10', 'vol_20', 'vol_30', 'vol_60', 'mom_10', 'mom_20', 'mom_30', 'mom_60', 'mom_90'], - time_varying_unknown_categoricals=[ ], - time_varying_unknown_reals=[ 'returns' ], - allow_missing_timesteps=True, - target_normalizer=GroupNormalizer( - groups=['ticker'], transformation="softplus"), - batch_size=128 -) - -dataset_options_fcn = dict( - time_idx= 'time_idx', - target= 'returns', - # weight="weight", - group_ids=[ 'ticker' ], - min_encoder_length = 36, # this is the look-back window, see https://github.com/jdb78/pytorch-forecasting/issues/448 - max_encoder_length = 36, - min_prediction_length = 1, - max_prediction_length = 1, - static_categoricals=[ ], - static_reals=[ ], - time_varying_known_categoricals=[ ], - time_varying_known_reals=[ ], - time_varying_unknown_categoricals=[ ], - time_varying_unknown_reals=[ 'returns' ], - allow_missing_timesteps=True, - target_normalizer=GroupNormalizer( - groups=['ticker'], transformation="softplus"), - batch_size=128 -) diff --git a/archive/lightning/run_pipeline.py b/archive/lightning/run_pipeline.py deleted file mode 100644 index 8d73c75..0000000 --- a/archive/lightning/run_pipeline.py +++ /dev/null @@ -1,41 +0,0 @@ - -#%% -from create_dataset import load_format_data, create_dataloaders -from train_predict import train_model, predict -from models.built_in_models import create_TemporalFusionTransformer -from models.custom_model import create_FullyConnectedModel -from options import training_options, model_options_tft, model_options_fcn, dataset_options_tft, dataset_options_fcn - -import warnings -warnings.filterwarnings("ignore") - -#%% -def run_pipeline(model_name, data_dir): - data = load_format_data(data_dir) - - dataset_options, model_options, _create_model = select_model(model_name) - training_dataset, train_dataloader, val_dataloader = create_dataloaders(data, dataset_options) - - model = _create_model( training_dataset, model_options ) - trainer = train_model(model, train_dataloader, val_dataloader, training_options) - predict(trainer, model, val_dataloader) - -#%% - -def select_model(model_name): - if model_name == "FullyConnectedLayer": - return dataset_options_fcn, model_options_fcn, create_FullyConnectedModel - elif model_name == "TemporalFusionTransformer": - return dataset_options_tft, model_options_tft, create_TemporalFusionTransformer - else: - assert False, "No such model exists." - - -#%% -run_pipeline("FullyConnectedLayer", '../data/') - -# #%% -# if __name__ == '__main__': -# run_pipeline("FullyConnectedLayer", '../data/') - - \ No newline at end of file diff --git a/archive/lightning/train_predict.py b/archive/lightning/train_predict.py deleted file mode 100644 index 65232fb..0000000 --- a/archive/lightning/train_predict.py +++ /dev/null @@ -1,67 +0,0 @@ -import pytorch_lightning as pl -from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor -import torch -from torch import nn - - -import warnings -warnings.filterwarnings("ignore") - -import sys -sys.path.insert(0, '..') - -from data_loader.load_data import load_files - - - - -def train_model(model, train_dataloader, val_dataloader, kwargs): - print() - print("Creating Trainer ===>", end=" ") - # define trainer with early stopping - early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min") - lr_logger = LearningRateMonitor() - - trainer = pl.Trainer( - **kwargs, - callbacks=[lr_logger, early_stop_callback], - ) - print("<=== Trainer Created") - print("Finding Optimal LR ===>", end=" ") - # find optimal learning rate (set limit_train_batches to 1.0 and log_interval = -1) - res = trainer.tuner.lr_find( - model, train_dataloader=train_dataloader, val_dataloaders=val_dataloader, early_stop_threshold=1000.0, max_lr=0.3, - ) - - print(f"<=== suggested learning rate: {res.suggestion()}") - fig = res.plot(show=True, suggest=True) - fig.show() - - print("Training the model ===>", end=" ") - # fit the model - trainer.fit( - model, train_dataloader=train_dataloader, val_dataloaders=val_dataloader, - ) - - print("<=== Training Finished") - return trainer - - -def predict(trainer, model, val_dataloader): - pass - # best_model_path = trainer.checkpoint_callback.best_model_path - # best_model = model.load_from_checkpoint(best_model_path) - - # # calcualte mean absolute error on validation set - # actuals = torch.cat([y[0] for x, y in iter(val_dataloader)]) - # predictions = best_model.predict(val_dataloader) - # (actuals - predictions).abs().mean() - - # raw_predictions, x = best_model.predict(val_dataloader, mode="raw", return_x=True) - # for idx in range(10): # plot 10 examples - # best_model.plot_prediction(x, raw_predictions, idx=idx, add_loss_to_title=True) - - # predictions, x = best_model.predict(val_dataloader, return_x=True) - # predictions_vs_actuals = best_model.calculate_prediction_actual_by_variable(x, predictions) - # best_model.plot_prediction_actual_by_variable(predictions_vs_actuals) - \ No newline at end of file diff --git a/archive/model_lightning.py b/archive/model_lightning.py deleted file mode 100644 index 608e230..0000000 --- a/archive/model_lightning.py +++ /dev/null @@ -1,60 +0,0 @@ - -import torch -from torch import nn -from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split -import pytorch_lightning as pl - - -class LitManualAutoEncoder(pl.LightningModule): - - def __init__(self): - super().__init__() - self.encoder = nn.Sequential(nn.Linear(28 * 28, 128), nn.ReLU(), nn.Linear(128, 3)) - self.decoder = nn.Sequential(nn.Linear(3, 128), nn.ReLU(), nn.Linear(128, 28 * 28)) - print("success") - - def training_step(self, batch, batch_idx): - # -------------------------- - # REPLACE WITH YOUR OWN - opt_a = self.optimizers() - - x, y = batch - x = x.view(x.size(0), -1) - z = self.encoder(x) - x_hat = self.decoder(z) - loss = F.mse_loss(x_hat, x) - - # backward acts like normal backward - self.manual_backward(loss, opt_a, retain_graph=True) - self.manual_backward(loss, opt_a) - opt_a.step() - opt_a.zero_grad() - - # -------------------------- - - def validation_step(self, batch, batch_idx): - # -------------------------- - # REPLACE WITH YOUR OWN - x, y = batch - x = x.view(x.size(0), -1) - z = self.encoder(x) - x_hat = self.decoder(z) - loss = F.mse_loss(x_hat, x) - self.log('val_loss', loss) - # -------------------------- - - def test_step(self, batch, batch_idx): - # -------------------------- - # REPLACE WITH YOUR OWN - x, y = batch - x = x.view(x.size(0), -1) - z = self.encoder(x) - x_hat = self.decoder(z) - loss = F.mse_loss(x_hat, x) - self.log('test_loss', loss) - # -------------------------- - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.parameters(), lr=1e-3) - return optimizer \ No newline at end of file diff --git a/archive/run_lightning.py b/archive/run_lightning.py deleted file mode 100644 index 04e09b2..0000000 --- a/archive/run_lightning.py +++ /dev/null @@ -1,48 +0,0 @@ -from data_loader.load_data import load_files -import pandas as pd -# from tensorflow import keras -from utils.normalize import normalize -# import tensorflow as tf -from utils.visualize import visualize_loss - -from torch.utils.data import DataLoader, random_split -from model_lightning import LitManualAutoEncoder -import pytorch_lightning as pl - -#%% -data = load_files('data/', False) -data.reset_index(drop=True, inplace=True) -data = data[[column for column in data.columns if not column.endswith('volume')]] - -data.head() - -#%% -ticker_to_predict = 'ETH_returns' - -learning_rate = 0.002 -batch_size = 64 -epochs = 100 - -split_fraction = 0.715 -train_split = int(split_fraction * int(data.shape[0])) - -past = 10 -future = 1 - -start = past + future -end = start + train_split - - -# train = DataLoader(train, batch_size=32) -# test = DataLoader(test, batch_size=32) -# val = DataLoader(val, batch_size=32) - - -# init model -ae = LitManualAutoEncoder() - -# Initialize a trainer -trainer = pl.Trainer(gpus=1, max_epochs=3, progress_bar_refresh_rate=20) - -# Train the model ⚡ -# trainer.fit(ae, train, val) \ No newline at end of file diff --git a/config/hashing.py b/config/hashing.py index 9872467..4f5bb31 100644 --- a/config/hashing.py +++ b/config/hashing.py @@ -1,18 +1,27 @@ - from data_loader.types import DataCollection + def hash_data_config(data_config: dict) -> str: - - def hash_data_collection(data_collection: DataCollection) -> str: return ''.join([a[0] + a[1] for a in data_collection]) - def hash_feature_extractors(feature_extractos) -> str: return ''.join([f[0] for f in feature_extractos]) - def to_str(x): return ''.join([str(i) for i in x]) - return '_'.join(to_str([ - hash_data_collection(data_config['assets']), - hash_data_collection(data_config['other_assets']), - hash_data_collection(data_config['exogenous_data']), - data_config['target_asset'][0] + data_config['target_asset'][1], - data_config['load_non_target_asset'], - hash_feature_extractors(data_config['own_features']), - hash_feature_extractors(data_config['other_features']), - hash_feature_extractors(data_config['exogenous_features']), - ])) + def hash_data_collection(data_collection: DataCollection) -> str: + return "".join([a[0] + a[1] for a in data_collection]) + + def hash_feature_extractors(feature_extractos) -> str: + return "".join([f[0] for f in feature_extractos]) + + def to_str(x): + return "".join([str(i) for i in x]) + + return "_".join( + to_str( + [ + hash_data_collection(data_config["assets"]), + hash_data_collection(data_config["other_assets"]), + hash_data_collection(data_config["exogenous_data"]), + data_config["target_asset"][0] + data_config["target_asset"][1], + data_config["load_non_target_asset"], + hash_feature_extractors(data_config["own_features"]), + hash_feature_extractors(data_config["other_features"]), + hash_feature_extractors(data_config["exogenous_features"]), + ] + ) + ) diff --git a/config/preprocess.py b/config/preprocess.py index efd4482..b6b63b8 100644 --- a/config/preprocess.py +++ b/config/preprocess.py @@ -1,6 +1,8 @@ from .types import Config, RawConfig from utils.helpers import flatten -from feature_extractors.feature_extractor_presets import presets as feature_extractor_presets +from feature_extractors.feature_extractor_presets import ( + presets as feature_extractor_presets, +) from models.model_map import get_model from data_loader.collections import data_collections from labeling.eventfilters_map import eventfilters_map @@ -8,6 +10,7 @@ from labeling.labellers_map import labellers_map from models.sklearn import SKLearnModel from sklearn.ensemble import VotingClassifier + def preprocess_config(raw_config: RawConfig) -> Config: config_dict = vars(raw_config) config_dict = __preprocess_model_config(config_dict) @@ -16,47 +19,73 @@ def preprocess_config(raw_config: RawConfig) -> Config: config_dict = __preprocess_event_filter_config(config_dict) config_dict = __preprocess_event_labeller_config(config_dict) - config_dict['no_of_classes'] = 'two' - config_dict['mode'] = 'training' + config_dict["no_of_classes"] = "two" + config_dict["mode"] = "training" config = Config(**config_dict) return config + def __preprocess_feature_extractors_config(data_dict: dict) -> dict: data_dict = data_dict.copy() - keys = ['own_features', 'other_features', 'exogenous_features'] + keys = ["own_features", "other_features", "exogenous_features"] for key in keys: preset_names = data_dict[key] - data_dict[key] = flatten([feature_extractor_presets[preset_name] for preset_name in preset_names]) + data_dict[key] = flatten( + [feature_extractor_presets[preset_name] for preset_name in preset_names] + ) return data_dict + def __preprocess_model_config(model_config: dict) -> dict: - directional_models = [get_model(model_name) for model_name in model_config['directional_models']] - model_config.pop('directional_models') - model_config['directional_model'] = SKLearnModel(VotingClassifier([(m.name, m)for m in directional_models], voting ='soft')) - if len(model_config['meta_models']) > 0: - meta_models = [get_model(model_name) for model_name in model_config['meta_models']] - model_config['meta_model'] = SKLearnModel(VotingClassifier([(m.name, m)for m in meta_models], voting ='soft')) - model_config.pop('meta_models') + directional_models = [ + get_model(model_name) for model_name in model_config["directional_models"] + ] + model_config.pop("directional_models") + model_config["directional_model"] = SKLearnModel( + VotingClassifier([(m.name, m) for m in directional_models], voting="soft") + ) + if len(model_config["meta_models"]) > 0: + meta_models = [ + get_model(model_name) for model_name in model_config["meta_models"] + ] + model_config["meta_model"] = SKLearnModel( + VotingClassifier([(m.name, m) for m in meta_models], voting="soft") + ) + model_config.pop("meta_models") return model_config + def __preprocess_data_collections_config(data_dict: dict) -> dict: - keys = ['assets', 'other_assets', 'exogenous_data'] + keys = ["assets", "other_assets", "exogenous_data"] for key in keys: preset_names = data_dict[key] - data_dict[key] = flatten([data_collections[preset_name] for preset_name in preset_names]) - target_asset = next(iter([asset for asset in data_dict['assets'] if asset[1] == data_dict['target_asset']]), None) - if target_asset is None: raise Exception('Target asset wasnt found in assets') - data_dict['target_asset'] = target_asset + data_dict[key] = flatten( + [data_collections[preset_name] for preset_name in preset_names] + ) + target_asset = next( + iter( + [ + asset + for asset in data_dict["assets"] + if asset[1] == data_dict["target_asset"] + ] + ), + None, + ) + if target_asset is None: + raise Exception("Target asset wasnt found in assets") + data_dict["target_asset"] = target_asset return data_dict + def __preprocess_event_filter_config(data_dict: dict) -> dict: - data_dict['event_filter'] = eventfilters_map[data_dict['event_filter']] + data_dict["event_filter"] = eventfilters_map[data_dict["event_filter"]] return data_dict + def __preprocess_event_labeller_config(config_dict: dict) -> dict: - config_dict['labeling'] = labellers_map[config_dict['labeling']](config_dict['forecasting_horizon']) + config_dict["labeling"] = labellers_map[config_dict["labeling"]]( + config_dict["forecasting_horizon"] + ) return config_dict - - - diff --git a/config/presets.py b/config/presets.py index a75ec94..74ecfb0 100644 --- a/config/presets.py +++ b/config/presets.py @@ -1,104 +1,101 @@ from .types import RawConfig, Config + def get_dev_config() -> RawConfig: - + classification_models = ["LogisticRegression_two_class"] return RawConfig( - directional_models_meta = False, - dimensionality_reduction = False, - n_features_to_select = 30, - expanding_window_base = False, - expanding_window_meta = False, - sliding_window_size_base = 380, - sliding_window_size_meta = 1, - retrain_every = 20, - scaler = 'minmax', # 'normalize' 'minmax' 'standardize' - - assets = ['daily_only_btc'], - target_asset = 'BTC_USD', - other_assets = [], - exogenous_data = [], - load_non_target_asset= True, - own_features = ['level_2', 'date_days'], - other_features = ['single_mom'], - exogenous_features = ['z_score'], - - directional_models = classification_models, - meta_models = [], - - event_filter = 'none', - labeling = 'two_class', - forecasting_horizon = 100, + directional_models_meta=False, + dimensionality_reduction=False, + n_features_to_select=30, + expanding_window_base=False, + expanding_window_meta=False, + sliding_window_size_base=380, + sliding_window_size_meta=1, + retrain_every=20, + scaler="minmax", # 'normalize' 'minmax' 'standardize' + assets=["daily_only_btc"], + target_asset="BTC_USD", + other_assets=[], + exogenous_data=[], + load_non_target_asset=True, + own_features=["level_2", "date_days"], + other_features=["single_mom"], + exogenous_features=["z_score"], + directional_models=classification_models, + meta_models=[], + event_filter="none", + labeling="two_class", + forecasting_horizon=100, ) def get_default_ensemble_config() -> RawConfig: - - classification_models = ["LogisticRegression_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"] - meta_models = ['LogisticRegression_two_class', 'LGBM'] + + classification_models = [ + "LogisticRegression_two_class", + "LDA", + "NB", + "RFC", + "XGB_two_class", + "LGBM", + "StaticMom", + ] + meta_models = ["LogisticRegression_two_class", "LGBM"] return RawConfig( - directional_models_meta = True, - dimensionality_reduction = False, - n_features_to_select = 30, - expanding_window_base = False, - expanding_window_meta = True, - sliding_window_size_base = 380, - sliding_window_size_meta = 240, - retrain_every = 10, - scaler = 'minmax', # 'normalize' 'minmax' 'standardize' - - assets = ['daily_crypto'], - target_asset = 'BTC_USD', - other_assets = ['daily_etf'], - exogenous_data = ['daily_glassnode'], - load_non_target_asset= True, - own_features = ['level_2', 'date_days', 'lags_up_to_5'], - other_features = ['level_2', 'lags_up_to_5'], - exogenous_features = ['z_score'], - - directional_models = classification_models, - meta_models = meta_models, - - event_filter = 'cusum_vol', - labeling = 'two_class', - forecasting_horizon = 100, + directional_models_meta=True, + dimensionality_reduction=False, + n_features_to_select=30, + expanding_window_base=False, + expanding_window_meta=True, + sliding_window_size_base=380, + sliding_window_size_meta=240, + retrain_every=10, + scaler="minmax", # 'normalize' 'minmax' 'standardize' + assets=["daily_crypto"], + target_asset="BTC_USD", + other_assets=["daily_etf"], + exogenous_data=["daily_glassnode"], + load_non_target_asset=True, + own_features=["level_2", "date_days", "lags_up_to_5"], + other_features=["level_2", "lags_up_to_5"], + exogenous_features=["z_score"], + directional_models=classification_models, + meta_models=meta_models, + event_filter="cusum_vol", + labeling="two_class", + forecasting_horizon=100, ) - def get_lightweight_ensemble_config() -> RawConfig: - - classification_models = ['LogisticRegression_two_class', 'LGBM'] - meta_models = ['LogisticRegression_two_class', 'LGBM'] + + classification_models = ["LogisticRegression_two_class", "LGBM"] + meta_models = ["LogisticRegression_two_class", "LGBM"] return RawConfig( - directional_models_meta = True, - dimensionality_reduction = True, - n_features_to_select = 30, - expanding_window_base = True, - expanding_window_meta = True, - sliding_window_size_base = 3800, - sliding_window_size_meta = 2400, - retrain_every = 1000, - scaler = 'minmax', # 'normalize' 'minmax' 'standardize' - - assets = ['fivemin_crypto'], - target_asset = 'BTC_USD', - other_assets = [], - exogenous_data = [], - load_non_target_asset= False, - own_features = ['level_1'], - other_features = [], - exogenous_features = [], - - directional_models = classification_models, - meta_models = meta_models, - - event_filter = 'cusum_fixed', - labeling = 'two_class', - forecasting_horizon = 50, + directional_models_meta=True, + dimensionality_reduction=True, + n_features_to_select=30, + expanding_window_base=True, + expanding_window_meta=True, + sliding_window_size_base=3800, + sliding_window_size_meta=2400, + retrain_every=1000, + scaler="minmax", # 'normalize' 'minmax' 'standardize' + assets=["fivemin_crypto"], + target_asset="BTC_USD", + other_assets=[], + exogenous_data=[], + load_non_target_asset=False, + own_features=["level_1"], + other_features=[], + exogenous_features=[], + directional_models=classification_models, + meta_models=meta_models, + event_filter="cusum_fixed", + labeling="two_class", + forecasting_horizon=50, ) - - diff --git a/config/types.py b/config/types.py index e6cd2d4..5cbcb2f 100644 --- a/config/types.py +++ b/config/types.py @@ -18,7 +18,7 @@ class RawConfig(BaseModel): sliding_window_size_base: int sliding_window_size_meta: int retrain_every: int - scaler: Literal['normalize', 'minmax', 'standardize'] + scaler: Literal["normalize", "minmax", "standardize"] assets: list[str] target_asset: str @@ -28,8 +28,8 @@ class RawConfig(BaseModel): own_features: list[str] other_features: list[str] exogenous_features: list[str] - event_filter: Literal['none', 'cusum_vol', 'cusum_fixed'] - labeling: Literal['two_class', 'three_class_balanced', 'three_class_imbalanced'] + event_filter: Literal["none", "cusum_vol", "cusum_fixed"] + labeling: Literal["two_class", "three_class_balanced", "three_class_imbalanced"] forecasting_horizon: int directional_models: list[str] @@ -46,7 +46,7 @@ class Config: sliding_window_size_base: int sliding_window_size_meta: int retrain_every: int - scaler: Literal['normalize', 'minmax', 'standardize'] + scaler: Literal["normalize", "minmax", "standardize"] assets: DataCollection target_asset: DataSource @@ -59,28 +59,27 @@ class Config: event_filter: EventFilter labeling: EventLabeller forecasting_horizon: int - no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'] + no_of_classes: Literal["two", "three-balanced", "three-imbalanced"] - mode: Literal['training', 'inference'] + mode: Literal["training", "inference"] directional_model: Model meta_model: Model - @validator('directional_model', 'meta_model') + @validator("directional_model", "meta_model") def check_model(cls, v): assert isinstance(v, BaseEstimator) return v - - @validator('event_filter') + + @validator("event_filter") def check_event_filter(cls, v): assert isinstance(v, EventFilter) return v - @validator('labeling') + @validator("labeling") def check_labeling(cls, v): assert isinstance(v, EventLabeller) return v + # class Config: # arbitrary_types_allowed = True - - diff --git a/data_loader/collections.py b/data_loader/collections.py index 9b583a5..e7afbed 100644 --- a/data_loader/collections.py +++ b/data_loader/collections.py @@ -1,51 +1,83 @@ from .types import Path, FileName, DataSource, DataCollection from utils.helpers import flatten + def transform_to_data_collection(path: str, file_names: list[str]) -> DataCollection: return list(zip([path] * len(file_names), file_names)) + __daily_etf = ["GLD", "IEF", "QQQ", "SPY", "TLT"] -__daily_crypto = ["ADA_USD", "BCH_USD", "BNB_USD", "BTC_USD", "DOT_USD", "ETC_USD", "ETH_USD", "FIL_USD", "LTC_USD", "SOL_USD", "THETA_USD", "TRX_USD", "UNI_USD", "XLM_USD", "XRP_USD", "XTZ_USD"] +__daily_crypto = [ + "ADA_USD", + "BCH_USD", + "BNB_USD", + "BTC_USD", + "DOT_USD", + "ETC_USD", + "ETH_USD", + "FIL_USD", + "LTC_USD", + "SOL_USD", + "THETA_USD", + "TRX_USD", + "UNI_USD", + "XLM_USD", + "XRP_USD", + "XTZ_USD", +] __daily_crypto_lightweight = ["ADA_USD", "BCH_USD"] -__5min_crypto = ["BTC_USD", "DASH_USD", "ETC_USD", "ETH_USD", "LTC_USD", "TRX_USD", "XLM_USD", "XMR_USD"] +__5min_crypto = [ + "BTC_USD", + "DASH_USD", + "ETC_USD", + "ETH_USD", + "LTC_USD", + "TRX_USD", + "XLM_USD", + "XMR_USD", +] -__daily_glassnode = ['rhodl_ratio', +__daily_glassnode = [ + "rhodl_ratio", # 'cvdd', - 'nvt_ratio', - 'nvt_signal', - 'velocity', - 'supply_adjusted_cdd', - 'binary_cdd', - 'supply_adjusted_dormancy', - 'puell_multiple', - 'asopr', - 'reserve_risk', - 'sopr', - 'cdd', - 'asol', - 'msol', - 'dormancy', - 'liveliness', - 'relative_unrealized_profit', - 'relative_unrealized_loss', - 'nupl', - 'sth_nupl', - 'lth_nupl', - 'ssr', - 'bvin', + "nvt_ratio", + "nvt_signal", + "velocity", + "supply_adjusted_cdd", + "binary_cdd", + "supply_adjusted_dormancy", + "puell_multiple", + "asopr", + "reserve_risk", + "sopr", + "cdd", + "asol", + "msol", + "dormancy", + "liveliness", + "relative_unrealized_profit", + "relative_unrealized_loss", + "nupl", + "sth_nupl", + "lth_nupl", + "ssr", + "bvin", # 'hash_rate' ] data_collections = dict( - daily_only_btc = transform_to_data_collection("data/daily_crypto", ['BTC_USD']), - daily_crypto = transform_to_data_collection("data/daily_crypto", __daily_crypto), - daily_crypto_lightweight = transform_to_data_collection("data/daily_crypto", __daily_crypto_lightweight), - daily_etf = transform_to_data_collection("data/daily_etf", __daily_etf), - fivemin_crypto = transform_to_data_collection("data/5min_crypto", __5min_crypto), - daily_glassnode =transform_to_data_collection("data/daily_glassnode", __daily_glassnode), + daily_only_btc=transform_to_data_collection("data/daily_crypto", ["BTC_USD"]), + daily_crypto=transform_to_data_collection("data/daily_crypto", __daily_crypto), + daily_crypto_lightweight=transform_to_data_collection( + "data/daily_crypto", __daily_crypto_lightweight + ), + daily_etf=transform_to_data_collection("data/daily_etf", __daily_etf), + fivemin_crypto=transform_to_data_collection("data/5min_crypto", __5min_crypto), + daily_glassnode=transform_to_data_collection( + "data/daily_glassnode", __daily_glassnode + ), ) - diff --git a/data_loader/get_prices.py b/data_loader/get_prices.py index c70ff58..7116497 100644 --- a/data_loader/get_prices.py +++ b/data_loader/get_prices.py @@ -3,45 +3,76 @@ import requests import pandas as pd # %% -def get_crypto_price_crypto_compare(symbol: str, exchange: str, days: int) -> pd.DataFrame: - api_url = f'https://min-api.cryptocompare.com/data/v2/histoday?fsym={symbol}&tsym={exchange}&limit={days}&api_key={CC_API_KEY}' +def get_crypto_price_crypto_compare( + symbol: str, exchange: str, days: int +) -> pd.DataFrame: + api_url = f"https://min-api.cryptocompare.com/data/v2/histoday?fsym={symbol}&tsym={exchange}&limit={days}&api_key={CC_API_KEY}" raw = requests.get(api_url).json() - df = pd.DataFrame(raw['Data']['Data'])[['time', 'high', 'low', 'open', 'close']].set_index('time') - df.index = pd.to_datetime(df.index, unit = 's') - df.sort_index(inplace = True, ascending= True) + df = pd.DataFrame(raw["Data"]["Data"])[ + ["time", "high", "low", "open", "close"] + ].set_index("time") + df.index = pd.to_datetime(df.index, unit="s") + df.sort_index(inplace=True, ascending=True) return df -ada = get_crypto_price_crypto_compare('ADA', 'USD', 1500) + +ada = get_crypto_price_crypto_compare("ADA", "USD", 1500) ada - -def get_crypto_price_av(symbol: str, exchange: str, start_date = None) -> pd.DataFrame: - api_url = f'https://www.alphavantage.co/query?function=DIGITAL_CURRENCY_DAILY&symbol={symbol}&market={exchange}&apikey={AV_API_KEY}' +def get_crypto_price_av(symbol: str, exchange: str, start_date=None) -> pd.DataFrame: + api_url = f"https://www.alphavantage.co/query?function=DIGITAL_CURRENCY_DAILY&symbol={symbol}&market={exchange}&apikey={AV_API_KEY}" raw_df = requests.get(api_url).json() - df = pd.DataFrame(raw_df['Time Series (Digital Currency Daily)']).T - df = df.rename(columns = {'1a. open (USD)': 'open', '2a. high (USD)': 'high', '3a. low (USD)': 'low', '4a. close (USD)': 'close', '5. volume': 'volume'}) + df = pd.DataFrame(raw_df["Time Series (Digital Currency Daily)"]).T + df = df.rename( + columns={ + "1a. open (USD)": "open", + "2a. high (USD)": "high", + "3a. low (USD)": "low", + "4a. close (USD)": "close", + "5. volume": "volume", + } + ) for i in df.columns: df[i] = df[i].astype(float) df.index = pd.to_datetime(df.index) - df = df.iloc[::-1].drop(['1b. open (USD)', '2b. high (USD)', '3b. low (USD)', '4b. close (USD)', '6. market cap (USD)'], axis = 1) + df = df.iloc[::-1].drop( + [ + "1b. open (USD)", + "2b. high (USD)", + "3b. low (USD)", + "4b. close (USD)", + "6. market cap (USD)", + ], + axis=1, + ) if start_date: df = df[df.index >= start_date] - df.sort_index(inplace = True, ascending= True) + df.sort_index(inplace=True, ascending=True) return df + def get_stock_price_av(symbol: str, start_date: str = None) -> pd.DataFrame: - api_url = f'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&outputsize=full&apikey={AV_API_KEY}' + api_url = f"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&outputsize=full&apikey={AV_API_KEY}" raw_df = requests.get(api_url).json() - df = pd.DataFrame(raw_df['Time Series (Daily)']).T - df = df.rename(columns = {'1. open': 'open', '2. high': 'high', '3. low': 'low', '5. adjusted close': 'close', '6. volume': 'volume'}) + df = pd.DataFrame(raw_df["Time Series (Daily)"]).T + df = df.rename( + columns={ + "1. open": "open", + "2. high": "high", + "3. low": "low", + "5. adjusted close": "close", + "6. volume": "volume", + } + ) for i in df.columns: df[i] = df[i].astype(float) df.index = pd.to_datetime(df.index) - df = df.iloc[::-1].drop(['4. close', '7. dividend amount', '8. split coefficient'], axis = 1) + df = df.iloc[::-1].drop( + ["4. close", "7. dividend amount", "8. split coefficient"], axis=1 + ) if start_date: df = df[df.index >= start_date] - df.sort_index(inplace = True, ascending= True) - df = df.rename_axis('time') + df.sort_index(inplace=True, ascending=True) + df = df.rename_axis("time") return df - diff --git a/data_loader/load.py b/data_loader/load.py index 93bf116..f14a410 100644 --- a/data_loader/load.py +++ b/data_loader/load.py @@ -10,8 +10,8 @@ import os from config.hashing import hash_data_config from .types import XDataFrame, ReturnSeries, ForwardReturnSeries from diskcache import Cache -cache = Cache(".cachedir/data") +cache = Cache(".cachedir/data") def load_data(**kwargs) -> tuple[XDataFrame, ReturnSeries]: @@ -23,15 +23,17 @@ def load_data(**kwargs) -> tuple[XDataFrame, ReturnSeries]: cache[hashed] = return_value return return_value -def __load_data(assets: DataCollection, - other_assets: DataCollection, - exogenous_data: DataCollection, - target_asset: DataSource, - load_non_target_asset: bool, - own_features: list[tuple[str, FeatureExtractor, list[int]]], - other_features: list[tuple[str, FeatureExtractor, list[int]]], - exogenous_features: list[tuple[str, FeatureExtractor, list[int]]], - ) -> tuple[XDataFrame, ReturnSeries]: + +def __load_data( + assets: DataCollection, + other_assets: DataCollection, + exogenous_data: DataCollection, + target_asset: DataSource, + load_non_target_asset: bool, + own_features: list[tuple[str, FeatureExtractor, list[int]]], + other_features: list[tuple[str, FeatureExtractor, list[int]]], + exogenous_features: list[tuple[str, FeatureExtractor, list[int]]], +) -> tuple[XDataFrame, ReturnSeries]: """ Loads asset data from the specified path. Returns: @@ -42,77 +44,96 @@ def __load_data(assets: DataCollection, target_file = [f for f in assets if f[1].startswith(target_asset[1])] assert len(target_file) == 1, "There should be exactly one target file" - other_files = [f for f in assets if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False] + other_files = [ + f + for f in assets + if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False + ] files = other_files + other_assets - - target_asset_future = [__load_df.remote( - data_source=data_source, - prefix=data_source[1], - returns='log_returns', - feature_extractors=own_features, - ) for data_source in target_file] + + target_asset_future = [ + __load_df.remote( + data_source=data_source, + prefix=data_source[1], + returns="log_returns", + feature_extractors=own_features, + ) + for data_source in target_file + ] target_asset_df = ray.get(target_asset_future) - + target_asset_only_returns_future = __load_df.remote( data_source=target_file[0], prefix=target_file[0][1], - returns='returns', + returns="returns", feature_extractors=[], ) df_target_asset_only_returns = ray.get(target_asset_only_returns_future) - asset_futures = [__load_df.remote( - data_source=data_source, - prefix=data_source[1], - returns='log_returns', - feature_extractors=other_features, - ) for data_source in files] + asset_futures = [ + __load_df.remote( + data_source=data_source, + prefix=data_source[1], + returns="log_returns", + feature_extractors=other_features, + ) + for data_source in files + ] asset_dfs = ray.get(asset_futures) - exogenous_futures = [__load_df.remote( - data_source=data_source, - prefix=data_source[1], - returns='none', - feature_extractors=exogenous_features, - ) for data_source in exogenous_data] + exogenous_futures = [ + __load_df.remote( + data_source=data_source, + prefix=data_source[1], + returns="none", + feature_extractors=exogenous_features, + ) + for data_source in exogenous_data + ] exogenous_dfs = ray.get(exogenous_futures) X = target_asset_df + asset_dfs + exogenous_dfs - X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.) + X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.0) X.index = pd.DatetimeIndex(X.index) - - ## Create target - returns = df_target_asset_only_returns[target_asset[1] + '_returns'] + + ## Create target + returns = df_target_asset_only_returns[target_asset[1] + "_returns"] returns.index = pd.DatetimeIndex(X.index) - - return X, returns + + return X, returns @ray.remote -def __load_df(data_source: DataSource, - prefix: str, - returns: Literal['none', 'price', 'returns', 'log_returns'], - feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]) -> pd.DataFrame: - df = pd.read_csv(os.path.join(data_source[0], data_source[1] + '.csv'), header=0, index_col=0).fillna(0) +def __load_df( + data_source: DataSource, + prefix: str, + returns: Literal["none", "price", "returns", "log_returns"], + feature_extractors: list[tuple[str, FeatureExtractor, list[int]]], +) -> pd.DataFrame: + df = pd.read_csv( + os.path.join(data_source[0], data_source[1] + ".csv"), header=0, index_col=0 + ).fillna(0) - if returns == 'log_returns': - df['returns'] = np.log(df['close']).diff(1) - elif returns == 'price': - df['returns'] = df['close'] - elif returns == 'returns': - df['returns'] = df['close'].pct_change() + if returns == "log_returns": + df["returns"] = np.log(df["close"]).diff(1) + elif returns == "price": + df["returns"] = df["close"] + elif returns == "returns": + df["returns"] = df["close"].pct_change() - df = __apply_feature_extractors(df, feature_extractors = feature_extractors) + df = __apply_feature_extractors(df, feature_extractors=feature_extractors) - df = df.replace([np.inf, -np.inf], 0.) - df = drop_columns_if_exist(df, ['open', 'high', 'low', 'close', 'volume']) - - df.columns = [prefix + "_" + c if 'date' not in c else c for c in df.columns] + df = df.replace([np.inf, -np.inf], 0.0) + df = drop_columns_if_exist(df, ["open", "high", "low", "close", "volume"]) + + df.columns = [prefix + "_" + c if "date" not in c else c for c in df.columns] return df -def __apply_feature_extractors(df: pd.DataFrame, feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]) -> pd.DataFrame: +def __apply_feature_extractors( + df: pd.DataFrame, feature_extractors: list[tuple[str, FeatureExtractor, list[int]]] +) -> pd.DataFrame: for name, extractor, periods in feature_extractors: for period in periods: @@ -120,20 +141,27 @@ def __apply_feature_extractors(df: pd.DataFrame, feature_extractors: list[tuple[ if type(features) == pd.DataFrame: df = pd.concat([df, features], axis=1) elif type(features) == pd.Series: - df[name + '_' + str(period)] = features + df[name + "_" + str(period)] = features else: - assert False, "Feature extractor must return a pd.DataFrame or pd.Series" + assert ( + False + ), "Feature extractor must return a pd.DataFrame or pd.Series" return df -def load_only_returns(assets: DataCollection, returns: Literal['price', 'returns']) -> pd.DataFrame: +def load_only_returns( + assets: DataCollection, returns: Literal["price", "returns"] +) -> pd.DataFrame: - assets_future = [__load_df.remote( - data_source=data_source, - prefix=data_source[1], - returns=returns, - feature_extractors=[], - ) for data_source in assets] + assets_future = [ + __load_df.remote( + data_source=data_source, + prefix=data_source[1], + returns=returns, + feature_extractors=[], + ) + for data_source in assets + ] dfs = ray.get(assets_future) dfs = pd.concat(dfs, axis=1) diff --git a/data_loader/process.py b/data_loader/process.py index 01bac1c..3116bc5 100644 --- a/data_loader/process.py +++ b/data_loader/process.py @@ -4,16 +4,21 @@ import warnings from utils.helpers import get_first_valid_return_index from data_loader.types import XDataFrame + def check_data(X: XDataFrame, config: Config) -> bool: - """ Returns True if data is valid, else returns False.""" - + """Returns True if data is valid, else returns False.""" + if has_enough_samples_to_train(X, config) == False: warnings.warn("Not enough samples to train") return False - + return True + def has_enough_samples_to_train(X: XDataFrame, config: Config) -> bool: - first_valid_index = get_first_valid_return_index(X.iloc[:,0]) + first_valid_index = get_first_valid_return_index(X.iloc[:, 0]) samples_to_train = len(X) - first_valid_index - return samples_to_train > config.sliding_window_size_base + config.sliding_window_size_meta + 100 \ No newline at end of file + return ( + samples_to_train + > config.sliding_window_size_base + config.sliding_window_size_meta + 100 + ) diff --git a/data_loader/types.py b/data_loader/types.py index aa1b798..807713a 100644 --- a/data_loader/types.py +++ b/data_loader/types.py @@ -8,4 +8,4 @@ DataCollection = list[DataSource] ReturnSeries = pd.Series ForwardReturnSeries = pd.Series XDataFrame = pd.DataFrame -ySeries = pd.Series \ No newline at end of file +ySeries = pd.Series diff --git a/data_loader/utils.py b/data_loader/utils.py index 16b707f..539567b 100644 --- a/data_loader/utils.py +++ b/data_loader/utils.py @@ -1,3 +1,5 @@ import pandas as pd -def deduplicate_indexes(df: pd.DataFrame) -> pd.DataFrame: return df[~df.index.duplicated(keep='last')] + +def deduplicate_indexes(df: pd.DataFrame) -> pd.DataFrame: + return df[~df.index.duplicated(keep="last")] diff --git a/docs/conf.py b/docs/conf.py index b22aa58..6bf3b7c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,17 +12,18 @@ # import os import sys -sys.path.insert(0, os.path.abspath('.')) + +sys.path.insert(0, os.path.abspath(".")) # -- Project information ----------------------------------------------------- -project = 'Drift' -copyright = '2022, Daniel Szemerey and Mark Szulyovszky' -author = 'Daniel Szemerey, Mark Szulyovszky' +project = "Drift" +copyright = "2022, Daniel Szemerey and Mark Szulyovszky" +author = "Daniel Szemerey, Mark Szulyovszky" # The full version, including alpha/beta/rc tags -release = '1.0' +release = "1.0" # -- General configuration --------------------------------------------------- @@ -30,17 +31,15 @@ release = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.autosectionlabel' -] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.autosectionlabel"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'sphinx-env'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "sphinx-env"] # -- Options for HTML output ------------------------------------------------- @@ -48,9 +47,9 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'sphinx-env'] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'furo' +html_theme = "furo" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] \ No newline at end of file +# html_static_path = ['_static'] diff --git a/environment.yml b/environment.yml index 1e1edf9..84683d1 100644 --- a/environment.yml +++ b/environment.yml @@ -9,6 +9,7 @@ dependencies: - python=3.9 - scikit-learn-intelex=2021.4.0 - pip: + - black - skorch - fracdiff - ray diff --git a/feature_extractors/feature_extractor_presets.py b/feature_extractors/feature_extractor_presets.py index 5be6094..1f51189 100644 --- a/feature_extractors/feature_extractor_presets.py +++ b/feature_extractors/feature_extractor_presets.py @@ -1,35 +1,55 @@ - from .types import FeatureExtractorConfig -from .feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_expanding_zscore, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead -from .fractional_differentiation import feature_fractional_differentiation, feature_fractional_differentiation_log +from .feature_extractors import ( + feature_lag, + feature_mom, + feature_ROC, + feature_RSI, + feature_STOD, + feature_STOK, + feature_expanding_zscore, + feature_vol, + feature_day_of_month, + feature_day_of_week, + feature_month, + feature_debug_future_lookahead, +) +from .fractional_differentiation import ( + feature_fractional_differentiation, + feature_fractional_differentiation_log, +) __presets = dict( - debug_future_lookahead = [('debug_future', feature_debug_future_lookahead, [1])], - single_mom = [('mom', feature_mom, [30])], - single_vol = [('vol', feature_vol, [30])], - mom = [('mom', feature_mom, [10, 20, 30, 60, 90])], - vol = [('vol', feature_vol, [10, 20, 30, 60])], - lags_up_to_5 = [('lag', feature_lag, [1,2,3,4,5])], - lags_up_to_10 = [('lag', feature_lag, [1,2,3,4,5,6,7,8,9,10])], - date_all = [ - ('day_of_week', feature_day_of_week, [0]), - ('day_of_month', feature_day_of_month, [0]), - ('month', feature_month, [0])], - date_days = [ - ('day_of_week', feature_day_of_week, [0]), - ('day_of_month', feature_day_of_month, [0]), + debug_future_lookahead=[("debug_future", feature_debug_future_lookahead, [1])], + single_mom=[("mom", feature_mom, [30])], + single_vol=[("vol", feature_vol, [30])], + mom=[("mom", feature_mom, [10, 20, 30, 60, 90])], + vol=[("vol", feature_vol, [10, 20, 30, 60])], + lags_up_to_5=[("lag", feature_lag, [1, 2, 3, 4, 5])], + lags_up_to_10=[("lag", feature_lag, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])], + date_all=[ + ("day_of_week", feature_day_of_week, [0]), + ("day_of_month", feature_day_of_month, [0]), + ("month", feature_month, [0]), ], - roc = [('roc', feature_ROC, [10, 30])], - rsi = [('rsi', feature_ROC, [10, 30, 100])], - stod = [('stod', feature_STOD, [10, 30, 200])], - stok = [('stok', feature_STOK, [10, 30, 200])], - fracdiff = [('fracdiff', feature_fractional_differentiation, [10, 30])], - fracdiff_log = [('fracdiff_log', feature_fractional_differentiation_log, [10, 30])], - z_score = [('z_score', feature_expanding_zscore, [10])], + date_days=[ + ("day_of_week", feature_day_of_week, [0]), + ("day_of_month", feature_day_of_month, [0]), + ], + roc=[("roc", feature_ROC, [10, 30])], + rsi=[("rsi", feature_ROC, [10, 30, 100])], + stod=[("stod", feature_STOD, [10, 30, 200])], + stok=[("stok", feature_STOK, [10, 30, 200])], + fracdiff=[("fracdiff", feature_fractional_differentiation, [10, 30])], + fracdiff_log=[("fracdiff_log", feature_fractional_differentiation_log, [10, 30])], + z_score=[("z_score", feature_expanding_zscore, [10])], ) presets = __presets | dict( - level_1 = __presets["mom"] + __presets["vol"], - level_2 = __presets["mom"] + __presets["vol"] + __presets["roc"] + __presets["rsi"] + __presets["stod"] + __presets["stok"], + level_1=__presets["mom"] + __presets["vol"], + level_2=__presets["mom"] + + __presets["vol"] + + __presets["roc"] + + __presets["rsi"] + + __presets["stod"] + + __presets["stok"], ) - diff --git a/feature_extractors/feature_extractors.py b/feature_extractors/feature_extractors.py index e09cd79..5491813 100644 --- a/feature_extractors/feature_extractors.py +++ b/feature_extractors/feature_extractors.py @@ -3,58 +3,85 @@ import numpy as np from feature_extractors.utils import get_close_low_high from feature_extractors.utils import apply_log_if_necessary_series + def feature_debug_future_lookahead(df: pd.DataFrame, period: int) -> pd.Series: - return df['returns'].shift(-period) + return df["returns"].shift(-period) + def feature_lag(df: pd.DataFrame, period: int) -> pd.Series: assert period > 0 - return df['returns'].shift(period) + return df["returns"].shift(period) + def feature_expanding_zscore(df: pd.DataFrame, period: int) -> pd.Series: - close = df['close'] + close = df["close"] return (close - close.expanding(period).mean()) / close.expanding(period).std() + def feature_day_of_week(df: pd.DataFrame, period: int) -> pd.DataFrame: - return pd.get_dummies(pd.DatetimeIndex(df.index).dayofweek, drop_first=True, prefix="date_day_week").set_index(df.index) + return pd.get_dummies( + pd.DatetimeIndex(df.index).dayofweek, drop_first=True, prefix="date_day_week" + ).set_index(df.index) + def feature_day_of_month(df: pd.DataFrame, period: int) -> pd.DataFrame: - return pd.get_dummies(pd.DatetimeIndex(df.index).day, drop_first=True, prefix="date_day_month").set_index(df.index) + return pd.get_dummies( + pd.DatetimeIndex(df.index).day, drop_first=True, prefix="date_day_month" + ).set_index(df.index) + def feature_month(df: pd.DataFrame, period: int) -> pd.DataFrame: - return pd.get_dummies(pd.DatetimeIndex(df.index).month, drop_first=True, prefix="date_month").set_index(df.index) + return pd.get_dummies( + pd.DatetimeIndex(df.index).month, drop_first=True, prefix="date_month" + ).set_index(df.index) + def feature_vol(df: pd.DataFrame, period: int) -> pd.Series: - return df['returns'].rolling(period).std() * (252**0.5) + return df["returns"].rolling(period).std() * (252**0.5) + def feature_mom(df: pd.DataFrame, period: int) -> pd.Series: - return np.log(df['close']).diff(period) + return np.log(df["close"]).diff(period) + def feature_STOK(df: pd.DataFrame, period: int) -> pd.Series: close, low, high = get_close_low_high(df) - STOK = ((close - low.rolling(period).min()) / (high.rolling(period).max() - low.rolling(period).min())) * 100 + STOK = ( + (close - low.rolling(period).min()) + / (high.rolling(period).max() - low.rolling(period).min()) + ) * 100 return apply_log_if_necessary_series(STOK, "stok") + def feature_STOD(df: pd.DataFrame, period: int) -> pd.Series: stok = feature_STOK(df, period) return stok.rolling(3).mean() + def feature_RSI(df: pd.DataFrame, period: int) -> pd.Series: - returns = df['returns'] + returns = df["returns"] delta = returns.diff().dropna() - u=delta*0 + u = delta * 0 d = u.copy() u[delta > 0] = delta[delta > 0] d[delta < 0] = -delta[delta < 0] - u[u.index[period-1]] = np.mean( u[:period] ) #first value is sum of avg gains u = u.drop(u.index[:(period-1)]) - d[d.index[period-1]] = np.mean( d[:period] ) #first value is sum of avg losses d = d.drop(d.index[:(period-1)]) - rs = u.ewm(com=period-1, adjust=False).mean() / \ - d.ewm(com=period-1, adjust=False).mean() - return apply_log_if_necessary_series(100-100/(1+rs), "rsi") + u[u.index[period - 1]] = np.mean( + u[:period] + ) # first value is sum of avg gains u = u.drop(u.index[:(period-1)]) + d[d.index[period - 1]] = np.mean( + d[:period] + ) # first value is sum of avg losses d = d.drop(d.index[:(period-1)]) + rs = ( + u.ewm(com=period - 1, adjust=False).mean() + / d.ewm(com=period - 1, adjust=False).mean() + ) + return apply_log_if_necessary_series(100 - 100 / (1 + rs), "rsi") + def feature_ROC(df: pd.DataFrame, period: int) -> pd.Series: - returns = df['returns'] + returns = df["returns"] M = returns.diff(period - 1) N = returns.shift(period - 1) - roc = pd.Series(((M / N) * 100), name = 'ROC_' + str(period)) - return apply_log_if_necessary_series(roc, "roc") \ No newline at end of file + roc = pd.Series(((M / N) * 100), name="ROC_" + str(period)) + return apply_log_if_necessary_series(roc, "roc") diff --git a/feature_extractors/fractional_differentiation.py b/feature_extractors/fractional_differentiation.py index c53df29..8a3b0f8 100644 --- a/feature_extractors/fractional_differentiation.py +++ b/feature_extractors/fractional_differentiation.py @@ -3,13 +3,14 @@ import pandas as pd import numpy as np from feature_extractors.utils import apply_log_if_necessary_series + def feature_fractional_differentiation(df: pd.DataFrame, period: int) -> pd.Series: - frac_diff = FracdiffStat(window = period) + frac_diff = FracdiffStat(window=period) input_series = df["close"].to_numpy().reshape(-1, 1) result = frac_diff.fit_transform(input_series) - return pd.Series(result.squeeze(), index = df.index) + return pd.Series(result.squeeze(), index=df.index) + def feature_fractional_differentiation_log(df: pd.DataFrame, period: int) -> pd.Series: series = feature_fractional_differentiation(df, period, is_log_return) return apply_log_if_necessary_series(series, "fracdiff") - \ No newline at end of file diff --git a/feature_extractors/ta.py b/feature_extractors/ta.py index d9e1573..a4d2f88 100644 --- a/feature_extractors/ta.py +++ b/feature_extractors/ta.py @@ -2,6 +2,7 @@ import pandas_ta as ta import pandas as pd from feature_extractors.utils import get_close_low_high + def feature_EBSW(df: pd.DataFrame, period: int) -> pd.Series: close, low, high = get_close_low_high(df) diff --git a/feature_extractors/types.py b/feature_extractors/types.py index 6b645b0..d32e3f1 100644 --- a/feature_extractors/types.py +++ b/feature_extractors/types.py @@ -6,4 +6,4 @@ IsLogReturn = bool FeatureExtractor = Callable[[pd.DataFrame, Period], Union[pd.DataFrame, pd.Series]] Name = str FeatureExtractorConfig = tuple[Name, FeatureExtractor, list[Period]] -ScalerTypes = Literal['normalize', 'minmax', 'standardize', 'none'] +ScalerTypes = Literal["normalize", "minmax", "standardize", "none"] diff --git a/feature_extractors/utils.py b/feature_extractors/utils.py index 6206ace..368e071 100644 --- a/feature_extractors/utils.py +++ b/feature_extractors/utils.py @@ -2,10 +2,11 @@ import pandas as pd import numpy as np from scipy.stats import shapiro + def get_close_low_high(df: pd.DataFrame) -> tuple[pd.Series, pd.Series, pd.Series]: - close = df['close'] - low = df['low'] - high = df['high'] + close = df["close"] + low = df["low"] + high = df["high"] return close, low, high diff --git a/labeling/event_filters/cusum.py b/labeling/event_filters/cusum.py index 9349ddd..ef42e1f 100644 --- a/labeling/event_filters/cusum.py +++ b/labeling/event_filters/cusum.py @@ -1,5 +1,3 @@ - - from numpy import float32 from ..types import EventFilter from data_loader.types import ReturnSeries @@ -7,8 +5,8 @@ import pandas as pd from numba import njit from numba.typed import List -class CUSUMVolatilityEventFilter(EventFilter): +class CUSUMVolatilityEventFilter(EventFilter): def __init__(self, vol_period: int): self.vol_period = vol_period @@ -36,34 +34,37 @@ class CUSUMVolatilityEventFilter(EventFilter): return pd.DatetimeIndex(filtered_indices) -class CUSUMFixedEventFilter(EventFilter): +class CUSUMFixedEventFilter(EventFilter): def __init__(self, threshold: float): self.threshold = threshold def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex: diffed_returns = returns.diff() - int_indicies = _process(List(diffed_returns.to_list()), abs(returns.mean()) * self.threshold) + int_indicies = _process( + List(diffed_returns.to_list()), abs(returns.mean()) * self.threshold + ) return pd.DatetimeIndex([returns.index[i] for i in int_indicies]) + @njit def _process(diffed_returns: List, threshold: float32) -> List: - pos_threshold: float32 = 0.0 # type: ignore - neg_threshold: float32 = 0.0 # type: ignore + pos_threshold: float32 = 0.0 # type: ignore + neg_threshold: float32 = 0.0 # type: ignore filtered_indicies = List() for index in range(1, len(diffed_returns[1:])): - pos_threshold, neg_threshold = ( # type: ignore - max(0, pos_threshold + diffed_returns[index]), # type: ignore - min(0, neg_threshold + diffed_returns[index]), # type: ignore + pos_threshold, neg_threshold = ( # type: ignore + max(0, pos_threshold + diffed_returns[index]), # type: ignore + min(0, neg_threshold + diffed_returns[index]), # type: ignore ) if neg_threshold < -threshold: - neg_threshold = 0.0 # type: ignore + neg_threshold = 0.0 # type: ignore filtered_indicies.append(index) elif pos_threshold > threshold: - pos_threshold = 0.0 # type: ignore + pos_threshold = 0.0 # type: ignore filtered_indicies.append(index) - return filtered_indicies \ No newline at end of file + return filtered_indicies diff --git a/labeling/event_filters/nofilter.py b/labeling/event_filters/nofilter.py index 50ad3da..4ac7afd 100644 --- a/labeling/event_filters/nofilter.py +++ b/labeling/event_filters/nofilter.py @@ -2,8 +2,7 @@ from ..types import EventFilter from data_loader.types import ReturnSeries import pandas as pd -class NoEventFilter(EventFilter): +class NoEventFilter(EventFilter): def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex: return returns.index - diff --git a/labeling/eventfilters_map.py b/labeling/eventfilters_map.py index 48c1569..d1c66b9 100644 --- a/labeling/eventfilters_map.py +++ b/labeling/eventfilters_map.py @@ -2,7 +2,7 @@ from .event_filters.nofilter import NoEventFilter from .event_filters.cusum import CUSUMVolatilityEventFilter, CUSUMFixedEventFilter eventfilters_map = dict( - none = NoEventFilter(), - cusum_vol = CUSUMVolatilityEventFilter(vol_period = 20), - cusum_fixed = CUSUMFixedEventFilter(threshold = 500) -) \ No newline at end of file + none=NoEventFilter(), + cusum_vol=CUSUMVolatilityEventFilter(vol_period=20), + cusum_fixed=CUSUMFixedEventFilter(threshold=500), +) diff --git a/labeling/labellers/fixed_time_three_class_balanced.py b/labeling/labellers/fixed_time_three_class_balanced.py index 26db4d9..fb1616a 100644 --- a/labeling/labellers/fixed_time_three_class_balanced.py +++ b/labeling/labellers/fixed_time_three_class_balanced.py @@ -3,6 +3,7 @@ from ..types import EventLabeller, EventsDataFrame import pandas as pd from .utils import create_forward_returns + class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller): time_horizon: int @@ -10,7 +11,9 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller): def __init__(self, time_horizon: int): self.time_horizon = time_horizon - def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]: + def label_events( + self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries + ) -> tuple[EventsDataFrame, ForwardReturnSeries]: forward_returns = create_forward_returns(returns, self.time_horizon) cutoff_point = returns.index[-self.time_horizon] @@ -18,7 +21,7 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller): event_candidates = forward_returns[event_start_times] def get_bins_threeway(x): - bins = pd.qcut(event_candidates, 3, retbins=True, duplicates = 'drop')[1] + bins = pd.qcut(event_candidates, 3, retbins=True, duplicates="drop")[1] if len(bins) != 4: # if we don't have enough data for the quantiles, we'll need to add hard-coded values @@ -26,6 +29,7 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller): upper_bound = bins[-1] bins = [lower_bound] + [-0.02, 0.02] + [upper_bound] return bins + bins = get_bins_threeway(event_candidates) def map_class_threeway(current_value): @@ -37,12 +41,17 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller): return 0 else: return 1 - labels = event_candidates.map(map_class_threeway) - - return (pd.DataFrame({ - 'start': event_start_times, - 'end': event_start_times + pd.Timedelta(days=self.time_horizon), - 'label': labels, - 'returns': forward_returns[event_start_times] - }), forward_returns[event_start_times]) + labels = event_candidates.map(map_class_threeway) + + return ( + pd.DataFrame( + { + "start": event_start_times, + "end": event_start_times + pd.Timedelta(days=self.time_horizon), + "label": labels, + "returns": forward_returns[event_start_times], + } + ), + forward_returns[event_start_times], + ) diff --git a/labeling/labellers/fixed_time_three_class_imbalanced.py b/labeling/labellers/fixed_time_three_class_imbalanced.py index f56e373..5073be5 100644 --- a/labeling/labellers/fixed_time_three_class_imbalanced.py +++ b/labeling/labellers/fixed_time_three_class_imbalanced.py @@ -2,6 +2,7 @@ from ..types import EventLabeller, EventsDataFrame, ReturnSeries, ForwardReturnS import pandas as pd from .utils import create_forward_returns + class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller): time_horizon: int @@ -9,7 +10,9 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller): def __init__(self, time_horizon: int): self.time_horizon = time_horizon - def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]: + def label_events( + self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries + ) -> tuple[EventsDataFrame, ForwardReturnSeries]: forward_returns = create_forward_returns(returns, self.time_horizon) cutoff_point = returns.index[-self.time_horizon] @@ -17,7 +20,7 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller): event_candidates = forward_returns[event_start_times] def get_bins_threeway(x): - bins = pd.qcut(event_candidates, 4, retbins=True, duplicates = 'drop')[1] + bins = pd.qcut(event_candidates, 4, retbins=True, duplicates="drop")[1] if len(bins) != 5: # if we don't have enough data for the quantiles, we'll need to add hard-coded values @@ -25,6 +28,7 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller): upper_bound = bins[-1] bins = [lower_bound] + [-0.02, 0.0, 0.02] + [upper_bound] return bins + bins = get_bins_threeway(event_candidates) def map_class_threeway(current_value): @@ -36,13 +40,17 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller): return 0 else: return 1 + labels = event_candidates.map(map_class_threeway) - - return (pd.DataFrame({ - 'start': event_start_times, - 'end': event_start_times + pd.Timedelta(days=self.time_horizon), - 'label': labels, - 'returns': forward_returns[event_start_times] - }), forward_returns[event_start_times]) - + return ( + pd.DataFrame( + { + "start": event_start_times, + "end": event_start_times + pd.Timedelta(days=self.time_horizon), + "label": labels, + "returns": forward_returns[event_start_times], + } + ), + forward_returns[event_start_times], + ) diff --git a/labeling/labellers/fixed_time_two_class.py b/labeling/labellers/fixed_time_two_class.py index 2b24443..5665dc4 100644 --- a/labeling/labellers/fixed_time_two_class.py +++ b/labeling/labellers/fixed_time_two_class.py @@ -2,6 +2,7 @@ from ..types import EventLabeller, EventsDataFrame, ReturnSeries, ForwardReturnS import pandas as pd from .utils import create_forward_returns + class FixedTimeHorionTwoClassEventLabeller(EventLabeller): time_horizon: int @@ -9,7 +10,9 @@ class FixedTimeHorionTwoClassEventLabeller(EventLabeller): def __init__(self, time_horizon: int): self.time_horizon = time_horizon - def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]: + def label_events( + self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries + ) -> tuple[EventsDataFrame, ForwardReturnSeries]: forward_returns = create_forward_returns(returns, self.time_horizon) cutoff_point = returns.index[-self.time_horizon] @@ -18,14 +21,17 @@ class FixedTimeHorionTwoClassEventLabeller(EventLabeller): def get_class_binary(x: float) -> int: return -1 if x <= 0.0 else 1 + labels = event_candidates.map(get_class_binary) - - return (pd.DataFrame({ - 'start': event_start_times, - 'end': event_start_times + pd.Timedelta(days=self.time_horizon), - 'label': labels, - 'returns': forward_returns[event_start_times] - }), forward_returns[event_start_times]) - - + return ( + pd.DataFrame( + { + "start": event_start_times, + "end": event_start_times + pd.Timedelta(days=self.time_horizon), + "label": labels, + "returns": forward_returns[event_start_times], + } + ), + forward_returns[event_start_times], + ) diff --git a/labeling/labellers/utils.py b/labeling/labellers/utils.py index 6e312f2..867bbe8 100644 --- a/labeling/labellers/utils.py +++ b/labeling/labellers/utils.py @@ -1,9 +1,10 @@ import pandas as pd from data_loader.types import ForwardReturnSeries + def create_forward_returns(series: pd.Series, period: int) -> ForwardReturnSeries: assert period > 0 indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=period) - + forward_returns = series.rolling(window=indexer).sum() - return forward_returns \ No newline at end of file + return forward_returns diff --git a/labeling/labellers_map.py b/labeling/labellers_map.py index b3176ad..22629dc 100644 --- a/labeling/labellers_map.py +++ b/labeling/labellers_map.py @@ -1,9 +1,13 @@ -from .labellers.fixed_time_three_class_balanced import FixedTimeHorionThreeClassBalancedEventLabeller -from .labellers.fixed_time_three_class_imbalanced import FixedTimeHorionThreeClassImbalancedEventLabeller +from .labellers.fixed_time_three_class_balanced import ( + FixedTimeHorionThreeClassBalancedEventLabeller, +) +from .labellers.fixed_time_three_class_imbalanced import ( + FixedTimeHorionThreeClassImbalancedEventLabeller, +) from .labellers.fixed_time_two_class import FixedTimeHorionTwoClassEventLabeller labellers_map = dict( - two_class = FixedTimeHorionTwoClassEventLabeller, - three_class_balanced = FixedTimeHorionThreeClassBalancedEventLabeller, - three_class_imbalanced = FixedTimeHorionThreeClassImbalancedEventLabeller -) \ No newline at end of file + two_class=FixedTimeHorionTwoClassEventLabeller, + three_class_balanced=FixedTimeHorionThreeClassBalancedEventLabeller, + three_class_imbalanced=FixedTimeHorionThreeClassImbalancedEventLabeller, +) diff --git a/labeling/process.py b/labeling/process.py index 9dd7a32..26f295f 100644 --- a/labeling/process.py +++ b/labeling/process.py @@ -1,19 +1,25 @@ from .types import EventFilter, EventLabeller, EventsDataFrame from data_loader.types import ForwardReturnSeries, XDataFrame, ReturnSeries, ySeries + def label_data( - event_filter: EventFilter, - event_labeller: EventLabeller, - X: XDataFrame, - returns: ReturnSeries) -> tuple[EventsDataFrame, XDataFrame, ySeries, ForwardReturnSeries]: + event_filter: EventFilter, + event_labeller: EventLabeller, + X: XDataFrame, + returns: ReturnSeries, +) -> tuple[EventsDataFrame, XDataFrame, ySeries, ForwardReturnSeries]: - event_start_times = event_filter.get_event_start_times(returns) - print("| Filtered out ", (1 - (len(event_start_times) / len(returns))) * 100, "% of timestamps" ) + event_start_times = event_filter.get_event_start_times(returns) + print( + "| Filtered out ", + (1 - (len(event_start_times) / len(returns))) * 100, + "% of timestamps", + ) - events, forward_returns = event_labeller.label_events(event_start_times, returns) + events, forward_returns = event_labeller.label_events(event_start_times, returns) - X = X.filter(items = events.index, axis = 0) - y = events['label'] - forward_returns = events['returns'] + X = X.filter(items=events.index, axis=0) + y = events["label"] + forward_returns = events["returns"] - return events, X, y, forward_returns \ No newline at end of file + return events, X, y, forward_returns diff --git a/labeling/types.py b/labeling/types.py index 12eedeb..f56bbd6 100644 --- a/labeling/types.py +++ b/labeling/types.py @@ -4,8 +4,8 @@ import pandas as pd import pandera as pa from pandera.typing import DataFrame, Series -class EventFilter(ABC): +class EventFilter(ABC): @abstractmethod def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex: raise NotImplementedError @@ -17,12 +17,13 @@ class EventSchema(pa.SchemaModel): label: Series[int] returns: Series[float] + EventsDataFrame = DataFrame[EventSchema] class EventLabeller(ABC): - @abstractmethod - def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]: + def label_events( + self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries + ) -> tuple[EventsDataFrame, ForwardReturnSeries]: raise NotImplementedError - diff --git a/models/base.py b/models/base.py index 7ef71aa..ffca8bd 100644 --- a/models/base.py +++ b/models/base.py @@ -3,12 +3,13 @@ from typing import Literal, Optional, Union from abc import ABC, abstractmethod import numpy as np + class Model(ABC): name: str = "" data_transformation: Literal["transformed", "original"] only_column: Optional[str] - predict_window_size: Literal['single_timestamp', 'window_size'] + predict_window_size: Literal["single_timestamp", "window_size"] @abstractmethod def fit(self, X: np.ndarray, y: np.ndarray) -> None: @@ -21,4 +22,3 @@ class Model(ABC): @abstractmethod def predict_proba(self, X: np.ndarray) -> np.ndarray: raise NotImplementedError - diff --git a/models/model_map.py b/models/model_map.py index afa6708..e8bb00c 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -3,49 +3,81 @@ from sklearnex.ensemble import RandomForestClassifier from sklearnex.ensemble import RandomForestRegressor from .base import Model -default_feature_selector_classification = SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)) +default_feature_selector_classification = SKLearnModel( + RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1) +) + def get_model(model_name: str) -> Model: - def set_name(model: Model) -> Model: model.name = model_name return model - if model_name == 'LogisticRegression_two_class': + if model_name == "LogisticRegression_two_class": from sklearn.linear_model import LogisticRegression - return set_name(SKLearnModel(LogisticRegression(C=10, random_state=1, solver='liblinear', max_iter=1000))) - elif model_name == 'LogisticRegression_three_class': + + return set_name( + SKLearnModel( + LogisticRegression( + C=10, random_state=1, solver="liblinear", max_iter=1000 + ) + ) + ) + elif model_name == "LogisticRegression_three_class": from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX - return set_name(SKLearnModel(LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1))) - elif model_name == 'LDA': + + return set_name( + SKLearnModel( + LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1) + ) + ) + elif model_name == "LDA": from sklearn.discriminant_analysis import LinearDiscriminantAnalysis + return set_name(SKLearnModel(LinearDiscriminantAnalysis())) - elif model_name == 'KNN': + elif model_name == "KNN": from sklearn.neighbors import KNeighborsClassifier + return set_name(SKLearnModel(KNeighborsClassifier())) - elif model_name == 'CART': + elif model_name == "CART": from sklearn.tree import DecisionTreeClassifier - return set_name(SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1))) - elif model_name == 'NB': + + return set_name( + SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)) + ) + elif model_name == "NB": from sklearn.naive_bayes import GaussianNB + return set_name(SKLearnModel(GaussianNB())) - elif model_name == 'AB': + elif model_name == "AB": from sklearn.ensemble import AdaBoostClassifier + return set_name(SKLearnModel(AdaBoostClassifier(n_estimators=15))) - elif model_name == 'RFC': - return set_name(SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1))) - elif model_name == 'SVC': + elif model_name == "RFC": + return set_name( + SKLearnModel( + RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1) + ) + ) + elif model_name == "SVC": from sklearn.svm import SVC - return set_name(SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True, random_state=1))) + + return set_name( + SKLearnModel(SVC(kernel="rbf", C=1e3, probability=True, random_state=1)) + ) # elif model_name == 'XGB_two_class': # from xgboost import XGBClassifier # from models.xgboost import XGBoostModel # return set_name(XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss'))) - elif model_name == 'LGBM': + elif model_name == "LGBM": from lightgbm import LGBMClassifier - return set_name(SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1))) - elif model_name == 'StaticMom': + + return set_name( + SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)) + ) + elif model_name == "StaticMom": from models.momentum import StaticMomentumModel + return set_name(StaticMomentumModel(allow_short=True)) else: - raise Exception(f'Model {model_name} not found') \ No newline at end of file + raise Exception(f"Model {model_name} not found") diff --git a/models/momentum.py b/models/momentum.py index 5a77a65..a58a304 100644 --- a/models/momentum.py +++ b/models/momentum.py @@ -3,14 +3,15 @@ import numpy as np from .base import Model from sklearn.base import BaseEstimator, ClassifierMixin -class StaticMomentumModel(BaseEstimator, ClassifierMixin, Model): - ''' - Model that uses only one feature: momentum. It's positive if momentum is greater than 0, otherwise it's negative. - ''' - data_transformation = 'original' - only_column = 'mom' - predict_window_size = 'single_timestamp' +class StaticMomentumModel(BaseEstimator, ClassifierMixin, Model): + """ + Model that uses only one feature: momentum. It's positive if momentum is greater than 0, otherwise it's negative. + """ + + data_transformation = "original" + only_column = "mom" + predict_window_size = "single_timestamp" def __init__(self, allow_short: bool) -> None: super().__init__() @@ -24,6 +25,6 @@ class StaticMomentumModel(BaseEstimator, ClassifierMixin, Model): negative_class = -1.0 if self.allow_short == True else 0.0 prediction = 1.0 if X[-1][0] > 0 else negative_class return np.array(prediction) - + def predict_proba(self, X) -> np.ndarray: return np.array([]) diff --git a/models/sklearn.py b/models/sklearn.py index a43df5a..667d0f9 100644 --- a/models/sklearn.py +++ b/models/sklearn.py @@ -4,20 +4,16 @@ from .base import Model import numpy as np - def SKLearnModel(instance) -> Model: - instance.data_transformation = 'transformed' + instance.data_transformation = "transformed" instance.only_column = None - instance.predict_window_size = 'single_timestamp' + instance.predict_window_size = "single_timestamp" instance.name = instance.__class__.__name__ return instance - # def predict(self, X) -> tuple[float, np.ndarray]: # pred = self.model.predict(X).item() # probability = self.model.predict_proba(X).squeeze() # return (pred, probability) - - \ No newline at end of file diff --git a/models/xgboost.py b/models/xgboost.py index eee592e..bbc1d66 100644 --- a/models/xgboost.py +++ b/models/xgboost.py @@ -9,7 +9,7 @@ # data_transformation = 'transformed' # only_column = None # predict_window_size = 'single_timestamp' - + # def fit(self, X: np.ndarray, y: np.ndarray) -> None: # def map_to_xgb(y): return np.array([1 if i == 1 else 0 for i in y]) # self.fit(X, map_to_xgb(y)) @@ -19,4 +19,3 @@ # probability = self.predict_proba(X).squeeze() # def map_from_xgb(y): return 1 if y == 1 else -1 # return (map_from_xgb(pred), probability) - diff --git a/mycodemod/replace_functions.py b/mycodemod/replace_functions.py index 377d454..8c2f6f3 100644 --- a/mycodemod/replace_functions.py +++ b/mycodemod/replace_functions.py @@ -12,23 +12,23 @@ class ReplaceFunctionCommand(VisitorBasedCodemodCommand): # Add a description so that future codemodders can see what this does. DESCRIPTION: str = "Replaces the body of a function with pass." - def __init__(self, context: CodemodContext) -> None: # Initialize the base class with context, and save our args. Remember, the # "dest" for each argument we added above must match a parameter name in # this init. super().__init__(context) - - def leave_FunctionDef(self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef) -> cst.FunctionDef: + def leave_FunctionDef( + self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef + ) -> cst.FunctionDef: functions_docstring = updated_node.get_docstring() docstring_should_be = '"""No docstring here yet."""' if functions_docstring is not None: docstring_should_be = '"""\n{}\n\n"""'.format(functions_docstring) replace_function = cst.FunctionDef( - name=updated_node.name, - params=updated_node.params,#cst.Parameters(), + name=updated_node.name, + params=updated_node.params, # cst.Parameters(), body=cst.IndentedBlock( body=[ cst.SimpleStatementLine( @@ -44,50 +44,47 @@ class ReplaceFunctionCommand(VisitorBasedCodemodCommand): ], leading_lines=[], trailing_whitespace=cst.TrailingWhitespace( - whitespace=cst.SimpleWhitespace( - value='', + whitespace=cst.SimpleWhitespace( + value="", ), comment=None, newline=cst.Newline( value=None, + ), ), - ), ), cst.SimpleStatementLine( - body=[ - cst.Pass(), - ], - ), - ] - ) + body=[ + cst.Pass(), + ], + ), + ] + ), ) - + return replace_function - - - - def leave_ClassDef(self, original_node: cst.ClassDef, updated_node: cst.ClassDef) -> cst.ClassDef: + def leave_ClassDef( + self, original_node: cst.ClassDef, updated_node: cst.ClassDef + ) -> cst.ClassDef: new_body = [] for body_item in updated_node.body.body: if type(body_item) is cst.FunctionDef: new_body.append(self.leave_FunctionDef(body_item, body_item)) - - return updated_node.with_changes( - body=cst.IndentedBlock(new_body) - ) - - def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module: - new_module_body=[] + + return updated_node.with_changes(body=cst.IndentedBlock(new_body)) + + def leave_Module( + self, original_node: cst.Module, updated_node: cst.Module + ) -> cst.Module: + new_module_body = [] for node in original_node.body: if type(node) is cst.FunctionDef: - new_module_body.append(self.leave_FunctionDef(node,node)) - - if type(node) is cst.ClassDef: - new_module_body.append(self.leave_ClassDef(node,node)) + new_module_body.append(self.leave_FunctionDef(node, node)) + + if type(node) is cst.ClassDef: + new_module_body.append(self.leave_ClassDef(node, node)) + + replace_function = cst.Module(body=new_module_body) - replace_function = cst.Module( - body= new_module_body - ) - return replace_function diff --git a/reporting/reporting.py b/reporting/reporting.py index 4ff31d9..3c889b1 100644 --- a/reporting/reporting.py +++ b/reporting/reporting.py @@ -1,29 +1,48 @@ from reporting.wandb import send_report_to_wandb import pandas as pd from utils.helpers import weighted_average -from config.types import Config +from config.types import Config from training.types import WeightsSeries, Stats -def report_results(directional_stats: Stats, output_stats: Stats, output_weights: WeightsSeries, config: Config, wandb, sweep: bool): + +def report_results( + directional_stats: Stats, + output_stats: Stats, + output_weights: WeightsSeries, + config: Config, + wandb, + sweep: bool, +): # Only send the results of the final model to wandb send_report_to_wandb(output_stats, wandb) - pd.Series(output_stats).to_csv('output/results.csv') + pd.Series(output_stats).to_csv("output/results.csv") - output_weights.rename(config.target_asset[1]).to_csv('output/predictions.csv') + output_weights.rename(config.target_asset[1]).to_csv("output/predictions.csv") print("\n--------\n") - print("Benchmark buy-and-hold sharpe: ", output_stats['benchmark_sharpe']) + print("Benchmark buy-and-hold sharpe: ", output_stats["benchmark_sharpe"]) - print("Level-1: Number of samples evaluated: ", directional_stats['no_of_samples']) - print("Mean Sharpe ratio for Level-1 models: ", round(directional_stats['sharpe'], 3)) - print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(directional_stats['prob_sharpe'], 3)) + print("Level-1: Number of samples evaluated: ", directional_stats["no_of_samples"]) + print( + "Mean Sharpe ratio for Level-1 models: ", round(directional_stats["sharpe"], 3) + ) + print( + "Mean Probabilistic Sharpe ratio for Level-1 models: ", + round(directional_stats["prob_sharpe"], 3), + ) - print("Level-2 (Ensemble): Number of samples evaluated: ", output_stats['no_of_samples']) - print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", output_stats['sharpe']) - print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", output_stats['prob_sharpe']) + print( + "Level-2 (Ensemble): Number of samples evaluated: ", + output_stats["no_of_samples"], + ) + print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", output_stats["sharpe"]) + print( + "Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", + output_stats["prob_sharpe"], + ) if sweep: if wandb.run is not None: - wandb.finish() \ No newline at end of file + wandb.finish() diff --git a/reporting/saving.py b/reporting/saving.py index a561977..ce414ab 100644 --- a/reporting/saving.py +++ b/reporting/saving.py @@ -6,33 +6,35 @@ import os import warnings from training.types import PipelineOutcome + def save_models(pipeline_outcome: PipelineOutcome, config: Config) -> None: dict_for_pickle = dict() - dict_for_pickle['config'] = config - dict_for_pickle['pipeline_outcome'] = pipeline_outcome - + dict_for_pickle["config"] = config + dict_for_pickle["pipeline_outcome"] = pipeline_outcome + date_string = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M") - - if not os.path.exists('output/models'): + + if not os.path.exists("output/models"): warnings.warn("No folder exists, creating one.") - os.makedirs('output/models') - - pickle.dump( dict_for_pickle, open( "output/models/{}.p".format(date_string), "wb" ) ) + os.makedirs("output/models") + + pickle.dump(dict_for_pickle, open("output/models/{}.p".format(date_string), "wb")) def load_models(file_name: Optional[str]) -> tuple[PipelineOutcome, Config]: - if file_name is None: - warnings.warn("No file name provided, will load latest models and configurations.") - files_in_directory:list = os.listdir('output/models') - + if file_name is None: + warnings.warn( + "No file name provided, will load latest models and configurations." + ) + files_in_directory: list = os.listdir("output/models") + assert len(files_in_directory) > 0, "No models found in output/models." file_name = sorted(files_in_directory)[-1] - - packacked_dict = pickle.load( open( "output/models/{}".format(file_name), "rb" ) ) - - config = packacked_dict.pop("config", None) - pipeline_outcome = packacked_dict.pop("pipeline_outcome", None) - - return pipeline_outcome, config + packacked_dict = pickle.load(open("output/models/{}".format(file_name), "rb")) + + config = packacked_dict.pop("config", None) + pipeline_outcome = packacked_dict.pop("pipeline_outcome", None) + + return pipeline_outcome, config diff --git a/reporting/wandb.py b/reporting/wandb.py index 2a848db..8465461 100644 --- a/reporting/wandb.py +++ b/reporting/wandb.py @@ -4,41 +4,49 @@ from typing import Optional from utils.helpers import weighted_average from training.types import Stats -def launch_wandb(project_name:str, default_config: RawConfig, sweep:bool=False) -> Optional[object]: + +def launch_wandb( + project_name: str, default_config: RawConfig, sweep: bool = False +) -> Optional[object]: from wandb_setup import get_wandb - wandb = get_wandb() + + wandb = get_wandb() if wandb is None: - raise Exception("Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)") + raise Exception( + "Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)" + ) elif sweep: - wandb.init(project=project_name, config = vars(default_config)) + wandb.init(project=project_name, config=vars(default_config)) return wandb else: - wandb.init(project=project_name, config = vars(default_config), reinit=True) + wandb.init(project=project_name, config=vars(default_config), reinit=True) return wandb - - -def override_config_with_wandb_values(wandb: Optional[object], raw_config: RawConfig) -> RawConfig: - if wandb is None: return raw_config + + +def override_config_with_wandb_values( + wandb: Optional[object], raw_config: RawConfig +) -> RawConfig: + if wandb is None: + return raw_config wandb_config: dict = wandb.config - + config_dict = vars(raw_config) for k in config_dict: config_dict[k] = wandb_config[k] return RawConfig(**config_dict) -def send_report_to_wandb(stats: Stats, wandb:Optional[object]): - if wandb is None: return + +def send_report_to_wandb(stats: Stats, wandb: Optional[object]): + if wandb is None: + return run = wandb.run run.save() for key, value in stats.items(): - run.log({ key: value }) + run.log({key: value}) run.finish() - - - \ No newline at end of file diff --git a/run_clear_cache.py b/run_clear_cache.py index 0b87e99..2818435 100644 --- a/run_clear_cache.py +++ b/run_clear_cache.py @@ -1,5 +1,6 @@ from diskcache import Cache + cache_1 = Cache(".cachedir/feature_selection") cache_2 = Cache(".cachedir/data") cache_1.clear() -cache_2.clear() \ No newline at end of file +cache_2.clear() diff --git a/run_evaluate_robustness.py b/run_evaluate_robustness.py index e511e8e..cbf6d88 100644 --- a/run_evaluate_robustness.py +++ b/run_evaluate_robustness.py @@ -5,15 +5,22 @@ import pandas as pd all_results = [] all_predictions = [] for index in range(6): - _, _, _, _, results_1, predictions_1, _ = run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, config= get_default_ensemble_config()) + _, _, _, _, results_1, predictions_1, _ = run_pipeline( + project_name="price-prediction", + with_wandb=False, + sweep=False, + config=get_default_ensemble_config(), + ) all_results.append(results_1) all_predictions.append(predictions_1) correlations = pd.Series() -for asset_name in [c for c in all_predictions[0].columns if 'ensemble' in c]: +for asset_name in [c for c in all_predictions[0].columns if "ensemble" in c]: - predictions_for_asset = pd.concat([preds[asset_name] for preds in all_predictions], axis=1) + predictions_for_asset = pd.concat( + [preds[asset_name] for preds in all_predictions], axis=1 + ) correlations[asset_name] = predictions_for_asset.corr().mean()[0] print("Correlation for asset ", asset_name, ": ", correlations[asset_name]) diff --git a/run_fetch_data_5min.py b/run_fetch_data_5min.py index 81ce663..4636584 100644 --- a/run_fetch_data_5min.py +++ b/run_fetch_data_5min.py @@ -7,31 +7,35 @@ from utils.resample import resample_ohlc base_url = "https://www.cryptodatadownload.com/cdd/" exchange_name = "Bitfinex" -period = "minute" # 1h +period = "minute" # 1h files_to_download = [ - exchange_name + '_TRXUSD_' + period + '.csv', - exchange_name + '_ETHUSD_' + period + '.csv', - exchange_name + '_XLMUSD_' + period + '.csv', - exchange_name + '_XMRUSD_' + period + '.csv', - exchange_name + '_LTCUSD_' + period + '.csv', - exchange_name + '_DASHUSD_' + period + '.csv', - exchange_name + '_BTCUSD_' + period + '.csv', - exchange_name + '_ETCUSD_' + period + '.csv', - ] + exchange_name + "_TRXUSD_" + period + ".csv", + exchange_name + "_ETHUSD_" + period + ".csv", + exchange_name + "_XLMUSD_" + period + ".csv", + exchange_name + "_XMRUSD_" + period + ".csv", + exchange_name + "_LTCUSD_" + period + ".csv", + exchange_name + "_DASHUSD_" + period + ".csv", + exchange_name + "_BTCUSD_" + period + ".csv", + exchange_name + "_ETCUSD_" + period + ".csv", +] for file in tqdm(files_to_download): data_location = base_url + file ssl._create_default_https_context = ssl._create_unverified_context - data = pd.read_csv(data_location, skiprows=1, index_col=1, parse_dates=True).drop(columns=['unix']) - volume_column_to_delete = [c for c in data.columns if c.startswith('Volume') and 'USD' not in c] - data.drop(volume_column_to_delete + ['symbol'], axis=1, inplace=True) - data.rename({'Volume USD': 'volume'}, axis=1, inplace=True) - data.index.rename('time', inplace=True) + data = pd.read_csv(data_location, skiprows=1, index_col=1, parse_dates=True).drop( + columns=["unix"] + ) + volume_column_to_delete = [ + c for c in data.columns if c.startswith("Volume") and "USD" not in c + ] + data.drop(volume_column_to_delete + ["symbol"], axis=1, inplace=True) + data.rename({"Volume USD": "volume"}, axis=1, inplace=True) + data.index.rename("time", inplace=True) data.sort_index(inplace=True) data = deduplicate_indexes(data) data.index = pd.to_datetime(data.index) - data = data.resample('1Min').ffill() - data = resample_ohlc(data, '5Min') - target_file = file.split('_')[1].replace('USD', '') + '_USD' + data = data.resample("1Min").ffill() + data = resample_ohlc(data, "5Min") + target_file = file.split("_")[1].replace("USD", "") + "_USD" data.to_csv(f"data/5min_crypto/{target_file}.csv") diff --git a/run_fetch_data_daily.py b/run_fetch_data_daily.py index 4fa95d2..10af6d6 100644 --- a/run_fetch_data_daily.py +++ b/run_fetch_data_daily.py @@ -3,7 +3,24 @@ import pandas as pd from data_loader.get_prices import get_crypto_price_crypto_compare, get_stock_price_av #%% -crypto_tickers = ["BTC", "ETH", "BNB", "ADA", "SOL", "XRP", "DOT", "LTC", "UNI", "TRX", "XLM", "BCH", "FIL", "ETC", "THETA", "XTZ"] +crypto_tickers = [ + "BTC", + "ETH", + "BNB", + "ADA", + "SOL", + "XRP", + "DOT", + "LTC", + "UNI", + "TRX", + "XLM", + "BCH", + "FIL", + "ETC", + "THETA", + "XTZ", +] etf_tickers = ["GLD", "IEF", "TLT", "SPY", "QQQ"] crypto_path = "data/daily_crypto" etf_path = "data/daily_etf" @@ -20,5 +37,3 @@ for src_ticker in crypto_tickers: df = get_crypto_price_crypto_compare(src_ticker, "USD", 1500) df.to_csv(f"{crypto_path}/{src_ticker}_USD.csv", index=True) - - diff --git a/run_fetch_data_glassnode.py b/run_fetch_data_glassnode.py index 953d487..d7decae 100644 --- a/run_fetch_data_glassnode.py +++ b/run_fetch_data_glassnode.py @@ -3,60 +3,63 @@ import pandas as pd from tqdm import tqdm from utils.glassnode import GlassnodeClient, Indicators, Mining -path = 'data/daily_glassnode/' -client = GlassnodeClient(asset='BTC', since='2014-01-01', until='2021-12-17') +path = "data/daily_glassnode/" +client = GlassnodeClient(asset="BTC", since="2014-01-01", until="2021-12-17") print("Client initiated") indicator_client = Indicators(client) -indicator_names = ['rhodl_ratio', - 'cvdd', - 'difficulty_ribbon_compression', - 'nvt_ratio', - 'nvt_signal', - 'velocity', - 'supply_adjusted_cdd', - 'binary_cdd', - 'supply_adjusted_dormancy', - 'puell_multiple', - 'asopr', - 'reserve_risk', - 'sopr', - 'cdd', - 'asol', - 'msol', - 'dormancy', - 'liveliness', - 'relative_unrealized_profit', - 'relative_unrealized_loss', - 'nupl', +indicator_names = [ + "rhodl_ratio", + "cvdd", + "difficulty_ribbon_compression", + "nvt_ratio", + "nvt_signal", + "velocity", + "supply_adjusted_cdd", + "binary_cdd", + "supply_adjusted_dormancy", + "puell_multiple", + "asopr", + "reserve_risk", + "sopr", + "cdd", + "asol", + "msol", + "dormancy", + "liveliness", + "relative_unrealized_profit", + "relative_unrealized_loss", + "nupl", # 'sth_nupl', # 'lth_nupl', - 'ssr', - 'bvin', + "ssr", + "bvin", ] + def process_df(df: pd.DataFrame) -> pd.DataFrame: - df.index.rename('time', inplace=True) + df.index.rename("time", inplace=True) if len(df.columns) != 1: - df = df[['v']] - df.rename(columns={df.columns[0]: 'close'}, inplace=True) + df = df[["v"]] + df.rename(columns={df.columns[0]: "close"}, inplace=True) df.sort_index(inplace=True) return df + for name in tqdm(indicator_names): method_to_call = getattr(indicator_client, name) df = method_to_call() df = process_df(df) - df.to_csv(path + name + '.csv') + df.to_csv(path + name + ".csv") # Mining data -mining_names = ['hash_rate'] +mining_names = ["hash_rate"] mining_client = Mining(client) for name in tqdm(mining_names): method_to_call = getattr(mining_client, name) df = method_to_call() df = process_df(df) - df.to_csv(path + name + '.csv') \ No newline at end of file + df.to_csv(path + name + ".csv") diff --git a/run_inference.py b/run_inference.py index b1dedd6..f326e7d 100644 --- a/run_inference.py +++ b/run_inference.py @@ -4,7 +4,11 @@ from reporting.saving import load_models from run_pipeline import run_pipeline from config.types import Config, RawConfig -from config.presets import get_dev_config, get_default_ensemble_config, get_lightweight_ensemble_config +from config.presets import ( + get_dev_config, + get_default_ensemble_config, + get_lightweight_ensemble_config, +) from labeling.process import label_data import pandas as pd @@ -12,44 +16,73 @@ from training.directional_training import train_directional_model from training.bet_sizing import bet_sizing_with_meta_model from training.types import PipelineOutcome -def run_inference(preload_models:bool, fallback_raw_config: RawConfig): + +def run_inference(preload_models: bool, fallback_raw_config: RawConfig): if preload_models: pipeline_outcome, config = load_models(None) else: - pipeline_outcome, config = run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, raw_config=fallback_raw_config) - - config.mode = 'inference' + pipeline_outcome, config = run_pipeline( + project_name="price-prediction", + with_wandb=False, + sweep=False, + raw_config=fallback_raw_config, + ) + + config.mode = "inference" __inference(config, pipeline_outcome) def __inference(config: Config, pipeline_outcome: PipelineOutcome): - + # 1. Load data, check for validity and process data X, returns = load_data( - assets = config.assets, - other_assets = config.other_assets, - exogenous_data = config.exogenous_data, - target_asset = config.target_asset, - load_non_target_asset = config.load_non_target_asset, - own_features = config.own_features, - other_features = config.other_features, - exogenous_features = config.exogenous_features, + assets=config.assets, + other_assets=config.other_assets, + exogenous_data=config.exogenous_data, + target_asset=config.target_asset, + load_non_target_asset=config.load_non_target_asset, + own_features=config.own_features, + other_features=config.other_features, + exogenous_features=config.exogenous_features, ) - assert check_data(X, config) == True, "Data is not valid. Cancelling Inference." + assert check_data(X, config) == True, "Data is not valid. Cancelling Inference." # 2. Filter for significant events when we want to trade, and label data - events, X, y, forward_returns = label_data(config.event_filter, config.labeling, X, returns) + events, X, y, forward_returns = label_data( + config.event_filter, config.labeling, X, returns + ) inference_from: pd.Timestamp = X.index[len(X.index) - 1] # 3. Train directional models - directional_training_outcome = train_directional_model(X, y, forward_returns, config, config.directional_model, from_index = inference_from, preloaded_training_step = pipeline_outcome.directional_training) - + directional_training_outcome = train_directional_model( + X, + y, + forward_returns, + config, + config.directional_model, + from_index=inference_from, + preloaded_training_step=pipeline_outcome.directional_training, + ) + # 4. Run bet sizing on primary model's output - bet_sizing_outcome = bet_sizing_with_meta_model(X, directional_training_outcome.training.predictions, y, forward_returns, config.meta_model, config, 'meta', from_index = inference_from, transformations_over_time = pipeline_outcome.bet_sizing.meta_transformations, preloaded_models = pipeline_outcome.bet_sizing.meta_training.model_over_time) + bet_sizing_outcome = bet_sizing_with_meta_model( + X, + directional_training_outcome.training.predictions, + y, + forward_returns, + config.meta_model, + config, + "meta", + from_index=inference_from, + transformations_over_time=pipeline_outcome.bet_sizing.meta_transformations, + preloaded_models=pipeline_outcome.bet_sizing.meta_training.model_over_time, + ) return PipelineOutcome(directional_training_outcome, bet_sizing_outcome) -if __name__ == '__main__': - run_inference(preload_models=True, fallback_raw_config=get_lightweight_ensemble_config()) \ No newline at end of file +if __name__ == "__main__": + run_inference( + preload_models=True, fallback_raw_config=get_lightweight_ensemble_config() + ) diff --git a/run_model_dev.py b/run_model_dev.py index 8d0a9d0..ea73ad1 100644 --- a/run_model_dev.py +++ b/run_model_dev.py @@ -2,4 +2,9 @@ from run_pipeline import run_pipeline from config.presets import get_dev_config -run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, raw_config=get_dev_config()) +run_pipeline( + project_name="price-prediction", + with_wandb=False, + sweep=False, + raw_config=get_dev_config(), +) diff --git a/run_pipeline.py b/run_pipeline.py index fe1059d..dec6769 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -18,54 +18,94 @@ from training.bet_sizing import bet_sizing_with_meta_model from training.types import PipelineOutcome import ray + ray.init() -def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, raw_config: RawConfig) -> tuple[PipelineOutcome, Config]: +def run_pipeline( + project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig +) -> tuple[PipelineOutcome, Config]: wandb, config = __setup_config(project_name, with_wandb, sweep, raw_config) - pipeline_outcome = __run_training(config) - report_results(pipeline_outcome.directional_training.training.stats, pipeline_outcome.get_output_stats(), pipeline_outcome.get_output_weights(), config, wandb, sweep) + pipeline_outcome = __run_training(config) + report_results( + pipeline_outcome.directional_training.training.stats, + pipeline_outcome.get_output_stats(), + pipeline_outcome.get_output_weights(), + config, + wandb, + sweep, + ) save_models(pipeline_outcome, config) return pipeline_outcome, config - -def __setup_config(project_name:str, with_wandb: bool, sweep: bool, raw_config: RawConfig) -> tuple[Optional[object], Config]: + +def __setup_config( + project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig +) -> tuple[Optional[object], Config]: wandb = None - if with_wandb: - wandb = launch_wandb(project_name=project_name, default_config=raw_config, sweep=sweep) + if with_wandb: + wandb = launch_wandb( + project_name=project_name, default_config=raw_config, sweep=sweep + ) raw_config = override_config_with_wandb_values(wandb, raw_config) config = preprocess_config(raw_config) return wandb, config - def __run_training(config: Config) -> PipelineOutcome: - + print("---> Load data, check for validity") X, returns = load_data( - assets = config.assets, - other_assets = config.other_assets, - exogenous_data = config.exogenous_data, - target_asset = config.target_asset, - load_non_target_asset = config.load_non_target_asset, - own_features = config.own_features, - other_features = config.other_features, - exogenous_features = config.exogenous_features, + assets=config.assets, + other_assets=config.other_assets, + exogenous_data=config.exogenous_data, + target_asset=config.target_asset, + load_non_target_asset=config.load_non_target_asset, + own_features=config.own_features, + other_features=config.other_features, + exogenous_features=config.exogenous_features, ) - assert check_data(X, config) == True, "Data is not valid." + assert check_data(X, config) == True, "Data is not valid." print("---> Filter for significant events when we want to trade, and label data") - events, X, y, forward_returns = label_data(config.event_filter, config.labeling, X, returns) + events, X, y, forward_returns = label_data( + config.event_filter, config.labeling, X, returns + ) print("---> Train directional models") - directional_training_outcome = train_directional_model(X, y, forward_returns, config, config.directional_model, from_index = None, preloaded_training_step = None) - + directional_training_outcome = train_directional_model( + X, + y, + forward_returns, + config, + config.directional_model, + from_index=None, + preloaded_training_step=None, + ) + print("---> Run bet sizing on directional model's output") - bet_sizing_outcomes = bet_sizing_with_meta_model(X, directional_training_outcome.training.predictions, y, forward_returns, config.meta_model, config, 'meta', None, None, None) + bet_sizing_outcomes = bet_sizing_with_meta_model( + X, + directional_training_outcome.training.predictions, + y, + forward_returns, + config.meta_model, + config, + "meta", + None, + None, + None, + ) return PipelineOutcome(directional_training_outcome, bet_sizing_outcomes) - -if __name__ == '__main__': - run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, raw_config=get_lightweight_ensemble_config()) \ No newline at end of file + + +if __name__ == "__main__": + run_pipeline( + project_name="price-prediction", + with_wandb=False, + sweep=False, + raw_config=get_lightweight_ensemble_config(), + ) diff --git a/run_portfolio_reporting.py b/run_portfolio_reporting.py index 1567472..d0a9181 100644 --- a/run_portfolio_reporting.py +++ b/run_portfolio_reporting.py @@ -11,10 +11,14 @@ import quantstats as qs from alphalens.utils import get_clean_factor_and_forward_returns -def fixed_weight(row: pd.Series, availability_row: pd.Series, allow_short: bool) -> pd.Series: + +def fixed_weight( + row: pd.Series, availability_row: pd.Series, allow_short: bool +) -> pd.Series: no_of_assets_available = availability_row.sum() unit = 1 / no_of_assets_available if allow_short: + def determine_pos(x): if x == 0.0 or np.isnan(x): return 0 @@ -22,18 +26,21 @@ def fixed_weight(row: pd.Series, availability_row: pd.Series, allow_short: bool) return 1 else: return -1 + row = row.apply(determine_pos) else: row = row.apply(lambda x: 1 if x > 0.0 else 0) return row * unit + def limit_weight(row: pd.Series) -> pd.Series: if row.sum() > 1: row = row / row.sum() elif row.sum() < -1: - row = row * (1. / abs(row.sum())) + row = row * (1.0 / abs(row.sum())) return row + def only_top_bottom_2(row: pd.Series) -> pd.Series: row = row.copy() row_sorted = row.sort_values() @@ -41,33 +48,36 @@ def only_top_bottom_2(row: pd.Series) -> pd.Series: top = row_sorted.iloc[-2:] middle = row_sorted.iloc[2:-2] for index, _ in middle.iteritems(): - row[index] = 0. + row[index] = 0.0 if bottom.sum() > 0: for index, _ in bottom.iteritems(): - row[index] = 0. + row[index] = 0.0 else: for index, _ in bottom.iteritems(): - row[index] = -.025 + row[index] = -0.025 if top.sum() < 0: for index, _ in top.iteritems(): - row[index] = 0. + row[index] = 0.0 else: for index, _ in top.iteritems(): - row[index] = .025 - + row[index] = 0.025 + return row - + + def equal_weight(row: pd.Series, availability: pd.Series) -> pd.Series: no_of_assets_available = availability.sum() unit = 1 / no_of_assets_available for index, _ in predictions.iteritems(): - row[index] = 0. if availability[index] == 0 else unit + row[index] = 0.0 if availability[index] == 0 else unit return row -def create_naive_portfolio_weights(predictions: pd.DataFrame, availability: pd.DataFrame, allow_short: bool) -> pd.DataFrame: +def create_naive_portfolio_weights( + predictions: pd.DataFrame, availability: pd.DataFrame, allow_short: bool +) -> pd.DataFrame: weights = predictions.copy() assert weights.shape[1] == availability.shape[1] for index, row in weights.iterrows(): @@ -79,14 +89,20 @@ def create_naive_portfolio_weights(predictions: pd.DataFrame, availability: pd.D weights.iloc[index] = row return weights -def create_quantile_weights(predictions: pd.DataFrame, availability: pd.DataFrame, allow_short: bool) -> pd.DataFrame: + +def create_quantile_weights( + predictions: pd.DataFrame, availability: pd.DataFrame, allow_short: bool +) -> pd.DataFrame: weights = predictions.copy() assert weights.shape[1] == availability.shape[1] quantiles = weights.copy() for column in weights.columns: - quantiles[column] = pd.qcut(weights[column], q=4, labels=False, duplicates='drop') - + quantiles[column] = pd.qcut( + weights[column], q=4, labels=False, duplicates="drop" + ) + for index, row in quantiles.iterrows(): + def only_select_bottom_top(x): if x == 0: return -1 @@ -94,6 +110,7 @@ def create_quantile_weights(predictions: pd.DataFrame, availability: pd.DataFram return 1 else: return 0 + row = row.apply(only_select_bottom_top) no_of_nonzero_predictions = row[row != 0].count() units = min(1 / no_of_nonzero_predictions, 0.25) @@ -101,49 +118,59 @@ def create_quantile_weights(predictions: pd.DataFrame, availability: pd.DataFram return weights -predictions = pd.read_csv('output/predictions.csv', index_col=0) +predictions = pd.read_csv("output/predictions.csv", index_col=0) predictions.index = pd.DatetimeIndex(predictions.index) -predictions.columns = ['_'.join(col.replace("model_", "").split("_")[:2]) for col in predictions.columns] +predictions.columns = [ + "_".join(col.replace("model_", "").split("_")[:2]) for col in predictions.columns +] first_index = get_first_valid_return_index(predictions[predictions.columns[0]]) predictions = predictions.iloc[first_index:] -close = load_only_returns(data_collections['daily_crypto'], 'price') +close = load_only_returns(data_collections["daily_crypto"], "price") close = close.iloc[first_index:] close.columns = [col.replace("_returns", "") for col in close.columns] close = close[predictions.columns] -close = close.filter(items = predictions.index, axis = 0) +close = close.filter(items=predictions.index, axis=0) availability = close.applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1) + def report_alphalens(): alpha_factors = predictions.copy() alpha_factors.index = close.index - alpha_factors_long = pd.melt(alpha_factors.reset_index(), id_vars=['time'], value_vars=alpha_factors.columns).set_index(['time', 'variable']) + alpha_factors_long = pd.melt( + alpha_factors.reset_index(), id_vars=["time"], value_vars=alpha_factors.columns + ).set_index(["time", "variable"]) factor_data = get_clean_factor_and_forward_returns( alpha_factors_long, close, quantiles=4, - periods=(1, 2, 3, 4, 5, 6, 10), - filter_zscore=None) + periods=(1, 2, 3, 4, 5, 6, 10), + filter_zscore=None, + ) # create_full_tear_sheet(factor_data, long_short=True) from matplotlib.backends.backend_pdf import PdfPages - mean_return_by_q_daily, std_err = alphalens.performance.mean_return_by_quantile(factor_data, by_date=True) - mean_return_by_q, std_err_by_q = alphalens.performance.mean_return_by_quantile(factor_data, by_group=False) + mean_return_by_q_daily, std_err = alphalens.performance.mean_return_by_quantile( + factor_data, by_date=True + ) + mean_return_by_q, std_err_by_q = alphalens.performance.mean_return_by_quantile( + factor_data, by_group=False + ) plot1 = alphalens.plotting.plot_quantile_returns_bar(mean_return_by_q) plot2 = alphalens.plotting.plot_quantile_returns_violin(mean_return_by_q_daily) - plot3 = alphalens.plotting.plot_cumulative_returns_by_quantile(mean_return_by_q_daily, period='D') + plot3 = alphalens.plotting.plot_cumulative_returns_by_quantile( + mean_return_by_q_daily, period="D" + ) - - with PdfPages('output/factors.pdf') as pdf: + with PdfPages("output/factors.pdf") as pdf: pdf.savefig(plot1.figure) pdf.savefig(plot2.figure) pdf.savefig(plot3.figure) - def report_backtest() -> vbt.Portfolio: weights = create_quantile_weights(predictions, availability, allow_short=True) @@ -151,7 +178,7 @@ def report_backtest() -> vbt.Portfolio: # rebalance every n days # weights.iloc[np.arange(len(weights)) % 2 != 0] = np.nan - weights.to_csv('output/weights.csv') + weights.to_csv("output/weights.csv") portfolio = vbt.Portfolio.from_orders( close=close, @@ -161,26 +188,27 @@ def report_backtest() -> vbt.Portfolio: cash_sharing=True, call_seq=CallSeqType.Auto, group_by=True, - freq='1D', + freq="1D", raise_reject=True, - fees=0.001, # assuming 0.1% fees (1.5x of FTX) - slippage= 0.002, # assuming 0.2% slippage (5x of avg. spread on FTX) + fees=0.001, # assuming 0.1% fees (1.5x of FTX) + slippage=0.002, # assuming 0.2% slippage (5x of avg. spread on FTX) seed=1, init_cash=1e5, - log=True + log=True, ) - qs.reports.full(portfolio.returns(), portfolio.benchmark_returns()) + qs.reports.full(portfolio.returns(), portfolio.benchmark_returns()) - qs.reports.html(portfolio.returns(), portfolio.benchmark_returns(), output='output/report.html') + qs.reports.html( + portfolio.returns(), portfolio.benchmark_returns(), output="output/report.html" + ) print(portfolio.stats()) return portfolio + portfolio = report_backtest() - - # from pypfopt import risk_models # from pypfopt import expected_returns # from pypfopt import EfficientFrontier diff --git a/run_sweep.py b/run_sweep.py index 1a4a5c1..7cd1b56 100644 --- a/run_sweep.py +++ b/run_sweep.py @@ -1,4 +1,9 @@ from run_pipeline import run_pipeline from config.presets import get_default_ensemble_config -run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True, raw_config= get_default_ensemble_config()) \ No newline at end of file +run_pipeline( + project_name="price-prediction", + with_wandb=True, + sweep=True, + raw_config=get_default_ensemble_config(), +) diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index dcc3317..f4cfde7 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -1,4 +1,3 @@ - import numpy as np import pandas as pd from training.walk_forward import walk_forward_train, walk_forward_inference @@ -8,10 +7,10 @@ from sklearn.base import BaseEstimator, ClassifierMixin no_of_rows = 100 + def __generate_even_odd_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]: - ''' Test data, where X[n][any_column] == 1 if n is even, else 0 - ''' - + """Test data, where X[n][any_column] == 1 if n is even, else 0""" + no_columns = 6 X = [[-1 if row % 2 == 0 else 1] * no_columns for row in range(no_of_rows)] assert X[0][0] == -1 @@ -20,7 +19,7 @@ def __generate_even_odd_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]: assert X[3][0] == 1 X = pd.DataFrame(X) - y = [-1 if (row+1) % 2 == 0 else 1 for row in range(no_of_rows)] + y = [-1 if (row + 1) % 2 == 0 else 1 for row in range(no_of_rows)] assert y[0] == 1 assert y[1] == -1 assert y[2] == 1 @@ -32,14 +31,14 @@ def __generate_even_odd_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]: class EvenOddStubModel(BaseEstimator, ClassifierMixin, Model): - ''' + """ A deteministic model that can predict the future with 100% accuracy It verifies that the X[n][any_column] == 1 if n is even, - ''' + """ data_transformation = "original" only_column = None - predict_window_size = 'single_timestamp' + predict_window_size = "single_timestamp" def __init__(self, window_length) -> None: super().__init__() @@ -59,12 +58,12 @@ class EvenOddStubModel(BaseEstimator, ClassifierMixin, Model): def test_evaluation(): X, y = __generate_even_odd_test_data(no_of_rows) - + window_length = 10 retrain_every = 10 - model = EvenOddStubModel(window_length = window_length) - + model = EvenOddStubModel(window_length=window_length) + model_over_time = walk_forward_train( model=model, X=X, @@ -74,20 +73,21 @@ def test_evaluation(): window_size=window_length, retrain_every=retrain_every, from_index=None, - transformations_over_time=[]) + transformations_over_time=[], + ) predictions, _ = walk_forward_inference( - model_name='test', + model_name="test", model_over_time=model_over_time, transformations_over_time=[], X=X, expanding_window=False, window_size=window_length, - retrain_every = retrain_every, + retrain_every=retrain_every, from_index=None, ) - + # verify if predictions are the same as y - for i in range(window_length+2, no_of_rows): + for i in range(window_length + 2, no_of_rows): assert predictions[i] == y[i] fake_forward_returns = y * 0.1 @@ -97,11 +97,8 @@ def test_evaluation(): forward_returns=fake_forward_returns, y_pred=processed_predictions_to_match_returns, y_true=y, - no_of_classes='two', - discretize=True + no_of_classes="two", + discretize=True, ) - assert result['accuracy'] == 100.0 - - - + assert result["accuracy"] == 100.0 diff --git a/tests/test_walk_forward.py b/tests/test_walk_forward.py index c0d381d..b7baf32 100644 --- a/tests/test_walk_forward.py +++ b/tests/test_walk_forward.py @@ -6,10 +6,10 @@ from sklearn.base import BaseEstimator, ClassifierMixin no_of_rows = 100 + def __generate_incremental_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]: - ''' Test data, where X[n][any_column] == y[n]+1 - ''' - + """Test data, where X[n][any_column] == y[n]+1""" + no_columns = 6 X = [[row] * no_columns for row in range(no_of_rows)] assert X[1][0] == 1 @@ -18,7 +18,7 @@ def __generate_incremental_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Serie assert X[4][0] == 4 X = pd.DataFrame(X) - y = [row+1 for row in range(no_of_rows)] + y = [row + 1 for row in range(no_of_rows)] assert y[1] == 2 assert y[2] == 3 assert y[3] == 4 @@ -27,17 +27,15 @@ def __generate_incremental_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Serie return X, y - class IncrementingStubModel(Model, BaseEstimator, ClassifierMixin): - ''' + """ A deteministic model that can predict the future with 100% accuracy It verifies that the X[n][any_column]+1 == y[n] - ''' + """ data_transformation = "original" only_column = None - predict_window_size = 'single_timestamp' - + predict_window_size = "single_timestamp" def __init__(self, window_length) -> None: super().__init__() @@ -53,14 +51,15 @@ class IncrementingStubModel(Model, BaseEstimator, ClassifierMixin): def predict_proba(self, X): return np.array([[row[0] + 1, 0] for row in X]) - + + def test_walk_forward_train_test(): X, y = __generate_incremental_test_data(no_of_rows) window_length = 10 retrain_every = 10 - model = IncrementingStubModel(window_length = window_length) + model = IncrementingStubModel(window_length=window_length) model_over_time = walk_forward_train( model=model, @@ -74,7 +73,7 @@ def test_walk_forward_train_test(): transformations_over_time=[], ) predictions, _ = walk_forward_inference( - model_name='test', + model_name="test", model_over_time=model_over_time, transformations_over_time=[], X=X, @@ -83,8 +82,7 @@ def test_walk_forward_train_test(): retrain_every=retrain_every, from_index=None, ) - - # verify if predictions are the same as y - for i in range(window_length+2, no_of_rows): - assert predictions[i] == y[i] + # verify if predictions are the same as y + for i in range(window_length + 2, no_of_rows): + assert predictions[i] == y[i] diff --git a/training/bet_sizing.py b/training/bet_sizing.py index bc1591b..4402972 100644 --- a/training/bet_sizing.py +++ b/training/bet_sizing.py @@ -13,75 +13,92 @@ from transformations.scaler import get_scaler from transformations.rfe import RFETransformation from transformations.pca import PCATransformation + def bet_sizing_with_meta_model( - X: XDataFrame, - input_predictions: pd.Series, - y: ySeries, - forward_returns: ForwardReturnSeries, - model: Model, - config: Config, - model_suffix: str, - from_index: Optional[pd.Timestamp], - transformations_over_time: Optional[TransformationsOverTime] = None, - preloaded_models: Optional[ModelOverTime] = None - ) -> BetSizingWithMetaOutcome: + X: XDataFrame, + input_predictions: pd.Series, + y: ySeries, + forward_returns: ForwardReturnSeries, + model: Model, + config: Config, + model_suffix: str, + from_index: Optional[pd.Timestamp], + transformations_over_time: Optional[TransformationsOverTime] = None, + preloaded_models: Optional[ModelOverTime] = None, +) -> BetSizingWithMetaOutcome: input_predictions.name = "model_predictions" - discretized_predictions = input_predictions.apply(discretize_threeway_threshold(0.33)) + discretized_predictions = input_predictions.apply( + discretize_threeway_threshold(0.33) + ) discretized_predictions.name = "model_discretized_predictions" - meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply(equal_except_nan, axis = 1) - meta_X = pd.concat([X, input_predictions, discretized_predictions], axis = 1) + meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply( + equal_except_nan, axis=1 + ) + meta_X = pd.concat([X, input_predictions, discretized_predictions], axis=1) if transformations_over_time is None: print("Preprocess transformations") transformations_over_time = walk_forward_process_transformations( - X = meta_X, - y = meta_y, - forward_returns = forward_returns, - expanding_window = config.expanding_window_meta, - window_size = config.sliding_window_size_meta, - retrain_every = config.retrain_every, - from_index = from_index, - transformations= [ + X=meta_X, + y=meta_y, + forward_returns=forward_returns, + expanding_window=config.expanding_window_meta, + window_size=config.sliding_window_size_meta, + retrain_every=config.retrain_every, + from_index=from_index, + transformations=[ get_scaler(config.scaler), - PCATransformation(ratio_components_to_keep=0.5, sliding_window_size=config.sliding_window_size_meta), - RFETransformation(n_feature_to_select=40, model=default_feature_selector_classification) + PCATransformation( + ratio_components_to_keep=0.5, + sliding_window_size=config.sliding_window_size_meta, + ), + RFETransformation( + n_feature_to_select=40, + model=default_feature_selector_classification, + ), ], ) meta_outcome = train_model( - ticker_to_predict = "prediction_correct", - X = meta_X, - y = meta_y, - forward_returns = forward_returns, - model = model, - expanding_window = config.expanding_window_meta, - sliding_window_size = config.sliding_window_size_meta, - retrain_every = config.retrain_every, - from_index = from_index, - no_of_classes = 'two', - level = 'meta', - output_stats = config.mode == 'training', - transformations_over_time = transformations_over_time, - model_over_time = preloaded_models, + ticker_to_predict="prediction_correct", + X=meta_X, + y=meta_y, + forward_returns=forward_returns, + model=model, + expanding_window=config.expanding_window_meta, + sliding_window_size=config.sliding_window_size_meta, + retrain_every=config.retrain_every, + from_index=from_index, + no_of_classes="two", + level="meta", + output_stats=config.mode == "training", + transformations_over_time=transformations_over_time, + model_over_time=preloaded_models, ) meta_predictions = meta_outcome.predictions - bet_size = meta_outcome.probabilities.iloc[:,1] + bet_size = meta_outcome.probabilities.iloc[:, 1] avg_predictions_with_sizing = input_predictions * meta_predictions * bet_size - if config.mode == 'training': + if config.mode == "training": stats = evaluate_predictions( - forward_returns = forward_returns, - y_pred = avg_predictions_with_sizing, - y_true = y, - no_of_classes = 'three-balanced', - discretize=False + forward_returns=forward_returns, + y_pred=avg_predictions_with_sizing, + y_true=y, + no_of_classes="three-balanced", + discretize=False, ) print(stats) else: stats = None model_id = "model_" + config.target_asset[1] + "_" + model_suffix - return BetSizingWithMetaOutcome(model_id, meta_outcome, transformations_over_time, avg_predictions_with_sizing, stats) + return BetSizingWithMetaOutcome( + model_id, + meta_outcome, + transformations_over_time, + avg_predictions_with_sizing, + stats, + ) diff --git a/training/directional_training.py b/training/directional_training.py index a4b4dcb..4bb2164 100644 --- a/training/directional_training.py +++ b/training/directional_training.py @@ -5,7 +5,7 @@ from training.train_model import train_model from training.walk_forward import walk_forward_process_transformations from typing import Optional -from config.types import Config +from config.types import Config from models.base import Model from models.model_map import default_feature_selector_classification @@ -13,53 +13,61 @@ from transformations.scaler import get_scaler from transformations.rfe import RFETransformation from transformations.pca import PCATransformation + def train_directional_model( - X: pd.DataFrame, - y: pd.Series, - forward_returns: pd.Series, - config: Config, - model: Model, - from_index: Optional[pd.Timestamp], - preloaded_training_step: Optional[DirectionalTrainingOutcome] = None, - ) -> DirectionalTrainingOutcome: + X: pd.DataFrame, + y: pd.Series, + forward_returns: pd.Series, + config: Config, + model: Model, + from_index: Optional[pd.Timestamp], + preloaded_training_step: Optional[DirectionalTrainingOutcome] = None, +) -> DirectionalTrainingOutcome: if preloaded_training_step is None: print("Preprocess transformations") transformations_over_time = walk_forward_process_transformations( - X = X, - y = y, - forward_returns = forward_returns, - expanding_window = config.expanding_window_base, - window_size = config.sliding_window_size_base, - retrain_every = config.retrain_every, - from_index = from_index, - transformations= [ + X=X, + y=y, + forward_returns=forward_returns, + expanding_window=config.expanding_window_base, + window_size=config.sliding_window_size_base, + retrain_every=config.retrain_every, + from_index=from_index, + transformations=[ get_scaler(config.scaler), - PCATransformation(ratio_components_to_keep=0.5, sliding_window_size=config.sliding_window_size_base), - RFETransformation(n_feature_to_select=40, model=default_feature_selector_classification) + PCATransformation( + ratio_components_to_keep=0.5, + sliding_window_size=config.sliding_window_size_base, + ), + RFETransformation( + n_feature_to_select=40, + model=default_feature_selector_classification, + ), ], ) else: transformations_over_time = preloaded_training_step.transformations training_outcome = train_model( - ticker_to_predict = config.target_asset[1], - X = X, - y = y, - forward_returns = forward_returns, - model = model, - expanding_window = config.expanding_window_base, - sliding_window_size = config.sliding_window_size_base, - retrain_every = config.retrain_every, - from_index = from_index, - no_of_classes = config.no_of_classes, - level = 'primary', - output_stats= config.mode == 'training', - transformations_over_time = transformations_over_time, - model_over_time = preloaded_training_step.training.model_over_time if preloaded_training_step else None + ticker_to_predict=config.target_asset[1], + X=X, + y=y, + forward_returns=forward_returns, + model=model, + expanding_window=config.expanding_window_base, + sliding_window_size=config.sliding_window_size_base, + retrain_every=config.retrain_every, + from_index=from_index, + no_of_classes=config.no_of_classes, + level="primary", + output_stats=config.mode == "training", + transformations_over_time=transformations_over_time, + model_over_time=preloaded_training_step.training.model_over_time + if preloaded_training_step + else None, ) - if config.mode == 'training': + if config.mode == "training": print(training_outcome.stats) return DirectionalTrainingOutcome(training_outcome, transformations_over_time) - diff --git a/training/train_model.py b/training/train_model.py index 59d4a44..c836fc9 100644 --- a/training/train_model.py +++ b/training/train_model.py @@ -1,69 +1,78 @@ import pandas as pd from typing import Literal, Optional -from training.walk_forward import walk_forward_train, walk_forward_inference, walk_forward_inference_batched +from training.walk_forward import ( + walk_forward_train, + walk_forward_inference, + walk_forward_inference_batched, +) from utils.evaluate import evaluate_predictions from models.base import Model from .types import ModelOverTime, TransformationsOverTime, TrainingOutcome + def train_model( - ticker_to_predict: str, - X: pd.DataFrame, - y: pd.Series, - forward_returns: pd.Series, - model: Model, - expanding_window: bool, - sliding_window_size: int, - retrain_every: int, - from_index: Optional[pd.Timestamp], - no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], - level: str, - output_stats: bool, - transformations_over_time: TransformationsOverTime, - model_over_time: Optional[ModelOverTime] - ) -> TrainingOutcome: + ticker_to_predict: str, + X: pd.DataFrame, + y: pd.Series, + forward_returns: pd.Series, + model: Model, + expanding_window: bool, + sliding_window_size: int, + retrain_every: int, + from_index: Optional[pd.Timestamp], + no_of_classes: Literal["two", "three-balanced", "three-imbalanced"], + level: str, + output_stats: bool, + transformations_over_time: TransformationsOverTime, + model_over_time: Optional[ModelOverTime], +) -> TrainingOutcome: if model_over_time is None: print("Train model") model_over_time = walk_forward_train( - model = model, - X = X, - y = y, - forward_returns = forward_returns, - expanding_window = expanding_window, - window_size = sliding_window_size, - retrain_every = retrain_every, - from_index = from_index, - transformations_over_time = transformations_over_time, + model=model, + X=X, + y=y, + forward_returns=forward_returns, + expanding_window=expanding_window, + window_size=sliding_window_size, + retrain_every=retrain_every, + from_index=from_index, + transformations_over_time=transformations_over_time, ) - levelname = ("_" + level) if level == 'meta' else "" + levelname = ("_" + level) if level == "meta" else "" if model_over_time is None: - model_id = "model_" + model.name + "_" + ticker_to_predict + levelname + model_id = "model_" + model.name + "_" + ticker_to_predict + levelname else: model_id = model_over_time.name - inference_function = walk_forward_inference if from_index is not None else walk_forward_inference_batched - predictions, probabilities = inference_function( - model_name = model_id, - model_over_time= model_over_time, - transformations_over_time = transformations_over_time, - X = X, - expanding_window = expanding_window, - window_size = sliding_window_size, - retrain_every = retrain_every, - from_index = from_index, + inference_function = ( + walk_forward_inference + if from_index is not None + else walk_forward_inference_batched ) - + predictions, probabilities = inference_function( + model_name=model_id, + model_over_time=model_over_time, + transformations_over_time=transformations_over_time, + X=X, + expanding_window=expanding_window, + window_size=sliding_window_size, + retrain_every=retrain_every, + from_index=from_index, + ) + assert len(predictions) == len(y) if output_stats: stats = evaluate_predictions( - forward_returns = forward_returns, - y_pred = predictions, - y_true = y, + forward_returns=forward_returns, + y_pred=predictions, + y_true=y, no_of_classes=no_of_classes, - discretize=True + discretize=True, ) else: stats = None - return TrainingOutcome(model_id, predictions, probabilities, stats, model_over_time) \ No newline at end of file + return TrainingOutcome(model_id, predictions, probabilities, stats, model_over_time) diff --git a/training/types.py b/training/types.py index 17a74f5..94a8b3f 100644 --- a/training/types.py +++ b/training/types.py @@ -19,6 +19,7 @@ class TrainingOutcome: stats: Optional[Stats] model_over_time: ModelOverTime + @dataclass class BetSizingWithMetaOutcome: model_id: str @@ -27,11 +28,13 @@ class BetSizingWithMetaOutcome: weights: WeightsSeries stats: Optional[Stats] + @dataclass class DirectionalTrainingOutcome: training: TrainingOutcome transformations: TransformationsOverTime - + + @dataclass class PipelineOutcome: directional_training: DirectionalTrainingOutcome @@ -41,4 +44,4 @@ class PipelineOutcome: return self.bet_sizing.weights def get_output_stats(self) -> Stats: - return self.bet_sizing.stats \ No newline at end of file + return self.bet_sizing.stats diff --git a/training/walk_forward/__init__.py b/training/walk_forward/__init__.py index 70759af..a031ed2 100644 --- a/training/walk_forward/__init__.py +++ b/training/walk_forward/__init__.py @@ -1,4 +1,4 @@ from .inference_batched import walk_forward_inference_batched from .inference import walk_forward_inference from .train import walk_forward_train -from .process_transformations import walk_forward_process_transformations \ No newline at end of file +from .process_transformations import walk_forward_process_transformations diff --git a/training/walk_forward/inference.py b/training/walk_forward/inference.py index 2fedf74..3af1883 100644 --- a/training/walk_forward/inference.py +++ b/training/walk_forward/inference.py @@ -1,49 +1,79 @@ import pandas as pd -from training.types import ModelOverTime, TransformationsOverTime, PredictionsSeries, ProbabilitiesDataFrame +from training.types import ( + ModelOverTime, + TransformationsOverTime, + PredictionsSeries, + ProbabilitiesDataFrame, +) from utils.helpers import get_first_valid_return_index from tqdm import tqdm from typing import Optional from data_loader.types import XDataFrame from utils.helpers import get_last_non_na_index + def walk_forward_inference( - model_name: str, - model_over_time: ModelOverTime, - transformations_over_time: TransformationsOverTime, - X: XDataFrame, - expanding_window: bool, - window_size: int, - retrain_every: int, - from_index: Optional[pd.Timestamp], - ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: - predictions = pd.Series(index=X.index, dtype='object').rename(model_name) + model_name: str, + model_over_time: ModelOverTime, + transformations_over_time: TransformationsOverTime, + X: XDataFrame, + expanding_window: bool, + window_size: int, + retrain_every: int, + from_index: Optional[pd.Timestamp], +) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: + predictions = pd.Series(index=X.index, dtype="object").rename(model_name) probabilities = pd.DataFrame(index=X.index) - inference_from = max(get_first_valid_return_index(model_over_time), get_first_valid_return_index(X.iloc[:,0])) if from_index is None else X.index.to_list().index(from_index) + inference_from = ( + max( + get_first_valid_return_index(model_over_time), + get_first_valid_return_index(X.iloc[:, 0]), + ) + if from_index is None + else X.index.to_list().index(from_index) + ) inference_till = X.shape[0] - model_index_offset = get_last_non_na_index(model_over_time, inference_from) if pd.isna(model_over_time[inference_from]) else 0 - first_model = model_over_time[inference_from - model_index_offset] if pd.isna(model_over_time[inference_from]) else model_over_time[inference_from] + model_index_offset = ( + get_last_non_na_index(model_over_time, inference_from) + if pd.isna(model_over_time[inference_from]) + else 0 + ) + first_model = ( + model_over_time[inference_from - model_index_offset] + if pd.isna(model_over_time[inference_from]) + else model_over_time[inference_from] + ) if first_model.only_column is not None: X = X[[column for column in X.columns if first_model.only_column in column]] - - if first_model.data_transformation == 'original': + + if first_model.data_transformation == "original": transformations_over_time = [] for index in tqdm(range(inference_from, inference_till)): - - last_model_index = index - ((index - inference_from) % retrain_every) - model_index_offset - train_window_start = X.index[inference_from] if expanding_window else X.index[index - window_size - 1] + + last_model_index = ( + index - ((index - inference_from) % retrain_every) - model_index_offset + ) + train_window_start = ( + X.index[inference_from] + if expanding_window + else X.index[index - window_size - 1] + ) current_model = model_over_time[X.index[last_model_index]] - current_transformations = [transformation_over_time[X.index[last_model_index]] for transformation_over_time in transformations_over_time] + current_transformations = [ + transformation_over_time[X.index[last_model_index]] + for transformation_over_time in transformations_over_time + ] - if current_model.predict_window_size == 'window_size': - next_timestep = X.loc[train_window_start:X.index[index]] - else: + if current_model.predict_window_size == "window_size": + next_timestep = X.loc[train_window_start : X.index[index]] + else: # we need to get a Dataframe out of it, since the transformation step always expects a 2D array, but it's equivalent to X.iloc[index] - next_timestep = X.loc[X.index[index]:X.index[index]] - + next_timestep = X.loc[X.index[index] : X.index[index]] + for transformation in current_transformations: next_timestep = transformation.transform(next_timestep) @@ -54,7 +84,9 @@ def walk_forward_inference( predictions[X.index[index]] = prediction if inference_from == index and len(probabilities.columns) != len(probs): - probabilities = probabilities.reindex(columns = ["prob_" + str(num) for num in range(0, len(probs.T))]) + probabilities = probabilities.reindex( + columns=["prob_" + str(num) for num in range(0, len(probs.T))] + ) probabilities.loc[X.index[index]] = probs return predictions, probabilities diff --git a/training/walk_forward/inference_batched.py b/training/walk_forward/inference_batched.py index 4013d8a..9631809 100644 --- a/training/walk_forward/inference_batched.py +++ b/training/walk_forward/inference_batched.py @@ -1,36 +1,64 @@ import pandas as pd -from training.types import ModelOverTime, TransformationsOverTime, PredictionsSeries, ProbabilitiesDataFrame +from training.types import ( + ModelOverTime, + TransformationsOverTime, + PredictionsSeries, + ProbabilitiesDataFrame, +) from utils.helpers import get_first_valid_return_index from tqdm import tqdm from typing import Optional from data_loader.types import XDataFrame from tqdm import tqdm -def walk_forward_inference_batched( - model_name: str, - model_over_time: ModelOverTime, - transformations_over_time: TransformationsOverTime, - X: XDataFrame, - expanding_window: bool, - window_size: int, - retrain_every: int, - from_index: Optional[pd.Timestamp], - ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: - predictions = pd.Series(index=X.index, dtype='object').rename(model_name) - probabilities = pd.DataFrame(index=X.index, columns=['0', '1']) - inference_from = max(get_first_valid_return_index(model_over_time), get_first_valid_return_index(X.iloc[:,0])) if from_index is None else X.index.to_list().index(from_index) +def walk_forward_inference_batched( + model_name: str, + model_over_time: ModelOverTime, + transformations_over_time: TransformationsOverTime, + X: XDataFrame, + expanding_window: bool, + window_size: int, + retrain_every: int, + from_index: Optional[pd.Timestamp], +) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: + predictions = pd.Series(index=X.index, dtype="object").rename(model_name) + probabilities = pd.DataFrame(index=X.index, columns=["0", "1"]) + + inference_from = ( + max( + get_first_valid_return_index(model_over_time), + get_first_valid_return_index(X.iloc[:, 0]), + ) + if from_index is None + else X.index.to_list().index(from_index) + ) inference_till = X.shape[0] first_model = model_over_time[inference_from] if first_model.only_column is not None: X = X[[column for column in X.columns if first_model.only_column in column]] - - if first_model.data_transformation == 'original': + + if first_model.data_transformation == "original": transformations_over_time = [] - batch_indices = range(inference_from, inference_till, retrain_every) if inference_till - inference_from > retrain_every else [inference_from] - batched_results = [__inference_from_window(index, index + retrain_every, X, model_over_time, transformations_over_time, expanding_window, window_size) for index in tqdm(batch_indices)] + batch_indices = ( + range(inference_from, inference_till, retrain_every) + if inference_till - inference_from > retrain_every + else [inference_from] + ) + batched_results = [ + __inference_from_window( + index, + index + retrain_every, + X, + model_over_time, + transformations_over_time, + expanding_window, + window_size, + ) + for index in tqdm(batch_indices) + ] for batch in batched_results: for index, prediction, probs in batch: predictions[X.index[index]] = prediction @@ -38,12 +66,24 @@ def walk_forward_inference_batched( return predictions, probabilities -def __inference_from_window(index_start: int, index_end: int, X: XDataFrame, model_over_time: ModelOverTime, transformations_over_time: TransformationsOverTime, expanding_window: bool, window_size: int) -> list[tuple[int, float, pd.Series]]: + +def __inference_from_window( + index_start: int, + index_end: int, + X: XDataFrame, + model_over_time: ModelOverTime, + transformations_over_time: TransformationsOverTime, + expanding_window: bool, + window_size: int, +) -> list[tuple[int, float, pd.Series]]: current_model = model_over_time[X.index[index_start]] - current_transformations = [transformation_over_time[X.index[index_start]] for transformation_over_time in transformations_over_time] + current_transformations = [ + transformation_over_time[X.index[index_start]] + for transformation_over_time in transformations_over_time + ] input_data = X.iloc[index_start:index_end] - + for transformation in current_transformations: input_data = transformation.transform(input_data) @@ -51,6 +91,9 @@ def __inference_from_window(index_start: int, index_end: int, X: XDataFrame, mod predictions = current_model.predict(input_data) probs = current_model.predict_proba(input_data) - results = [(index_start + index, predictions[index], probs[index]) for index in range(len(predictions))] - - return results \ No newline at end of file + results = [ + (index_start + index, predictions[index], probs[index]) + for index in range(len(predictions)) + ] + + return results diff --git a/training/walk_forward/inference_parallel.py b/training/walk_forward/inference_parallel.py index 775f669..eeaa0dd 100644 --- a/training/walk_forward/inference_parallel.py +++ b/training/walk_forward/inference_parallel.py @@ -1,6 +1,11 @@ import pandas as pd from models.base import Model -from training.types import ModelOverTime, TransformationsOverTime, PredictionsSeries, ProbabilitiesDataFrame +from training.types import ( + ModelOverTime, + TransformationsOverTime, + PredictionsSeries, + ProbabilitiesDataFrame, +) from transformations.base import Transformation from utils.helpers import get_first_valid_return_index from tqdm import tqdm @@ -8,31 +13,54 @@ from typing import Optional from data_loader.types import XDataFrame import ray + def walk_forward_inference( - model_name: str, - model_over_time: ModelOverTime, - transformations_over_time: TransformationsOverTime, - X: XDataFrame, - expanding_window: bool, - window_size: int, - retrain_every: int, - from_index: Optional[pd.Timestamp], - ) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: - predictions = pd.Series(index=X.index, dtype='object').rename(model_name) + model_name: str, + model_over_time: ModelOverTime, + transformations_over_time: TransformationsOverTime, + X: XDataFrame, + expanding_window: bool, + window_size: int, + retrain_every: int, + from_index: Optional[pd.Timestamp], +) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]: + predictions = pd.Series(index=X.index, dtype="object").rename(model_name) probabilities = pd.DataFrame(index=X.index) - inference_from = max(get_first_valid_return_index(model_over_time), get_first_valid_return_index(X.iloc[:,0])) if from_index is None else X.index.to_list().index(from_index) + inference_from = ( + max( + get_first_valid_return_index(model_over_time), + get_first_valid_return_index(X.iloc[:, 0]), + ) + if from_index is None + else X.index.to_list().index(from_index) + ) inference_till = X.shape[0] first_model = model_over_time[inference_from] if first_model.only_column is not None: X = X[[column for column in X.columns if first_model.only_column in column]] - - if first_model.data_transformation == 'original': + + if first_model.data_transformation == "original": transformations_over_time = [] batch_size = int((inference_till - inference_from) / 10) - batched_results = ray.get([__inference_from_window.remote(index, index + batch_size, inference_from, retrain_every, X, model_over_time, transformations_over_time, expanding_window, window_size) for index in range(inference_from, inference_till)]) + batched_results = ray.get( + [ + __inference_from_window.remote( + index, + index + batch_size, + inference_from, + retrain_every, + X, + model_over_time, + transformations_over_time, + expanding_window, + window_size, + ) + for index in range(inference_from, inference_till) + ] + ) for batch in batched_results: for index, prediction, probs in batch: predictions[X.index[index]] = prediction @@ -40,23 +68,41 @@ def walk_forward_inference( return predictions, probabilities + @ray.remote -def __inference_from_window(index_start: int, index_end: int, inference_from: int, retrain_every: int, X: XDataFrame, model_over_time: ModelOverTime, transformations_over_time: TransformationsOverTime, expanding_window: bool, window_size: int) -> list[tuple[int, float, pd.Series]]: - +def __inference_from_window( + index_start: int, + index_end: int, + inference_from: int, + retrain_every: int, + X: XDataFrame, + model_over_time: ModelOverTime, + transformations_over_time: TransformationsOverTime, + expanding_window: bool, + window_size: int, +) -> list[tuple[int, float, pd.Series]]: + results = [] for index in range(index_start, index_end): last_model_index = index - ((index - inference_from) % retrain_every) - train_window_start = X.index[inference_from] if expanding_window else X.index[index - window_size - 1] + train_window_start = ( + X.index[inference_from] + if expanding_window + else X.index[index - window_size - 1] + ) current_model = model_over_time[X.index[last_model_index]] - current_transformations = [transformation_over_time[X.index[last_model_index]] for transformation_over_time in transformations_over_time] + current_transformations = [ + transformation_over_time[X.index[last_model_index]] + for transformation_over_time in transformations_over_time + ] - if current_model.predict_window_size == 'window_size': - next_timestep = X.loc[train_window_start:X.index[index]] - else: + if current_model.predict_window_size == "window_size": + next_timestep = X.loc[train_window_start : X.index[index]] + else: # we need to get a Dataframe out of it, since the transformation step always expects a 2D array, but it's equivalent to X.iloc[index] - next_timestep = X.loc[X.index[index]:X.index[index]] - + next_timestep = X.loc[X.index[index] : X.index[index]] + for transformation in current_transformations: next_timestep = transformation.transform(next_timestep) @@ -64,5 +110,5 @@ def __inference_from_window(index_start: int, index_end: int, inference_from: in prediction, probs = current_model.predict(next_timestep) results.append((index, prediction, probs)) - - return results \ No newline at end of file + + return results diff --git a/training/walk_forward/process_transformations.py b/training/walk_forward/process_transformations.py index d0e815c..cc376ff 100644 --- a/training/walk_forward/process_transformations.py +++ b/training/walk_forward/process_transformations.py @@ -8,40 +8,63 @@ from data_loader.types import ForwardReturnSeries, XDataFrame, ySeries def walk_forward_process_transformations( - X: XDataFrame, - y: ySeries, - forward_returns: ForwardReturnSeries, - expanding_window: bool, - window_size: int, - retrain_every: int, - from_index: Optional[pd.Timestamp], - transformations: list[Transformation], - ) -> TransformationsOverTime: - transformations_over_time = [pd.Series(index=y.index, dtype='object').rename(t.get_name()) for t in transformations] + X: XDataFrame, + y: ySeries, + forward_returns: ForwardReturnSeries, + expanding_window: bool, + window_size: int, + retrain_every: int, + from_index: Optional[pd.Timestamp], + transformations: list[Transformation], +) -> TransformationsOverTime: + transformations_over_time = [ + pd.Series(index=y.index, dtype="object").rename(t.get_name()) + for t in transformations + ] - first_nonzero_return = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y)) - train_from = first_nonzero_return + window_size + 1 if from_index is None else X.index.to_list().index(from_index) + first_nonzero_return = max( + get_first_valid_return_index(forward_returns), + get_first_valid_return_index(X.iloc[:, 0]), + get_first_valid_return_index(y), + ) + train_from = ( + first_nonzero_return + window_size + 1 + if from_index is None + else X.index.to_list().index(from_index) + ) train_till = len(y) iterations_before_retrain = 0 - - for index in tqdm(range(train_from, train_till)): - train_window_start = X.index[first_nonzero_return] if expanding_window else X.index[index - window_size - 1] - if iterations_before_retrain <= 0 or pd.isna(transformations_over_time[0][index-1]): + for index in tqdm(range(train_from, train_till)): + train_window_start = ( + X.index[first_nonzero_return] + if expanding_window + else X.index[index - window_size - 1] + ) + + if iterations_before_retrain <= 0 or pd.isna( + transformations_over_time[0][index - 1] + ): train_window_end = X.index[index - 1] - + X_expanding_window = X[train_window_start:train_window_end] y_expanding_window = y[train_window_start:train_window_end] current_transformations = [t.clone() for t in transformations] - for transformation_index, transformation in enumerate(current_transformations): - X_expanding_window = transformation.fit_transform(X_expanding_window, y_expanding_window) + for transformation_index, transformation in enumerate( + current_transformations + ): + X_expanding_window = transformation.fit_transform( + X_expanding_window, y_expanding_window + ) iterations_before_retrain = retrain_every for transformation_index, transformation in enumerate(current_transformations): - transformations_over_time[transformation_index][X.index[index]] = transformation + transformations_over_time[transformation_index][ + X.index[index] + ] = transformation iterations_before_retrain -= 1 diff --git a/training/walk_forward/process_transformations_parallel.py b/training/walk_forward/process_transformations_parallel.py index b8f1f07..0d4b2a2 100644 --- a/training/walk_forward/process_transformations_parallel.py +++ b/training/walk_forward/process_transformations_parallel.py @@ -8,33 +8,72 @@ from data_loader.types import ForwardReturnSeries, XDataFrame, ySeries import ray from utils.parallel import parallel_compute_with_bar -def walk_forward_process_transformations( - X: XDataFrame, - y: ySeries, - forward_returns: ForwardReturnSeries, - expanding_window: bool, - window_size: int, - retrain_every: int, - from_index: Optional[pd.Timestamp], - transformations: list[Transformation], - ) -> TransformationsOverTime: - transformations_over_time = [pd.Series(index=y.index).rename(t.get_name()) for t in transformations] - first_nonzero_return = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y)) - train_from = first_nonzero_return + window_size + 1 if from_index is None else X.index.to_list().index(from_index) +def walk_forward_process_transformations( + X: XDataFrame, + y: ySeries, + forward_returns: ForwardReturnSeries, + expanding_window: bool, + window_size: int, + retrain_every: int, + from_index: Optional[pd.Timestamp], + transformations: list[Transformation], +) -> TransformationsOverTime: + transformations_over_time = [ + pd.Series(index=y.index).rename(t.get_name()) for t in transformations + ] + + first_nonzero_return = max( + get_first_valid_return_index(forward_returns), + get_first_valid_return_index(X.iloc[:, 0]), + get_first_valid_return_index(y), + ) + train_from = ( + first_nonzero_return + window_size + 1 + if from_index is None + else X.index.to_list().index(from_index) + ) train_till = len(y) - - processed_transformations = parallel_compute_with_bar([preprocess_transformations_window.remote(X, y, expanding_window, window_size, transformations, first_nonzero_return, index) for index in range(train_from, train_till, retrain_every)]) - + + processed_transformations = parallel_compute_with_bar( + [ + preprocess_transformations_window.remote( + X, + y, + expanding_window, + window_size, + transformations, + first_nonzero_return, + index, + ) + for index in range(train_from, train_till, retrain_every) + ] + ) + for transformation, index_time in processed_transformations: for transformation_index, transformation in enumerate(transformation): - transformations_over_time[transformation_index][X.index[index_time]] = transformation + transformations_over_time[transformation_index][ + X.index[index_time] + ] = transformation return transformations_over_time + @ray.remote -def preprocess_transformations_window(X: XDataFrame, y: ySeries, expanding_window: bool, window_size: int, transformations: list[Transformation], first_nonzero_return: int, index: int) -> tuple[list[Transformation], int]: - train_window_start = X.index[first_nonzero_return] if expanding_window else X.index[index - window_size - 1] +def preprocess_transformations_window( + X: XDataFrame, + y: ySeries, + expanding_window: bool, + window_size: int, + transformations: list[Transformation], + first_nonzero_return: int, + index: int, +) -> tuple[list[Transformation], int]: + train_window_start = ( + X.index[first_nonzero_return] + if expanding_window + else X.index[index - window_size - 1] + ) train_window_end = X.index[index - 1] X_expanding_window = X[train_window_start:train_window_end] @@ -42,6 +81,8 @@ def preprocess_transformations_window(X: XDataFrame, y: ySeries, expanding_windo current_transformations = [t.clone() for t in transformations] for transformation in current_transformations: - X_expanding_window = transformation.fit_transform(X_expanding_window, y_expanding_window) + X_expanding_window = transformation.fit_transform( + X_expanding_window, y_expanding_window + ) - return (current_transformations, index) \ No newline at end of file + return (current_transformations, index) diff --git a/training/walk_forward/train.py b/training/walk_forward/train.py index 3d85297..c72db88 100644 --- a/training/walk_forward/train.py +++ b/training/walk_forward/train.py @@ -7,39 +7,55 @@ from typing import Optional from data_loader.types import ForwardReturnSeries, XDataFrame, ySeries from copy import deepcopy -def walk_forward_train( - model: Model, - X: XDataFrame, - y: ySeries, - forward_returns: ForwardReturnSeries, - expanding_window: bool, - window_size: int, - retrain_every: int, - from_index: Optional[pd.Timestamp], - transformations_over_time: TransformationsOverTime, - ) -> ModelOverTime: - models_over_time = pd.Series(index=y.index, dtype='object').rename(model.name) - first_nonzero_return = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y)) - train_from = first_nonzero_return + window_size + 1 if from_index is None else X.index.to_list().index(from_index) +def walk_forward_train( + model: Model, + X: XDataFrame, + y: ySeries, + forward_returns: ForwardReturnSeries, + expanding_window: bool, + window_size: int, + retrain_every: int, + from_index: Optional[pd.Timestamp], + transformations_over_time: TransformationsOverTime, +) -> ModelOverTime: + models_over_time = pd.Series(index=y.index, dtype="object").rename(model.name) + + first_nonzero_return = max( + get_first_valid_return_index(forward_returns), + get_first_valid_return_index(X.iloc[:, 0]), + get_first_valid_return_index(y), + ) + train_from = ( + first_nonzero_return + window_size + 1 + if from_index is None + else X.index.to_list().index(from_index) + ) train_till = len(y) if model.only_column is not None: X = X[[column for column in X.columns if model.only_column in column]] - - if model.data_transformation == 'original': + + if model.data_transformation == "original": transformations_over_time = [] - + for index in tqdm(range(train_from, train_till, retrain_every)): - train_window_start = X.index[first_nonzero_return] if expanding_window else X.index[index - window_size - 1] + train_window_start = ( + X.index[first_nonzero_return] + if expanding_window + else X.index[index - window_size - 1] + ) train_window_end = X.index[index - 1] - current_transformations = [transformation_over_time[index] for transformation_over_time in transformations_over_time] + current_transformations = [ + transformation_over_time[index] + for transformation_over_time in transformations_over_time + ] X_slice = X[train_window_start:train_window_end] for transformation in current_transformations: X_slice = transformation.transform(X_slice) - + X_slice = X_slice.to_numpy() y_slice = y[train_window_start:train_window_end].to_numpy() @@ -48,4 +64,4 @@ def walk_forward_train( models_over_time[X.index[index]] = current_model - return models_over_time \ No newline at end of file + return models_over_time diff --git a/training/walk_forward/train_parallel.py b/training/walk_forward/train_parallel.py index f6ffee7..da2143c 100644 --- a/training/walk_forward/train_parallel.py +++ b/training/walk_forward/train_parallel.py @@ -9,46 +9,86 @@ import ray from utils.parallel import parallel_compute_with_bar from copy import deepcopy + def walk_forward_train( - model: Model, - X: XDataFrame, - y: ySeries, - forward_returns: ForwardReturnSeries, - expanding_window: bool, - window_size: int, - retrain_every: int, - from_index: Optional[pd.Timestamp], - transformations_over_time: TransformationsOverTime, - ) -> ModelOverTime: + model: Model, + X: XDataFrame, + y: ySeries, + forward_returns: ForwardReturnSeries, + expanding_window: bool, + window_size: int, + retrain_every: int, + from_index: Optional[pd.Timestamp], + transformations_over_time: TransformationsOverTime, +) -> ModelOverTime: models_over_time = pd.Series(index=y.index).rename(model.name) - first_nonzero_return = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y)) - train_from = first_nonzero_return + window_size + 1 if from_index is None else X.index.to_list().index(from_index) + first_nonzero_return = max( + get_first_valid_return_index(forward_returns), + get_first_valid_return_index(X.iloc[:, 0]), + get_first_valid_return_index(y), + ) + train_from = ( + first_nonzero_return + window_size + 1 + if from_index is None + else X.index.to_list().index(from_index) + ) train_till = len(y) if model.only_column is not None: X = X[[column for column in X.columns if model.only_column in column]] - - if model.data_transformation == 'original': + + if model.data_transformation == "original": transformations_over_time = [] - - models = parallel_compute_with_bar([train_on_window.remote(index, first_nonzero_return, window_size, X, y, model, expanding_window, transformations_over_time) for index in tqdm(range(train_from, train_till, retrain_every))]) - for index, current_model in models: + + models = parallel_compute_with_bar( + [ + train_on_window.remote( + index, + first_nonzero_return, + window_size, + X, + y, + model, + expanding_window, + transformations_over_time, + ) + for index in tqdm(range(train_from, train_till, retrain_every)) + ] + ) + for index, current_model in models: models_over_time[X.index[index]] = current_model return models_over_time + @ray.remote -def train_on_window(index: int, first_nonzero_return: int, window_size: int, X: XDataFrame, y: ySeries, model: Model, expanding_window: bool, transformations_over_time: TransformationsOverTime) -> tuple[int, Model]: - train_window_start = X.index[first_nonzero_return] if expanding_window else X.index[index - window_size - 1] +def train_on_window( + index: int, + first_nonzero_return: int, + window_size: int, + X: XDataFrame, + y: ySeries, + model: Model, + expanding_window: bool, + transformations_over_time: TransformationsOverTime, +) -> tuple[int, Model]: + train_window_start = ( + X.index[first_nonzero_return] + if expanding_window + else X.index[index - window_size - 1] + ) train_window_end = X.index[index - 1] - current_transformations = [transformation_over_time[index] for transformation_over_time in transformations_over_time] + current_transformations = [ + transformation_over_time[index] + for transformation_over_time in transformations_over_time + ] X_slice = X[train_window_start:train_window_end] for transformation in current_transformations: X_slice = transformation.transform(X_slice) - + X_slice = X_slice.to_numpy() y_slice = y[train_window_start:train_window_end].to_numpy() diff --git a/transformations/base.py b/transformations/base.py index fd39d68..ee24ee4 100644 --- a/transformations/base.py +++ b/transformations/base.py @@ -3,14 +3,16 @@ from typing import Literal, Optional, Union from abc import ABC, abstractmethod import pandas as pd -class Transformation(ABC): +class Transformation(ABC): @abstractmethod def fit(self, X: pd.DataFrame, y: Optional[pd.Series]) -> None: raise NotImplementedError @abstractmethod - def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> pd.DataFrame: + def fit_transform( + self, X: pd.DataFrame, y: Optional[pd.Series] = None + ) -> pd.DataFrame: raise NotImplementedError @abstractmethod @@ -24,5 +26,3 @@ class Transformation(ABC): @abstractmethod def get_name(self) -> str: raise NotImplementedError - - diff --git a/transformations/pca.py b/transformations/pca.py index e358802..5589631 100644 --- a/transformations/pca.py +++ b/transformations/pca.py @@ -5,6 +5,7 @@ from copy import deepcopy from sklearn.decomposition import PCA import pandas as pd + class PCATransformation(Transformation): pca: PCA @@ -14,16 +15,23 @@ class PCATransformation(Transformation): self.sliding_window_size = sliding_window_size def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> None: - self.pca = PCA(n_components = min(int(len(X.columns) * self.ratio_components_to_keep), self.sliding_window_size)) + self.pca = PCA( + n_components=min( + int(len(X.columns) * self.ratio_components_to_keep), + self.sliding_window_size, + ) + ) self.pca.fit(X, y) - - def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> pd.DataFrame: + + def fit_transform( + self, X: pd.DataFrame, y: Optional[pd.Series] = None + ) -> pd.DataFrame: self.fit(X, y) return self.transform(X) def transform(self, X: pd.DataFrame) -> pd.DataFrame: - X = pd.DataFrame(self.pca.transform(X), index = X.index) - X.columns = ['PCA_' + str(i) for i in range(1, len(X.columns)+1)] + X = pd.DataFrame(self.pca.transform(X), index=X.index) + X.columns = ["PCA_" + str(i) for i in range(1, len(X.columns) + 1)] return X def clone(self) -> PCATransformation: @@ -31,9 +39,3 @@ class PCATransformation(Transformation): def get_name(self) -> str: return "PCA" - - - - - - diff --git a/transformations/rfe.py b/transformations/rfe.py index cc5cbb3..e3a233a 100644 --- a/transformations/rfe.py +++ b/transformations/rfe.py @@ -6,37 +6,37 @@ from sklearn.feature_selection import RFE import pandas as pd from models.sklearn import SKLearnModel + class RFETransformation(Transformation): rfe: RFE n_feature_to_select: int - def __init__(self, n_feature_to_select: int, model: SKLearnModel, step = 0.1): + def __init__(self, n_feature_to_select: int, model: SKLearnModel, step=0.1): self.n_feature_to_keep = n_feature_to_select self.model = model - self.rfe = RFE(model, n_features_to_select= n_feature_to_select, step=step) + self.rfe = RFE(model, n_features_to_select=n_feature_to_select, step=step) def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> None: - if self.rfe is None: return + if self.rfe is None: + return self.rfe.fit(X, y) - def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> pd.DataFrame: - if self.rfe is None: return X + def fit_transform( + self, X: pd.DataFrame, y: Optional[pd.Series] = None + ) -> pd.DataFrame: + if self.rfe is None: + return X self.fit(X, y) return self.transform(X) def transform(self, X: pd.DataFrame) -> pd.DataFrame: - if self.rfe is None: return X - return pd.DataFrame(X[X.columns[self.rfe.support_]], index= X.index) + if self.rfe is None: + return X + return pd.DataFrame(X[X.columns[self.rfe.support_]], index=X.index) def clone(self) -> RFETransformation: return deepcopy(self) def get_name(self) -> str: return "RFE" - - - - - - diff --git a/transformations/scaler.py b/transformations/scaler.py index 72f64f8..3069905 100644 --- a/transformations/scaler.py +++ b/transformations/scaler.py @@ -2,14 +2,15 @@ from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler from .sklearn import SKLearnTransformation from typing import Literal -ScalerTypes = Literal['normalize', 'minmax', 'standardize'] +ScalerTypes = Literal["normalize", "minmax", "standardize"] + def get_scaler(type: ScalerTypes) -> SKLearnTransformation: - if type == 'normalize': + if type == "normalize": return SKLearnTransformation(Normalizer()) - elif type == 'minmax': - return SKLearnTransformation(MinMaxScaler(feature_range= (-1, 1))) - elif type == 'standardize': + elif type == "minmax": + return SKLearnTransformation(MinMaxScaler(feature_range=(-1, 1))) + elif type == "standardize": return SKLearnTransformation(StandardScaler()) else: - raise Exception("Scaler type not supported") \ No newline at end of file + raise Exception("Scaler type not supported") diff --git a/transformations/sklearn.py b/transformations/sklearn.py index 7940218..b59045a 100644 --- a/transformations/sklearn.py +++ b/transformations/sklearn.py @@ -4,6 +4,7 @@ from typing import Literal, Optional, Union from sklearn.base import clone, BaseEstimator import pandas as pd + class SKLearnTransformation(Transformation): transformer: BaseEstimator @@ -13,22 +14,18 @@ class SKLearnTransformation(Transformation): def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> None: self.transformer.fit(X, y) - + def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series]) -> pd.DataFrame: self.fit(X, y) return self.transform(X) def transform(self, X: pd.DataFrame) -> pd.DataFrame: - return pd.DataFrame(self.transformer.transform(X), index = X.index, columns = X.columns) + return pd.DataFrame( + self.transformer.transform(X), index=X.index, columns=X.columns + ) def clone(self) -> SKLearnTransformation: return SKLearnTransformation(clone(self.transformer)) def get_name(self) -> str: return self.transformer.__class__.__name__ - - - - - - diff --git a/utils/evaluate.py b/utils/evaluate.py index ddee5a2..22ec5a6 100644 --- a/utils/evaluate.py +++ b/utils/evaluate.py @@ -8,95 +8,125 @@ import numpy as np from data_loader.types import ForwardReturnSeries, ySeries from training.types import Stats, WeightsSeries -def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.002) -> pd.Series: - delta_pos = signal.diff(1).abs().fillna(0.) + +def backtest( + returns: pd.Series, signal: pd.Series, transaction_cost=0.002 +) -> pd.Series: + delta_pos = signal.diff(1).abs().fillna(0.0) costs = transaction_cost * delta_pos return (signal * returns) - costs -def __preprocess(forward_returns: ForwardReturnSeries, y_pred: pd.Series, y_true: pd.Series, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame: - y_pred.name = 'y_pred' - forward_returns.name = 'forward_returns' - df = pd.concat([y_pred, forward_returns],axis=1).dropna() +def __preprocess( + forward_returns: ForwardReturnSeries, + y_pred: pd.Series, + y_true: pd.Series, + no_of_classes: Literal["two", "three-balanced", "three-imbalanced"], + discretize: bool, +) -> pd.DataFrame: + y_pred.name = "y_pred" + forward_returns.name = "forward_returns" + df = pd.concat([y_pred, forward_returns], axis=1).dropna() discretize_func = get_discretize_function(no_of_classes) # make sure that we evaluate binary/three-way predictions even if the model is a regression - df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred - df['sign_true'] = y_true + df["sign_pred"] = df.y_pred.apply(discretize_func) if discretize else df.y_pred + df["sign_true"] = y_true - df['result'] = backtest(df.forward_returns, df.sign_pred) + df["result"] = backtest(df.forward_returns, df.sign_pred) return df + def evaluate_predictions( - forward_returns: ForwardReturnSeries, - y_pred: WeightsSeries, - y_true: ySeries, - no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], - discretize: bool = False, - ) -> Stats: + forward_returns: ForwardReturnSeries, + y_pred: WeightsSeries, + y_true: ySeries, + no_of_classes: Literal["two", "three-balanced", "three-imbalanced"], + discretize: bool = False, +) -> Stats: # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size) - evaluate_from = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(y_pred)) - + evaluate_from = max( + get_first_valid_return_index(forward_returns), + get_first_valid_return_index(y_pred), + ) + forward_returns = pd.Series(forward_returns[evaluate_from:]) y_pred = pd.Series(y_pred[evaluate_from:]) df = __preprocess(forward_returns, y_pred, y_true, no_of_classes, discretize) - + scorecard = dict() - + def count_non_zero(series: pd.Series) -> int: return len(series[series != 0]) - no_of_samples = count_non_zero(df.y_pred) - scorecard['no_of_samples'] = no_of_samples - sharpe = sharpe_ratio(df.result + 1e-20) - scorecard['sharpe'] = sharpe - benchmark_sharpe = sharpe_ratio(df.forward_returns) - scorecard['benchmark_sharpe'] = benchmark_sharpe - scorecard['prob_sharpe'] = probabilistic_sharpe_ratio(sharpe, benchmark_sharpe, no_of_samples) - scorecard['sortino'] = sortino(df.result) - scorecard['skew'] = skew(df.result) - labels = [1, -1] if no_of_classes == 'two' else [1, -1, 0] - avg_type = 'weighted' if no_of_classes == 'two' else 'macro' + no_of_samples = count_non_zero(df.y_pred) + scorecard["no_of_samples"] = no_of_samples + sharpe = sharpe_ratio(df.result + 1e-20) + scorecard["sharpe"] = sharpe + benchmark_sharpe = sharpe_ratio(df.forward_returns) + scorecard["benchmark_sharpe"] = benchmark_sharpe + scorecard["prob_sharpe"] = probabilistic_sharpe_ratio( + sharpe, benchmark_sharpe, no_of_samples + ) + scorecard["sortino"] = sortino(df.result) + scorecard["skew"] = skew(df.result) + + labels = [1, -1] if no_of_classes == "two" else [1, -1, 0] + avg_type = "weighted" if no_of_classes == "two" else "macro" if discretize == True: - scorecard['accuracy'] = accuracy_score(df.sign_true, df.sign_pred) * 100 - scorecard['recall'] = recall_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type) - scorecard['precision'] = precision_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type) - scorecard['f1_score'] = f1_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type) - scorecard['edge'] = df.result.mean() - scorecard['noise'] = df.y_pred.diff().abs().mean() - scorecard['edge_to_noise'] = scorecard['edge'] / (scorecard['noise'] + 0.00001) - + scorecard["accuracy"] = accuracy_score(df.sign_true, df.sign_pred) * 100 + scorecard["recall"] = recall_score( + df.sign_true, df.sign_pred, labels=labels, average=avg_type + ) + scorecard["precision"] = precision_score( + df.sign_true, df.sign_pred, labels=labels, average=avg_type + ) + scorecard["f1_score"] = f1_score( + df.sign_true, df.sign_pred, labels=labels, average=avg_type + ) + scorecard["edge"] = df.result.mean() + scorecard["noise"] = df.y_pred.diff().abs().mean() + scorecard["edge_to_noise"] = scorecard["edge"] / (scorecard["noise"] + 0.00001) + if discretize == True: for index, row in df.sign_true.value_counts().iteritems(): - scorecard['sign_true_ratio_' + str(index)] = row / len(df.sign_true) - + scorecard["sign_true_ratio_" + str(index)] = row / len(df.sign_true) + for index, row in df.sign_pred.value_counts().iteritems(): - scorecard['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred) + scorecard["sign_pred_ratio_" + str(index)] = row / len(df.sign_pred) scorecard = {k: round(float(v), 3) for k, v in scorecard.items()} - return scorecard + return scorecard + + +def __discretize_binary(x): + return 1 if x > 0 else -1 + + +def __discretize_threeway(x): + return 0 if x == 0 else 1 if x > 0 else -1 -def __discretize_binary(x): return 1 if x > 0 else -1 -def __discretize_threeway(x): return 0 if x == 0 else 1 if x > 0 else -1 def discretize_threeway_threshold(threshold: float) -> Callable: def discretize(current_value): lower_threshold = -threshold upper_threshold = threshold if np.isnan(current_value): - return np.nan + return np.nan elif current_value <= lower_threshold: return -1 elif current_value > lower_threshold and current_value < upper_threshold: return 0 else: return 1 + return discretize -def get_discretize_function(no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']) -> Callable: - return __discretize_binary if no_of_classes == 'two' else __discretize_threeway - +def get_discretize_function( + no_of_classes: Literal["two", "three-balanced", "three-imbalanced"] +) -> Callable: + return __discretize_binary if no_of_classes == "two" else __discretize_threeway diff --git a/utils/glassnode/blockchain.py b/utils/glassnode/blockchain.py index 4791ca9..f8a754c 100644 --- a/utils/glassnode/blockchain.py +++ b/utils/glassnode/blockchain.py @@ -4,49 +4,50 @@ from .client import GlassnodeClient class Blockchain: """ - Blockchain class. + Blockchain class. - Methods - ------- - __init__(glassnode_client): - Constructs a Blockchain object. - utxos_total(): - Returns the total number of UTXOs in the network. - utxos_created(): - Returns the number of created unspent transaction outputs. - utxos_spent(): - Returns the number of spent transaction outputs. - utxo_value_created_total(): - Returns the total amount of coins in newly created UTXOs. - utxo_value_spent_total(): - Returns the total amount of coins in spent transaction outputs. - utxo_value_created_mean(): - Returns the mean amount of coins in newly created UTXOs. - utxo_value_spent_mean(): - Returns the mean amount of coins in spent transaction outputs. - utxo_value_created_median(): - Returns the median amount of coins in newly created UTXOs. - utxo_value_spent_median(): - Returns the median amount of coins in spent transaction outputs. - utxos_in_profit(): - Returns the number of unspent transaction outputs in profit. - utxos_in_loss(): - Returns the number of unspent transaction outputs in loss. - percent_utxos_in_profit(): - Returns the percentage of unspent transaction outputs in profit. - block_heights(): - Returns the block height. - blocks_mined(): - Returns the number of blocks mined. - block_interval_mean(): - Returns the mean time (in seconds) between mined blocks. - block_interval_median(): - Returns the median time (in seconds) between mined blocks. - block_size_mean(): - Returns the mean size of all blocks created within the time period (in bytes). - block_size_total(): - Returns the total size of all blocks created within the time period (in bytes). + Methods + ------- + __init__(glassnode_client): + Constructs a Blockchain object. + utxos_total(): + Returns the total number of UTXOs in the network. + utxos_created(): + Returns the number of created unspent transaction outputs. + utxos_spent(): + Returns the number of spent transaction outputs. + utxo_value_created_total(): + Returns the total amount of coins in newly created UTXOs. + utxo_value_spent_total(): + Returns the total amount of coins in spent transaction outputs. + utxo_value_created_mean(): + Returns the mean amount of coins in newly created UTXOs. + utxo_value_spent_mean(): + Returns the mean amount of coins in spent transaction outputs. + utxo_value_created_median(): + Returns the median amount of coins in newly created UTXOs. + utxo_value_spent_median(): + Returns the median amount of coins in spent transaction outputs. + utxos_in_profit(): + Returns the number of unspent transaction outputs in profit. + utxos_in_loss(): + Returns the number of unspent transaction outputs in loss. + percent_utxos_in_profit(): + Returns the percentage of unspent transaction outputs in profit. + block_heights(): + Returns the block height. + blocks_mined(): + Returns the number of blocks mined. + block_interval_mean(): + Returns the mean time (in seconds) between mined blocks. + block_interval_median(): + Returns the median time (in seconds) between mined blocks. + block_size_mean(): + Returns the mean size of all blocks created within the time period (in bytes). + block_size_total(): + Returns the total size of all blocks created within the time period (in bytes). """ + def __init__(self, glassnode_client: GlassnodeClient): self._gc = glassnode_client @@ -57,7 +58,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_count' + endpoint = "/v1/metrics/blockchain/utxo_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -70,7 +71,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_created_count' + endpoint = "/v1/metrics/blockchain/utxo_created_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -83,7 +84,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_spent_count' + endpoint = "/v1/metrics/blockchain/utxo_spent_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -96,7 +97,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_created_value_sum' + endpoint = "/v1/metrics/blockchain/utxo_created_value_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -109,7 +110,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_spent_value_sum' + endpoint = "/v1/metrics/blockchain/utxo_spent_value_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -122,7 +123,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_created_value_mean' + endpoint = "/v1/metrics/blockchain/utxo_created_value_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -135,7 +136,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_spent_value_mean' + endpoint = "/v1/metrics/blockchain/utxo_spent_value_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -148,7 +149,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_created_value_median' + endpoint = "/v1/metrics/blockchain/utxo_created_value_median" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -161,7 +162,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_spent_value_median' + endpoint = "/v1/metrics/blockchain/utxo_spent_value_median" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -174,7 +175,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_profit_count' + endpoint = "/v1/metrics/blockchain/utxo_profit_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -187,7 +188,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_loss_count' + endpoint = "/v1/metrics/blockchain/utxo_loss_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -200,7 +201,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/utxo_profit_relative' + endpoint = "/v1/metrics/blockchain/utxo_profit_relative" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -213,7 +214,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/block_height' + endpoint = "/v1/metrics/blockchain/block_height" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -226,7 +227,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/block_count' + endpoint = "/v1/metrics/blockchain/block_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -239,7 +240,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/block_interval_mean' + endpoint = "/v1/metrics/blockchain/block_interval_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -252,7 +253,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/block_interval_median' + endpoint = "/v1/metrics/blockchain/block_interval_median" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -265,7 +266,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/block_size_mean' + endpoint = "/v1/metrics/blockchain/block_size_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -278,7 +279,7 @@ class Blockchain: :return: DataFrame """ - endpoint = '/v1/metrics/blockchain/block_size_sum' + endpoint = "/v1/metrics/blockchain/block_size_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() diff --git a/utils/glassnode/client.py b/utils/glassnode/client.py index fbb48a6..df1218d 100644 --- a/utils/glassnode/client.py +++ b/utils/glassnode/client.py @@ -6,13 +6,13 @@ from .endpoints import Endpoints class GlassnodeClient: def __init__( - self, - api_key=None, - asset='BTC', - resolution='24h', - currency='native', - since=None, - until=None + self, + api_key=None, + asset="BTC", + resolution="24h", + currency="native", + since=None, + until=None, ): """ Glassnode API client. @@ -25,11 +25,11 @@ class GlassnodeClient: """ if api_key: self._api_key = api_key - elif 'GLASSNODE_API_KEY' in os.environ: - self._api_key = os.environ.get('GLASSNODE_API_KEY') + elif "GLASSNODE_API_KEY" in os.environ: + self._api_key = os.environ.get("GLASSNODE_API_KEY") else: # API key is required for every endpoint! - print(f'\033[91m ERROR: Glassnode API key required!\033[0m') + print(f"\033[91m ERROR: Glassnode API key required!\033[0m") sys.exit() self.endpoints = Endpoints() @@ -53,27 +53,27 @@ class GlassnodeClient: def __prepare_request_params(self, params): p = dict() - p['api_key'] = self._api_key - p['a'] = self._asset - p['i'] = self._resolution + p["api_key"] = self._api_key + p["a"] = self._asset + p["i"] = self._resolution if self._since is not None: try: - p['s'] = unix_timestamp(self._since) + p["s"] = unix_timestamp(self._since) except Exception as e: print(e) if self._until is not None: try: - p['u'] = unix_timestamp(self._until) + p["u"] = unix_timestamp(self._until) except Exception as e: print(e) # Set domain specific query parameters if available if params: - if 'e' in p: - p['e'] = params['e'] - if 'm' in p: - p['miner'] = params['m'] + if "e" in p: + p["e"] = params["e"] + if "m" in p: + p["miner"] = params["m"] return p diff --git a/utils/glassnode/derivatives.py b/utils/glassnode/derivatives.py index 2bfd38e..98ccc7d 100644 --- a/utils/glassnode/derivatives.py +++ b/utils/glassnode/derivatives.py @@ -4,45 +4,46 @@ from .client import GlassnodeClient class Derivatives: """ - Derivatives class. + Derivatives class. - Methods - ------- - __init__(glassnode_client): - Constructs a Derivatives object. - futures_perpetual_funding_rate([exchange]): - Returns the average funding rate (in %) set by exchanges for perpetual futures contracts. - futures_perpetual_funding_rate_all(): - Returns the average funding rate (in %) set by exchanges for perpetual futures contracts. - futures_volume([exchange]): - Returns the total volume traded in futures contracts in the last 24 hours. - futures_volume_latest_24h(): - Returns the total volume traded in futures contracts per exchange over the last 24 hours. - futures_volume_stacked(): - Returns the total volume traded in futures contracts in the last 24 hours. - futures_volume_perpetual([exchange]): - Returns The total volume traded in perpetual futures contracts in the last 24 hours. - futures_volume_perpetual_stacked(): - Returns the total volume traded in perpetual futures contracts in the last 24 hours. - futures_open_interest([exchange]): - Returns the total amount of funds allocated in open futures contracts. - futures_open_interest_current(): - Returns the current amount of allocated funds in futures contracts per exchange. - futures_open_interest_perpetual([exchange]): - Returns the total amount of funds allocated in open perpetual futures contracts. - futures_open_interest_perpetual_stacked(): - Returns the total amount of funds allocated in open perpetual futures contracts. - futures_open_interest_stacked(): - Returns the total amount of funds allocated in open futures contracts. - futures_long_liquidations([exchange]): - Returns the sum liquidated volume from long positions in futures contracts. - futures_long_liquidations_mean([exchange]): - Returns the mean liquidated volume from long positions in futures contracts. - futures_short_liquidations([exchange]): - Returns the sum liquidated volume from short positions in futures contracts. - futures_short_liquidations_mean([exchange]): - Returns the mean liquidated volume from short positions in futures contracts. + Methods + ------- + __init__(glassnode_client): + Constructs a Derivatives object. + futures_perpetual_funding_rate([exchange]): + Returns the average funding rate (in %) set by exchanges for perpetual futures contracts. + futures_perpetual_funding_rate_all(): + Returns the average funding rate (in %) set by exchanges for perpetual futures contracts. + futures_volume([exchange]): + Returns the total volume traded in futures contracts in the last 24 hours. + futures_volume_latest_24h(): + Returns the total volume traded in futures contracts per exchange over the last 24 hours. + futures_volume_stacked(): + Returns the total volume traded in futures contracts in the last 24 hours. + futures_volume_perpetual([exchange]): + Returns The total volume traded in perpetual futures contracts in the last 24 hours. + futures_volume_perpetual_stacked(): + Returns the total volume traded in perpetual futures contracts in the last 24 hours. + futures_open_interest([exchange]): + Returns the total amount of funds allocated in open futures contracts. + futures_open_interest_current(): + Returns the current amount of allocated funds in futures contracts per exchange. + futures_open_interest_perpetual([exchange]): + Returns the total amount of funds allocated in open perpetual futures contracts. + futures_open_interest_perpetual_stacked(): + Returns the total amount of funds allocated in open perpetual futures contracts. + futures_open_interest_stacked(): + Returns the total amount of funds allocated in open futures contracts. + futures_long_liquidations([exchange]): + Returns the sum liquidated volume from long positions in futures contracts. + futures_long_liquidations_mean([exchange]): + Returns the mean liquidated volume from long positions in futures contracts. + futures_short_liquidations([exchange]): + Returns the sum liquidated volume from short positions in futures contracts. + futures_short_liquidations_mean([exchange]): + Returns the mean liquidated volume from short positions in futures contracts. """ + def __init__(self, glassnode_client: GlassnodeClient): self._gc = glassnode_client @@ -51,11 +52,11 @@ class Derivatives: The average funding rate (in %) set by exchanges for perpetual futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_funding_rate_perpetual' + endpoint = "/v1/metrics/derivatives/futures_funding_rate_perpetual" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) @dataframe_with_inner_object def futures_perpetual_funding_rate_all(self) -> pd.DataFrame: @@ -63,7 +64,7 @@ class Derivatives: The average funding rate (in %) set by exchanges for perpetual futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_funding_rate_perpetual_all' + endpoint = "/v1/metrics/derivatives/futures_funding_rate_perpetual_all" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -74,11 +75,11 @@ class Derivatives: The total volume traded in futures contracts in the last 24 hours. `View in Studio `_ """ - url = '/v1/metrics/derivatives/futures_volume_daily_sum' + url = "/v1/metrics/derivatives/futures_volume_daily_sum" if not is_supported_by_endpoint(self._gc, url): return pd.DataFrame() - return response_to_dataframe(self._gc.get(url, {'e': exchange})) + return response_to_dataframe(self._gc.get(url, {"e": exchange})) # TODO: Unpack inner object from response def futures_volume_latest_24h(self) -> pd.DataFrame: @@ -87,7 +88,7 @@ class Derivatives: Values are updated every 10 min. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_volume_daily_latest' + endpoint = "/v1/metrics/derivatives/futures_volume_daily_latest" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -99,7 +100,7 @@ class Derivatives: The total volume traded in futures contracts in the last 24 hours. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_volume_daily_sum_all' + endpoint = "/v1/metrics/derivatives/futures_volume_daily_sum_all" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -110,11 +111,11 @@ class Derivatives: The total volume traded in perpetual (non-expiring) futures contracts in the last 24 hours. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_volume_daily_perpetual_sum' + endpoint = "/v1/metrics/derivatives/futures_volume_daily_perpetual_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) @dataframe_with_inner_object def futures_volume_perpetual_stacked(self) -> pd.DataFrame: @@ -122,7 +123,7 @@ class Derivatives: The total volume traded in perpetual (non-expiring) futures contracts in the last 24 hours. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_volume_daily_perpetual_sum_all' + endpoint = "/v1/metrics/derivatives/futures_volume_daily_perpetual_sum_all" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -133,11 +134,11 @@ class Derivatives: The total amount of funds allocated in open futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_open_interest_sum' + endpoint = "/v1/metrics/derivatives/futures_open_interest_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) # TODO: Unpack inner object from response def futures_open_interest_current(self) -> pd.DataFrame: @@ -145,7 +146,7 @@ class Derivatives: The current amount of allocated funds in futures contracts per exchange.Values are updated every 10 min. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_open_interest_latest' + endpoint = "/v1/metrics/derivatives/futures_open_interest_latest" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -156,11 +157,11 @@ class Derivatives: The total amount of funds allocated in open perpetual (non-expiring) futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_open_interest_perpetual_sum' + endpoint = "/v1/metrics/derivatives/futures_open_interest_perpetual_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) @dataframe_with_inner_object def futures_open_interest_perpetual_stacked(self) -> pd.DataFrame: @@ -168,7 +169,7 @@ class Derivatives: The total amount of funds allocated in open perpetual (non-expiring) futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_open_interest_perpetual_sum_all' + endpoint = "/v1/metrics/derivatives/futures_open_interest_perpetual_sum_all" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -180,7 +181,7 @@ class Derivatives: The total amount of funds allocated in open futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_open_interest_sum_all' + endpoint = "/v1/metrics/derivatives/futures_open_interest_sum_all" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -191,49 +192,49 @@ class Derivatives: The sum liquidated volume from long positions in futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_liquidated_volume_long_sum' + endpoint = "/v1/metrics/derivatives/futures_liquidated_volume_long_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) def futures_long_liquidations_mean(self, exchange: str = None) -> pd.DataFrame: """ The mean liquidated volume from long positions in futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_liquidated_volume_long_mean' + endpoint = "/v1/metrics/derivatives/futures_liquidated_volume_long_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) def futures_short_liquidations(self, exchange: str = None) -> pd.DataFrame: """ The sum liquidated volume from short positions in futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_liquidated_volume_short_sum' + endpoint = "/v1/metrics/derivatives/futures_liquidated_volume_short_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) def futures_short_liquidations_mean(self, exchange: str = None) -> pd.DataFrame: """ The mean liquidated volume from short positions in futures contracts. `View in Studio `_ """ - endpoint = '/v1/metrics/derivatives/futures_liquidated_volume_short_mean' + endpoint = "/v1/metrics/derivatives/futures_liquidated_volume_short_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) def futures_estimated_leverage_ratio(self, exchange: str = None) -> pd.DataFrame: - endpoint = '/v1/metrics/derivatives/futures_estimated_leverage_ratio' + endpoint = "/v1/metrics/derivatives/futures_estimated_leverage_ratio" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) diff --git a/utils/glassnode/distribution.py b/utils/glassnode/distribution.py index 06eb12e..a893e61 100644 --- a/utils/glassnode/distribution.py +++ b/utils/glassnode/distribution.py @@ -3,33 +3,34 @@ from .utils import * class Distribution: """ - Distribution class. + Distribution class. - Methods - ------- - __init__(glassnode_client): - Constructs a Distribution object. - exchange_balance_total(exchange): - Returns the total amount of coins held on exchange addresses. - exchange_balance_percent(exchange): - Returns the percent supply held on exchange addresses. - exchange_balance_stacked(): - Returns the total amount of coins held on exchange addresses. - miner_balance(): - Returns the total supply held in miner addresses. - miner_balance_stacked(): - Returns the total supply held in miner addresses. - balance_miners_change(): - Returns 30d change of the supply held in miner addresses. - supply_top_one_pct_addresses(): - Returns the percentage of supply held by the top 1% addresses. - gini_coefficient(): - Returns gini coefficient data. - herfindahl_index(): - Returns herfindahl index data. - supply_in_smart_contracts(): - Returns percent of total supply that is held in smart contracts. + Methods + ------- + __init__(glassnode_client): + Constructs a Distribution object. + exchange_balance_total(exchange): + Returns the total amount of coins held on exchange addresses. + exchange_balance_percent(exchange): + Returns the percent supply held on exchange addresses. + exchange_balance_stacked(): + Returns the total amount of coins held on exchange addresses. + miner_balance(): + Returns the total supply held in miner addresses. + miner_balance_stacked(): + Returns the total supply held in miner addresses. + balance_miners_change(): + Returns 30d change of the supply held in miner addresses. + supply_top_one_pct_addresses(): + Returns the percentage of supply held by the top 1% addresses. + gini_coefficient(): + Returns gini coefficient data. + herfindahl_index(): + Returns herfindahl index data. + supply_in_smart_contracts(): + Returns percent of total supply that is held in smart contracts. """ + def __init__(self, glassnode_client): self._gc = glassnode_client @@ -41,11 +42,11 @@ class Distribution: :return: A DataFrame with exchange balance data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/balance_exchanges' + endpoint = "/v1/metrics/distribution/balance_exchanges" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) def exchange_balance_percent(self, exchange=None) -> pd.DataFrame: """ @@ -55,11 +56,11 @@ class Distribution: :return: A DataFrame with exchange balance data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/balance_exchanges_relative' + endpoint = "/v1/metrics/distribution/balance_exchanges_relative" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'e': exchange})) + return response_to_dataframe(self._gc.get(endpoint, {"e": exchange})) def exchange_balance_stacked(self) -> pd.DataFrame: """ @@ -69,7 +70,7 @@ class Distribution: :return: A DataFrame with stacked exchange balance data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/balance_exchanges_all' + endpoint = "/v1/metrics/distribution/balance_exchanges_all" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -83,7 +84,7 @@ class Distribution: :return: A DataFrame miner balance data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/balance_miners_sum' + endpoint = "/v1/metrics/distribution/balance_miners_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -97,7 +98,7 @@ class Distribution: :return: A DataFrame with stacked miner balance data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/balance_miners_all' + endpoint = "/v1/metrics/distribution/balance_miners_all" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -105,13 +106,13 @@ class Distribution: def balance_miners_change(self) -> pd.DataFrame: """ - The 30d change of the supply held in miner addresses. + The 30d change of the supply held in miner addresses. `View in Studio `_ :return: A DataFrame with 30d change of the supply held in miner addresses. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/balance_miners_change' + endpoint = "/v1/metrics/distribution/balance_miners_change" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -125,7 +126,7 @@ class Distribution: :return: A DataFrame with top 1% supply data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/balance_1pct_holders' + endpoint = "/v1/metrics/distribution/balance_1pct_holders" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -139,7 +140,7 @@ class Distribution: :return: A DataFrame Gini Coefficient data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/gini' + endpoint = "/v1/metrics/distribution/gini" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -153,7 +154,7 @@ class Distribution: :return: A DataFrame Herfindahl index data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/herfindahl' + endpoint = "/v1/metrics/distribution/herfindahl" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -167,7 +168,7 @@ class Distribution: :return: A DataFrame smart contracts supply data. :rtype: DataFrame """ - endpoint = '/v1/metrics/distribution/supply_contracts' + endpoint = "/v1/metrics/distribution/supply_contracts" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() diff --git a/utils/glassnode/endpoints.py b/utils/glassnode/endpoints.py index 65bba1c..ff10466 100644 --- a/utils/glassnode/endpoints.py +++ b/utils/glassnode/endpoints.py @@ -3,11 +3,11 @@ from .utils import fetch def create_endpoints_dict(endpoints): return { - endpoint['path']: { - 'assets': {asset['symbol']: asset['tags'] for asset in endpoint['assets']}, - 'currencies': endpoint['currencies'], - 'resolutions': endpoint['resolutions'], - 'formats': endpoint['formats'] + endpoint["path"]: { + "assets": {asset["symbol"]: asset["tags"] for asset in endpoint["assets"]}, + "currencies": endpoint["currencies"], + "resolutions": endpoint["resolutions"], + "formats": endpoint["formats"], } for endpoint in endpoints } @@ -37,7 +37,9 @@ class Endpoints(metaclass=MetaEndpoints): @endpoints.setter def endpoints(self, api_key): - self._endpoints = create_endpoints_dict(fetch('/v2/metrics/endpoints', {'api_key': api_key})) + self._endpoints = create_endpoints_dict( + fetch("/v2/metrics/endpoints", {"api_key": api_key}) + ) def query(self, path): return self._endpoints[path] diff --git a/utils/glassnode/entities.py b/utils/glassnode/entities.py index 94ad366..b855be8 100644 --- a/utils/glassnode/entities.py +++ b/utils/glassnode/entities.py @@ -4,49 +4,50 @@ from .client import GlassnodeClient class Entities: """ - Entities class. + Entities class. - Methods - ------- - __init__(glassnode_client): - Constructs a Entities object. - sending_entities(): - Returns the number of unique entities that were active as a sender. - receiving_entities(): - Returns the number of unique entities that were active as a receiver. - active_entities(): - Returns the number of unique entities that were active either as a sender or receiver. - new_entities(): - Returns The number of unique entities that appeared for the first time in a transaction. - entities_net_growth(): - Returns the net growth of unique entities in the network. - number_of_whales(): - Returns the number of unique entities holding at least 1k coins. - supply_balance_less_0001(): - Returns the total circulating supply held by entities with a balance lower than 0.001 coins. - supply_balance_0001_001(): - Returns the total circulating supply held by entities with a balance between 0.001 and 0.01 coins. - supply_balance_001_01(): - Returns the total circulating supply held by entities with a balance between 0.01 and 0.1 coins. - supply_balance_01_1(): - Returns the total circulating supply held by entities with a balance between 0.1 and 1 coins. - supply_balance_1_10(): - Returns the total circulating supply held by entities with a balance between 1 and 10 coins. - supply_balance_10_100(): - Returns the total circulating supply held by entities with a balance between 10 and 100 coins. - supply_balance_100_1k(): - Returns the total circulating supply held by entities with a balance between 100 and 1,000 coins. - supply_balance_1k_10k(): - Returns the total circulating supply held by entities with a balance between 1,000 and 10,000 coins. - supply_balance_10k_100k(): - Returns the total circulating supply held by entities with a balance between 10,000 and 100,000 coins. - supply_balance_more_100k(): - Returns the total circulating supply held by entities with a balance of at least 100,000 coins. - entities_supply_distribution(): - Returns relative distribution of the circulating supply held by entities with specific balance bands. - percent_entities_in_profit(): - Returns the percentage of entities in the network that are currently in profit. + Methods + ------- + __init__(glassnode_client): + Constructs a Entities object. + sending_entities(): + Returns the number of unique entities that were active as a sender. + receiving_entities(): + Returns the number of unique entities that were active as a receiver. + active_entities(): + Returns the number of unique entities that were active either as a sender or receiver. + new_entities(): + Returns The number of unique entities that appeared for the first time in a transaction. + entities_net_growth(): + Returns the net growth of unique entities in the network. + number_of_whales(): + Returns the number of unique entities holding at least 1k coins. + supply_balance_less_0001(): + Returns the total circulating supply held by entities with a balance lower than 0.001 coins. + supply_balance_0001_001(): + Returns the total circulating supply held by entities with a balance between 0.001 and 0.01 coins. + supply_balance_001_01(): + Returns the total circulating supply held by entities with a balance between 0.01 and 0.1 coins. + supply_balance_01_1(): + Returns the total circulating supply held by entities with a balance between 0.1 and 1 coins. + supply_balance_1_10(): + Returns the total circulating supply held by entities with a balance between 1 and 10 coins. + supply_balance_10_100(): + Returns the total circulating supply held by entities with a balance between 10 and 100 coins. + supply_balance_100_1k(): + Returns the total circulating supply held by entities with a balance between 100 and 1,000 coins. + supply_balance_1k_10k(): + Returns the total circulating supply held by entities with a balance between 1,000 and 10,000 coins. + supply_balance_10k_100k(): + Returns the total circulating supply held by entities with a balance between 10,000 and 100,000 coins. + supply_balance_more_100k(): + Returns the total circulating supply held by entities with a balance of at least 100,000 coins. + entities_supply_distribution(): + Returns relative distribution of the circulating supply held by entities with specific balance bands. + percent_entities_in_profit(): + Returns the percentage of entities in the network that are currently in profit. """ + def __init__(self, glassnode_client: GlassnodeClient): self._gc = glassnode_client @@ -55,7 +56,7 @@ class Entities: The number of unique entities that were active as a sender. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/sending_count' + endpoint = "/v1/metrics/entities/sending_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -66,7 +67,7 @@ class Entities: The number of unique entities that were active as a receiver. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/receiving_count' + endpoint = "/v1/metrics/entities/receiving_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -77,7 +78,7 @@ class Entities: The number of unique entities that were active either as a sender or receiver. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/active_count' + endpoint = "/v1/metrics/entities/active_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -89,7 +90,7 @@ class Entities: in a transaction of the native coin in the network. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/new_count' + endpoint = "/v1/metrics/entities/new_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -100,7 +101,7 @@ class Entities: The net growth of unique entities in the network. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/net_growth_count' + endpoint = "/v1/metrics/entities/net_growth_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -111,7 +112,7 @@ class Entities: The number of unique entities holding at least 1k coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/min_1k_count' + endpoint = "/v1/metrics/entities/min_1k_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -122,7 +123,7 @@ class Entities: The total circulating supply held by entities with a balance lower than 0.001 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_less_0001' + endpoint = "/v1/metrics/entities/supply_balance_less_0001" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -133,7 +134,7 @@ class Entities: The total circulating supply held by entities with a balance between 0.001 and 0.01 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_0001_001' + endpoint = "/v1/metrics/entities/supply_balance_0001_001" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -144,7 +145,7 @@ class Entities: The total circulating supply held by entities with a balance between 0.01 and 0.1 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_001_01' + endpoint = "/v1/metrics/entities/supply_balance_001_01" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -155,7 +156,7 @@ class Entities: The total circulating supply held by entities with a balance between 0.1 and 1 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_01_1' + endpoint = "/v1/metrics/entities/supply_balance_01_1" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -166,7 +167,7 @@ class Entities: The total circulating supply held by entities with a balance between 1 and 10 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_1_10' + endpoint = "/v1/metrics/entities/supply_balance_1_10" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -177,7 +178,7 @@ class Entities: The total circulating supply held by entities with a balance between 10 and 100 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_10_100' + endpoint = "/v1/metrics/entities/supply_balance_10_100" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -188,7 +189,7 @@ class Entities: The total circulating supply held by entities with a balance between 100 and 1,000 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_100_1k' + endpoint = "/v1/metrics/entities/supply_balance_100_1k" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -199,7 +200,7 @@ class Entities: The total circulating supply held by entities with a balance between 1,000 and 10,000 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_1k_10k' + endpoint = "/v1/metrics/entities/supply_balance_1k_10k" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -210,7 +211,7 @@ class Entities: The total circulating supply held by entities with a balance between 10,000 and 100,000 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_10k_100k' + endpoint = "/v1/metrics/entities/supply_balance_10k_100k" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -221,7 +222,7 @@ class Entities: The total circulating supply held by entities with a balance of at least 100,000 coins. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_balance_more_100k' + endpoint = "/v1/metrics/entities/supply_balance_more_100k" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -233,7 +234,7 @@ class Entities: Relative distribution of the circulating supply held by entities with specific balance bands. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/supply_distribution_relative' + endpoint = "/v1/metrics/entities/supply_distribution_relative" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -244,8 +245,8 @@ class Entities: The percentage of entities in the network that are currently in profit. `View in Studio `_ """ - endpoint = '/v1/metrics/entities/profit_relative' + endpoint = "/v1/metrics/entities/profit_relative" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint)) \ No newline at end of file + return response_to_dataframe(self._gc.get(endpoint)) diff --git a/utils/glassnode/eth2.py b/utils/glassnode/eth2.py index 1ba99f8..0752d88 100644 --- a/utils/glassnode/eth2.py +++ b/utils/glassnode/eth2.py @@ -3,27 +3,28 @@ from .utils import * class ETH2: """ - ETH 2.0 class. + ETH 2.0 class. - Methods - ------- - __init__(glassnode_client): - Constructs an ETH2 object. - new_deposits(): - Returns the number transactions depositing 32 ETH to the ETH2 deposit contract. - new_value_staked(): - Returns the amount of ETH transferred to the ETH2 deposit contract. - new_validators(): - Returns the number of new validators depositing 32 ETH to the ETH2 deposit contract. - total_number_of_deposits(): - Returns the total number of transactions to the ETH2 deposit contract. - total_value_staked(): - Returns the amount of ETH deposited to the ETH2 deposit contract. - total_number_of_validators(): - Returns the total number of unique validators. - phase_zero_staking_goal(): - Returns the percentage of the Phase 0 staking goal. + Methods + ------- + __init__(glassnode_client): + Constructs an ETH2 object. + new_deposits(): + Returns the number transactions depositing 32 ETH to the ETH2 deposit contract. + new_value_staked(): + Returns the amount of ETH transferred to the ETH2 deposit contract. + new_validators(): + Returns the number of new validators depositing 32 ETH to the ETH2 deposit contract. + total_number_of_deposits(): + Returns the total number of transactions to the ETH2 deposit contract. + total_value_staked(): + Returns the amount of ETH deposited to the ETH2 deposit contract. + total_number_of_validators(): + Returns the total number of unique validators. + phase_zero_staking_goal(): + Returns the percentage of the Phase 0 staking goal. """ + def __init__(self, glassnode_client): self._gc = glassnode_client @@ -35,7 +36,7 @@ class ETH2: :return: A DataFrame with ETH2 deposit data. :rtype: DataFrame """ - endpoint = '/v1/metrics/eth2/staking_deposits_count' + endpoint = "/v1/metrics/eth2/staking_deposits_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -49,7 +50,7 @@ class ETH2: :return: A DataFrame with staked value data. :rtype: DataFrame """ - endpoint = '/v1/metrics/eth2/staking_volume_sum' + endpoint = "/v1/metrics/eth2/staking_volume_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -63,7 +64,7 @@ class ETH2: :return: A DataFrame with new validators data. :rtype: DataFrame """ - endpoint = '/v1/metrics/eth2/staking_validators_count' + endpoint = "/v1/metrics/eth2/staking_validators_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -77,7 +78,7 @@ class ETH2: :return: A DataFrame with ETH2 deposit data. :rtype: DataFrame """ - endpoint = '/v1/metrics/eth2/staking_total_deposits_count' + endpoint = "/v1/metrics/eth2/staking_total_deposits_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -92,7 +93,7 @@ class ETH2: :return: A DataFrame with ETH2 deposit data. :rtype: DataFrame """ - endpoint = '/v1/metrics/eth2/staking_total_volume_sum' + endpoint = "/v1/metrics/eth2/staking_total_volume_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -106,7 +107,7 @@ class ETH2: :return: A DataFrame with ETH2 deposit data. :rtype: DataFrame """ - endpoint = '/v1/metrics/eth2/staking_total_validators_count' + endpoint = "/v1/metrics/eth2/staking_total_validators_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -120,7 +121,7 @@ class ETH2: :return: A DataFrame with staking goal data. :rtype: DataFrame """ - endpoint = '/v1/metrics/eth2/staking_phase_0_goal_percent' + endpoint = "/v1/metrics/eth2/staking_phase_0_goal_percent" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() diff --git a/utils/glassnode/fees.py b/utils/glassnode/fees.py index a1300e3..b45e8c3 100644 --- a/utils/glassnode/fees.py +++ b/utils/glassnode/fees.py @@ -4,41 +4,42 @@ from .client import GlassnodeClient class Fees: """ - Fees class. + Fees class. - Methods - ------- - __init__(glassnode_client): - Constructs a Mining object. - fee_ratio_multiple(): - Returns the Fee Ratio Multiple (FRM). - fees_total(): - Returns the total amount of fees paid to miners. - fees_mean(): - Returns the mean fee per transaction. - fees_median(): - Returns the median fee per transaction. - gas_used_total(): - Returns the total amount of gas used in all transactions. - gas_used_mean(): - Returns the mean amount of gas used per transaction. - gas_used_median(): - Returns the median amount of gas used per transaction. - gas_price_mean(): - Returns the mean gas price paid per transaction. - gas_price_median(): - Returns the median gas price paid per transaction. - transaction_gas_limit_mean(): - Returns the mean gas limit per transaction. - transaction_gas_limit_median(): - Returns the median gas limit per transaction. - exchange_fees_total(): - Returns the total amount of fees paid in transactions related to on-chain exchange activity. - exchange_fees_mean(): - Returns the mean amount of fees paid in transactions related to on-chain exchange activity. - exchange_fees_dominance(): - Returns the exchange fee dominance. + Methods + ------- + __init__(glassnode_client): + Constructs a Mining object. + fee_ratio_multiple(): + Returns the Fee Ratio Multiple (FRM). + fees_total(): + Returns the total amount of fees paid to miners. + fees_mean(): + Returns the mean fee per transaction. + fees_median(): + Returns the median fee per transaction. + gas_used_total(): + Returns the total amount of gas used in all transactions. + gas_used_mean(): + Returns the mean amount of gas used per transaction. + gas_used_median(): + Returns the median amount of gas used per transaction. + gas_price_mean(): + Returns the mean gas price paid per transaction. + gas_price_median(): + Returns the median gas price paid per transaction. + transaction_gas_limit_mean(): + Returns the mean gas limit per transaction. + transaction_gas_limit_median(): + Returns the median gas limit per transaction. + exchange_fees_total(): + Returns the total amount of fees paid in transactions related to on-chain exchange activity. + exchange_fees_mean(): + Returns the mean amount of fees paid in transactions related to on-chain exchange activity. + exchange_fees_dominance(): + Returns the exchange fee dominance. """ + def __init__(self, glassnode_client: GlassnodeClient): self._gc = glassnode_client @@ -48,7 +49,7 @@ class Fees: and gives an assessment how secure a chain is once block rewards disappear. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/fee_ratio_multiple' + endpoint = "/v1/metrics/fees/fee_ratio_multiple" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -59,7 +60,7 @@ class Fees: The total amount of fees paid to miners. Issued (minted) coins are not included. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/volume_sum' + endpoint = "/v1/metrics/fees/volume_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -70,7 +71,7 @@ class Fees: The mean fee per transaction. Issued (minted) coins are not included. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/volume_mean' + endpoint = "/v1/metrics/fees/volume_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -81,7 +82,7 @@ class Fees: The median fee per transaction. Issued (minted) coins are not included. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/volume_median' + endpoint = "/v1/metrics/fees/volume_median" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -92,7 +93,7 @@ class Fees: The total amount of gas used in all transactions. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/gas_used_sum' + endpoint = "/v1/metrics/fees/gas_used_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -103,7 +104,7 @@ class Fees: The mean amount of gas used per transaction. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/gas_used_mean' + endpoint = "/v1/metrics/fees/gas_used_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -114,7 +115,7 @@ class Fees: The median amount of gas used per transaction. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/gas_used_median' + endpoint = "/v1/metrics/fees/gas_used_median" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -125,7 +126,7 @@ class Fees: The mean gas price paid per transaction. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/gas_price_mean' + endpoint = "/v1/metrics/fees/gas_price_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -136,7 +137,7 @@ class Fees: The median gas price paid per transaction. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/gas_price_median' + endpoint = "/v1/metrics/fees/gas_price_median" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -147,7 +148,7 @@ class Fees: The mean gas limit per transaction. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/gas_limit_tx_mean' + endpoint = "/v1/metrics/fees/gas_limit_tx_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -158,7 +159,7 @@ class Fees: The median gas limit per transaction. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/gas_limit_tx_median' + endpoint = "/v1/metrics/fees/gas_limit_tx_median" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -170,7 +171,7 @@ class Fees: The total amount of fees paid in transactions related to on-chain exchange activity. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/exchanges_sum' + endpoint = "/v1/metrics/fees/exchanges_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -182,7 +183,7 @@ class Fees: The mean amount of fees paid in transactions related to on-chain exchange activity. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/exchanges_mean' + endpoint = "/v1/metrics/fees/exchanges_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -195,7 +196,7 @@ class Fees: paid in transactions related to on-chain exchange activity. `View in Studio `_ """ - endpoint = '/v1/metrics/fees/exchanges_relative' + endpoint = "/v1/metrics/fees/exchanges_relative" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() diff --git a/utils/glassnode/indicators.py b/utils/glassnode/indicators.py index 19c53c2..8bf9a93 100644 --- a/utils/glassnode/indicators.py +++ b/utils/glassnode/indicators.py @@ -11,7 +11,7 @@ class Indicators: The Realized HODL Ratio is a market indicator that uses a ratio of the Realized Cap HODL Waves. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/rhodl_ratio' + endpoint = "/v1/metrics/indicators/rhodl_ratio" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -23,7 +23,7 @@ class Indicators: the market age (in days). Historically, CVDD has been an accurate indicator for global Bitcoin market bottoms. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/cvdd' + endpoint = "/v1/metrics/indicators/cvdd" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -37,7 +37,7 @@ class Indicators: the worst of the miner capitulation is over when the 30d MA of the hash rate crosses above the 60d MA. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/hash_ribbon' + endpoint = "/v1/metrics/indicators/hash_ribbon" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -50,7 +50,7 @@ class Indicators: of the Bitcoin mining difficulty to create the ribbon. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/difficulty_ribbon' + endpoint = "/v1/metrics/indicators/difficulty_ribbon" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -62,7 +62,7 @@ class Indicators: standard deviation to quantify compression of the Difficulty Ribbon. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/difficulty_ribbon_compression' + endpoint = "/v1/metrics/indicators/difficulty_ribbon_compression" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -74,7 +74,7 @@ class Indicators: the market cap by the transferred on-chain volume measured in USD. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/nvt' + endpoint = "/v1/metrics/indicators/nvt" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -85,7 +85,7 @@ class Indicators: The NVT Signal (NVTS) is a modified version of the original NVT Ratio. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/nvts' + endpoint = "/v1/metrics/indicators/nvts" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -97,7 +97,7 @@ class Indicators: by dividing the on-chain transaction volume (in USD) by the market cap, i.e. the inverse of the NVT ratio. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/velocity' + endpoint = "/v1/metrics/indicators/velocity" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -108,7 +108,7 @@ class Indicators: Adjusted Coin Days Destroyed simply divides CDD by the circulating supply. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/cdd_supply_adjusted' + endpoint = "/v1/metrics/indicators/cdd_supply_adjusted" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -119,7 +119,7 @@ class Indicators: Binary Coin Days Destroyed is computed by thresholding Adjusted CDD by its average over time. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/cdd_supply_adjusted_binary' + endpoint = "/v1/metrics/indicators/cdd_supply_adjusted_binary" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -131,7 +131,7 @@ class Indicators: and is defined as the ratio of coin days destroyed and total transfer volume. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/average_dormancy_supply_adjusted' + endpoint = "/v1/metrics/indicators/average_dormancy_supply_adjusted" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -144,7 +144,7 @@ class Indicators: 0 and the current ATH in 100 equally-spaced partitions. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/spent_output_price_distribution_ath' + endpoint = "/v1/metrics/indicators/spent_output_price_distribution_ath" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -157,7 +157,7 @@ class Indicators: and creating 50 equally-spaced bucket each above and below the current price in steps of +/- 2%. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/spent_output_price_distribution_percent' + endpoint = "/v1/metrics/indicators/spent_output_price_distribution_percent" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -169,7 +169,7 @@ class Indicators: by the 365-day moving average of daily issuance value. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/puell_multiple' + endpoint = "/v1/metrics/indicators/puell_multiple" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -180,7 +180,7 @@ class Indicators: Adjusted SOPR is SOPR ignoring all outputs with a lifespan of less than 1 hour. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/sopr_adjusted' + endpoint = "/v1/metrics/indicators/sopr_adjusted" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -192,7 +192,7 @@ class Indicators: When confidence is low and price is high then risk/reward is unattractive at that time (Reserve Risk is high). `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/reserve_risk' + endpoint = "/v1/metrics/indicators/reserve_risk" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -204,7 +204,7 @@ class Indicators: younger than 155 days and serves as an indicator to assess the behaviour of short term investors. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/sopr_less_155' + endpoint = "/v1/metrics/indicators/sopr_less_155" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -216,7 +216,7 @@ class Indicators: of at least 155 days and serves as an indicator to assess the behaviour of long term investors. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/sopr_more_155' + endpoint = "/v1/metrics/indicators/sopr_more_155" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -227,7 +227,7 @@ class Indicators: HODLer Net Position Change shows the monthly position change of long term investors. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/hodler_net_position_change' + endpoint = "/v1/metrics/indicators/hodler_net_position_change" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -238,7 +238,7 @@ class Indicators: Lost or HODLed Bitcoins indicates moves of large and old stashes. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/hodled_lost_coins' + endpoint = "/v1/metrics/indicators/hodled_lost_coins" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -250,7 +250,7 @@ class Indicators: divided by the value at creation (USD) of a spent output. Or simply: price sold / price paid. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/sopr' + endpoint = "/v1/metrics/indicators/sopr" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -262,7 +262,7 @@ class Indicators: in a transaction and multiplying it by the number of days it has been since those coins were last spent. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/cdd' + endpoint = "/v1/metrics/indicators/cdd" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -274,7 +274,7 @@ class Indicators: Outputs with a lifespan of less than 1h are discarded. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/asol' + endpoint = "/v1/metrics/indicators/asol" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -286,7 +286,7 @@ class Indicators: Outputs with a lifespan of less than 1h are discarded. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/msol' + endpoint = "/v1/metrics/indicators/msol" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -298,7 +298,7 @@ class Indicators: and is defined as the ratio of coin days destroyed and total transfer volume. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/average_dormancy' + endpoint = "/v1/metrics/indicators/average_dormancy" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -310,7 +310,7 @@ class Indicators: Liveliness increases as long term holder liquidate positions and decreases while they accumulate to HODL. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/liveliness' + endpoint = "/v1/metrics/indicators/liveliness" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -322,7 +322,7 @@ class Indicators: whose price at realisation time was lower than the current price normalised by the market cap. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/unrealized_profit' + endpoint = "/v1/metrics/indicators/unrealized_profit" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -334,7 +334,7 @@ class Indicators: whose price at realisation time was higher than the current price normalised by the market cap. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/unrealized_loss' + endpoint = "/v1/metrics/indicators/unrealized_loss" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -345,7 +345,7 @@ class Indicators: Net Unrealized Profit/Loss (NUPL) is the difference between Relative Unrealized Profit/Loss. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/net_unrealized_profit_loss' + endpoint = "/v1/metrics/indicators/net_unrealized_profit_loss" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -357,7 +357,7 @@ class Indicators: younger than 155 days and serves as an indicator to assess the behaviour of short term investors. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/nupl_less_155' + endpoint = "/v1/metrics/indicators/nupl_less_155" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -369,7 +369,7 @@ class Indicators: with a lifespan of at least 155 days and serves as an indicator to assess the behaviour of long term investors. `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/nupl_more_155' + endpoint = "/v1/metrics/indicators/nupl_more_155" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -378,28 +378,26 @@ class Indicators: @dataframe_with_inner_object def ssr(self) -> pd.DataFrame: """ - The Stablecoin Supply Ratio (SSR) is the ratio between Bitcoin supply and the supply of stablecoins denoted - in BTC, or: Bitcoin Marketcap / Stablecoin Marketcap. We use the following stablecoins for the supply: USDT, - TUSD, USDC, PAX, GUSD, DAI, SAI, and BUSD. When the SSR is low, the current stablecoin supply has more "buying power" - to purchase BTC. It serves as a proxy for the supply/demand mechanics between BTC and USD. For more information see - this article (https://medium.com/@glassnode/stablecoins-buying-power-over-bitcoin-3475c0d8779d). + The Stablecoin Supply Ratio (SSR) is the ratio between Bitcoin supply and the supply of stablecoins denoted + in BTC, or: Bitcoin Marketcap / Stablecoin Marketcap. We use the following stablecoins for the supply: USDT, + TUSD, USDC, PAX, GUSD, DAI, SAI, and BUSD. When the SSR is low, the current stablecoin supply has more "buying power" + to purchase BTC. It serves as a proxy for the supply/demand mechanics between BTC and USD. For more information see + this article (https://medium.com/@glassnode/stablecoins-buying-power-over-bitcoin-3475c0d8779d). `View in Studio `_ """ - endpoint = '/v1/metrics/indicators/ssr' + endpoint = "/v1/metrics/indicators/ssr" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() return response_to_dataframe(self._gc.get(endpoint)) - def bvin(self): - ''' + """ The Bitcoin Volatility Index (BVIN) is an implied volatility index that also represents the fair value of a bitcoin variance swap. The index is calculated by CryptoCompare using options data from Deribit and has been developed in collaboration with Carol Alexander and Arben Imeraj at the University of Sussex Business School. The index is suitable for use as a settlement price for bitcoin volatility futures. For more information on the methodology please see Alexander and Imeraj (2020). - ''' - endpoint = '/v1/metrics/indicators/bvin' + """ + endpoint = "/v1/metrics/indicators/bvin" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() return response_to_dataframe(self._gc.get(endpoint)) - diff --git a/utils/glassnode/market.py b/utils/glassnode/market.py index 37d9b10..6d77221 100644 --- a/utils/glassnode/market.py +++ b/utils/glassnode/market.py @@ -3,33 +3,34 @@ from .utils import * class Market: """ - Market class. + Market class. - Methods - ------- - __init__(glassnode_client): - Constructs a Market object. - price(): - Returns the asset's price in USD. - price_ohlc(): - Returns OHLC candlestick data. - price_drawdown_from_ath(): - Returns the percent drawdown from previous all-time high. - marketcap(): - Returns the market capitalization of the asset. - mvrv_ratio(): - Returns MVRV ratio. - realized_cap(): - Returns realized cap data. - mvrv_z_score(): - Returns MVRV Z-Score. - sth_mvrv(): - Returns Short Term Holder MVRV data. - lth_mvrv(): - Returns Long Term Holder MVRV data. - realized_price(): - Returns realized price data. + Methods + ------- + __init__(glassnode_client): + Constructs a Market object. + price(): + Returns the asset's price in USD. + price_ohlc(): + Returns OHLC candlestick data. + price_drawdown_from_ath(): + Returns the percent drawdown from previous all-time high. + marketcap(): + Returns the market capitalization of the asset. + mvrv_ratio(): + Returns MVRV ratio. + realized_cap(): + Returns realized cap data. + mvrv_z_score(): + Returns MVRV Z-Score. + sth_mvrv(): + Returns Short Term Holder MVRV data. + lth_mvrv(): + Returns Long Term Holder MVRV data. + realized_price(): + Returns realized price data. """ + def __init__(self, glassnode_client): self._gc = glassnode_client @@ -41,7 +42,7 @@ class Market: :return: A DataFrame containing the asset's price data. :rtype: DataFrame """ - endpoint = '/v1/metrics/market/price_usd' + endpoint = "/v1/metrics/market/price_usd" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -56,7 +57,7 @@ class Market: :return: A DataFrame containing OHLC candlestick data. :rtype: DataFrame """ - endpoint = '/v1/metrics/market/price_usd_ohlc' + endpoint = "/v1/metrics/market/price_usd_ohlc" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -70,7 +71,7 @@ class Market: :return: A DataFrame containing the percent drawdown data. :rtype: DataFrame """ - endpoint = '/v1/metrics/market/price_drawdown_relative' + endpoint = "/v1/metrics/market/price_drawdown_relative" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -84,7 +85,7 @@ class Market: :return: DataFrame """ - endpoint = '/v1/metrics/market/marketcap_usd' + endpoint = "/v1/metrics/market/marketcap_usd" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -98,7 +99,7 @@ class Market: :return: DataFrame """ - endpoint = '/v1/metrics/market/mvrv' + endpoint = "/v1/metrics/market/mvrv" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -112,7 +113,7 @@ class Market: :return: DataFrame """ - endpoint = '/v1/metrics/market/marketcap_realized_usd' + endpoint = "/v1/metrics/market/marketcap_realized_usd" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -125,7 +126,7 @@ class Market: :return: DataFrame """ - endpoint = '/v1/metrics/market/mvrv_z_score' + endpoint = "/v1/metrics/market/mvrv_z_score" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -139,7 +140,7 @@ class Market: :return: DataFrame """ - endpoint = '/v1/metrics/market/mvrv_less_155' + endpoint = "/v1/metrics/market/mvrv_less_155" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -153,7 +154,7 @@ class Market: :return: DataFrame """ - endpoint = '/v1/metrics/market/mvrv_more_155' + endpoint = "/v1/metrics/market/mvrv_more_155" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -166,7 +167,7 @@ class Market: :return: DataFrame """ - endpoint = '/v1/metrics/market/price_realized_usd' + endpoint = "/v1/metrics/market/price_realized_usd" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() diff --git a/utils/glassnode/mining.py b/utils/glassnode/mining.py index ae826a9..dfab4ed 100644 --- a/utils/glassnode/mining.py +++ b/utils/glassnode/mining.py @@ -3,33 +3,34 @@ from .utils import * class Mining: """ - Mining class. + Mining class. - Methods - ------- - __init__(glassnode_client): - Constructs a Mining object. - difficulty(): - Returns difficulty to mine a block. - hash_rate(): - Returns hash rate. - miner_revenue_total(): - Returns the total miner revenue. - miner_revenue_fees(): - Returns the percentage of miner revenue derived from fees. - miner_revenue_block_rewards(): - Returns the total amount of newly minted coins. - miner_outflow_multiple(): - Returns the miner outflow multiple. - thermocap(): - Returns Thermocap data. - market_cap_to_thermocap_ratio(): - Returns the Marketcap to Thermocap Ratio. - miner_unspent_supply(): - Returns unspent miner supply. - miner_names(): - Returns miner names for a mining endpoint. + Methods + ------- + __init__(glassnode_client): + Constructs a Mining object. + difficulty(): + Returns difficulty to mine a block. + hash_rate(): + Returns hash rate. + miner_revenue_total(): + Returns the total miner revenue. + miner_revenue_fees(): + Returns the percentage of miner revenue derived from fees. + miner_revenue_block_rewards(): + Returns the total amount of newly minted coins. + miner_outflow_multiple(): + Returns the miner outflow multiple. + thermocap(): + Returns Thermocap data. + market_cap_to_thermocap_ratio(): + Returns the Marketcap to Thermocap Ratio. + miner_unspent_supply(): + Returns unspent miner supply. + miner_names(): + Returns miner names for a mining endpoint. """ + def __init__(self, glassnode_client): self._gc = glassnode_client @@ -41,7 +42,7 @@ class Mining: :return: A DataFrame with the latest difficulty data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/difficulty_latest' + endpoint = "/v1/metrics/mining/difficulty_latest" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -55,7 +56,7 @@ class Mining: :return: A DataFrame with hash rate data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/hash_rate_mean' + endpoint = "/v1/metrics/mining/hash_rate_mean" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -69,11 +70,11 @@ class Mining: :return: A DataFrame with total revenue data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/revenue_sum' + endpoint = "/v1/metrics/mining/revenue_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'m': miner})) + return response_to_dataframe(self._gc.get(endpoint, {"m": miner})) def miner_revenue_fees(self) -> pd.DataFrame: """ @@ -83,7 +84,7 @@ class Mining: :return: A DataFrame with revenue fees data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/revenue_from_fees' + endpoint = "/v1/metrics/mining/revenue_from_fees" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -97,11 +98,11 @@ class Mining: :return: A DataFrame with revenue block rewards data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/volume_mined_sum' + endpoint = "/v1/metrics/mining/volume_mined_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'m': miner})) + return response_to_dataframe(self._gc.get(endpoint, {"m": miner})) def miner_outflow_multiple(self, miner=None) -> pd.DataFrame: """ @@ -112,11 +113,11 @@ class Mining: :return: A DataFrame MOM data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/miners_outflow_multiple' + endpoint = "/v1/metrics/mining/miners_outflow_multiple" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() - return response_to_dataframe(self._gc.get(endpoint, {'m': miner})) + return response_to_dataframe(self._gc.get(endpoint, {"m": miner})) def thermocap(self) -> pd.DataFrame: """ @@ -127,7 +128,7 @@ class Mining: :return: A DataFrame with thermocap data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/thermocap' + endpoint = "/v1/metrics/mining/thermocap" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -142,7 +143,7 @@ class Mining: :return: A DataFrame with M/T ratio data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/marketcap_thermocap_ratio' + endpoint = "/v1/metrics/mining/marketcap_thermocap_ratio" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -156,13 +157,13 @@ class Mining: :return: A DataFrame with unspent miner supply data. :rtype: DataFrame """ - endpoint = '/v1/metrics/mining/miners_unspent_supply' + endpoint = "/v1/metrics/mining/miners_unspent_supply" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() return response_to_dataframe(self._gc.get(endpoint)) - def miner_names(self, endpoint='revenue_sum') -> list: + def miner_names(self, endpoint="revenue_sum") -> list: """ Returns a list of miner names for a mining endpoint. @@ -170,5 +171,5 @@ class Mining: :return: A List with miner names. :rtype: List """ - miners = self._gc.get(f'/v1/metrics/mining/{endpoint}/miners') + miners = self._gc.get(f"/v1/metrics/mining/{endpoint}/miners") return miners[self._gc.asset] diff --git a/utils/glassnode/protocols.py b/utils/glassnode/protocols.py index b4480ae..e6bcf76 100644 --- a/utils/glassnode/protocols.py +++ b/utils/glassnode/protocols.py @@ -3,20 +3,21 @@ from .utils import * class Protocols: """ - Protocols class. + Protocols class. - Methods - ------- - __init__(glassnode_client): - Constructs a Protocols object. - uniswap_transactions(): - Returns the total number of transactions - that contains an interaction within Uniswap contracts. - uniswap_liquidity(): - Returns the current liquidity on Uniswap. - uniswap_volume(): - Returns the total volume traded on Uniswap. + Methods + ------- + __init__(glassnode_client): + Constructs a Protocols object. + uniswap_transactions(): + Returns the total number of transactions + that contains an interaction within Uniswap contracts. + uniswap_liquidity(): + Returns the current liquidity on Uniswap. + uniswap_volume(): + Returns the total volume traded on Uniswap. """ + def __init__(self, glassnode_client): self._gc = glassnode_client self._endpoints = self._gc.endpoints @@ -30,7 +31,7 @@ class Protocols: :return: A DataFrame containing Uniswap transactions data. :rtype: DataFrame """ - endpoint = '/v1/metrics/protocols/uniswap_transaction_count' + endpoint = "/v1/metrics/protocols/uniswap_transaction_count" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -44,7 +45,7 @@ class Protocols: :return: A DataFrame containing Uniswap liquidity data. :rtype: DataFrame """ - endpoint = '/v1/metrics/protocols/uniswap_liquidity_latest' + endpoint = "/v1/metrics/protocols/uniswap_liquidity_latest" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -58,7 +59,7 @@ class Protocols: :return: A DataFrame containing Uniswap volume data. :rtype: DataFrame """ - endpoint = '/v1/metrics/protocols/uniswap_volume_sum' + endpoint = "/v1/metrics/protocols/uniswap_volume_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() diff --git a/utils/glassnode/supply.py b/utils/glassnode/supply.py index 24c5c0f..f1d3443 100644 --- a/utils/glassnode/supply.py +++ b/utils/glassnode/supply.py @@ -4,85 +4,86 @@ from .client import GlassnodeClient class Supply: """ - Supply class. + Supply class. - Methods - ------- - __init__(glassnode_client): - Constructs a Supply object. - liquid_illiquid_supply(): - Returns the total supply held by illiquid, liquid, and highly liquid entities. - liquid_supply_change(): - Returns the monthly (30d) net change of supply held by liquid and highly liquid entities. - illiquid_supply_change(): - Returns the monthly (30d) net change of supply held by illiquid entities. - circulating_supply(): - Returns the total amount of all coins ever created/issued. - issuance(): - Returns the total amount of new coins added to the current supply. - inflation_rate(): - Returns the yearly inflation rate. - supply_last_active_less_24h(): - Returns the amount of circulating supply last moved in the last 24 hours. - supply_last_active_1d_1w(): - Returns the amount of circulating supply last moved between 1 day and 1 week ago. - supply_last_active_1w_1m(): - Returns the amount of circulating supply last moved between 1 week and 1 month ago. - supply_last_active_1m_3m(): - Returns the amount of circulating supply last moved between 1 month and 3 months ago. - supply_last_active_3m_6m(): - Returns the amount of circulating supply last moved between 3 months and 6 months ago. - supply_last_active_6m_12m(): - Returns the amount of circulating supply last moved between 6 months and 12 months ago. - supply_last_active_1y_2y(): - Returns the amount of circulating supply last moved between 1 year and 6 years ago. - supply_last_active_2y_3y(): - Returns the amount of circulating supply last moved between 2 years and 3 years ago. - supply_last_active_3y_5y(): - Returns the amount of circulating supply last moved between 3 years and 5 years ago. - supply_last_active_5y_7y(): - Returns the amount of circulating supply last moved between 5 years and 7 years ago. - supply_last_active_7y_10y(): - Returns the amount of circulating supply last moved between 7 years and 10 years ago. - supply_last_active_more_10y(): - Returns the amount of circulating supply last moved more than 10 years ago. - hodl_waves(): - Returns a bundle of all active supply age bands, aka HODL waves. - supply_last_active_more_1y_ago(): - Returns the percent of circulating supply that has not moved in at least 1 year. - supply_last_active_more_2y_ago(): - Returns the percent of circulating supply that has not moved in at least 2 years. - supply_last_active_more_3y_ago(): - Returns the percent of circulating supply that has not moved in at least 3 years. - supply_last_active_more_5y_ago(): - Returns the percent of circulating supply that has not moved in at least 5 years. - realized_cap_hodl_waves(): - Returns HODL waves weighted by Realized Price. - adjusted_supply(): - Returns the circulating supply adjusted by accounting for lost coins. - supply_in_profit(): - Returns the circulating supply in profit. - supply_in_loss(): - Returns the circulating supply in loss. - supply_in_profit_relative(): - Returns the percentage of circulating supply in profit. - short_term_holder_supply(): - Returns the total amount of circulating supply held by short-term holders. - long_term_holder_supply(): - Returns the total amount of circulating supply held by long-term holders. - short_term_holder_supply_in_loss(): - Returns the total amount of circulating supply that is currently at loss and held by short-term holders. - long_term_holder_supply_in_loss(): - Returns the total amount of circulating supply that is currently at loss and held by long-term holders. - short_term_holder_supply_in_profit(): - Returns the total amount of circulating supply that is currently in profit and held by short-term holders. - long_term_holder_supply_in_profit(): - Returns the total amount of circulating supply that is currently in profit and held by long-term holders. - relative_long_short_term_holder_supply(): - Returns the relative amount of circulating supply of held by long- and short-term holders in profit/loss. - long_term_holder_position_change(): - Returns the monthly net position change of long-term holders. + Methods + ------- + __init__(glassnode_client): + Constructs a Supply object. + liquid_illiquid_supply(): + Returns the total supply held by illiquid, liquid, and highly liquid entities. + liquid_supply_change(): + Returns the monthly (30d) net change of supply held by liquid and highly liquid entities. + illiquid_supply_change(): + Returns the monthly (30d) net change of supply held by illiquid entities. + circulating_supply(): + Returns the total amount of all coins ever created/issued. + issuance(): + Returns the total amount of new coins added to the current supply. + inflation_rate(): + Returns the yearly inflation rate. + supply_last_active_less_24h(): + Returns the amount of circulating supply last moved in the last 24 hours. + supply_last_active_1d_1w(): + Returns the amount of circulating supply last moved between 1 day and 1 week ago. + supply_last_active_1w_1m(): + Returns the amount of circulating supply last moved between 1 week and 1 month ago. + supply_last_active_1m_3m(): + Returns the amount of circulating supply last moved between 1 month and 3 months ago. + supply_last_active_3m_6m(): + Returns the amount of circulating supply last moved between 3 months and 6 months ago. + supply_last_active_6m_12m(): + Returns the amount of circulating supply last moved between 6 months and 12 months ago. + supply_last_active_1y_2y(): + Returns the amount of circulating supply last moved between 1 year and 6 years ago. + supply_last_active_2y_3y(): + Returns the amount of circulating supply last moved between 2 years and 3 years ago. + supply_last_active_3y_5y(): + Returns the amount of circulating supply last moved between 3 years and 5 years ago. + supply_last_active_5y_7y(): + Returns the amount of circulating supply last moved between 5 years and 7 years ago. + supply_last_active_7y_10y(): + Returns the amount of circulating supply last moved between 7 years and 10 years ago. + supply_last_active_more_10y(): + Returns the amount of circulating supply last moved more than 10 years ago. + hodl_waves(): + Returns a bundle of all active supply age bands, aka HODL waves. + supply_last_active_more_1y_ago(): + Returns the percent of circulating supply that has not moved in at least 1 year. + supply_last_active_more_2y_ago(): + Returns the percent of circulating supply that has not moved in at least 2 years. + supply_last_active_more_3y_ago(): + Returns the percent of circulating supply that has not moved in at least 3 years. + supply_last_active_more_5y_ago(): + Returns the percent of circulating supply that has not moved in at least 5 years. + realized_cap_hodl_waves(): + Returns HODL waves weighted by Realized Price. + adjusted_supply(): + Returns the circulating supply adjusted by accounting for lost coins. + supply_in_profit(): + Returns the circulating supply in profit. + supply_in_loss(): + Returns the circulating supply in loss. + supply_in_profit_relative(): + Returns the percentage of circulating supply in profit. + short_term_holder_supply(): + Returns the total amount of circulating supply held by short-term holders. + long_term_holder_supply(): + Returns the total amount of circulating supply held by long-term holders. + short_term_holder_supply_in_loss(): + Returns the total amount of circulating supply that is currently at loss and held by short-term holders. + long_term_holder_supply_in_loss(): + Returns the total amount of circulating supply that is currently at loss and held by long-term holders. + short_term_holder_supply_in_profit(): + Returns the total amount of circulating supply that is currently in profit and held by short-term holders. + long_term_holder_supply_in_profit(): + Returns the total amount of circulating supply that is currently in profit and held by long-term holders. + relative_long_short_term_holder_supply(): + Returns the relative amount of circulating supply of held by long- and short-term holders in profit/loss. + long_term_holder_position_change(): + Returns the monthly net position change of long-term holders. """ + def __init__(self, glassnode_client: GlassnodeClient): self._gc = glassnode_client @@ -93,7 +94,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/liquid_illiquid_sum' + endpoint = "/v1/metrics/supply/liquid_illiquid_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -106,7 +107,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/liquid_change' + endpoint = "/v1/metrics/supply/liquid_change" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -119,7 +120,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/illiquid_change' + endpoint = "/v1/metrics/supply/illiquid_change" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -132,7 +133,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/current' + endpoint = "/v1/metrics/supply/current" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -146,7 +147,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/issued' + endpoint = "/v1/metrics/supply/issued" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -160,7 +161,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/inflation_rate' + endpoint = "/v1/metrics/supply/inflation_rate" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -173,7 +174,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_24h' + endpoint = "/v1/metrics/supply/active_24h" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -186,7 +187,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_1d_1w' + endpoint = "/v1/metrics/supply/active_1d_1w" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -199,7 +200,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_1w_1m' + endpoint = "/v1/metrics/supply/active_1w_1m" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -212,7 +213,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_1m_3m' + endpoint = "/v1/metrics/supply/active_1m_3m" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -225,7 +226,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_3m_6m' + endpoint = "/v1/metrics/supply/active_3m_6m" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -238,7 +239,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_6m_12m' + endpoint = "/v1/metrics/supply/active_6m_12m" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -251,7 +252,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_1y_2y' + endpoint = "/v1/metrics/supply/active_1y_2y" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -264,7 +265,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_2y_3y' + endpoint = "/v1/metrics/supply/active_2y_3y" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -277,7 +278,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_3y_5y' + endpoint = "/v1/metrics/supply/active_3y_5y" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -290,7 +291,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_5y_7y' + endpoint = "/v1/metrics/supply/active_5y_7y" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -303,7 +304,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_7y_10y' + endpoint = "/v1/metrics/supply/active_7y_10y" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -316,7 +317,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_more_10y' + endpoint = "/v1/metrics/supply/active_more_10y" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -329,7 +330,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/hodl_waves' + endpoint = "/v1/metrics/supply/hodl_waves" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -342,7 +343,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_more_1y_percent' + endpoint = "/v1/metrics/supply/active_more_1y_percent" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -355,7 +356,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_more_2y_percent' + endpoint = "/v1/metrics/supply/active_more_2y_percent" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -368,7 +369,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_more_3y_percent' + endpoint = "/v1/metrics/supply/active_more_3y_percent" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -381,7 +382,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/active_more_5y_percent' + endpoint = "/v1/metrics/supply/active_more_5y_percent" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -394,7 +395,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/rcap_hodl_waves' + endpoint = "/v1/metrics/supply/rcap_hodl_waves" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -407,7 +408,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/current_adjusted' + endpoint = "/v1/metrics/supply/current_adjusted" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -421,7 +422,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/profit_sum' + endpoint = "/v1/metrics/supply/profit_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -435,7 +436,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/loss_sum' + endpoint = "/v1/metrics/supply/loss_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -448,7 +449,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/profit_relative' + endpoint = "/v1/metrics/supply/profit_relative" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -461,7 +462,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/sth_sum' + endpoint = "/v1/metrics/supply/sth_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -474,7 +475,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/lth_sum' + endpoint = "/v1/metrics/supply/lth_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -487,7 +488,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/sth_loss_sum' + endpoint = "/v1/metrics/supply/sth_loss_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -500,7 +501,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/lth_loss_sum' + endpoint = "/v1/metrics/supply/lth_loss_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -513,7 +514,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/sth_profit_sum' + endpoint = "/v1/metrics/supply/sth_profit_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -526,7 +527,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/lth_profit_sum' + endpoint = "/v1/metrics/supply/lth_profit_sum" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -539,7 +540,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/lth_sth_profit_loss_relative' + endpoint = "/v1/metrics/supply/lth_sth_profit_loss_relative" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() @@ -553,7 +554,7 @@ class Supply: :rtype: DataFrame """ - endpoint = '/v1/metrics/supply/lth_net_change' + endpoint = "/v1/metrics/supply/lth_net_change" if not is_supported_by_endpoint(self._gc, endpoint): return pd.DataFrame() diff --git a/utils/glassnode/utils.py b/utils/glassnode/utils.py index abe9168..3bd62dd 100644 --- a/utils/glassnode/utils.py +++ b/utils/glassnode/utils.py @@ -18,11 +18,11 @@ def unix_timestamp(date_str): def is_supported_by_endpoint(glassnode_client, url): path = glassnode_client.endpoints.query(url) - if glassnode_client.asset not in path['assets']: - print(f'{url} metric is not available for {glassnode_client.asset}') + if glassnode_client.asset not in path["assets"]: + print(f"{url} metric is not available for {glassnode_client.asset}") return False - if glassnode_client.resolution not in path['resolutions']: - print(f'{url} metric is not available for {glassnode_client.resolution}') + if glassnode_client.resolution not in path["resolutions"]: + print(f"{url} metric is not available for {glassnode_client.resolution}") return False return True @@ -36,8 +36,8 @@ def response_to_dataframe(response): """ try: df = pd.DataFrame(response) - df.set_index('t', inplace=True) - df.index = pd.to_datetime(df.index, unit='s') + df.set_index("t", inplace=True) + df.index = pd.to_datetime(df.index, unit="s") df.index.name = None df.sort_index(ascending=False, inplace=True) return df @@ -48,7 +48,8 @@ def response_to_dataframe(response): def dataframe_with_inner_object(func): def wrapper(*args, **kwargs): df = func(*args, **kwargs) - return pd.concat([df.drop(['o'], axis=1), df['o'].apply(pd.Series)], axis=1) + return pd.concat([df.drop(["o"], axis=1), df["o"].apply(pd.Series)], axis=1) + return wrapper @@ -60,7 +61,7 @@ def fetch(endpoint, params=None): :param endpoint: Endpoint url corresponding to some metric (ex. '/v1/metrics/market/price_usd') :return: DataFrame of {'t' : datetime, 'v' : 'metric-value'} pairs """ - r = requests.get(f'https://api.glassnode.com{endpoint}', params=params, stream=True) + r = requests.get(f"https://api.glassnode.com{endpoint}", params=params, stream=True) try: r.raise_for_status() except requests.exceptions.HTTPError as e: diff --git a/utils/hashing.py b/utils/hashing.py index 1a5704e..8fb225b 100644 --- a/utils/hashing.py +++ b/utils/hashing.py @@ -1,10 +1,12 @@ from hashlib import sha256 import pandas as pd + def hash_df(df: pd.DataFrame) -> str: s = str(df.columns) + str(df.index) + str(df.values) return sha256(s.encode()).hexdigest() + def hash_series(df: pd.Series) -> str: s = str(df.name) + str(df.index) + str(df.values) - return sha256(s.encode()).hexdigest() \ No newline at end of file + return sha256(s.encode()).hexdigest() diff --git a/utils/helpers.py b/utils/helpers.py index 6a639a8..d17a159 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -5,11 +5,19 @@ import string import random from itertools import dropwhile + def get_files_from_dir(path: str) -> list[str]: - return [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')] + return [ + f + for f in os.listdir(path) + if os.path.isfile(os.path.join(path, f)) and not f.startswith(".") + ] + def get_first_valid_return_index(series: pd.Series) -> int: - double_nested_results = np.where(np.logical_and(series != 0, np.logical_not(pd.isna(series)))) + double_nested_results = np.where( + np.logical_and(series != 0, np.logical_not(pd.isna(series))) + ) if len(double_nested_results) == 0: return 0 nested_result = double_nested_results[0] @@ -17,42 +25,56 @@ def get_first_valid_return_index(series: pd.Series) -> int: return 0 return nested_result[0] + def get_last_non_na_index(series: pd.Series, index: int) -> int: - return next(dropwhile(lambda x: pd.isna(x[1]), enumerate(reversed(series[:index+1]))))[0] + return next( + dropwhile(lambda x: pd.isna(x[1]), enumerate(reversed(series[: index + 1]))) + )[0] def flatten(list_of_lists: list) -> list: return [item for sublist in list_of_lists for item in sublist] + def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.Series: if df.shape[1] == 0: return df - mean_df = df.iloc[:,0] + mean_df = df.iloc[:, 0] weights = df.loc[weights_source] for i, row in df.iterrows(): - if i == weights_source: continue + if i == weights_source: + continue mean_df.loc[i] = (row * weights).sum() / df.loc[weights_source].sum() return mean_df + def drop_columns_if_exist(df: pd.DataFrame, columns: list) -> pd.DataFrame: for column in columns: if column in df.columns: df = df.drop(column, axis=1) return df + def random_string(n: int) -> str: - return ''.join(random.choices(string.ascii_uppercase + string.digits, k=n)) + return "".join(random.choices(string.ascii_uppercase + string.digits, k=n)) + def equal_except_nan(row: pd.Series): if np.isnan(row.iloc[0]) or np.isnan(row.iloc[1]): return np.nan if row.iloc[0] == row.iloc[1]: - return 1. + return 1.0 else: - return 0. + return 0.0 -def drop_until_first_valid_index(df: pd.DataFrame, series: pd.Series) -> tuple[pd.DataFrame, pd.Series]: - first_valid_index = max(get_first_valid_return_index(df.iloc[:,0]), get_first_valid_return_index(series)) - return df.iloc[first_valid_index:], series.iloc[first_valid_index:] \ No newline at end of file + +def drop_until_first_valid_index( + df: pd.DataFrame, series: pd.Series +) -> tuple[pd.DataFrame, pd.Series]: + first_valid_index = max( + get_first_valid_return_index(df.iloc[:, 0]), + get_first_valid_return_index(series), + ) + return df.iloc[first_valid_index:], series.iloc[first_valid_index:] diff --git a/utils/metrics.py b/utils/metrics.py index 7f6933d..afd2c6d 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -3,6 +3,7 @@ import pandas as pd import scipy.stats as ss import numpy as np + def timing_of_flattening_and_flips(target_positions: pd.Series) -> pd.DatetimeIndex: """ Advances in Financial Machine Learning, Snippet 14.1, page 197 @@ -15,8 +16,12 @@ def timing_of_flattening_and_flips(target_positions: pd.Series) -> pd.DatetimeIn :return: (pd.DatetimeIndex) Timestamps of trades flattening, flipping and last bet """ - empty_positions = target_positions[(target_positions == 0)].index # Empty positions index - previous_positions = target_positions.shift(1) # Timestamps pointing at previous positions + empty_positions = target_positions[ + (target_positions == 0) + ].index # Empty positions index + previous_positions = target_positions.shift( + 1 + ) # Timestamps pointing at previous positions # Index of positions where previous one wasn't empty previous_positions = previous_positions[(previous_positions != 0)].index @@ -30,8 +35,12 @@ def timing_of_flattening_and_flips(target_positions: pd.Series) -> pd.DatetimeIn # FLIPS - if current position has another direction compared to the next flips = multiplied_posions[(multiplied_posions < 0)].index flips_and_flattenings = flattening.union(flips).sort_values() - if target_positions.index[-1] not in flips_and_flattenings: # Appending with last bet - flips_and_flattenings = flips_and_flattenings.append(target_positions.index[-1:]) + if ( + target_positions.index[-1] not in flips_and_flattenings + ): # Appending with last bet + flips_and_flattenings = flips_and_flattenings.append( + target_positions.index[-1:] + ) return flips_and_flattenings @@ -50,19 +59,23 @@ def average_holding_period(target_positions: pd.Series) -> float: :return: (float) Estimated average holding period, NaN if zero or unpredicted """ - holding_period = pd.DataFrame(columns=['holding_time', 'weight']) + holding_period = pd.DataFrame(columns=["holding_time", "weight"]) entry_time = 0 position_difference = target_positions.diff() # Time elapsed from the starting time for each position - time_difference = (target_positions.index - target_positions.index[0]) / np.timedelta64(1, 'D') + time_difference = ( + target_positions.index - target_positions.index[0] + ) / np.timedelta64(1, "D") for i in range(1, target_positions.size): # Increased or unchanged position if float(position_difference.iloc[i] * target_positions.iloc[i - 1]) >= 0: if float(target_positions.iloc[i]) != 0: # And not an empty position - entry_time = (entry_time * target_positions.iloc[i - 1] + - time_difference[i] * position_difference.iloc[i]) / target_positions.iloc[i] + entry_time = ( + entry_time * target_positions.iloc[i - 1] + + time_difference[i] * position_difference.iloc[i] + ) / target_positions.iloc[i] # Decreased if float(position_difference.iloc[i] * target_positions.iloc[i - 1]) < 0: @@ -71,19 +84,25 @@ def average_holding_period(target_positions: pd.Series) -> float: # Flip of a position if float(target_positions.iloc[i] * target_positions.iloc[i - 1]) < 0: weight = abs(target_positions.iloc[i - 1]) - holding_period.loc[target_positions.index[i], ['holding_time', 'weight']] = (hold_time, weight) + holding_period.loc[ + target_positions.index[i], ["holding_time", "weight"] + ] = (hold_time, weight) entry_time = time_difference[i] # Reset entry time # Only a part of position is closed else: weight = abs(position_difference.iloc[i]) - holding_period.loc[target_positions.index[i], ['holding_time', 'weight']] = (hold_time, weight) + holding_period.loc[ + target_positions.index[i], ["holding_time", "weight"] + ] = (hold_time, weight) - if float(holding_period['weight'].sum()) > 0: # If there were closed trades at all - avg_holding_period = float((holding_period['holding_time'] * \ - holding_period['weight']).sum() / holding_period['weight'].sum()) + if float(holding_period["weight"].sum()) > 0: # If there were closed trades at all + avg_holding_period = float( + (holding_period["holding_time"] * holding_period["weight"]).sum() + / holding_period["weight"].sum() + ) else: - avg_holding_period = float('nan') + avg_holding_period = float("nan") return avg_holding_period @@ -99,15 +118,15 @@ def bets_concentration(returns: pd.Series) -> float: """ if returns.size <= 2: - return float('nan') # If less than 3 bets + return float("nan") # If less than 3 bets weights = returns / returns.sum() # Weights of each bet - hhi = (weights ** 2).sum() # Herfindahl-Hirschman Index for weights + hhi = (weights**2).sum() # Herfindahl-Hirschman Index for weights hhi = float((hhi - returns.size ** (-1)) / (1 - returns.size ** (-1))) return hhi -def all_bets_concentration(returns: pd.Series, frequency: str = 'M') -> tuple: +def all_bets_concentration(returns: pd.Series, frequency: str = "M") -> tuple: """ Advances in Financial Machine Learning, Snippet 14.3, page 201 Given a pd.Series of returns, derives concentration of positive returns, negative returns @@ -131,7 +150,9 @@ def all_bets_concentration(returns: pd.Series, frequency: str = 'M') -> tuple: negative_concentration = bets_concentration(returns[returns < 0]) # Concentration of bets/time period (month by default) - time_concentration = bets_concentration(returns.groupby(pd.Grouper(freq=frequency)).count()) + time_concentration = bets_concentration( + returns.groupby(pd.Grouper(freq=frequency)).count() + ) return (positive_concentration, negative_concentration, time_concentration) @@ -157,35 +178,42 @@ def drawdown_and_time_under_water(returns: pd.Series, dollars: bool = False) -> :return: (tuple of pd.Series) Series of drawdowns and time under water """ - frame = returns.to_frame('pnl') - frame['hwm'] = returns.expanding().max() # Adding high watermarks as column + frame = returns.to_frame("pnl") + frame["hwm"] = returns.expanding().max() # Adding high watermarks as column # Grouped as min returns by high watermarks - high_watermarks = frame.groupby('hwm').min().reset_index() - high_watermarks.columns = ['hwm', 'min'] + high_watermarks = frame.groupby("hwm").min().reset_index() + high_watermarks.columns = ["hwm", "min"] # Time high watermark occurred - high_watermarks.index = frame['hwm'].drop_duplicates(keep='first').index + high_watermarks.index = frame["hwm"].drop_duplicates(keep="first").index # Picking ones that had a drawdown after high watermark - high_watermarks = high_watermarks[high_watermarks['hwm'] > high_watermarks['min']] + high_watermarks = high_watermarks[high_watermarks["hwm"] > high_watermarks["min"]] if dollars: - drawdown = high_watermarks['hwm'] - high_watermarks['min'] + drawdown = high_watermarks["hwm"] - high_watermarks["min"] else: - drawdown = 1 - high_watermarks['min'] / high_watermarks['hwm'] + drawdown = 1 - high_watermarks["min"] / high_watermarks["hwm"] - time_under_water = ((high_watermarks.index[1:] - high_watermarks.index[:-1]) / np.timedelta64(1, 'Y')).values + time_under_water = ( + (high_watermarks.index[1:] - high_watermarks.index[:-1]) + / np.timedelta64(1, "Y") + ).values # Adding also period from last High watermark to last return observed. - time_under_water = np.append(time_under_water, - (returns.index[-1] - high_watermarks.index[-1]) / np.timedelta64(1, 'Y')) + time_under_water = np.append( + time_under_water, + (returns.index[-1] - high_watermarks.index[-1]) / np.timedelta64(1, "Y"), + ) time_under_water = pd.Series(time_under_water, index=high_watermarks.index) return drawdown, time_under_water -def sharpe_ratio(returns: pd.Series, entries_per_year: int = 252, risk_free_rate: float = 0) -> float: +def sharpe_ratio( + returns: pd.Series, entries_per_year: int = 252, risk_free_rate: float = 0 +) -> float: """ Calculates annualized Sharpe ratio for pd.Series of normal or log returns. Risk_free_rate should be given for the same period the returns are given. @@ -197,12 +225,18 @@ def sharpe_ratio(returns: pd.Series, entries_per_year: int = 252, risk_free_rate :return: (float) Annualized Sharpe ratio """ - sharpe_r = (returns.mean() - risk_free_rate) / returns.std() * (entries_per_year) ** (1 / 2) + sharpe_r = ( + (returns.mean() - risk_free_rate) + / returns.std() + * (entries_per_year) ** (1 / 2) + ) return sharpe_r -def information_ratio(returns: pd.Series, benchmark: float = 0, entries_per_year: int = 252) -> float: +def information_ratio( + returns: pd.Series, benchmark: float = 0, entries_per_year: int = 252 +) -> float: """ Calculates annualized information ratio for pd.Series of normal or log returns. Benchmark should be provided as a return for the same time period as that between @@ -223,8 +257,13 @@ def information_ratio(returns: pd.Series, benchmark: float = 0, entries_per_year return information_r -def probabilistic_sharpe_ratio(observed_sr: float, benchmark_sr: float, number_of_returns: int, - skewness_of_returns: float = 0, kurtosis_of_returns: float = 3) -> float: +def probabilistic_sharpe_ratio( + observed_sr: float, + benchmark_sr: float, + number_of_returns: int, + skewness_of_returns: float = 0, + kurtosis_of_returns: float = 3, +) -> float: """ Calculates the probabilistic Sharpe ratio (PSR) that provides an adjusted estimate of SR, by removing the inflationary effect caused by short series with skewed and/or @@ -241,30 +280,47 @@ def probabilistic_sharpe_ratio(observed_sr: float, benchmark_sr: float, number_o :return: (float) Probabilistic Sharpe ratio """ - test_value = ((observed_sr - benchmark_sr) * np.sqrt(number_of_returns - 1)) / \ - ((1 - skewness_of_returns * observed_sr + (kurtosis_of_returns - 1) / \ - 4 * observed_sr ** 2)**(1 / 2)) + test_value = ((observed_sr - benchmark_sr) * np.sqrt(number_of_returns - 1)) / ( + ( + 1 + - skewness_of_returns * observed_sr + + (kurtosis_of_returns - 1) / 4 * observed_sr**2 + ) + ** (1 / 2) + ) if np.isnan(test_value): - warnings.warn('Test value is nan. Please check the input values.', UserWarning) + warnings.warn("Test value is nan. Please check the input values.", UserWarning) return test_value if isinstance(test_value, complex): - warnings.warn('Output is a complex number. You may want to check the input skewness (too high), ' - 'kurtosis (too low), or observed_sr values.', UserWarning) + warnings.warn( + "Output is a complex number. You may want to check the input skewness (too high), " + "kurtosis (too low), or observed_sr values.", + UserWarning, + ) if np.isinf(test_value): - warnings.warn('Test value is infinite. You may want to check the input skewness, ' - 'kurtosis, or observed_sr values.', UserWarning) + warnings.warn( + "Test value is infinite. You may want to check the input skewness, " + "kurtosis, or observed_sr values.", + UserWarning, + ) probab_sr = ss.norm.cdf(test_value) return probab_sr -def deflated_sharpe_ratio(observed_sr: float, sr_estimates: list, number_of_returns: int, - skewness_of_returns: float = 0, kurtosis_of_returns: float = 3, - estimates_param: bool = False, benchmark_out: bool = False) -> float: +def deflated_sharpe_ratio( + observed_sr: float, + sr_estimates: list, + number_of_returns: int, + skewness_of_returns: float = 0, + kurtosis_of_returns: float = 3, + estimates_param: bool = False, + benchmark_out: bool = False, +) -> float: """ Calculates the deflated Sharpe ratio (DSR) - a PSR where the rejection threshold is adjusted to reflect the multiplicity of trials. DSR is estimated as PSR[SR∗], where @@ -290,18 +346,25 @@ def deflated_sharpe_ratio(observed_sr: float, sr_estimates: list, number_of_retu # Calculating benchmark_SR from the parameters of estimates if estimates_param: - benchmark_sr = sr_estimates[0] * \ - ((1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / sr_estimates[1]) + - np.euler_gamma * ss.norm.ppf(1 - 1 / sr_estimates[1] * np.e ** (-1))) + benchmark_sr = sr_estimates[0] * ( + (1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / sr_estimates[1]) + + np.euler_gamma * ss.norm.ppf(1 - 1 / sr_estimates[1] * np.e ** (-1)) + ) # Calculating benchmark_SR from a list of estimates else: - benchmark_sr = np.array(sr_estimates).std() * \ - ((1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / len(sr_estimates)) + - np.euler_gamma * ss.norm.ppf(1 - 1 / len(sr_estimates) * np.e ** (-1))) + benchmark_sr = np.array(sr_estimates).std() * ( + (1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / len(sr_estimates)) + + np.euler_gamma * ss.norm.ppf(1 - 1 / len(sr_estimates) * np.e ** (-1)) + ) - deflated_sr = probabilistic_sharpe_ratio(observed_sr, benchmark_sr, number_of_returns, - skewness_of_returns, kurtosis_of_returns) + deflated_sr = probabilistic_sharpe_ratio( + observed_sr, + benchmark_sr, + number_of_returns, + skewness_of_returns, + kurtosis_of_returns, + ) if benchmark_out: return benchmark_sr @@ -309,10 +372,13 @@ def deflated_sharpe_ratio(observed_sr: float, sr_estimates: list, number_of_retu return deflated_sr -def minimum_track_record_length(observed_sr: float, benchmark_sr: float, - skewness_of_returns: float = 0, - kurtosis_of_returns: float = 3, - alpha: float = 0.05) -> float: +def minimum_track_record_length( + observed_sr: float, + benchmark_sr: float, + skewness_of_returns: float = 0, + kurtosis_of_returns: float = 3, + alpha: float = 0.05, +) -> float: """ Calculates the minimum track record length (MinTRL) - "How long should a track record be in order to have statistical confidence that its Sharpe ratio is above @@ -329,8 +395,10 @@ def minimum_track_record_length(observed_sr: float, benchmark_sr: float, :return: (float) Minimum number of track records """ - track_rec_length = 1 + (1 - skewness_of_returns * observed_sr + - (kurtosis_of_returns - 1) / 4 * observed_sr ** 2) * \ - (ss.norm.ppf(1 - alpha) / (observed_sr - benchmark_sr)) ** (2) + track_rec_length = 1 + ( + 1 + - skewness_of_returns * observed_sr + + (kurtosis_of_returns - 1) / 4 * observed_sr**2 + ) * (ss.norm.ppf(1 - alpha) / (observed_sr - benchmark_sr)) ** (2) - return track_rec_length \ No newline at end of file + return track_rec_length diff --git a/utils/parallel.py b/utils/parallel.py index 9890e87..5804be4 100644 --- a/utils/parallel.py +++ b/utils/parallel.py @@ -1,8 +1,8 @@ from tqdm import tqdm import ray -def parallel_compute_with_bar(computations) -> list: +def parallel_compute_with_bar(computations) -> list: def to_iterator(obj_ids): while obj_ids: done, obj_ids = ray.wait(obj_ids) @@ -12,4 +12,4 @@ def parallel_compute_with_bar(computations) -> list: for x in tqdm(to_iterator(computations), total=len(computations)): ret.append(x) - return ret \ No newline at end of file + return ret diff --git a/utils/resample.py b/utils/resample.py index ba98692..3324faf 100644 --- a/utils/resample.py +++ b/utils/resample.py @@ -1,9 +1,10 @@ import pandas as pd + def resample_ohlc(df, period): output = pd.DataFrame() - output['open'] = df.open.resample(period).first() - output['high'] = df.high.resample(period).max() - output['low'] = df.low.resample(period).min() - output['close'] = df.close.resample(period).last() - return output \ No newline at end of file + output["open"] = df.open.resample(period).first() + output["high"] = df.high.resample(period).max() + output["low"] = df.low.resample(period).min() + output["close"] = df.close.resample(period).last() + return output diff --git a/utils/sktime.py b/utils/sktime.py index 8c84956..8722b27 100644 --- a/utils/sktime.py +++ b/utils/sktime.py @@ -1,17 +1,18 @@ import pandas as pd import numpy as np + def from_df_to_sktime_data(x: pd.DataFrame) -> pd.DataFrame: instance_list = [] instance_list.append([]) - + x_data = pd.DataFrame(dtype=np.float32) - + for i in range(len(x.index)): instance_list[0].append(pd.Series(x.iloc[i])) - + # only dim_0 for univariate time series for dim in range(len(instance_list)): - x_data['dim_' + str(dim)] = instance_list[dim] - - return x_data \ No newline at end of file + x_data["dim_" + str(dim)] = instance_list[dim] + + return x_data diff --git a/utils/technical_indicators.py b/utils/technical_indicators.py index 943354e..faf882d 100644 --- a/utils/technical_indicators.py +++ b/utils/technical_indicators.py @@ -1,29 +1,43 @@ import pandas as pd import numpy as np + def STOK(close, low, high, n): - STOK = ((close - low.rolling(n).min()) / (high.rolling(n).max() - low.rolling(n).min())) * 100 + STOK = ( + (close - low.rolling(n).min()) / (high.rolling(n).max() - low.rolling(n).min()) + ) * 100 return STOK + def STOD(close, low, high, n): - STOK = ((close - low.rolling(n).min()) / (high.rolling(n).max() - low.rolling(n).min())) * 100 + STOK = ( + (close - low.rolling(n).min()) / (high.rolling(n).max() - low.rolling(n).min()) + ) * 100 STOD = STOK.rolling(3).mean() return STOD + def RSI(series, period): delta = series.diff().dropna() - u=delta*0 + u = delta * 0 d = u.copy() u[delta > 0] = delta[delta > 0] d[delta < 0] = -delta[delta < 0] - u[u.index[period-1]] = np.mean( u[:period] ) #first value is sum of avg gains u = u.drop(u.index[:(period-1)]) - d[d.index[period-1]] = np.mean( d[:period] ) #first value is sum of avg losses d = d.drop(d.index[:(period-1)]) - rs = u.ewm(com=period-1, adjust=False).mean() / \ - d.ewm(com=period-1, adjust=False).mean() - return 100-100/(1+rs) + u[u.index[period - 1]] = np.mean( + u[:period] + ) # first value is sum of avg gains u = u.drop(u.index[:(period-1)]) + d[d.index[period - 1]] = np.mean( + d[:period] + ) # first value is sum of avg losses d = d.drop(d.index[:(period-1)]) + rs = ( + u.ewm(com=period - 1, adjust=False).mean() + / d.ewm(com=period - 1, adjust=False).mean() + ) + return 100 - 100 / (1 + rs) + def ROC(df, n): M = df.diff(n - 1) N = df.shift(n - 1) - ROC = pd.Series(((M / N) * 100), name = 'ROC_' + str(n)) - return ROC \ No newline at end of file + ROC = pd.Series(((M / N) * 100), name="ROC_" + str(n)) + return ROC diff --git a/wandb_setup.py b/wandb_setup.py index b5857cd..5d967b9 100644 --- a/wandb_setup.py +++ b/wandb_setup.py @@ -4,14 +4,15 @@ import os def get_wandb(): from dotenv import load_dotenv + load_dotenv() - - ''' 0. Login to Weights and Biases ''' - wsb_token = os.environ.get('WANDB_API_KEY') - if wsb_token: + + """ 0. Login to Weights and Biases """ + wsb_token = os.environ.get("WANDB_API_KEY") + if wsb_token: wandb.login(key=wsb_token) return wandb - else: return None #wandb.login() - - # return wandb + else: + return None # wandb.login() + # return wandb