gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-01 21:27:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(Linter): reformatted code with black (#211)

Browse Source 
* chore(Linter): reformatted code with black

* Create black.yaml
This commit is contained in:
Mark Aron Szulyovszky
2022-02-17 19:22:17 +01:00
committed by GitHub
parent f3fee4a4e1
commit 8dd2d88740
101 changed files with 2596 additions and 2320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/black.yaml
+10
View File
@@ -0,0 +1,10 @@
name: Lint
on: push
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: psf/black@stable
.vscode/settings.json
Vendored
+10
View File
@@ -0,0 +1,10 @@
{
"python.testing.pytestArgs": [
"."
],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true,
"python.linting.enabled": true,
"python.formatting.provider": "black",
"editor.formatOnSave": true,
}
archive/forecasting.py
-119
View File
@@ -1,119 +0,0 @@
#%%
import pytorch_lightning as pl
from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
from pytorch_forecasting import TimeSeriesDataSet, TemporalFusionTransformer
from pytorch_forecasting.data import GroupNormalizer
from pytorch_forecasting.metrics import QuantileLoss
import sys
sys.path.insert(0, '..')
from data_loader import load_files
print("success")
#%%
# load data
data = load_files('../data/', add_features=True, log_returns=False, narrow_format=True)
# need to treat time as an independent column
data = data.reset_index().rename({'index':'time'}, axis = 'columns')
# we need a `time_idx` column for pytorch-forecasting, so we convert the time column to a time index by encoding the dates as consecutive days from the first date.
data['time_idx'] = (data['time']-data['time'].min()).astype('timedelta64[D]').astype(int)+1
# volume needs some love before we can use it
data.drop(columns=['volume'], inplace=True)
data['month'] = data['month'].astype(str)
data['day_month'] = data['day_month'].astype(str)
data['day_week'] = data['day_week'].astype(str)
#%%
# define dataset
max_encoder_length = 36 # this is the look-back window, see https://github.com/jdb78/pytorch-forecasting/issues/448
max_prediction_length = 6
# training_cutoff = "YYYY-MM-DD" # day for cutoff
# training_cutoff = data["time_idx"].max() - max_prediction_length
#%%
data.head()
data.describe()
#%%
training = TimeSeriesDataSet(
data, # data[lambda x: x.date < training_cutoff],
time_idx= 'time_idx',
target= 'returns',
# weight="weight",
group_ids=[ 'ticker' ],
min_encoder_length = max_encoder_length,##max_encoder_length//2,
max_encoder_length = max_encoder_length,
min_prediction_length = 1,
max_prediction_length = 1,#max_prediction_length,
static_categoricals=[ ],
static_reals=[ ],
time_varying_known_categoricals=[ 'day_month', 'day_week', 'month' ],
time_varying_known_reals=[ 'vol_10', 'vol_20', 'vol_30', 'vol_60', 'mom_10', 'mom_20', 'mom_30', 'mom_60', 'mom_90'],
time_varying_unknown_categoricals=[ ],
time_varying_unknown_reals=[ 'returns' ],
allow_missing_timesteps=True,
target_normalizer=GroupNormalizer(
groups=['ticker'], transformation="softplus")
)
#%%
print(training.index.time)
print(training.index.time.max())
#%%
# create validation and training dataset
validation = TimeSeriesDataSet.from_dataset(training, data, predict=True, stop_randomization=True)
batch_size = 128
train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0)
val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=0)
# define trainer with early stopping
early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min")
lr_logger = LearningRateMonitor()
trainer = pl.Trainer(
max_epochs=100,
gpus=0,
gradient_clip_val=0.1,
limit_train_batches=30,
callbacks=[lr_logger, early_stop_callback],
)
# create the model
tft = TemporalFusionTransformer.from_dataset(
training,
learning_rate=0.03,
hidden_size=32,
attention_head_size=1,
dropout=0.1,
hidden_continuous_size=16,
output_size=7,
loss=QuantileLoss(),
log_interval=2,
reduce_on_plateau_patience=4
)
print(f"Number of parameters in network: {tft.size()/1e3:.1f}k")
# find optimal learning rate (set limit_train_batches to 1.0 and log_interval = -1)
res = trainer.tuner.lr_find(
tft, train_dataloader=train_dataloader, val_dataloaders=val_dataloader, early_stop_threshold=1000.0, max_lr=0.3,
)
print(f"suggested learning rate: {res.suggestion()}")
fig = res.plot(show=True, suggest=True)
fig.show()
# fit the model
trainer.fit(
tft, train_dataloader=train_dataloader, val_dataloaders=val_dataloader,
)
# %%
archive/lightning/create_dataset.py
-64
View File
@@ -1,64 +0,0 @@
import pytorch_lightning as pl
from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
from pytorch_forecasting import TimeSeriesDataSet, TemporalFusionTransformer
from pytorch_forecasting.data import GroupNormalizer
from pytorch_forecasting.metrics import QuantileLoss
import sys
sys.path.insert(0, '..')
from data_loader.load_data import load_files
print("success")
def load_format_data(data_dir):
print("Data Starting ===>", end=" ")
# load data
data = load_files(data_dir, add_features=True, log_returns=False, narrow_format=True)
# need to treat time as an independent column
data = data.reset_index().rename({'index':'time'}, axis = 'columns')
# we need a `time_idx` column for pytorch-forecasting, so we convert the time column to a time index by encoding the dates as consecutive days from the first date.
data['time_idx'] = (data['time']-data['time'].min()).astype('timedelta64[D]').astype(int)+1
# volume needs some love before we can use it
data.drop(columns=['volume'], inplace=True)
data['month'] = data['month'].astype(str)
data['day_month'] = data['day_month'].astype(str)
data['day_week'] = data['day_week'].astype(str)
print("<=== Data Loaded")
print(data.head(3))
print(data.describe())
print("")
return data
def create_dataloaders(data, kwargs):
print("DataLoader Starting ===>", end=" ")
# training_cutoff = "YYYY-MM-DD" # day for cutoff
# training_cutoff = data["time_idx"].max() - max_prediction_length
batch_size = kwargs['batch_size']
del kwargs['batch_size']
training_dataset = TimeSeriesDataSet(
data, # data[lambda x: x.date < training_cutoff],
**kwargs
)
#%%
# create validation and training dataset
validation = TimeSeriesDataSet.from_dataset(training_dataset, data, predict=True, stop_randomization=True)
train_dataloader = training_dataset.to_dataloader(train=True, batch_size=batch_size, num_workers=0)
val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=0)
print("<=== DataLoader Created")
return training_dataset, train_dataloader, val_dataloader
archive/lightning/define_model.py
View File
archive/lightning/models/built_in_models.py
-19
View File
@@ -1,19 +0,0 @@
from pytorch_forecasting import TemporalFusionTransformer
from pytorch_forecasting.metrics import QuantileLoss
def create_TemporalFusionTransformer(training_dataset, model_options):
# create the model
tft = TemporalFusionTransformer.from_dataset(
training_dataset,
learning_rate=0.03,
hidden_size=32,
attention_head_size=1,
dropout=0.1,
hidden_continuous_size=16,
output_size=7,
loss=QuantileLoss(),
log_interval=2,
reduce_on_plateau_patience=4
)
print(f"Number of parameters in network: {tft.size()/1e3:.1f}k")
return tft
archive/lightning/models/custom_model.py
-98
View File
@@ -1,98 +0,0 @@
#%%
import warnings
from typing import Dict
warnings.filterwarnings("ignore")
import torch
from torch import nn
from pytorch_forecasting.models import BaseModel
from pytorch_forecasting import TimeSeriesDataSet
#%%
class FullyConnectedModule(nn.Module):
def __init__(self, input_size: int, output_size: int, hidden_size: int, n_hidden_layers: int):
super().__init__()
# input layer
module_list = [nn.Linear(input_size, hidden_size), nn.ReLU()]
# hidden layers
for _ in range(n_hidden_layers):
module_list.extend([nn.Linear(hidden_size, hidden_size), nn.ReLU()])
# output layer
module_list.append(nn.Linear(hidden_size, output_size))
self.sequential = nn.Sequential(*module_list)
def forward(self, x: torch.Tensor) -> torch.Tensor:
# x of shape: batch_size x n_timesteps_in
# output of shape batch_size x n_timesteps_out
return self.sequential(x)
#%%
class FullyConnectedModel(BaseModel):
def __init__(self, input_size: int, output_size: int, hidden_size: int, n_hidden_layers: int, **kwargs):
# saves arguments in signature to `.hparams` attribute, mandatory call - do not skip this
self.save_hyperparameters()
# pass additional arguments to BaseModel.__init__, mandatory call - do not skip this
super().__init__(**kwargs)
self.network = FullyConnectedModule(
input_size=self.hparams.input_size,
output_size=self.hparams.output_size,
hidden_size=self.hparams.hidden_size,
n_hidden_layers=self.hparams.n_hidden_layers,
)
def forward(self, x: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
# x is a batch generated based on the TimeSeriesDataset
network_input = x["encoder_cont"].squeeze(-1)
prediction = self.network(network_input)
# rescale predictions into target space
prediction = self.transform_output(prediction, target_scale=x["target_scale"])
# We need to return a dictionary that at least contains the prediction
# The parameter can be directly forwarded from the input.
# The conversion to a named tuple can be directly achieved with the `to_network_output` function.
return self.to_network_output(prediction=prediction)
@classmethod
def from_dataset(cls, dataset: TimeSeriesDataSet, **kwargs):
new_kwargs = {
"output_size": dataset.max_prediction_length,
"input_size": dataset.max_encoder_length,
}
new_kwargs.update(kwargs) # use to pass real hyperparameters and override defaults set by dataset
# example for dataset validation
assert dataset.max_prediction_length == dataset.min_prediction_length, "Decoder only supports a fixed length"
assert dataset.min_encoder_length == dataset.max_encoder_length, "Encoder only supports a fixed length"
assert (
len(dataset.time_varying_known_categoricals) == 0
and len(dataset.time_varying_known_reals) == 0
and len(dataset.time_varying_unknown_categoricals) == 0
and len(dataset.static_categoricals) == 0
and len(dataset.static_reals) == 0
and len(dataset.time_varying_unknown_reals) == 1
and dataset.time_varying_unknown_reals[0] == dataset.target
), "Only covariate should be the target in 'time_varying_unknown_reals'"
return super().from_dataset(dataset, **new_kwargs)
def calculate_prediction_actual_by_variable(x, train):
pass
# %%
def create_FullyConnectedModel(training_dataset, kwargs):
model = FullyConnectedModel.from_dataset(training_dataset, **kwargs)
model.summarize("full") # print model summary
model.hparams
return model
archive/lightning/options.py
-63
View File
@@ -1,63 +0,0 @@
from pytorch_forecasting.data import GroupNormalizer
from pytorch_forecasting.models.temporal_fusion_transformer import TemporalFusionTransformer
from lightning.models.custom_model import FullyConnectedModel
training_options = dict(
max_epochs=100,
gpus=0,
gradient_clip_val=0.1,
limit_train_batches=30
)
model_options_fcn = dict(
hidden_size=64,
n_hidden_layers=2,
)
model_options_tft = dict(
hidden_size=64,
n_hidden_layers=2,
)
dataset_options_tft = dict(
time_idx= 'time_idx',
target= 'returns',
# weight="weight",
group_ids=[ 'ticker' ],
min_encoder_length = 36, # this is the look-back window, see https://github.com/jdb78/pytorch-forecasting/issues/448
max_encoder_length = 36,
min_prediction_length = 1,
max_prediction_length = 1,
static_categoricals=[ ],
static_reals=[ ],
time_varying_known_categoricals=[ 'day_month', 'day_week', 'month' ],
time_varying_known_reals=[ 'vol_10', 'vol_20', 'vol_30', 'vol_60', 'mom_10', 'mom_20', 'mom_30', 'mom_60', 'mom_90'],
time_varying_unknown_categoricals=[ ],
time_varying_unknown_reals=[ 'returns' ],
allow_missing_timesteps=True,
target_normalizer=GroupNormalizer(
groups=['ticker'], transformation="softplus"),
batch_size=128
)
dataset_options_fcn = dict(
time_idx= 'time_idx',
target= 'returns',
# weight="weight",
group_ids=[ 'ticker' ],
min_encoder_length = 36, # this is the look-back window, see https://github.com/jdb78/pytorch-forecasting/issues/448
max_encoder_length = 36,
min_prediction_length = 1,
max_prediction_length = 1,
static_categoricals=[ ],
static_reals=[ ],
time_varying_known_categoricals=[ ],
time_varying_known_reals=[ ],
time_varying_unknown_categoricals=[ ],
time_varying_unknown_reals=[ 'returns' ],
allow_missing_timesteps=True,
target_normalizer=GroupNormalizer(
groups=['ticker'], transformation="softplus"),
batch_size=128
)
archive/lightning/run_pipeline.py
-41
View File
@@ -1,41 +0,0 @@
#%%
from create_dataset import load_format_data, create_dataloaders
from train_predict import train_model, predict
from models.built_in_models import create_TemporalFusionTransformer
from models.custom_model import create_FullyConnectedModel
from options import training_options, model_options_tft, model_options_fcn, dataset_options_tft, dataset_options_fcn
import warnings
warnings.filterwarnings("ignore")
#%%
def run_pipeline(model_name, data_dir):
data = load_format_data(data_dir)
dataset_options, model_options, _create_model = select_model(model_name)
training_dataset, train_dataloader, val_dataloader = create_dataloaders(data, dataset_options)
model = _create_model( training_dataset, model_options )
trainer = train_model(model, train_dataloader, val_dataloader, training_options)
predict(trainer, model, val_dataloader)
#%%
def select_model(model_name):
if model_name == "FullyConnectedLayer":
return dataset_options_fcn, model_options_fcn, create_FullyConnectedModel
elif model_name == "TemporalFusionTransformer":
return dataset_options_tft, model_options_tft, create_TemporalFusionTransformer
else:
assert False, "No such model exists."
#%%
run_pipeline("FullyConnectedLayer", '../data/')
# #%%
# if __name__ == '__main__':
# run_pipeline("FullyConnectedLayer", '../data/')
archive/lightning/train_predict.py
-67
View File
@@ -1,67 +0,0 @@
import pytorch_lightning as pl
from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
import torch
from torch import nn
import warnings
warnings.filterwarnings("ignore")
import sys
sys.path.insert(0, '..')
from data_loader.load_data import load_files
def train_model(model, train_dataloader, val_dataloader, kwargs):
print()
print("Creating Trainer ===>", end=" ")
# define trainer with early stopping
early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min")
lr_logger = LearningRateMonitor()
trainer = pl.Trainer(
**kwargs,
callbacks=[lr_logger, early_stop_callback],
)
print("<=== Trainer Created")
print("Finding Optimal LR ===>", end=" ")
# find optimal learning rate (set limit_train_batches to 1.0 and log_interval = -1)
res = trainer.tuner.lr_find(
model, train_dataloader=train_dataloader, val_dataloaders=val_dataloader, early_stop_threshold=1000.0, max_lr=0.3,
)
print(f"<=== suggested learning rate: {res.suggestion()}")
fig = res.plot(show=True, suggest=True)
fig.show()
print("Training the model ===>", end=" ")
# fit the model
trainer.fit(
model, train_dataloader=train_dataloader, val_dataloaders=val_dataloader,
)
print("<=== Training Finished")
return trainer
def predict(trainer, model, val_dataloader):
pass
# best_model_path = trainer.checkpoint_callback.best_model_path
# best_model = model.load_from_checkpoint(best_model_path)
# # calcualte mean absolute error on validation set
# actuals = torch.cat([y[0] for x, y in iter(val_dataloader)])
# predictions = best_model.predict(val_dataloader)
# (actuals - predictions).abs().mean()
# raw_predictions, x = best_model.predict(val_dataloader, mode="raw", return_x=True)
# for idx in range(10): # plot 10 examples
# best_model.plot_prediction(x, raw_predictions, idx=idx, add_loss_to_title=True)
# predictions, x = best_model.predict(val_dataloader, return_x=True)
# predictions_vs_actuals = best_model.calculate_prediction_actual_by_variable(x, predictions)
# best_model.plot_prediction_actual_by_variable(predictions_vs_actuals)
archive/model_lightning.py
-60
View File
@@ -1,60 +0,0 @@
import torch
from torch import nn
from torch.nn import functional as F
from torch.utils.data import DataLoader, random_split
import pytorch_lightning as pl
class LitManualAutoEncoder(pl.LightningModule):
def __init__(self):
super().__init__()
self.encoder = nn.Sequential(nn.Linear(28 * 28, 128), nn.ReLU(), nn.Linear(128, 3))
self.decoder = nn.Sequential(nn.Linear(3, 128), nn.ReLU(), nn.Linear(128, 28 * 28))
print("success")
def training_step(self, batch, batch_idx):
# --------------------------
# REPLACE WITH YOUR OWN
opt_a = self.optimizers()
x, y = batch
x = x.view(x.size(0), -1)
z = self.encoder(x)
x_hat = self.decoder(z)
loss = F.mse_loss(x_hat, x)
# backward acts like normal backward
self.manual_backward(loss, opt_a, retain_graph=True)
self.manual_backward(loss, opt_a)
opt_a.step()
opt_a.zero_grad()
# --------------------------
def validation_step(self, batch, batch_idx):
# --------------------------
# REPLACE WITH YOUR OWN
x, y = batch
x = x.view(x.size(0), -1)
z = self.encoder(x)
x_hat = self.decoder(z)
loss = F.mse_loss(x_hat, x)
self.log('val_loss', loss)
# --------------------------
def test_step(self, batch, batch_idx):
# --------------------------
# REPLACE WITH YOUR OWN
x, y = batch
x = x.view(x.size(0), -1)
z = self.encoder(x)
x_hat = self.decoder(z)
loss = F.mse_loss(x_hat, x)
self.log('test_loss', loss)
# --------------------------
def configure_optimizers(self):
optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
return optimizer
archive/run_lightning.py
-48
View File
@@ -1,48 +0,0 @@
from data_loader.load_data import load_files
import pandas as pd
# from tensorflow import keras
from utils.normalize import normalize
# import tensorflow as tf
from utils.visualize import visualize_loss
from torch.utils.data import DataLoader, random_split
from model_lightning import LitManualAutoEncoder
import pytorch_lightning as pl
#%%
data = load_files('data/', False)
data.reset_index(drop=True, inplace=True)
data = data[[column for column in data.columns if not column.endswith('volume')]]
data.head()
#%%
ticker_to_predict = 'ETH_returns'
learning_rate = 0.002
batch_size = 64
epochs = 100
split_fraction = 0.715
train_split = int(split_fraction * int(data.shape[0]))
past = 10
future = 1
start = past + future
end = start + train_split
# train = DataLoader(train, batch_size=32)
# test = DataLoader(test, batch_size=32)
# val = DataLoader(val, batch_size=32)
# init model
ae = LitManualAutoEncoder()
# Initialize a trainer
trainer = pl.Trainer(gpus=1, max_epochs=3, progress_bar_refresh_rate=20)
# Train the model ⚡
# trainer.fit(ae, train, val)
config/hashing.py
+24 -15
View File
@@ -1,18 +1,27 @@
from data_loader.types import DataCollection
def hash_data_config(data_config: dict) -> str:
def hash_data_collection(data_collection: DataCollection) -> str: return ''.join([a[0] + a[1] for a in data_collection])
def hash_feature_extractors(feature_extractos) -> str: return ''.join([f[0] for f in feature_extractos])
def to_str(x): return ''.join([str(i) for i in x])
return '_'.join(to_str([
hash_data_collection(data_config['assets']),
hash_data_collection(data_config['other_assets']),
hash_data_collection(data_config['exogenous_data']),
data_config['target_asset'][0] + data_config['target_asset'][1],
data_config['load_non_target_asset'],
hash_feature_extractors(data_config['own_features']),
hash_feature_extractors(data_config['other_features']),
hash_feature_extractors(data_config['exogenous_features']),
]))
def hash_data_collection(data_collection: DataCollection) -> str:
return "".join([a[0] + a[1] for a in data_collection])
def hash_feature_extractors(feature_extractos) -> str:
return "".join([f[0] for f in feature_extractos])
def to_str(x):
return "".join([str(i) for i in x])
return "_".join(
to_str(
[
hash_data_collection(data_config["assets"]),
hash_data_collection(data_config["other_assets"]),
hash_data_collection(data_config["exogenous_data"]),
data_config["target_asset"][0] + data_config["target_asset"][1],
data_config["load_non_target_asset"],
hash_feature_extractors(data_config["own_features"]),
hash_feature_extractors(data_config["other_features"]),
hash_feature_extractors(data_config["exogenous_features"]),
]
)
)
config/preprocess.py
+51 -22
View File
@@ -1,6 +1,8 @@
from .types import Config, RawConfig
from utils.helpers import flatten
from feature_extractors.feature_extractor_presets import presets as feature_extractor_presets
from feature_extractors.feature_extractor_presets import (
presets as feature_extractor_presets,
)
from models.model_map import get_model
from data_loader.collections import data_collections
from labeling.eventfilters_map import eventfilters_map
@@ -8,6 +10,7 @@ from labeling.labellers_map import labellers_map
from models.sklearn import SKLearnModel
from sklearn.ensemble import VotingClassifier
def preprocess_config(raw_config: RawConfig) -> Config:
config_dict = vars(raw_config)
config_dict = __preprocess_model_config(config_dict)
@@ -16,47 +19,73 @@ def preprocess_config(raw_config: RawConfig) -> Config:
config_dict = __preprocess_event_filter_config(config_dict)
config_dict = __preprocess_event_labeller_config(config_dict)
config_dict['no_of_classes'] = 'two'
config_dict['mode'] = 'training'
config_dict["no_of_classes"] = "two"
config_dict["mode"] = "training"
config = Config(**config_dict)
return config
def __preprocess_feature_extractors_config(data_dict: dict) -> dict:
data_dict = data_dict.copy()
keys = ['own_features', 'other_features', 'exogenous_features']
keys = ["own_features", "other_features", "exogenous_features"]
for key in keys:
preset_names = data_dict[key]
data_dict[key] = flatten([feature_extractor_presets[preset_name] for preset_name in preset_names])
data_dict[key] = flatten(
[feature_extractor_presets[preset_name] for preset_name in preset_names]
)
return data_dict
def __preprocess_model_config(model_config: dict) -> dict:
directional_models = [get_model(model_name) for model_name in model_config['directional_models']]
model_config.pop('directional_models')
model_config['directional_model'] = SKLearnModel(VotingClassifier([(m.name, m)for m in directional_models], voting ='soft'))
if len(model_config['meta_models']) > 0:
meta_models = [get_model(model_name) for model_name in model_config['meta_models']]
model_config['meta_model'] = SKLearnModel(VotingClassifier([(m.name, m)for m in meta_models], voting ='soft'))
model_config.pop('meta_models')
directional_models = [
get_model(model_name) for model_name in model_config["directional_models"]
]
model_config.pop("directional_models")
model_config["directional_model"] = SKLearnModel(
VotingClassifier([(m.name, m) for m in directional_models], voting="soft")
)
if len(model_config["meta_models"]) > 0:
meta_models = [
get_model(model_name) for model_name in model_config["meta_models"]
]
model_config["meta_model"] = SKLearnModel(
VotingClassifier([(m.name, m) for m in meta_models], voting="soft")
)
model_config.pop("meta_models")
return model_config
def __preprocess_data_collections_config(data_dict: dict) -> dict:
keys = ['assets', 'other_assets', 'exogenous_data']
keys = ["assets", "other_assets", "exogenous_data"]
for key in keys:
preset_names = data_dict[key]
data_dict[key] = flatten([data_collections[preset_name] for preset_name in preset_names])
target_asset = next(iter([asset for asset in data_dict['assets'] if asset[1] == data_dict['target_asset']]), None)
if target_asset is None: raise Exception('Target asset wasnt found in assets')
data_dict['target_asset'] = target_asset
data_dict[key] = flatten(
[data_collections[preset_name] for preset_name in preset_names]
)
target_asset = next(
iter(
[
asset
for asset in data_dict["assets"]
if asset[1] == data_dict["target_asset"]
]
),
None,
)
if target_asset is None:
raise Exception("Target asset wasnt found in assets")
data_dict["target_asset"] = target_asset
return data_dict
def __preprocess_event_filter_config(data_dict: dict) -> dict:
data_dict['event_filter'] = eventfilters_map[data_dict['event_filter']]
data_dict["event_filter"] = eventfilters_map[data_dict["event_filter"]]
return data_dict
def __preprocess_event_labeller_config(config_dict: dict) -> dict:
config_dict['labeling'] = labellers_map[config_dict['labeling']](config_dict['forecasting_horizon'])
config_dict["labeling"] = labellers_map[config_dict["labeling"]](
config_dict["forecasting_horizon"]
)
return config_dict
config/presets.py
+82 -85
View File
@@ -1,104 +1,101 @@
from .types import RawConfig, Config
def get_dev_config() -> RawConfig:
classification_models = ["LogisticRegression_two_class"]
return RawConfig(
directional_models_meta = False,
dimensionality_reduction = False,
n_features_to_select = 30,
expanding_window_base = False,
expanding_window_meta = False,
sliding_window_size_base = 380,
sliding_window_size_meta = 1,
retrain_every = 20,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
assets = ['daily_only_btc'],
target_asset = 'BTC_USD',
other_assets = [],
exogenous_data = [],
load_non_target_asset= True,
own_features = ['level_2', 'date_days'],
other_features = ['single_mom'],
exogenous_features = ['z_score'],
directional_models = classification_models,
meta_models = [],
event_filter = 'none',
labeling = 'two_class',
forecasting_horizon = 100,
directional_models_meta=False,
dimensionality_reduction=False,
n_features_to_select=30,
expanding_window_base=False,
expanding_window_meta=False,
sliding_window_size_base=380,
sliding_window_size_meta=1,
retrain_every=20,
scaler="minmax", # 'normalize' 'minmax' 'standardize'
assets=["daily_only_btc"],
target_asset="BTC_USD",
other_assets=[],
exogenous_data=[],
load_non_target_asset=True,
own_features=["level_2", "date_days"],
other_features=["single_mom"],
exogenous_features=["z_score"],
directional_models=classification_models,
meta_models=[],
event_filter="none",
labeling="two_class",
forecasting_horizon=100,
)
def get_default_ensemble_config() -> RawConfig:
classification_models = ["LogisticRegression_two_class", "LDA", "NB", "RFC", "XGB_two_class", "LGBM", "StaticMom"]
meta_models = ['LogisticRegression_two_class', 'LGBM']
classification_models = [
"LogisticRegression_two_class",
"LDA",
"NB",
"RFC",
"XGB_two_class",
"LGBM",
"StaticMom",
]
meta_models = ["LogisticRegression_two_class", "LGBM"]
return RawConfig(
directional_models_meta = True,
dimensionality_reduction = False,
n_features_to_select = 30,
expanding_window_base = False,
expanding_window_meta = True,
sliding_window_size_base = 380,
sliding_window_size_meta = 240,
retrain_every = 10,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
assets = ['daily_crypto'],
target_asset = 'BTC_USD',
other_assets = ['daily_etf'],
exogenous_data = ['daily_glassnode'],
load_non_target_asset= True,
own_features = ['level_2', 'date_days', 'lags_up_to_5'],
other_features = ['level_2', 'lags_up_to_5'],
exogenous_features = ['z_score'],
directional_models = classification_models,
meta_models = meta_models,
event_filter = 'cusum_vol',
labeling = 'two_class',
forecasting_horizon = 100,
directional_models_meta=True,
dimensionality_reduction=False,
n_features_to_select=30,
expanding_window_base=False,
expanding_window_meta=True,
sliding_window_size_base=380,
sliding_window_size_meta=240,
retrain_every=10,
scaler="minmax", # 'normalize' 'minmax' 'standardize'
assets=["daily_crypto"],
target_asset="BTC_USD",
other_assets=["daily_etf"],
exogenous_data=["daily_glassnode"],
load_non_target_asset=True,
own_features=["level_2", "date_days", "lags_up_to_5"],
other_features=["level_2", "lags_up_to_5"],
exogenous_features=["z_score"],
directional_models=classification_models,
meta_models=meta_models,
event_filter="cusum_vol",
labeling="two_class",
forecasting_horizon=100,
)
def get_lightweight_ensemble_config() -> RawConfig:
classification_models = ['LogisticRegression_two_class', 'LGBM']
meta_models = ['LogisticRegression_two_class', 'LGBM']
classification_models = ["LogisticRegression_two_class", "LGBM"]
meta_models = ["LogisticRegression_two_class", "LGBM"]
return RawConfig(
directional_models_meta = True,
dimensionality_reduction = True,
n_features_to_select = 30,
expanding_window_base = True,
expanding_window_meta = True,
sliding_window_size_base = 3800,
sliding_window_size_meta = 2400,
retrain_every = 1000,
scaler = 'minmax', # 'normalize' 'minmax' 'standardize'
assets = ['fivemin_crypto'],
target_asset = 'BTC_USD',
other_assets = [],
exogenous_data = [],
load_non_target_asset= False,
own_features = ['level_1'],
other_features = [],
exogenous_features = [],
directional_models = classification_models,
meta_models = meta_models,
event_filter = 'cusum_fixed',
labeling = 'two_class',
forecasting_horizon = 50,
directional_models_meta=True,
dimensionality_reduction=True,
n_features_to_select=30,
expanding_window_base=True,
expanding_window_meta=True,
sliding_window_size_base=3800,
sliding_window_size_meta=2400,
retrain_every=1000,
scaler="minmax", # 'normalize' 'minmax' 'standardize'
assets=["fivemin_crypto"],
target_asset="BTC_USD",
other_assets=[],
exogenous_data=[],
load_non_target_asset=False,
own_features=["level_1"],
other_features=[],
exogenous_features=[],
directional_models=classification_models,
meta_models=meta_models,
event_filter="cusum_fixed",
labeling="two_class",
forecasting_horizon=50,
)
config/types.py
+11 -12
View File
@@ -18,7 +18,7 @@ class RawConfig(BaseModel):
sliding_window_size_base: int
sliding_window_size_meta: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
scaler: Literal["normalize", "minmax", "standardize"]
assets: list[str]
target_asset: str
@@ -28,8 +28,8 @@ class RawConfig(BaseModel):
own_features: list[str]
other_features: list[str]
exogenous_features: list[str]
event_filter: Literal['none', 'cusum_vol', 'cusum_fixed']
labeling: Literal['two_class', 'three_class_balanced', 'three_class_imbalanced']
event_filter: Literal["none", "cusum_vol", "cusum_fixed"]
labeling: Literal["two_class", "three_class_balanced", "three_class_imbalanced"]
forecasting_horizon: int
directional_models: list[str]
@@ -46,7 +46,7 @@ class Config:
sliding_window_size_base: int
sliding_window_size_meta: int
retrain_every: int
scaler: Literal['normalize', 'minmax', 'standardize']
scaler: Literal["normalize", "minmax", "standardize"]
assets: DataCollection
target_asset: DataSource
@@ -59,28 +59,27 @@ class Config:
event_filter: EventFilter
labeling: EventLabeller
forecasting_horizon: int
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"]
mode: Literal['training', 'inference']
mode: Literal["training", "inference"]
directional_model: Model
meta_model: Model
@validator('directional_model', 'meta_model')
@validator("directional_model", "meta_model")
def check_model(cls, v):
assert isinstance(v, BaseEstimator)
return v
@validator('event_filter')
@validator("event_filter")
def check_event_filter(cls, v):
assert isinstance(v, EventFilter)
return v
@validator('labeling')
@validator("labeling")
def check_labeling(cls, v):
assert isinstance(v, EventLabeller)
return v
# class Config:
# arbitrary_types_allowed = True
data_loader/collections.py
+64 -32
View File
@@ -1,51 +1,83 @@
from .types import Path, FileName, DataSource, DataCollection
from utils.helpers import flatten
def transform_to_data_collection(path: str, file_names: list[str]) -> DataCollection:
return list(zip([path] * len(file_names), file_names))
__daily_etf = ["GLD", "IEF", "QQQ", "SPY", "TLT"]
__daily_crypto = ["ADA_USD", "BCH_USD", "BNB_USD", "BTC_USD", "DOT_USD", "ETC_USD", "ETH_USD", "FIL_USD", "LTC_USD", "SOL_USD", "THETA_USD", "TRX_USD", "UNI_USD", "XLM_USD", "XRP_USD", "XTZ_USD"]
__daily_crypto = [
"ADA_USD",
"BCH_USD",
"BNB_USD",
"BTC_USD",
"DOT_USD",
"ETC_USD",
"ETH_USD",
"FIL_USD",
"LTC_USD",
"SOL_USD",
"THETA_USD",
"TRX_USD",
"UNI_USD",
"XLM_USD",
"XRP_USD",
"XTZ_USD",
]
__daily_crypto_lightweight = ["ADA_USD", "BCH_USD"]
__5min_crypto = ["BTC_USD", "DASH_USD", "ETC_USD", "ETH_USD", "LTC_USD", "TRX_USD", "XLM_USD", "XMR_USD"]
__5min_crypto = [
"BTC_USD",
"DASH_USD",
"ETC_USD",
"ETH_USD",
"LTC_USD",
"TRX_USD",
"XLM_USD",
"XMR_USD",
]
__daily_glassnode = ['rhodl_ratio',
__daily_glassnode = [
"rhodl_ratio",
# 'cvdd',
'nvt_ratio',
'nvt_signal',
'velocity',
'supply_adjusted_cdd',
'binary_cdd',
'supply_adjusted_dormancy',
'puell_multiple',
'asopr',
'reserve_risk',
'sopr',
'cdd',
'asol',
'msol',
'dormancy',
'liveliness',
'relative_unrealized_profit',
'relative_unrealized_loss',
'nupl',
'sth_nupl',
'lth_nupl',
'ssr',
'bvin',
"nvt_ratio",
"nvt_signal",
"velocity",
"supply_adjusted_cdd",
"binary_cdd",
"supply_adjusted_dormancy",
"puell_multiple",
"asopr",
"reserve_risk",
"sopr",
"cdd",
"asol",
"msol",
"dormancy",
"liveliness",
"relative_unrealized_profit",
"relative_unrealized_loss",
"nupl",
"sth_nupl",
"lth_nupl",
"ssr",
"bvin",
# 'hash_rate'
]
data_collections = dict(
daily_only_btc = transform_to_data_collection("data/daily_crypto", ['BTC_USD']),
daily_crypto = transform_to_data_collection("data/daily_crypto", __daily_crypto),
daily_crypto_lightweight = transform_to_data_collection("data/daily_crypto", __daily_crypto_lightweight),
daily_etf = transform_to_data_collection("data/daily_etf", __daily_etf),
fivemin_crypto = transform_to_data_collection("data/5min_crypto", __5min_crypto),
daily_glassnode =transform_to_data_collection("data/daily_glassnode", __daily_glassnode),
daily_only_btc=transform_to_data_collection("data/daily_crypto", ["BTC_USD"]),
daily_crypto=transform_to_data_collection("data/daily_crypto", __daily_crypto),
daily_crypto_lightweight=transform_to_data_collection(
"data/daily_crypto", __daily_crypto_lightweight
),
daily_etf=transform_to_data_collection("data/daily_etf", __daily_etf),
fivemin_crypto=transform_to_data_collection("data/5min_crypto", __5min_crypto),
daily_glassnode=transform_to_data_collection(
"data/daily_glassnode", __daily_glassnode
),
)
data_loader/get_prices.py
+51 -20
View File
@@ -3,45 +3,76 @@ import requests
import pandas as pd
# %%
def get_crypto_price_crypto_compare(symbol: str, exchange: str, days: int) -> pd.DataFrame:
api_url = f'https://min-api.cryptocompare.com/data/v2/histoday?fsym={symbol}&tsym={exchange}&limit={days}&api_key={CC_API_KEY}'
def get_crypto_price_crypto_compare(
symbol: str, exchange: str, days: int
) -> pd.DataFrame:
api_url = f"https://min-api.cryptocompare.com/data/v2/histoday?fsym={symbol}&tsym={exchange}&limit={days}&api_key={CC_API_KEY}"
raw = requests.get(api_url).json()
df = pd.DataFrame(raw['Data']['Data'])[['time', 'high', 'low', 'open', 'close']].set_index('time')
df.index = pd.to_datetime(df.index, unit = 's')
df.sort_index(inplace = True, ascending= True)
df = pd.DataFrame(raw["Data"]["Data"])[
["time", "high", "low", "open", "close"]
].set_index("time")
df.index = pd.to_datetime(df.index, unit="s")
df.sort_index(inplace=True, ascending=True)
return df
ada = get_crypto_price_crypto_compare('ADA', 'USD', 1500)
ada = get_crypto_price_crypto_compare("ADA", "USD", 1500)
ada
def get_crypto_price_av(symbol: str, exchange: str, start_date = None) -> pd.DataFrame:
api_url = f'https://www.alphavantage.co/query?function=DIGITAL_CURRENCY_DAILY&symbol={symbol}&market={exchange}&apikey={AV_API_KEY}'
def get_crypto_price_av(symbol: str, exchange: str, start_date=None) -> pd.DataFrame:
api_url = f"https://www.alphavantage.co/query?function=DIGITAL_CURRENCY_DAILY&symbol={symbol}&market={exchange}&apikey={AV_API_KEY}"
raw_df = requests.get(api_url).json()
df = pd.DataFrame(raw_df['Time Series (Digital Currency Daily)']).T
df = df.rename(columns = {'1a. open (USD)': 'open', '2a. high (USD)': 'high', '3a. low (USD)': 'low', '4a. close (USD)': 'close', '5. volume': 'volume'})
df = pd.DataFrame(raw_df["Time Series (Digital Currency Daily)"]).T
df = df.rename(
columns={
"1a. open (USD)": "open",
"2a. high (USD)": "high",
"3a. low (USD)": "low",
"4a. close (USD)": "close",
"5. volume": "volume",
}
)
for i in df.columns:
df[i] = df[i].astype(float)
df.index = pd.to_datetime(df.index)
df = df.iloc[::-1].drop(['1b. open (USD)', '2b. high (USD)', '3b. low (USD)', '4b. close (USD)', '6. market cap (USD)'], axis = 1)
df = df.iloc[::-1].drop(
[
"1b. open (USD)",
"2b. high (USD)",
"3b. low (USD)",
"4b. close (USD)",
"6. market cap (USD)",
],
axis=1,
)
if start_date:
df = df[df.index >= start_date]
df.sort_index(inplace = True, ascending= True)
df.sort_index(inplace=True, ascending=True)
return df
def get_stock_price_av(symbol: str, start_date: str = None) -> pd.DataFrame:
api_url = f'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&outputsize=full&apikey={AV_API_KEY}'
api_url = f"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&outputsize=full&apikey={AV_API_KEY}"
raw_df = requests.get(api_url).json()
df = pd.DataFrame(raw_df['Time Series (Daily)']).T
df = df.rename(columns = {'1. open': 'open', '2. high': 'high', '3. low': 'low', '5. adjusted close': 'close', '6. volume': 'volume'})
df = pd.DataFrame(raw_df["Time Series (Daily)"]).T
df = df.rename(
columns={
"1. open": "open",
"2. high": "high",
"3. low": "low",
"5. adjusted close": "close",
"6. volume": "volume",
}
)
for i in df.columns:
df[i] = df[i].astype(float)
df.index = pd.to_datetime(df.index)
df = df.iloc[::-1].drop(['4. close', '7. dividend amount', '8. split coefficient'], axis = 1)
df = df.iloc[::-1].drop(
["4. close", "7. dividend amount", "8. split coefficient"], axis=1
)
if start_date:
df = df[df.index >= start_date]
df.sort_index(inplace = True, ascending= True)
df = df.rename_axis('time')
df.sort_index(inplace=True, ascending=True)
df = df.rename_axis("time")
return df
data_loader/load.py
+92 -64
View File
@@ -10,8 +10,8 @@ import os
from config.hashing import hash_data_config
from .types import XDataFrame, ReturnSeries, ForwardReturnSeries
from diskcache import Cache
cache = Cache(".cachedir/data")
cache = Cache(".cachedir/data")
def load_data(**kwargs) -> tuple[XDataFrame, ReturnSeries]:
@@ -23,15 +23,17 @@ def load_data(**kwargs) -> tuple[XDataFrame, ReturnSeries]:
cache[hashed] = return_value
return return_value
def __load_data(assets: DataCollection,
other_assets: DataCollection,
exogenous_data: DataCollection,
target_asset: DataSource,
load_non_target_asset: bool,
own_features: list[tuple[str, FeatureExtractor, list[int]]],
other_features: list[tuple[str, FeatureExtractor, list[int]]],
exogenous_features: list[tuple[str, FeatureExtractor, list[int]]],
) -> tuple[XDataFrame, ReturnSeries]:
def __load_data(
assets: DataCollection,
other_assets: DataCollection,
exogenous_data: DataCollection,
target_asset: DataSource,
load_non_target_asset: bool,
own_features: list[tuple[str, FeatureExtractor, list[int]]],
other_features: list[tuple[str, FeatureExtractor, list[int]]],
exogenous_features: list[tuple[str, FeatureExtractor, list[int]]],
) -> tuple[XDataFrame, ReturnSeries]:
"""
Loads asset data from the specified path.
Returns:
@@ -42,77 +44,96 @@ def __load_data(assets: DataCollection,
target_file = [f for f in assets if f[1].startswith(target_asset[1])]
assert len(target_file) == 1, "There should be exactly one target file"
other_files = [f for f in assets if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False]
other_files = [
f
for f in assets
if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False
]
files = other_files + other_assets
target_asset_future = [__load_df.remote(
data_source=data_source,
prefix=data_source[1],
returns='log_returns',
feature_extractors=own_features,
) for data_source in target_file]
target_asset_future = [
__load_df.remote(
data_source=data_source,
prefix=data_source[1],
returns="log_returns",
feature_extractors=own_features,
)
for data_source in target_file
]
target_asset_df = ray.get(target_asset_future)
target_asset_only_returns_future = __load_df.remote(
data_source=target_file[0],
prefix=target_file[0][1],
returns='returns',
returns="returns",
feature_extractors=[],
)
df_target_asset_only_returns = ray.get(target_asset_only_returns_future)
asset_futures = [__load_df.remote(
data_source=data_source,
prefix=data_source[1],
returns='log_returns',
feature_extractors=other_features,
) for data_source in files]
asset_futures = [
__load_df.remote(
data_source=data_source,
prefix=data_source[1],
returns="log_returns",
feature_extractors=other_features,
)
for data_source in files
]
asset_dfs = ray.get(asset_futures)
exogenous_futures = [__load_df.remote(
data_source=data_source,
prefix=data_source[1],
returns='none',
feature_extractors=exogenous_features,
) for data_source in exogenous_data]
exogenous_futures = [
__load_df.remote(
data_source=data_source,
prefix=data_source[1],
returns="none",
feature_extractors=exogenous_features,
)
for data_source in exogenous_data
]
exogenous_dfs = ray.get(exogenous_futures)
X = target_asset_df + asset_dfs + exogenous_dfs
X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.)
X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.0)
X.index = pd.DatetimeIndex(X.index)
## Create target
returns = df_target_asset_only_returns[target_asset[1] + '_returns']
## Create target
returns = df_target_asset_only_returns[target_asset[1] + "_returns"]
returns.index = pd.DatetimeIndex(X.index)
return X, returns
return X, returns
@ray.remote
def __load_df(data_source: DataSource,
prefix: str,
returns: Literal['none', 'price', 'returns', 'log_returns'],
feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]) -> pd.DataFrame:
df = pd.read_csv(os.path.join(data_source[0], data_source[1] + '.csv'), header=0, index_col=0).fillna(0)
def __load_df(
data_source: DataSource,
prefix: str,
returns: Literal["none", "price", "returns", "log_returns"],
feature_extractors: list[tuple[str, FeatureExtractor, list[int]]],
) -> pd.DataFrame:
df = pd.read_csv(
os.path.join(data_source[0], data_source[1] + ".csv"), header=0, index_col=0
).fillna(0)
if returns == 'log_returns':
df['returns'] = np.log(df['close']).diff(1)
elif returns == 'price':
df['returns'] = df['close']
elif returns == 'returns':
df['returns'] = df['close'].pct_change()
if returns == "log_returns":
df["returns"] = np.log(df["close"]).diff(1)
elif returns == "price":
df["returns"] = df["close"]
elif returns == "returns":
df["returns"] = df["close"].pct_change()
df = __apply_feature_extractors(df, feature_extractors = feature_extractors)
df = __apply_feature_extractors(df, feature_extractors=feature_extractors)
df = df.replace([np.inf, -np.inf], 0.)
df = drop_columns_if_exist(df, ['open', 'high', 'low', 'close', 'volume'])
df.columns = [prefix + "_" + c if 'date' not in c else c for c in df.columns]
df = df.replace([np.inf, -np.inf], 0.0)
df = drop_columns_if_exist(df, ["open", "high", "low", "close", "volume"])
df.columns = [prefix + "_" + c if "date" not in c else c for c in df.columns]
return df
def __apply_feature_extractors(df: pd.DataFrame, feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]) -> pd.DataFrame:
def __apply_feature_extractors(
df: pd.DataFrame, feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]
) -> pd.DataFrame:
for name, extractor, periods in feature_extractors:
for period in periods:
@@ -120,20 +141,27 @@ def __apply_feature_extractors(df: pd.DataFrame, feature_extractors: list[tuple[
if type(features) == pd.DataFrame:
df = pd.concat([df, features], axis=1)
elif type(features) == pd.Series:
df[name + '_' + str(period)] = features
df[name + "_" + str(period)] = features
else:
assert False, "Feature extractor must return a pd.DataFrame or pd.Series"
assert (
False
), "Feature extractor must return a pd.DataFrame or pd.Series"
return df
def load_only_returns(assets: DataCollection, returns: Literal['price', 'returns']) -> pd.DataFrame:
def load_only_returns(
assets: DataCollection, returns: Literal["price", "returns"]
) -> pd.DataFrame:
assets_future = [__load_df.remote(
data_source=data_source,
prefix=data_source[1],
returns=returns,
feature_extractors=[],
) for data_source in assets]
assets_future = [
__load_df.remote(
data_source=data_source,
prefix=data_source[1],
returns=returns,
feature_extractors=[],
)
for data_source in assets
]
dfs = ray.get(assets_future)
dfs = pd.concat(dfs, axis=1)
data_loader/process.py
+10 -5
View File
@@ -4,16 +4,21 @@ import warnings
from utils.helpers import get_first_valid_return_index
from data_loader.types import XDataFrame
def check_data(X: XDataFrame, config: Config) -> bool:
""" Returns True if data is valid, else returns False."""
"""Returns True if data is valid, else returns False."""
if has_enough_samples_to_train(X, config) == False:
warnings.warn("Not enough samples to train")
return False
return True
def has_enough_samples_to_train(X: XDataFrame, config: Config) -> bool:
first_valid_index = get_first_valid_return_index(X.iloc[:,0])
first_valid_index = get_first_valid_return_index(X.iloc[:, 0])
samples_to_train = len(X) - first_valid_index
return samples_to_train > config.sliding_window_size_base + config.sliding_window_size_meta + 100
return (
samples_to_train
> config.sliding_window_size_base + config.sliding_window_size_meta + 100
)
data_loader/types.py
+1 -1
View File
@@ -8,4 +8,4 @@ DataCollection = list[DataSource]
ReturnSeries = pd.Series
ForwardReturnSeries = pd.Series
XDataFrame = pd.DataFrame
ySeries = pd.Series
ySeries = pd.Series
data_loader/utils.py
+3 -1
View File
@@ -1,3 +1,5 @@
import pandas as pd
def deduplicate_indexes(df: pd.DataFrame) -> pd.DataFrame: return df[~df.index.duplicated(keep='last')]
def deduplicate_indexes(df: pd.DataFrame) -> pd.DataFrame:
return df[~df.index.duplicated(keep="last")]
docs/conf.py
+11 -12
View File
@@ -12,17 +12,18 @@
#
import os
import sys
sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath("."))
# -- Project information -----------------------------------------------------
project = 'Drift'
copyright = '2022, Daniel Szemerey and Mark Szulyovszky'
author = 'Daniel Szemerey, Mark Szulyovszky'
project = "Drift"
copyright = "2022, Daniel Szemerey and Mark Szulyovszky"
author = "Daniel Szemerey, Mark Szulyovszky"
# The full version, including alpha/beta/rc tags
release = '1.0'
release = "1.0"
# -- General configuration ---------------------------------------------------
@@ -30,17 +31,15 @@ release = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc',
'sphinx.ext.autosectionlabel'
]
extensions = ["sphinx.ext.autodoc", "sphinx.ext.autosectionlabel"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'sphinx-env']
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "sphinx-env"]
# -- Options for HTML output -------------------------------------------------
@@ -48,9 +47,9 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'sphinx-env']
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'furo'
html_theme = "furo"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ['_static']
# html_static_path = ['_static']
environment.yml
+1
View File
@@ -9,6 +9,7 @@ dependencies:
- python=3.9
- scikit-learn-intelex=2021.4.0
- pip:
- black
- skorch
- fracdiff
- ray
feature_extractors/feature_extractor_presets.py
+47 -27
View File
@@ -1,35 +1,55 @@
from .types import FeatureExtractorConfig
from .feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_expanding_zscore, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
from .fractional_differentiation import feature_fractional_differentiation, feature_fractional_differentiation_log
from .feature_extractors import (
feature_lag,
feature_mom,
feature_ROC,
feature_RSI,
feature_STOD,
feature_STOK,
feature_expanding_zscore,
feature_vol,
feature_day_of_month,
feature_day_of_week,
feature_month,
feature_debug_future_lookahead,
)
from .fractional_differentiation import (
feature_fractional_differentiation,
feature_fractional_differentiation_log,
)
__presets = dict(
debug_future_lookahead = [('debug_future', feature_debug_future_lookahead, [1])],
single_mom = [('mom', feature_mom, [30])],
single_vol = [('vol', feature_vol, [30])],
mom = [('mom', feature_mom, [10, 20, 30, 60, 90])],
vol = [('vol', feature_vol, [10, 20, 30, 60])],
lags_up_to_5 = [('lag', feature_lag, [1,2,3,4,5])],
lags_up_to_10 = [('lag', feature_lag, [1,2,3,4,5,6,7,8,9,10])],
date_all = [
('day_of_week', feature_day_of_week, [0]),
('day_of_month', feature_day_of_month, [0]),
('month', feature_month, [0])],
date_days = [
('day_of_week', feature_day_of_week, [0]),
('day_of_month', feature_day_of_month, [0]),
debug_future_lookahead=[("debug_future", feature_debug_future_lookahead, [1])],
single_mom=[("mom", feature_mom, [30])],
single_vol=[("vol", feature_vol, [30])],
mom=[("mom", feature_mom, [10, 20, 30, 60, 90])],
vol=[("vol", feature_vol, [10, 20, 30, 60])],
lags_up_to_5=[("lag", feature_lag, [1, 2, 3, 4, 5])],
lags_up_to_10=[("lag", feature_lag, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])],
date_all=[
("day_of_week", feature_day_of_week, [0]),
("day_of_month", feature_day_of_month, [0]),
("month", feature_month, [0]),
],
roc = [('roc', feature_ROC, [10, 30])],
rsi = [('rsi', feature_ROC, [10, 30, 100])],
stod = [('stod', feature_STOD, [10, 30, 200])],
stok = [('stok', feature_STOK, [10, 30, 200])],
fracdiff = [('fracdiff', feature_fractional_differentiation, [10, 30])],
fracdiff_log = [('fracdiff_log', feature_fractional_differentiation_log, [10, 30])],
z_score = [('z_score', feature_expanding_zscore, [10])],
date_days=[
("day_of_week", feature_day_of_week, [0]),
("day_of_month", feature_day_of_month, [0]),
],
roc=[("roc", feature_ROC, [10, 30])],
rsi=[("rsi", feature_ROC, [10, 30, 100])],
stod=[("stod", feature_STOD, [10, 30, 200])],
stok=[("stok", feature_STOK, [10, 30, 200])],
fracdiff=[("fracdiff", feature_fractional_differentiation, [10, 30])],
fracdiff_log=[("fracdiff_log", feature_fractional_differentiation_log, [10, 30])],
z_score=[("z_score", feature_expanding_zscore, [10])],
)
presets = __presets | dict(
level_1 = __presets["mom"] + __presets["vol"],
level_2 = __presets["mom"] + __presets["vol"] + __presets["roc"] + __presets["rsi"] + __presets["stod"] + __presets["stok"],
level_1=__presets["mom"] + __presets["vol"],
level_2=__presets["mom"]
+ __presets["vol"]
+ __presets["roc"]
+ __presets["rsi"]
+ __presets["stod"]
+ __presets["stok"],
)
feature_extractors/feature_extractors.py
+46 -19
View File
@@ -3,58 +3,85 @@ import numpy as np
from feature_extractors.utils import get_close_low_high
from feature_extractors.utils import apply_log_if_necessary_series
def feature_debug_future_lookahead(df: pd.DataFrame, period: int) -> pd.Series:
return df['returns'].shift(-period)
return df["returns"].shift(-period)
def feature_lag(df: pd.DataFrame, period: int) -> pd.Series:
assert period > 0
return df['returns'].shift(period)
return df["returns"].shift(period)
def feature_expanding_zscore(df: pd.DataFrame, period: int) -> pd.Series:
close = df['close']
close = df["close"]
return (close - close.expanding(period).mean()) / close.expanding(period).std()
def feature_day_of_week(df: pd.DataFrame, period: int) -> pd.DataFrame:
return pd.get_dummies(pd.DatetimeIndex(df.index).dayofweek, drop_first=True, prefix="date_day_week").set_index(df.index)
return pd.get_dummies(
pd.DatetimeIndex(df.index).dayofweek, drop_first=True, prefix="date_day_week"
).set_index(df.index)
def feature_day_of_month(df: pd.DataFrame, period: int) -> pd.DataFrame:
return pd.get_dummies(pd.DatetimeIndex(df.index).day, drop_first=True, prefix="date_day_month").set_index(df.index)
return pd.get_dummies(
pd.DatetimeIndex(df.index).day, drop_first=True, prefix="date_day_month"
).set_index(df.index)
def feature_month(df: pd.DataFrame, period: int) -> pd.DataFrame:
return pd.get_dummies(pd.DatetimeIndex(df.index).month, drop_first=True, prefix="date_month").set_index(df.index)
return pd.get_dummies(
pd.DatetimeIndex(df.index).month, drop_first=True, prefix="date_month"
).set_index(df.index)
def feature_vol(df: pd.DataFrame, period: int) -> pd.Series:
return df['returns'].rolling(period).std() * (252**0.5)
return df["returns"].rolling(period).std() * (252**0.5)
def feature_mom(df: pd.DataFrame, period: int) -> pd.Series:
return np.log(df['close']).diff(period)
return np.log(df["close"]).diff(period)
def feature_STOK(df: pd.DataFrame, period: int) -> pd.Series:
close, low, high = get_close_low_high(df)
STOK = ((close - low.rolling(period).min()) / (high.rolling(period).max() - low.rolling(period).min())) * 100
STOK = (
(close - low.rolling(period).min())
/ (high.rolling(period).max() - low.rolling(period).min())
) * 100
return apply_log_if_necessary_series(STOK, "stok")
def feature_STOD(df: pd.DataFrame, period: int) -> pd.Series:
stok = feature_STOK(df, period)
return stok.rolling(3).mean()
def feature_RSI(df: pd.DataFrame, period: int) -> pd.Series:
returns = df['returns']
returns = df["returns"]
delta = returns.diff().dropna()
u=delta*0
u = delta * 0
d = u.copy()
u[delta > 0] = delta[delta > 0]
d[delta < 0] = -delta[delta < 0]
u[u.index[period-1]] = np.mean( u[:period] ) #first value is sum of avg gains u = u.drop(u.index[:(period-1)])
d[d.index[period-1]] = np.mean( d[:period] ) #first value is sum of avg losses d = d.drop(d.index[:(period-1)])
rs = u.ewm(com=period-1, adjust=False).mean() / \
d.ewm(com=period-1, adjust=False).mean()
return apply_log_if_necessary_series(100-100/(1+rs), "rsi")
u[u.index[period - 1]] = np.mean(
u[:period]
) # first value is sum of avg gains u = u.drop(u.index[:(period-1)])
d[d.index[period - 1]] = np.mean(
d[:period]
) # first value is sum of avg losses d = d.drop(d.index[:(period-1)])
rs = (
u.ewm(com=period - 1, adjust=False).mean()
/ d.ewm(com=period - 1, adjust=False).mean()
)
return apply_log_if_necessary_series(100 - 100 / (1 + rs), "rsi")
def feature_ROC(df: pd.DataFrame, period: int) -> pd.Series:
returns = df['returns']
returns = df["returns"]
M = returns.diff(period - 1)
N = returns.shift(period - 1)
roc = pd.Series(((M / N) * 100), name = 'ROC_' + str(period))
return apply_log_if_necessary_series(roc, "roc")
roc = pd.Series(((M / N) * 100), name="ROC_" + str(period))
return apply_log_if_necessary_series(roc, "roc")
feature_extractors/fractional_differentiation.py
+4 -3
View File
@@ -3,13 +3,14 @@ import pandas as pd
import numpy as np
from feature_extractors.utils import apply_log_if_necessary_series
def feature_fractional_differentiation(df: pd.DataFrame, period: int) -> pd.Series:
frac_diff = FracdiffStat(window = period)
frac_diff = FracdiffStat(window=period)
input_series = df["close"].to_numpy().reshape(-1, 1)
result = frac_diff.fit_transform(input_series)
return pd.Series(result.squeeze(), index = df.index)
return pd.Series(result.squeeze(), index=df.index)
def feature_fractional_differentiation_log(df: pd.DataFrame, period: int) -> pd.Series:
series = feature_fractional_differentiation(df, period, is_log_return)
return apply_log_if_necessary_series(series, "fracdiff")
feature_extractors/ta.py
+1
View File
@@ -2,6 +2,7 @@ import pandas_ta as ta
import pandas as pd
from feature_extractors.utils import get_close_low_high
def feature_EBSW(df: pd.DataFrame, period: int) -> pd.Series:
close, low, high = get_close_low_high(df)
feature_extractors/types.py
+1 -1
View File
@@ -6,4 +6,4 @@ IsLogReturn = bool
FeatureExtractor = Callable[[pd.DataFrame, Period], Union[pd.DataFrame, pd.Series]]
Name = str
FeatureExtractorConfig = tuple[Name, FeatureExtractor, list[Period]]
ScalerTypes = Literal['normalize', 'minmax', 'standardize', 'none']
ScalerTypes = Literal["normalize", "minmax", "standardize", "none"]
feature_extractors/utils.py
+4 -3
View File
@@ -2,10 +2,11 @@ import pandas as pd
import numpy as np
from scipy.stats import shapiro
def get_close_low_high(df: pd.DataFrame) -> tuple[pd.Series, pd.Series, pd.Series]:
close = df['close']
low = df['low']
high = df['high']
close = df["close"]
low = df["low"]
high = df["high"]
return close, low, high
labeling/event_filters/cusum.py
+14 -13
View File
@@ -1,5 +1,3 @@
from numpy import float32
from ..types import EventFilter
from data_loader.types import ReturnSeries
@@ -7,8 +5,8 @@ import pandas as pd
from numba import njit
from numba.typed import List
class CUSUMVolatilityEventFilter(EventFilter):
class CUSUMVolatilityEventFilter(EventFilter):
def __init__(self, vol_period: int):
self.vol_period = vol_period
@@ -36,34 +34,37 @@ class CUSUMVolatilityEventFilter(EventFilter):
return pd.DatetimeIndex(filtered_indices)
class CUSUMFixedEventFilter(EventFilter):
class CUSUMFixedEventFilter(EventFilter):
def __init__(self, threshold: float):
self.threshold = threshold
def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex:
diffed_returns = returns.diff()
int_indicies = _process(List(diffed_returns.to_list()), abs(returns.mean()) * self.threshold)
int_indicies = _process(
List(diffed_returns.to_list()), abs(returns.mean()) * self.threshold
)
return pd.DatetimeIndex([returns.index[i] for i in int_indicies])
@njit
def _process(diffed_returns: List, threshold: float32) -> List:
pos_threshold: float32 = 0.0 # type: ignore
neg_threshold: float32 = 0.0 # type: ignore
pos_threshold: float32 = 0.0 # type: ignore
neg_threshold: float32 = 0.0 # type: ignore
filtered_indicies = List()
for index in range(1, len(diffed_returns[1:])):
pos_threshold, neg_threshold = ( # type: ignore
max(0, pos_threshold + diffed_returns[index]), # type: ignore
min(0, neg_threshold + diffed_returns[index]), # type: ignore
pos_threshold, neg_threshold = ( # type: ignore
max(0, pos_threshold + diffed_returns[index]), # type: ignore
min(0, neg_threshold + diffed_returns[index]), # type: ignore
)
if neg_threshold < -threshold:
neg_threshold = 0.0 # type: ignore
neg_threshold = 0.0 # type: ignore
filtered_indicies.append(index)
elif pos_threshold > threshold:
pos_threshold = 0.0 # type: ignore
pos_threshold = 0.0 # type: ignore
filtered_indicies.append(index)
return filtered_indicies
return filtered_indicies
labeling/event_filters/nofilter.py
+1 -2
View File
@@ -2,8 +2,7 @@ from ..types import EventFilter
from data_loader.types import ReturnSeries
import pandas as pd
class NoEventFilter(EventFilter):
class NoEventFilter(EventFilter):
def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex:
return returns.index
labeling/eventfilters_map.py
+4 -4
View File
@@ -2,7 +2,7 @@ from .event_filters.nofilter import NoEventFilter
from .event_filters.cusum import CUSUMVolatilityEventFilter, CUSUMFixedEventFilter
eventfilters_map = dict(
none = NoEventFilter(),
cusum_vol = CUSUMVolatilityEventFilter(vol_period = 20),
cusum_fixed = CUSUMFixedEventFilter(threshold = 500)
)
none=NoEventFilter(),
cusum_vol=CUSUMVolatilityEventFilter(vol_period=20),
cusum_fixed=CUSUMFixedEventFilter(threshold=500),
)
labeling/labellers/fixed_time_three_class_balanced.py
+19 -10
View File
@@ -3,6 +3,7 @@ from ..types import EventLabeller, EventsDataFrame
import pandas as pd
from .utils import create_forward_returns
class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
time_horizon: int
@@ -10,7 +11,9 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
def __init__(self, time_horizon: int):
self.time_horizon = time_horizon
def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]:
def label_events(
self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
) -> tuple[EventsDataFrame, ForwardReturnSeries]:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
@@ -18,7 +21,7 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
event_candidates = forward_returns[event_start_times]
def get_bins_threeway(x):
bins = pd.qcut(event_candidates, 3, retbins=True, duplicates = 'drop')[1]
bins = pd.qcut(event_candidates, 3, retbins=True, duplicates="drop")[1]
if len(bins) != 4:
# if we don't have enough data for the quantiles, we'll need to add hard-coded values
@@ -26,6 +29,7 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
upper_bound = bins[-1]
bins = [lower_bound] + [-0.02, 0.02] + [upper_bound]
return bins
bins = get_bins_threeway(event_candidates)
def map_class_threeway(current_value):
@@ -37,12 +41,17 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
return 0
else:
return 1
labels = event_candidates.map(map_class_threeway)
return (pd.DataFrame({
'start': event_start_times,
'end': event_start_times + pd.Timedelta(days=self.time_horizon),
'label': labels,
'returns': forward_returns[event_start_times]
}), forward_returns[event_start_times])
labels = event_candidates.map(map_class_threeway)
return (
pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(days=self.time_horizon),
"label": labels,
"returns": forward_returns[event_start_times],
}
),
forward_returns[event_start_times],
)
labeling/labellers/fixed_time_three_class_imbalanced.py
+18 -10
View File
@@ -2,6 +2,7 @@ from ..types import EventLabeller, EventsDataFrame, ReturnSeries, ForwardReturnS
import pandas as pd
from .utils import create_forward_returns
class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
time_horizon: int
@@ -9,7 +10,9 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
def __init__(self, time_horizon: int):
self.time_horizon = time_horizon
def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]:
def label_events(
self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
) -> tuple[EventsDataFrame, ForwardReturnSeries]:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
@@ -17,7 +20,7 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
event_candidates = forward_returns[event_start_times]
def get_bins_threeway(x):
bins = pd.qcut(event_candidates, 4, retbins=True, duplicates = 'drop')[1]
bins = pd.qcut(event_candidates, 4, retbins=True, duplicates="drop")[1]
if len(bins) != 5:
# if we don't have enough data for the quantiles, we'll need to add hard-coded values
@@ -25,6 +28,7 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
upper_bound = bins[-1]
bins = [lower_bound] + [-0.02, 0.0, 0.02] + [upper_bound]
return bins
bins = get_bins_threeway(event_candidates)
def map_class_threeway(current_value):
@@ -36,13 +40,17 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
return 0
else:
return 1
labels = event_candidates.map(map_class_threeway)
return (pd.DataFrame({
'start': event_start_times,
'end': event_start_times + pd.Timedelta(days=self.time_horizon),
'label': labels,
'returns': forward_returns[event_start_times]
}), forward_returns[event_start_times])
return (
pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(days=self.time_horizon),
"label": labels,
"returns": forward_returns[event_start_times],
}
),
forward_returns[event_start_times],
)
labeling/labellers/fixed_time_two_class.py
+16 -10
View File
@@ -2,6 +2,7 @@ from ..types import EventLabeller, EventsDataFrame, ReturnSeries, ForwardReturnS
import pandas as pd
from .utils import create_forward_returns
class FixedTimeHorionTwoClassEventLabeller(EventLabeller):
time_horizon: int
@@ -9,7 +10,9 @@ class FixedTimeHorionTwoClassEventLabeller(EventLabeller):
def __init__(self, time_horizon: int):
self.time_horizon = time_horizon
def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]:
def label_events(
self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
) -> tuple[EventsDataFrame, ForwardReturnSeries]:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
@@ -18,14 +21,17 @@ class FixedTimeHorionTwoClassEventLabeller(EventLabeller):
def get_class_binary(x: float) -> int:
return -1 if x <= 0.0 else 1
labels = event_candidates.map(get_class_binary)
return (pd.DataFrame({
'start': event_start_times,
'end': event_start_times + pd.Timedelta(days=self.time_horizon),
'label': labels,
'returns': forward_returns[event_start_times]
}), forward_returns[event_start_times])
return (
pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(days=self.time_horizon),
"label": labels,
"returns": forward_returns[event_start_times],
}
),
forward_returns[event_start_times],
)
labeling/labellers/utils.py
+3 -2
View File
@@ -1,9 +1,10 @@
import pandas as pd
from data_loader.types import ForwardReturnSeries
def create_forward_returns(series: pd.Series, period: int) -> ForwardReturnSeries:
assert period > 0
indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=period)
forward_returns = series.rolling(window=indexer).sum()
return forward_returns
return forward_returns
labeling/labellers_map.py
+10 -6
View File
@@ -1,9 +1,13 @@
from .labellers.fixed_time_three_class_balanced import FixedTimeHorionThreeClassBalancedEventLabeller
from .labellers.fixed_time_three_class_imbalanced import FixedTimeHorionThreeClassImbalancedEventLabeller
from .labellers.fixed_time_three_class_balanced import (
FixedTimeHorionThreeClassBalancedEventLabeller,
)
from .labellers.fixed_time_three_class_imbalanced import (
FixedTimeHorionThreeClassImbalancedEventLabeller,
)
from .labellers.fixed_time_two_class import FixedTimeHorionTwoClassEventLabeller
labellers_map = dict(
two_class = FixedTimeHorionTwoClassEventLabeller,
three_class_balanced = FixedTimeHorionThreeClassBalancedEventLabeller,
three_class_imbalanced = FixedTimeHorionThreeClassImbalancedEventLabeller
)
two_class=FixedTimeHorionTwoClassEventLabeller,
three_class_balanced=FixedTimeHorionThreeClassBalancedEventLabeller,
three_class_imbalanced=FixedTimeHorionThreeClassImbalancedEventLabeller,
)
labeling/process.py
+17 -11
View File
@@ -1,19 +1,25 @@
from .types import EventFilter, EventLabeller, EventsDataFrame
from data_loader.types import ForwardReturnSeries, XDataFrame, ReturnSeries, ySeries
def label_data(
event_filter: EventFilter,
event_labeller: EventLabeller,
X: XDataFrame,
returns: ReturnSeries) -> tuple[EventsDataFrame, XDataFrame, ySeries, ForwardReturnSeries]:
event_filter: EventFilter,
event_labeller: EventLabeller,
X: XDataFrame,
returns: ReturnSeries,
) -> tuple[EventsDataFrame, XDataFrame, ySeries, ForwardReturnSeries]:
event_start_times = event_filter.get_event_start_times(returns)
print("| Filtered out ", (1 - (len(event_start_times) / len(returns))) * 100, "% of timestamps" )
event_start_times = event_filter.get_event_start_times(returns)
print(
"| Filtered out ",
(1 - (len(event_start_times) / len(returns))) * 100,
"% of timestamps",
)
events, forward_returns = event_labeller.label_events(event_start_times, returns)
events, forward_returns = event_labeller.label_events(event_start_times, returns)
X = X.filter(items = events.index, axis = 0)
y = events['label']
forward_returns = events['returns']
X = X.filter(items=events.index, axis=0)
y = events["label"]
forward_returns = events["returns"]
return events, X, y, forward_returns
return events, X, y, forward_returns
labeling/types.py
+5 -4
View File
@@ -4,8 +4,8 @@ import pandas as pd
import pandera as pa
from pandera.typing import DataFrame, Series
class EventFilter(ABC):
class EventFilter(ABC):
@abstractmethod
def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex:
raise NotImplementedError
@@ -17,12 +17,13 @@ class EventSchema(pa.SchemaModel):
label: Series[int]
returns: Series[float]
EventsDataFrame = DataFrame[EventSchema]
class EventLabeller(ABC):
@abstractmethod
def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]:
def label_events(
self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
) -> tuple[EventsDataFrame, ForwardReturnSeries]:
raise NotImplementedError
models/base.py
+2 -2
View File
@@ -3,12 +3,13 @@ from typing import Literal, Optional, Union
from abc import ABC, abstractmethod
import numpy as np
class Model(ABC):
name: str = ""
data_transformation: Literal["transformed", "original"]
only_column: Optional[str]
predict_window_size: Literal['single_timestamp', 'window_size']
predict_window_size: Literal["single_timestamp", "window_size"]
@abstractmethod
def fit(self, X: np.ndarray, y: np.ndarray) -> None:
@@ -21,4 +22,3 @@ class Model(ABC):
@abstractmethod
def predict_proba(self, X: np.ndarray) -> np.ndarray:
raise NotImplementedError
models/model_map.py
+52 -20
View File
@@ -3,49 +3,81 @@ from sklearnex.ensemble import RandomForestClassifier
from sklearnex.ensemble import RandomForestRegressor
from .base import Model
default_feature_selector_classification = SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1))
default_feature_selector_classification = SKLearnModel(
RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)
)
def get_model(model_name: str) -> Model:
def set_name(model: Model) -> Model:
model.name = model_name
return model
if model_name == 'LogisticRegression_two_class':
if model_name == "LogisticRegression_two_class":
from sklearn.linear_model import LogisticRegression
return set_name(SKLearnModel(LogisticRegression(C=10, random_state=1, solver='liblinear', max_iter=1000)))
elif model_name == 'LogisticRegression_three_class':
return set_name(
SKLearnModel(
LogisticRegression(
C=10, random_state=1, solver="liblinear", max_iter=1000
)
)
)
elif model_name == "LogisticRegression_three_class":
from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX
return set_name(SKLearnModel(LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1)))
elif model_name == 'LDA':
return set_name(
SKLearnModel(
LogisticRegression_EX(C=10, random_state=1, max_iter=1000, n_jobs=-1)
)
)
elif model_name == "LDA":
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
return set_name(SKLearnModel(LinearDiscriminantAnalysis()))
elif model_name == 'KNN':
elif model_name == "KNN":
from sklearn.neighbors import KNeighborsClassifier
return set_name(SKLearnModel(KNeighborsClassifier()))
elif model_name == 'CART':
elif model_name == "CART":
from sklearn.tree import DecisionTreeClassifier
return set_name(SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1)))
elif model_name == 'NB':
return set_name(
SKLearnModel(DecisionTreeClassifier(max_depth=15, random_state=1))
)
elif model_name == "NB":
from sklearn.naive_bayes import GaussianNB
return set_name(SKLearnModel(GaussianNB()))
elif model_name == 'AB':
elif model_name == "AB":
from sklearn.ensemble import AdaBoostClassifier
return set_name(SKLearnModel(AdaBoostClassifier(n_estimators=15)))
elif model_name == 'RFC':
return set_name(SKLearnModel(RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)))
elif model_name == 'SVC':
elif model_name == "RFC":
return set_name(
SKLearnModel(
RandomForestClassifier(n_jobs=-1, max_depth=20, random_state=1)
)
)
elif model_name == "SVC":
from sklearn.svm import SVC
return set_name(SKLearnModel(SVC(kernel='rbf', C=1e3, probability=True, random_state=1)))
return set_name(
SKLearnModel(SVC(kernel="rbf", C=1e3, probability=True, random_state=1))
)
# elif model_name == 'XGB_two_class':
# from xgboost import XGBClassifier
# from models.xgboost import XGBoostModel
# return set_name(XGBoostModel(XGBClassifier(n_jobs=-1, max_depth = 20, random_state=1, objective='binary:logistic', use_label_encoder= False, eval_metric='mlogloss')))
elif model_name == 'LGBM':
elif model_name == "LGBM":
from lightgbm import LGBMClassifier
return set_name(SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1)))
elif model_name == 'StaticMom':
return set_name(
SKLearnModel(LGBMClassifier(n_jobs=-1, max_depth=20, random_state=1))
)
elif model_name == "StaticMom":
from models.momentum import StaticMomentumModel
return set_name(StaticMomentumModel(allow_short=True))
else:
raise Exception(f'Model {model_name} not found')
raise Exception(f"Model {model_name} not found")
models/momentum.py
+9 -8
View File
@@ -3,14 +3,15 @@ import numpy as np
from .base import Model
from sklearn.base import BaseEstimator, ClassifierMixin
class StaticMomentumModel(BaseEstimator, ClassifierMixin, Model):
'''
Model that uses only one feature: momentum. It's positive if momentum is greater than 0, otherwise it's negative.
'''
data_transformation = 'original'
only_column = 'mom'
predict_window_size = 'single_timestamp'
class StaticMomentumModel(BaseEstimator, ClassifierMixin, Model):
"""
Model that uses only one feature: momentum. It's positive if momentum is greater than 0, otherwise it's negative.
"""
data_transformation = "original"
only_column = "mom"
predict_window_size = "single_timestamp"
def __init__(self, allow_short: bool) -> None:
super().__init__()
@@ -24,6 +25,6 @@ class StaticMomentumModel(BaseEstimator, ClassifierMixin, Model):
negative_class = -1.0 if self.allow_short == True else 0.0
prediction = 1.0 if X[-1][0] > 0 else negative_class
return np.array(prediction)
def predict_proba(self, X) -> np.ndarray:
return np.array([])
models/sklearn.py
+2 -6
View File
@@ -4,20 +4,16 @@ from .base import Model
import numpy as np
def SKLearnModel(instance) -> Model:
instance.data_transformation = 'transformed'
instance.data_transformation = "transformed"
instance.only_column = None
instance.predict_window_size = 'single_timestamp'
instance.predict_window_size = "single_timestamp"
instance.name = instance.__class__.__name__
return instance
# def predict(self, X) -> tuple[float, np.ndarray]:
# pred = self.model.predict(X).item()
# probability = self.model.predict_proba(X).squeeze()
# return (pred, probability)
models/xgboost.py
+1 -2
View File
@@ -9,7 +9,7 @@
# data_transformation = 'transformed'
# only_column = None
# predict_window_size = 'single_timestamp'
# def fit(self, X: np.ndarray, y: np.ndarray) -> None:
# def map_to_xgb(y): return np.array([1 if i == 1 else 0 for i in y])
# self.fit(X, map_to_xgb(y))
@@ -19,4 +19,3 @@
# probability = self.predict_proba(X).squeeze()
# def map_from_xgb(y): return 1 if y == 1 else -1
# return (map_from_xgb(pred), probability)
mycodemod/replace_functions.py
+31 -34
View File
@@ -12,23 +12,23 @@ class ReplaceFunctionCommand(VisitorBasedCodemodCommand):
# Add a description so that future codemodders can see what this does.
DESCRIPTION: str = "Replaces the body of a function with pass."
def __init__(self, context: CodemodContext) -> None:
# Initialize the base class with context, and save our args. Remember, the
# "dest" for each argument we added above must match a parameter name in
# this init.
super().__init__(context)
def leave_FunctionDef(self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef) -> cst.FunctionDef:
def leave_FunctionDef(
self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef
) -> cst.FunctionDef:
functions_docstring = updated_node.get_docstring()
docstring_should_be = '"""No docstring here yet."""'
if functions_docstring is not None:
docstring_should_be = '"""\n{}\n\n"""'.format(functions_docstring)
replace_function = cst.FunctionDef(
name=updated_node.name,
params=updated_node.params,#cst.Parameters(),
name=updated_node.name,
params=updated_node.params, # cst.Parameters(),
body=cst.IndentedBlock(
body=[
cst.SimpleStatementLine(
@@ -44,50 +44,47 @@ class ReplaceFunctionCommand(VisitorBasedCodemodCommand):
],
leading_lines=[],
trailing_whitespace=cst.TrailingWhitespace(
whitespace=cst.SimpleWhitespace(
value='',
whitespace=cst.SimpleWhitespace(
value="",
),
comment=None,
newline=cst.Newline(
value=None,
),
),
),
),
cst.SimpleStatementLine(
body=[
cst.Pass(),
],
),
]
)
body=[
cst.Pass(),
],
),
]
),
)
return replace_function
def leave_ClassDef(self, original_node: cst.ClassDef, updated_node: cst.ClassDef) -> cst.ClassDef:
def leave_ClassDef(
self, original_node: cst.ClassDef, updated_node: cst.ClassDef
) -> cst.ClassDef:
new_body = []
for body_item in updated_node.body.body:
if type(body_item) is cst.FunctionDef:
new_body.append(self.leave_FunctionDef(body_item, body_item))
return updated_node.with_changes(
body=cst.IndentedBlock(new_body)
)
def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module:
new_module_body=[]
return updated_node.with_changes(body=cst.IndentedBlock(new_body))
def leave_Module(
self, original_node: cst.Module, updated_node: cst.Module
) -> cst.Module:
new_module_body = []
for node in original_node.body:
if type(node) is cst.FunctionDef:
new_module_body.append(self.leave_FunctionDef(node,node))
if type(node) is cst.ClassDef:
new_module_body.append(self.leave_ClassDef(node,node))
new_module_body.append(self.leave_FunctionDef(node, node))
if type(node) is cst.ClassDef:
new_module_body.append(self.leave_ClassDef(node, node))
replace_function = cst.Module(body=new_module_body)
replace_function = cst.Module(
body= new_module_body
)
return replace_function
reporting/reporting.py
+31 -12
View File
@@ -1,29 +1,48 @@
from reporting.wandb import send_report_to_wandb
import pandas as pd
from utils.helpers import weighted_average
from config.types import Config
from config.types import Config
from training.types import WeightsSeries, Stats
def report_results(directional_stats: Stats, output_stats: Stats, output_weights: WeightsSeries, config: Config, wandb, sweep: bool):
def report_results(
directional_stats: Stats,
output_stats: Stats,
output_weights: WeightsSeries,
config: Config,
wandb,
sweep: bool,
):
# Only send the results of the final model to wandb
send_report_to_wandb(output_stats, wandb)
pd.Series(output_stats).to_csv('output/results.csv')
pd.Series(output_stats).to_csv("output/results.csv")
output_weights.rename(config.target_asset[1]).to_csv('output/predictions.csv')
output_weights.rename(config.target_asset[1]).to_csv("output/predictions.csv")
print("\n--------\n")
print("Benchmark buy-and-hold sharpe: ", output_stats['benchmark_sharpe'])
print("Benchmark buy-and-hold sharpe: ", output_stats["benchmark_sharpe"])
print("Level-1: Number of samples evaluated: ", directional_stats['no_of_samples'])
print("Mean Sharpe ratio for Level-1 models: ", round(directional_stats['sharpe'], 3))
print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(directional_stats['prob_sharpe'], 3))
print("Level-1: Number of samples evaluated: ", directional_stats["no_of_samples"])
print(
"Mean Sharpe ratio for Level-1 models: ", round(directional_stats["sharpe"], 3)
)
print(
"Mean Probabilistic Sharpe ratio for Level-1 models: ",
round(directional_stats["prob_sharpe"], 3),
)
print("Level-2 (Ensemble): Number of samples evaluated: ", output_stats['no_of_samples'])
print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", output_stats['sharpe'])
print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", output_stats['prob_sharpe'])
print(
"Level-2 (Ensemble): Number of samples evaluated: ",
output_stats["no_of_samples"],
)
print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", output_stats["sharpe"])
print(
"Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ",
output_stats["prob_sharpe"],
)
if sweep:
if wandb.run is not None:
wandb.finish()
wandb.finish()
reporting/saving.py
+21 -19
View File
@@ -6,33 +6,35 @@ import os
import warnings
from training.types import PipelineOutcome
def save_models(pipeline_outcome: PipelineOutcome, config: Config) -> None:
dict_for_pickle = dict()
dict_for_pickle['config'] = config
dict_for_pickle['pipeline_outcome'] = pipeline_outcome
dict_for_pickle["config"] = config
dict_for_pickle["pipeline_outcome"] = pipeline_outcome
date_string = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")
if not os.path.exists('output/models'):
if not os.path.exists("output/models"):
warnings.warn("No folder exists, creating one.")
os.makedirs('output/models')
pickle.dump( dict_for_pickle, open( "output/models/{}.p".format(date_string), "wb" ) )
os.makedirs("output/models")
pickle.dump(dict_for_pickle, open("output/models/{}.p".format(date_string), "wb"))
def load_models(file_name: Optional[str]) -> tuple[PipelineOutcome, Config]:
if file_name is None:
warnings.warn("No file name provided, will load latest models and configurations.")
files_in_directory:list = os.listdir('output/models')
if file_name is None:
warnings.warn(
"No file name provided, will load latest models and configurations."
)
files_in_directory: list = os.listdir("output/models")
assert len(files_in_directory) > 0, "No models found in output/models."
file_name = sorted(files_in_directory)[-1]
packacked_dict = pickle.load( open( "output/models/{}".format(file_name), "rb" ) )
config = packacked_dict.pop("config", None)
pipeline_outcome = packacked_dict.pop("pipeline_outcome", None)
return pipeline_outcome, config
packacked_dict = pickle.load(open("output/models/{}".format(file_name), "rb"))
config = packacked_dict.pop("config", None)
pipeline_outcome = packacked_dict.pop("pipeline_outcome", None)
return pipeline_outcome, config
reporting/wandb.py
+24 -16
View File
@@ -4,41 +4,49 @@ from typing import Optional
from utils.helpers import weighted_average
from training.types import Stats
def launch_wandb(project_name:str, default_config: RawConfig, sweep:bool=False) -> Optional[object]:
def launch_wandb(
project_name: str, default_config: RawConfig, sweep: bool = False
) -> Optional[object]:
from wandb_setup import get_wandb
wandb = get_wandb()
wandb = get_wandb()
if wandb is None:
raise Exception("Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)")
raise Exception(
"Wandb can not be initalized, the environment variable WANDB_API_KEY is missing (can also use .env file)"
)
elif sweep:
wandb.init(project=project_name, config = vars(default_config))
wandb.init(project=project_name, config=vars(default_config))
return wandb
else:
wandb.init(project=project_name, config = vars(default_config), reinit=True)
wandb.init(project=project_name, config=vars(default_config), reinit=True)
return wandb
def override_config_with_wandb_values(wandb: Optional[object], raw_config: RawConfig) -> RawConfig:
if wandb is None: return raw_config
def override_config_with_wandb_values(
wandb: Optional[object], raw_config: RawConfig
) -> RawConfig:
if wandb is None:
return raw_config
wandb_config: dict = wandb.config
config_dict = vars(raw_config)
for k in config_dict:
config_dict[k] = wandb_config[k]
return RawConfig(**config_dict)
def send_report_to_wandb(stats: Stats, wandb:Optional[object]):
if wandb is None: return
def send_report_to_wandb(stats: Stats, wandb: Optional[object]):
if wandb is None:
return
run = wandb.run
run.save()
for key, value in stats.items():
run.log({ key: value })
run.log({key: value})
run.finish()
run_clear_cache.py
+2 -1
View File
@@ -1,5 +1,6 @@
from diskcache import Cache
cache_1 = Cache(".cachedir/feature_selection")
cache_2 = Cache(".cachedir/data")
cache_1.clear()
cache_2.clear()
cache_2.clear()
run_evaluate_robustness.py
+10 -3
View File
@@ -5,15 +5,22 @@ import pandas as pd
all_results = []
all_predictions = []
for index in range(6):
_, _, _, _, results_1, predictions_1, _ = run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, config= get_default_ensemble_config())
_, _, _, _, results_1, predictions_1, _ = run_pipeline(
project_name="price-prediction",
with_wandb=False,
sweep=False,
config=get_default_ensemble_config(),
)
all_results.append(results_1)
all_predictions.append(predictions_1)
correlations = pd.Series()
for asset_name in [c for c in all_predictions[0].columns if 'ensemble' in c]:
for asset_name in [c for c in all_predictions[0].columns if "ensemble" in c]:
predictions_for_asset = pd.concat([preds[asset_name] for preds in all_predictions], axis=1)
predictions_for_asset = pd.concat(
[preds[asset_name] for preds in all_predictions], axis=1
)
correlations[asset_name] = predictions_for_asset.corr().mean()[0]
print("Correlation for asset ", asset_name, ": ", correlations[asset_name])
run_fetch_data_5min.py
+22 -18
View File
@@ -7,31 +7,35 @@ from utils.resample import resample_ohlc
base_url = "https://www.cryptodatadownload.com/cdd/"
exchange_name = "Bitfinex"
period = "minute" # 1h
period = "minute" # 1h
files_to_download = [
exchange_name + '_TRXUSD_' + period + '.csv',
exchange_name + '_ETHUSD_' + period + '.csv',
exchange_name + '_XLMUSD_' + period + '.csv',
exchange_name + '_XMRUSD_' + period + '.csv',
exchange_name + '_LTCUSD_' + period + '.csv',
exchange_name + '_DASHUSD_' + period + '.csv',
exchange_name + '_BTCUSD_' + period + '.csv',
exchange_name + '_ETCUSD_' + period + '.csv',
]
exchange_name + "_TRXUSD_" + period + ".csv",
exchange_name + "_ETHUSD_" + period + ".csv",
exchange_name + "_XLMUSD_" + period + ".csv",
exchange_name + "_XMRUSD_" + period + ".csv",
exchange_name + "_LTCUSD_" + period + ".csv",
exchange_name + "_DASHUSD_" + period + ".csv",
exchange_name + "_BTCUSD_" + period + ".csv",
exchange_name + "_ETCUSD_" + period + ".csv",
]
for file in tqdm(files_to_download):
data_location = base_url + file
ssl._create_default_https_context = ssl._create_unverified_context
data = pd.read_csv(data_location, skiprows=1, index_col=1, parse_dates=True).drop(columns=['unix'])
volume_column_to_delete = [c for c in data.columns if c.startswith('Volume') and 'USD' not in c]
data.drop(volume_column_to_delete + ['symbol'], axis=1, inplace=True)
data.rename({'Volume USD': 'volume'}, axis=1, inplace=True)
data.index.rename('time', inplace=True)
data = pd.read_csv(data_location, skiprows=1, index_col=1, parse_dates=True).drop(
columns=["unix"]
)
volume_column_to_delete = [
c for c in data.columns if c.startswith("Volume") and "USD" not in c
]
data.drop(volume_column_to_delete + ["symbol"], axis=1, inplace=True)
data.rename({"Volume USD": "volume"}, axis=1, inplace=True)
data.index.rename("time", inplace=True)
data.sort_index(inplace=True)
data = deduplicate_indexes(data)
data.index = pd.to_datetime(data.index)
data = data.resample('1Min').ffill()
data = resample_ohlc(data, '5Min')
target_file = file.split('_')[1].replace('USD', '') + '_USD'
data = data.resample("1Min").ffill()
data = resample_ohlc(data, "5Min")
target_file = file.split("_")[1].replace("USD", "") + "_USD"
data.to_csv(f"data/5min_crypto/{target_file}.csv")
run_fetch_data_daily.py
+18 -3
View File
@@ -3,7 +3,24 @@ import pandas as pd
from data_loader.get_prices import get_crypto_price_crypto_compare, get_stock_price_av
#%%
crypto_tickers = ["BTC", "ETH", "BNB", "ADA", "SOL", "XRP", "DOT", "LTC", "UNI", "TRX", "XLM", "BCH", "FIL", "ETC", "THETA", "XTZ"]
crypto_tickers = [
"BTC",
"ETH",
"BNB",
"ADA",
"SOL",
"XRP",
"DOT",
"LTC",
"UNI",
"TRX",
"XLM",
"BCH",
"FIL",
"ETC",
"THETA",
"XTZ",
]
etf_tickers = ["GLD", "IEF", "TLT", "SPY", "QQQ"]
crypto_path = "data/daily_crypto"
etf_path = "data/daily_etf"
@@ -20,5 +37,3 @@ for src_ticker in crypto_tickers:
df = get_crypto_price_crypto_compare(src_ticker, "USD", 1500)
df.to_csv(f"{crypto_path}/{src_ticker}_USD.csv", index=True)
run_fetch_data_glassnode.py
+34 -31
View File
@@ -3,60 +3,63 @@ import pandas as pd
from tqdm import tqdm
from utils.glassnode import GlassnodeClient, Indicators, Mining
path = 'data/daily_glassnode/'
client = GlassnodeClient(asset='BTC', since='2014-01-01', until='2021-12-17')
path = "data/daily_glassnode/"
client = GlassnodeClient(asset="BTC", since="2014-01-01", until="2021-12-17")
print("Client initiated")
indicator_client = Indicators(client)
indicator_names = ['rhodl_ratio',
'cvdd',
'difficulty_ribbon_compression',
'nvt_ratio',
'nvt_signal',
'velocity',
'supply_adjusted_cdd',
'binary_cdd',
'supply_adjusted_dormancy',
'puell_multiple',
'asopr',
'reserve_risk',
'sopr',
'cdd',
'asol',
'msol',
'dormancy',
'liveliness',
'relative_unrealized_profit',
'relative_unrealized_loss',
'nupl',
indicator_names = [
"rhodl_ratio",
"cvdd",
"difficulty_ribbon_compression",
"nvt_ratio",
"nvt_signal",
"velocity",
"supply_adjusted_cdd",
"binary_cdd",
"supply_adjusted_dormancy",
"puell_multiple",
"asopr",
"reserve_risk",
"sopr",
"cdd",
"asol",
"msol",
"dormancy",
"liveliness",
"relative_unrealized_profit",
"relative_unrealized_loss",
"nupl",
# 'sth_nupl',
# 'lth_nupl',
'ssr',
'bvin',
"ssr",
"bvin",
]
def process_df(df: pd.DataFrame) -> pd.DataFrame:
df.index.rename('time', inplace=True)
df.index.rename("time", inplace=True)
if len(df.columns) != 1:
df = df[['v']]
df.rename(columns={df.columns[0]: 'close'}, inplace=True)
df = df[["v"]]
df.rename(columns={df.columns[0]: "close"}, inplace=True)
df.sort_index(inplace=True)
return df
for name in tqdm(indicator_names):
method_to_call = getattr(indicator_client, name)
df = method_to_call()
df = process_df(df)
df.to_csv(path + name + '.csv')
df.to_csv(path + name + ".csv")
# Mining data
mining_names = ['hash_rate']
mining_names = ["hash_rate"]
mining_client = Mining(client)
for name in tqdm(mining_names):
method_to_call = getattr(mining_client, name)
df = method_to_call()
df = process_df(df)
df.to_csv(path + name + '.csv')
df.to_csv(path + name + ".csv")
run_inference.py
+54 -21
View File
@@ -4,7 +4,11 @@ from reporting.saving import load_models
from run_pipeline import run_pipeline
from config.types import Config, RawConfig
from config.presets import get_dev_config, get_default_ensemble_config, get_lightweight_ensemble_config
from config.presets import (
get_dev_config,
get_default_ensemble_config,
get_lightweight_ensemble_config,
)
from labeling.process import label_data
import pandas as pd
@@ -12,44 +16,73 @@ from training.directional_training import train_directional_model
from training.bet_sizing import bet_sizing_with_meta_model
from training.types import PipelineOutcome
def run_inference(preload_models:bool, fallback_raw_config: RawConfig):
def run_inference(preload_models: bool, fallback_raw_config: RawConfig):
if preload_models:
pipeline_outcome, config = load_models(None)
else:
pipeline_outcome, config = run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, raw_config=fallback_raw_config)
config.mode = 'inference'
pipeline_outcome, config = run_pipeline(
project_name="price-prediction",
with_wandb=False,
sweep=False,
raw_config=fallback_raw_config,
)
config.mode = "inference"
__inference(config, pipeline_outcome)
def __inference(config: Config, pipeline_outcome: PipelineOutcome):
# 1. Load data, check for validity and process data
X, returns = load_data(
assets = config.assets,
other_assets = config.other_assets,
exogenous_data = config.exogenous_data,
target_asset = config.target_asset,
load_non_target_asset = config.load_non_target_asset,
own_features = config.own_features,
other_features = config.other_features,
exogenous_features = config.exogenous_features,
assets=config.assets,
other_assets=config.other_assets,
exogenous_data=config.exogenous_data,
target_asset=config.target_asset,
load_non_target_asset=config.load_non_target_asset,
own_features=config.own_features,
other_features=config.other_features,
exogenous_features=config.exogenous_features,
)
assert check_data(X, config) == True, "Data is not valid. Cancelling Inference."
assert check_data(X, config) == True, "Data is not valid. Cancelling Inference."
# 2. Filter for significant events when we want to trade, and label data
events, X, y, forward_returns = label_data(config.event_filter, config.labeling, X, returns)
events, X, y, forward_returns = label_data(
config.event_filter, config.labeling, X, returns
)
inference_from: pd.Timestamp = X.index[len(X.index) - 1]
# 3. Train directional models
directional_training_outcome = train_directional_model(X, y, forward_returns, config, config.directional_model, from_index = inference_from, preloaded_training_step = pipeline_outcome.directional_training)
directional_training_outcome = train_directional_model(
X,
y,
forward_returns,
config,
config.directional_model,
from_index=inference_from,
preloaded_training_step=pipeline_outcome.directional_training,
)
# 4. Run bet sizing on primary model's output
bet_sizing_outcome = bet_sizing_with_meta_model(X, directional_training_outcome.training.predictions, y, forward_returns, config.meta_model, config, 'meta', from_index = inference_from, transformations_over_time = pipeline_outcome.bet_sizing.meta_transformations, preloaded_models = pipeline_outcome.bet_sizing.meta_training.model_over_time)
bet_sizing_outcome = bet_sizing_with_meta_model(
X,
directional_training_outcome.training.predictions,
y,
forward_returns,
config.meta_model,
config,
"meta",
from_index=inference_from,
transformations_over_time=pipeline_outcome.bet_sizing.meta_transformations,
preloaded_models=pipeline_outcome.bet_sizing.meta_training.model_over_time,
)
return PipelineOutcome(directional_training_outcome, bet_sizing_outcome)
if __name__ == '__main__':
run_inference(preload_models=True, fallback_raw_config=get_lightweight_ensemble_config())
if __name__ == "__main__":
run_inference(
preload_models=True, fallback_raw_config=get_lightweight_ensemble_config()
)
run_model_dev.py
+6 -1
View File
@@ -2,4 +2,9 @@ from run_pipeline import run_pipeline
from config.presets import get_dev_config
run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, raw_config=get_dev_config())
run_pipeline(
project_name="price-prediction",
with_wandb=False,
sweep=False,
raw_config=get_dev_config(),
)
run_pipeline.py
+65 -25
View File
@@ -18,54 +18,94 @@ from training.bet_sizing import bet_sizing_with_meta_model
from training.types import PipelineOutcome
import ray
ray.init()
def run_pipeline(project_name:str, with_wandb: bool, sweep: bool, raw_config: RawConfig) -> tuple[PipelineOutcome, Config]:
def run_pipeline(
project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig
) -> tuple[PipelineOutcome, Config]:
wandb, config = __setup_config(project_name, with_wandb, sweep, raw_config)
pipeline_outcome = __run_training(config)
report_results(pipeline_outcome.directional_training.training.stats, pipeline_outcome.get_output_stats(), pipeline_outcome.get_output_weights(), config, wandb, sweep)
pipeline_outcome = __run_training(config)
report_results(
pipeline_outcome.directional_training.training.stats,
pipeline_outcome.get_output_stats(),
pipeline_outcome.get_output_weights(),
config,
wandb,
sweep,
)
save_models(pipeline_outcome, config)
return pipeline_outcome, config
def __setup_config(project_name:str, with_wandb: bool, sweep: bool, raw_config: RawConfig) -> tuple[Optional[object], Config]:
def __setup_config(
project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig
) -> tuple[Optional[object], Config]:
wandb = None
if with_wandb:
wandb = launch_wandb(project_name=project_name, default_config=raw_config, sweep=sweep)
if with_wandb:
wandb = launch_wandb(
project_name=project_name, default_config=raw_config, sweep=sweep
)
raw_config = override_config_with_wandb_values(wandb, raw_config)
config = preprocess_config(raw_config)
return wandb, config
def __run_training(config: Config) -> PipelineOutcome:
print("---> Load data, check for validity")
X, returns = load_data(
assets = config.assets,
other_assets = config.other_assets,
exogenous_data = config.exogenous_data,
target_asset = config.target_asset,
load_non_target_asset = config.load_non_target_asset,
own_features = config.own_features,
other_features = config.other_features,
exogenous_features = config.exogenous_features,
assets=config.assets,
other_assets=config.other_assets,
exogenous_data=config.exogenous_data,
target_asset=config.target_asset,
load_non_target_asset=config.load_non_target_asset,
own_features=config.own_features,
other_features=config.other_features,
exogenous_features=config.exogenous_features,
)
assert check_data(X, config) == True, "Data is not valid."
assert check_data(X, config) == True, "Data is not valid."
print("---> Filter for significant events when we want to trade, and label data")
events, X, y, forward_returns = label_data(config.event_filter, config.labeling, X, returns)
events, X, y, forward_returns = label_data(
config.event_filter, config.labeling, X, returns
)
print("---> Train directional models")
directional_training_outcome = train_directional_model(X, y, forward_returns, config, config.directional_model, from_index = None, preloaded_training_step = None)
directional_training_outcome = train_directional_model(
X,
y,
forward_returns,
config,
config.directional_model,
from_index=None,
preloaded_training_step=None,
)
print("---> Run bet sizing on directional model's output")
bet_sizing_outcomes = bet_sizing_with_meta_model(X, directional_training_outcome.training.predictions, y, forward_returns, config.meta_model, config, 'meta', None, None, None)
bet_sizing_outcomes = bet_sizing_with_meta_model(
X,
directional_training_outcome.training.predictions,
y,
forward_returns,
config.meta_model,
config,
"meta",
None,
None,
None,
)
return PipelineOutcome(directional_training_outcome, bet_sizing_outcomes)
if __name__ == '__main__':
run_pipeline(project_name='price-prediction', with_wandb = False, sweep = False, raw_config=get_lightweight_ensemble_config())
if __name__ == "__main__":
run_pipeline(
project_name="price-prediction",
with_wandb=False,
sweep=False,
raw_config=get_lightweight_ensemble_config(),
)
run_portfolio_reporting.py
+64 -36
View File
@@ -11,10 +11,14 @@ import quantstats as qs
from alphalens.utils import get_clean_factor_and_forward_returns
def fixed_weight(row: pd.Series, availability_row: pd.Series, allow_short: bool) -> pd.Series:
def fixed_weight(
row: pd.Series, availability_row: pd.Series, allow_short: bool
) -> pd.Series:
no_of_assets_available = availability_row.sum()
unit = 1 / no_of_assets_available
if allow_short:
def determine_pos(x):
if x == 0.0 or np.isnan(x):
return 0
@@ -22,18 +26,21 @@ def fixed_weight(row: pd.Series, availability_row: pd.Series, allow_short: bool)
return 1
else:
return -1
row = row.apply(determine_pos)
else:
row = row.apply(lambda x: 1 if x > 0.0 else 0)
return row * unit
def limit_weight(row: pd.Series) -> pd.Series:
if row.sum() > 1:
row = row / row.sum()
elif row.sum() < -1:
row = row * (1. / abs(row.sum()))
row = row * (1.0 / abs(row.sum()))
return row
def only_top_bottom_2(row: pd.Series) -> pd.Series:
row = row.copy()
row_sorted = row.sort_values()
@@ -41,33 +48,36 @@ def only_top_bottom_2(row: pd.Series) -> pd.Series:
top = row_sorted.iloc[-2:]
middle = row_sorted.iloc[2:-2]
for index, _ in middle.iteritems():
row[index] = 0.
row[index] = 0.0
if bottom.sum() > 0:
for index, _ in bottom.iteritems():
row[index] = 0.
row[index] = 0.0
else:
for index, _ in bottom.iteritems():
row[index] = -.025
row[index] = -0.025
if top.sum() < 0:
for index, _ in top.iteritems():
row[index] = 0.
row[index] = 0.0
else:
for index, _ in top.iteritems():
row[index] = .025
row[index] = 0.025
return row
def equal_weight(row: pd.Series, availability: pd.Series) -> pd.Series:
no_of_assets_available = availability.sum()
unit = 1 / no_of_assets_available
for index, _ in predictions.iteritems():
row[index] = 0. if availability[index] == 0 else unit
row[index] = 0.0 if availability[index] == 0 else unit
return row
def create_naive_portfolio_weights(predictions: pd.DataFrame, availability: pd.DataFrame, allow_short: bool) -> pd.DataFrame:
def create_naive_portfolio_weights(
predictions: pd.DataFrame, availability: pd.DataFrame, allow_short: bool
) -> pd.DataFrame:
weights = predictions.copy()
assert weights.shape[1] == availability.shape[1]
for index, row in weights.iterrows():
@@ -79,14 +89,20 @@ def create_naive_portfolio_weights(predictions: pd.DataFrame, availability: pd.D
weights.iloc[index] = row
return weights
def create_quantile_weights(predictions: pd.DataFrame, availability: pd.DataFrame, allow_short: bool) -> pd.DataFrame:
def create_quantile_weights(
predictions: pd.DataFrame, availability: pd.DataFrame, allow_short: bool
) -> pd.DataFrame:
weights = predictions.copy()
assert weights.shape[1] == availability.shape[1]
quantiles = weights.copy()
for column in weights.columns:
quantiles[column] = pd.qcut(weights[column], q=4, labels=False, duplicates='drop')
quantiles[column] = pd.qcut(
weights[column], q=4, labels=False, duplicates="drop"
)
for index, row in quantiles.iterrows():
def only_select_bottom_top(x):
if x == 0:
return -1
@@ -94,6 +110,7 @@ def create_quantile_weights(predictions: pd.DataFrame, availability: pd.DataFram
return 1
else:
return 0
row = row.apply(only_select_bottom_top)
no_of_nonzero_predictions = row[row != 0].count()
units = min(1 / no_of_nonzero_predictions, 0.25)
@@ -101,49 +118,59 @@ def create_quantile_weights(predictions: pd.DataFrame, availability: pd.DataFram
return weights
predictions = pd.read_csv('output/predictions.csv', index_col=0)
predictions = pd.read_csv("output/predictions.csv", index_col=0)
predictions.index = pd.DatetimeIndex(predictions.index)
predictions.columns = ['_'.join(col.replace("model_", "").split("_")[:2]) for col in predictions.columns]
predictions.columns = [
"_".join(col.replace("model_", "").split("_")[:2]) for col in predictions.columns
]
first_index = get_first_valid_return_index(predictions[predictions.columns[0]])
predictions = predictions.iloc[first_index:]
close = load_only_returns(data_collections['daily_crypto'], 'price')
close = load_only_returns(data_collections["daily_crypto"], "price")
close = close.iloc[first_index:]
close.columns = [col.replace("_returns", "") for col in close.columns]
close = close[predictions.columns]
close = close.filter(items = predictions.index, axis = 0)
close = close.filter(items=predictions.index, axis=0)
availability = close.applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1)
def report_alphalens():
alpha_factors = predictions.copy()
alpha_factors.index = close.index
alpha_factors_long = pd.melt(alpha_factors.reset_index(), id_vars=['time'], value_vars=alpha_factors.columns).set_index(['time', 'variable'])
alpha_factors_long = pd.melt(
alpha_factors.reset_index(), id_vars=["time"], value_vars=alpha_factors.columns
).set_index(["time", "variable"])
factor_data = get_clean_factor_and_forward_returns(
alpha_factors_long,
close,
quantiles=4,
periods=(1, 2, 3, 4, 5, 6, 10),
filter_zscore=None)
periods=(1, 2, 3, 4, 5, 6, 10),
filter_zscore=None,
)
# create_full_tear_sheet(factor_data, long_short=True)
from matplotlib.backends.backend_pdf import PdfPages
mean_return_by_q_daily, std_err = alphalens.performance.mean_return_by_quantile(factor_data, by_date=True)
mean_return_by_q, std_err_by_q = alphalens.performance.mean_return_by_quantile(factor_data, by_group=False)
mean_return_by_q_daily, std_err = alphalens.performance.mean_return_by_quantile(
factor_data, by_date=True
)
mean_return_by_q, std_err_by_q = alphalens.performance.mean_return_by_quantile(
factor_data, by_group=False
)
plot1 = alphalens.plotting.plot_quantile_returns_bar(mean_return_by_q)
plot2 = alphalens.plotting.plot_quantile_returns_violin(mean_return_by_q_daily)
plot3 = alphalens.plotting.plot_cumulative_returns_by_quantile(mean_return_by_q_daily, period='D')
plot3 = alphalens.plotting.plot_cumulative_returns_by_quantile(
mean_return_by_q_daily, period="D"
)
with PdfPages('output/factors.pdf') as pdf:
with PdfPages("output/factors.pdf") as pdf:
pdf.savefig(plot1.figure)
pdf.savefig(plot2.figure)
pdf.savefig(plot3.figure)
def report_backtest() -> vbt.Portfolio:
weights = create_quantile_weights(predictions, availability, allow_short=True)
@@ -151,7 +178,7 @@ def report_backtest() -> vbt.Portfolio:
# rebalance every n days
# weights.iloc[np.arange(len(weights)) % 2 != 0] = np.nan
weights.to_csv('output/weights.csv')
weights.to_csv("output/weights.csv")
portfolio = vbt.Portfolio.from_orders(
close=close,
@@ -161,26 +188,27 @@ def report_backtest() -> vbt.Portfolio:
cash_sharing=True,
call_seq=CallSeqType.Auto,
group_by=True,
freq='1D',
freq="1D",
raise_reject=True,
fees=0.001, # assuming 0.1% fees (1.5x of FTX)
slippage= 0.002, # assuming 0.2% slippage (5x of avg. spread on FTX)
fees=0.001, # assuming 0.1% fees (1.5x of FTX)
slippage=0.002, # assuming 0.2% slippage (5x of avg. spread on FTX)
seed=1,
init_cash=1e5,
log=True
log=True,
)
qs.reports.full(portfolio.returns(), portfolio.benchmark_returns())
qs.reports.full(portfolio.returns(), portfolio.benchmark_returns())
qs.reports.html(portfolio.returns(), portfolio.benchmark_returns(), output='output/report.html')
qs.reports.html(
portfolio.returns(), portfolio.benchmark_returns(), output="output/report.html"
)
print(portfolio.stats())
return portfolio
portfolio = report_backtest()
# from pypfopt import risk_models
# from pypfopt import expected_returns
# from pypfopt import EfficientFrontier
run_sweep.py
+6 -1
View File
@@ -1,4 +1,9 @@
from run_pipeline import run_pipeline
from config.presets import get_default_ensemble_config
run_pipeline(project_name='price-prediction', with_wandb = True, sweep = True, raw_config= get_default_ensemble_config())
run_pipeline(
project_name="price-prediction",
with_wandb=True,
sweep=True,
raw_config=get_default_ensemble_config(),
)
tests/test_evaluation.py
+19 -22
View File
@@ -1,4 +1,3 @@
import numpy as np
import pandas as pd
from training.walk_forward import walk_forward_train, walk_forward_inference
@@ -8,10 +7,10 @@ from sklearn.base import BaseEstimator, ClassifierMixin
no_of_rows = 100
def __generate_even_odd_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]:
''' Test data, where X[n][any_column] == 1 if n is even, else 0
'''
"""Test data, where X[n][any_column] == 1 if n is even, else 0"""
no_columns = 6
X = [[-1 if row % 2 == 0 else 1] * no_columns for row in range(no_of_rows)]
assert X[0][0] == -1
@@ -20,7 +19,7 @@ def __generate_even_odd_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]:
assert X[3][0] == 1
X = pd.DataFrame(X)
y = [-1 if (row+1) % 2 == 0 else 1 for row in range(no_of_rows)]
y = [-1 if (row + 1) % 2 == 0 else 1 for row in range(no_of_rows)]
assert y[0] == 1
assert y[1] == -1
assert y[2] == 1
@@ -32,14 +31,14 @@ def __generate_even_odd_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]:
class EvenOddStubModel(BaseEstimator, ClassifierMixin, Model):
'''
"""
A deteministic model that can predict the future with 100% accuracy
It verifies that the X[n][any_column] == 1 if n is even,
'''
"""
data_transformation = "original"
only_column = None
predict_window_size = 'single_timestamp'
predict_window_size = "single_timestamp"
def __init__(self, window_length) -> None:
super().__init__()
@@ -59,12 +58,12 @@ class EvenOddStubModel(BaseEstimator, ClassifierMixin, Model):
def test_evaluation():
X, y = __generate_even_odd_test_data(no_of_rows)
window_length = 10
retrain_every = 10
model = EvenOddStubModel(window_length = window_length)
model = EvenOddStubModel(window_length=window_length)
model_over_time = walk_forward_train(
model=model,
X=X,
@@ -74,20 +73,21 @@ def test_evaluation():
window_size=window_length,
retrain_every=retrain_every,
from_index=None,
transformations_over_time=[])
transformations_over_time=[],
)
predictions, _ = walk_forward_inference(
model_name='test',
model_name="test",
model_over_time=model_over_time,
transformations_over_time=[],
X=X,
expanding_window=False,
window_size=window_length,
retrain_every = retrain_every,
retrain_every=retrain_every,
from_index=None,
)
# verify if predictions are the same as y
for i in range(window_length+2, no_of_rows):
for i in range(window_length + 2, no_of_rows):
assert predictions[i] == y[i]
fake_forward_returns = y * 0.1
@@ -97,11 +97,8 @@ def test_evaluation():
forward_returns=fake_forward_returns,
y_pred=processed_predictions_to_match_returns,
y_true=y,
no_of_classes='two',
discretize=True
no_of_classes="two",
discretize=True,
)
assert result['accuracy'] == 100.0
assert result["accuracy"] == 100.0
tests/test_walk_forward.py
+14 -16
View File
@@ -6,10 +6,10 @@ from sklearn.base import BaseEstimator, ClassifierMixin
no_of_rows = 100
def __generate_incremental_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]:
''' Test data, where X[n][any_column] == y[n]+1
'''
"""Test data, where X[n][any_column] == y[n]+1"""
no_columns = 6
X = [[row] * no_columns for row in range(no_of_rows)]
assert X[1][0] == 1
@@ -18,7 +18,7 @@ def __generate_incremental_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Serie
assert X[4][0] == 4
X = pd.DataFrame(X)
y = [row+1 for row in range(no_of_rows)]
y = [row + 1 for row in range(no_of_rows)]
assert y[1] == 2
assert y[2] == 3
assert y[3] == 4
@@ -27,17 +27,15 @@ def __generate_incremental_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Serie
return X, y
class IncrementingStubModel(Model, BaseEstimator, ClassifierMixin):
'''
"""
A deteministic model that can predict the future with 100% accuracy
It verifies that the X[n][any_column]+1 == y[n]
'''
"""
data_transformation = "original"
only_column = None
predict_window_size = 'single_timestamp'
predict_window_size = "single_timestamp"
def __init__(self, window_length) -> None:
super().__init__()
@@ -53,14 +51,15 @@ class IncrementingStubModel(Model, BaseEstimator, ClassifierMixin):
def predict_proba(self, X):
return np.array([[row[0] + 1, 0] for row in X])
def test_walk_forward_train_test():
X, y = __generate_incremental_test_data(no_of_rows)
window_length = 10
retrain_every = 10
model = IncrementingStubModel(window_length = window_length)
model = IncrementingStubModel(window_length=window_length)
model_over_time = walk_forward_train(
model=model,
@@ -74,7 +73,7 @@ def test_walk_forward_train_test():
transformations_over_time=[],
)
predictions, _ = walk_forward_inference(
model_name='test',
model_name="test",
model_over_time=model_over_time,
transformations_over_time=[],
X=X,
@@ -83,8 +82,7 @@ def test_walk_forward_train_test():
retrain_every=retrain_every,
from_index=None,
)
# verify if predictions are the same as y
for i in range(window_length+2, no_of_rows):
assert predictions[i] == y[i]
# verify if predictions are the same as y
for i in range(window_length + 2, no_of_rows):
assert predictions[i] == y[i]
training/bet_sizing.py
+63 -46
View File
@@ -13,75 +13,92 @@ from transformations.scaler import get_scaler
from transformations.rfe import RFETransformation
from transformations.pca import PCATransformation
def bet_sizing_with_meta_model(
X: XDataFrame,
input_predictions: pd.Series,
y: ySeries,
forward_returns: ForwardReturnSeries,
model: Model,
config: Config,
model_suffix: str,
from_index: Optional[pd.Timestamp],
transformations_over_time: Optional[TransformationsOverTime] = None,
preloaded_models: Optional[ModelOverTime] = None
) -> BetSizingWithMetaOutcome:
X: XDataFrame,
input_predictions: pd.Series,
y: ySeries,
forward_returns: ForwardReturnSeries,
model: Model,
config: Config,
model_suffix: str,
from_index: Optional[pd.Timestamp],
transformations_over_time: Optional[TransformationsOverTime] = None,
preloaded_models: Optional[ModelOverTime] = None,
) -> BetSizingWithMetaOutcome:
input_predictions.name = "model_predictions"
discretized_predictions = input_predictions.apply(discretize_threeway_threshold(0.33))
discretized_predictions = input_predictions.apply(
discretize_threeway_threshold(0.33)
)
discretized_predictions.name = "model_discretized_predictions"
meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply(equal_except_nan, axis = 1)
meta_X = pd.concat([X, input_predictions, discretized_predictions], axis = 1)
meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply(
equal_except_nan, axis=1
)
meta_X = pd.concat([X, input_predictions, discretized_predictions], axis=1)
if transformations_over_time is None:
print("Preprocess transformations")
transformations_over_time = walk_forward_process_transformations(
X = meta_X,
y = meta_y,
forward_returns = forward_returns,
expanding_window = config.expanding_window_meta,
window_size = config.sliding_window_size_meta,
retrain_every = config.retrain_every,
from_index = from_index,
transformations= [
X=meta_X,
y=meta_y,
forward_returns=forward_returns,
expanding_window=config.expanding_window_meta,
window_size=config.sliding_window_size_meta,
retrain_every=config.retrain_every,
from_index=from_index,
transformations=[
get_scaler(config.scaler),
PCATransformation(ratio_components_to_keep=0.5, sliding_window_size=config.sliding_window_size_meta),
RFETransformation(n_feature_to_select=40, model=default_feature_selector_classification)
PCATransformation(
ratio_components_to_keep=0.5,
sliding_window_size=config.sliding_window_size_meta,
),
RFETransformation(
n_feature_to_select=40,
model=default_feature_selector_classification,
),
],
)
meta_outcome = train_model(
ticker_to_predict = "prediction_correct",
X = meta_X,
y = meta_y,
forward_returns = forward_returns,
model = model,
expanding_window = config.expanding_window_meta,
sliding_window_size = config.sliding_window_size_meta,
retrain_every = config.retrain_every,
from_index = from_index,
no_of_classes = 'two',
level = 'meta',
output_stats = config.mode == 'training',
transformations_over_time = transformations_over_time,
model_over_time = preloaded_models,
ticker_to_predict="prediction_correct",
X=meta_X,
y=meta_y,
forward_returns=forward_returns,
model=model,
expanding_window=config.expanding_window_meta,
sliding_window_size=config.sliding_window_size_meta,
retrain_every=config.retrain_every,
from_index=from_index,
no_of_classes="two",
level="meta",
output_stats=config.mode == "training",
transformations_over_time=transformations_over_time,
model_over_time=preloaded_models,
)
meta_predictions = meta_outcome.predictions
bet_size = meta_outcome.probabilities.iloc[:,1]
bet_size = meta_outcome.probabilities.iloc[:, 1]
avg_predictions_with_sizing = input_predictions * meta_predictions * bet_size
if config.mode == 'training':
if config.mode == "training":
stats = evaluate_predictions(
forward_returns = forward_returns,
y_pred = avg_predictions_with_sizing,
y_true = y,
no_of_classes = 'three-balanced',
discretize=False
forward_returns=forward_returns,
y_pred=avg_predictions_with_sizing,
y_true=y,
no_of_classes="three-balanced",
discretize=False,
)
print(stats)
else:
stats = None
model_id = "model_" + config.target_asset[1] + "_" + model_suffix
return BetSizingWithMetaOutcome(model_id, meta_outcome, transformations_over_time, avg_predictions_with_sizing, stats)
return BetSizingWithMetaOutcome(
model_id,
meta_outcome,
transformations_over_time,
avg_predictions_with_sizing,
stats,
)
training/directional_training.py
+43 -35
View File
@@ -5,7 +5,7 @@ from training.train_model import train_model
from training.walk_forward import walk_forward_process_transformations
from typing import Optional
from config.types import Config
from config.types import Config
from models.base import Model
from models.model_map import default_feature_selector_classification
@@ -13,53 +13,61 @@ from transformations.scaler import get_scaler
from transformations.rfe import RFETransformation
from transformations.pca import PCATransformation
def train_directional_model(
X: pd.DataFrame,
y: pd.Series,
forward_returns: pd.Series,
config: Config,
model: Model,
from_index: Optional[pd.Timestamp],
preloaded_training_step: Optional[DirectionalTrainingOutcome] = None,
) -> DirectionalTrainingOutcome:
X: pd.DataFrame,
y: pd.Series,
forward_returns: pd.Series,
config: Config,
model: Model,
from_index: Optional[pd.Timestamp],
preloaded_training_step: Optional[DirectionalTrainingOutcome] = None,
) -> DirectionalTrainingOutcome:
if preloaded_training_step is None:
print("Preprocess transformations")
transformations_over_time = walk_forward_process_transformations(
X = X,
y = y,
forward_returns = forward_returns,
expanding_window = config.expanding_window_base,
window_size = config.sliding_window_size_base,
retrain_every = config.retrain_every,
from_index = from_index,
transformations= [
X=X,
y=y,
forward_returns=forward_returns,
expanding_window=config.expanding_window_base,
window_size=config.sliding_window_size_base,
retrain_every=config.retrain_every,
from_index=from_index,
transformations=[
get_scaler(config.scaler),
PCATransformation(ratio_components_to_keep=0.5, sliding_window_size=config.sliding_window_size_base),
RFETransformation(n_feature_to_select=40, model=default_feature_selector_classification)
PCATransformation(
ratio_components_to_keep=0.5,
sliding_window_size=config.sliding_window_size_base,
),
RFETransformation(
n_feature_to_select=40,
model=default_feature_selector_classification,
),
],
)
else:
transformations_over_time = preloaded_training_step.transformations
training_outcome = train_model(
ticker_to_predict = config.target_asset[1],
X = X,
y = y,
forward_returns = forward_returns,
model = model,
expanding_window = config.expanding_window_base,
sliding_window_size = config.sliding_window_size_base,
retrain_every = config.retrain_every,
from_index = from_index,
no_of_classes = config.no_of_classes,
level = 'primary',
output_stats= config.mode == 'training',
transformations_over_time = transformations_over_time,
model_over_time = preloaded_training_step.training.model_over_time if preloaded_training_step else None
ticker_to_predict=config.target_asset[1],
X=X,
y=y,
forward_returns=forward_returns,
model=model,
expanding_window=config.expanding_window_base,
sliding_window_size=config.sliding_window_size_base,
retrain_every=config.retrain_every,
from_index=from_index,
no_of_classes=config.no_of_classes,
level="primary",
output_stats=config.mode == "training",
transformations_over_time=transformations_over_time,
model_over_time=preloaded_training_step.training.model_over_time
if preloaded_training_step
else None,
)
if config.mode == 'training':
if config.mode == "training":
print(training_outcome.stats)
return DirectionalTrainingOutcome(training_outcome, transformations_over_time)
training/train_model.py
+52 -43
View File
@@ -1,69 +1,78 @@
import pandas as pd
from typing import Literal, Optional
from training.walk_forward import walk_forward_train, walk_forward_inference, walk_forward_inference_batched
from training.walk_forward import (
walk_forward_train,
walk_forward_inference,
walk_forward_inference_batched,
)
from utils.evaluate import evaluate_predictions
from models.base import Model
from .types import ModelOverTime, TransformationsOverTime, TrainingOutcome
def train_model(
ticker_to_predict: str,
X: pd.DataFrame,
y: pd.Series,
forward_returns: pd.Series,
model: Model,
expanding_window: bool,
sliding_window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
level: str,
output_stats: bool,
transformations_over_time: TransformationsOverTime,
model_over_time: Optional[ModelOverTime]
) -> TrainingOutcome:
ticker_to_predict: str,
X: pd.DataFrame,
y: pd.Series,
forward_returns: pd.Series,
model: Model,
expanding_window: bool,
sliding_window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
level: str,
output_stats: bool,
transformations_over_time: TransformationsOverTime,
model_over_time: Optional[ModelOverTime],
) -> TrainingOutcome:
if model_over_time is None:
print("Train model")
model_over_time = walk_forward_train(
model = model,
X = X,
y = y,
forward_returns = forward_returns,
expanding_window = expanding_window,
window_size = sliding_window_size,
retrain_every = retrain_every,
from_index = from_index,
transformations_over_time = transformations_over_time,
model=model,
X=X,
y=y,
forward_returns=forward_returns,
expanding_window=expanding_window,
window_size=sliding_window_size,
retrain_every=retrain_every,
from_index=from_index,
transformations_over_time=transformations_over_time,
)
levelname = ("_" + level) if level == 'meta' else ""
levelname = ("_" + level) if level == "meta" else ""
if model_over_time is None:
model_id = "model_" + model.name + "_" + ticker_to_predict + levelname
model_id = "model_" + model.name + "_" + ticker_to_predict + levelname
else:
model_id = model_over_time.name
inference_function = walk_forward_inference if from_index is not None else walk_forward_inference_batched
predictions, probabilities = inference_function(
model_name = model_id,
model_over_time= model_over_time,
transformations_over_time = transformations_over_time,
X = X,
expanding_window = expanding_window,
window_size = sliding_window_size,
retrain_every = retrain_every,
from_index = from_index,
inference_function = (
walk_forward_inference
if from_index is not None
else walk_forward_inference_batched
)
predictions, probabilities = inference_function(
model_name=model_id,
model_over_time=model_over_time,
transformations_over_time=transformations_over_time,
X=X,
expanding_window=expanding_window,
window_size=sliding_window_size,
retrain_every=retrain_every,
from_index=from_index,
)
assert len(predictions) == len(y)
if output_stats:
stats = evaluate_predictions(
forward_returns = forward_returns,
y_pred = predictions,
y_true = y,
forward_returns=forward_returns,
y_pred=predictions,
y_true=y,
no_of_classes=no_of_classes,
discretize=True
discretize=True,
)
else:
stats = None
return TrainingOutcome(model_id, predictions, probabilities, stats, model_over_time)
return TrainingOutcome(model_id, predictions, probabilities, stats, model_over_time)
training/types.py
+5 -2
View File
@@ -19,6 +19,7 @@ class TrainingOutcome:
stats: Optional[Stats]
model_over_time: ModelOverTime
@dataclass
class BetSizingWithMetaOutcome:
model_id: str
@@ -27,11 +28,13 @@ class BetSizingWithMetaOutcome:
weights: WeightsSeries
stats: Optional[Stats]
@dataclass
class DirectionalTrainingOutcome:
training: TrainingOutcome
transformations: TransformationsOverTime
@dataclass
class PipelineOutcome:
directional_training: DirectionalTrainingOutcome
@@ -41,4 +44,4 @@ class PipelineOutcome:
return self.bet_sizing.weights
def get_output_stats(self) -> Stats:
return self.bet_sizing.stats
return self.bet_sizing.stats
training/walk_forward/__init__.py
+1 -1
View File
@@ -1,4 +1,4 @@
from .inference_batched import walk_forward_inference_batched
from .inference import walk_forward_inference
from .train import walk_forward_train
from .process_transformations import walk_forward_process_transformations
from .process_transformations import walk_forward_process_transformations
training/walk_forward/inference.py
+58 -26
View File
@@ -1,49 +1,79 @@
import pandas as pd
from training.types import ModelOverTime, TransformationsOverTime, PredictionsSeries, ProbabilitiesDataFrame
from training.types import (
ModelOverTime,
TransformationsOverTime,
PredictionsSeries,
ProbabilitiesDataFrame,
)
from utils.helpers import get_first_valid_return_index
from tqdm import tqdm
from typing import Optional
from data_loader.types import XDataFrame
from utils.helpers import get_last_non_na_index
def walk_forward_inference(
model_name: str,
model_over_time: ModelOverTime,
transformations_over_time: TransformationsOverTime,
X: XDataFrame,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
predictions = pd.Series(index=X.index, dtype='object').rename(model_name)
model_name: str,
model_over_time: ModelOverTime,
transformations_over_time: TransformationsOverTime,
X: XDataFrame,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
predictions = pd.Series(index=X.index, dtype="object").rename(model_name)
probabilities = pd.DataFrame(index=X.index)
inference_from = max(get_first_valid_return_index(model_over_time), get_first_valid_return_index(X.iloc[:,0])) if from_index is None else X.index.to_list().index(from_index)
inference_from = (
max(
get_first_valid_return_index(model_over_time),
get_first_valid_return_index(X.iloc[:, 0]),
)
if from_index is None
else X.index.to_list().index(from_index)
)
inference_till = X.shape[0]
model_index_offset = get_last_non_na_index(model_over_time, inference_from) if pd.isna(model_over_time[inference_from]) else 0
first_model = model_over_time[inference_from - model_index_offset] if pd.isna(model_over_time[inference_from]) else model_over_time[inference_from]
model_index_offset = (
get_last_non_na_index(model_over_time, inference_from)
if pd.isna(model_over_time[inference_from])
else 0
)
first_model = (
model_over_time[inference_from - model_index_offset]
if pd.isna(model_over_time[inference_from])
else model_over_time[inference_from]
)
if first_model.only_column is not None:
X = X[[column for column in X.columns if first_model.only_column in column]]
if first_model.data_transformation == 'original':
if first_model.data_transformation == "original":
transformations_over_time = []
for index in tqdm(range(inference_from, inference_till)):
last_model_index = index - ((index - inference_from) % retrain_every) - model_index_offset
train_window_start = X.index[inference_from] if expanding_window else X.index[index - window_size - 1]
last_model_index = (
index - ((index - inference_from) % retrain_every) - model_index_offset
)
train_window_start = (
X.index[inference_from]
if expanding_window
else X.index[index - window_size - 1]
)
current_model = model_over_time[X.index[last_model_index]]
current_transformations = [transformation_over_time[X.index[last_model_index]] for transformation_over_time in transformations_over_time]
current_transformations = [
transformation_over_time[X.index[last_model_index]]
for transformation_over_time in transformations_over_time
]
if current_model.predict_window_size == 'window_size':
next_timestep = X.loc[train_window_start:X.index[index]]
else:
if current_model.predict_window_size == "window_size":
next_timestep = X.loc[train_window_start : X.index[index]]
else:
# we need to get a Dataframe out of it, since the transformation step always expects a 2D array, but it's equivalent to X.iloc[index]
next_timestep = X.loc[X.index[index]:X.index[index]]
next_timestep = X.loc[X.index[index] : X.index[index]]
for transformation in current_transformations:
next_timestep = transformation.transform(next_timestep)
@@ -54,7 +84,9 @@ def walk_forward_inference(
predictions[X.index[index]] = prediction
if inference_from == index and len(probabilities.columns) != len(probs):
probabilities = probabilities.reindex(columns = ["prob_" + str(num) for num in range(0, len(probs.T))])
probabilities = probabilities.reindex(
columns=["prob_" + str(num) for num in range(0, len(probs.T))]
)
probabilities.loc[X.index[index]] = probs
return predictions, probabilities
training/walk_forward/inference_batched.py
+67 -24
View File
@@ -1,36 +1,64 @@
import pandas as pd
from training.types import ModelOverTime, TransformationsOverTime, PredictionsSeries, ProbabilitiesDataFrame
from training.types import (
ModelOverTime,
TransformationsOverTime,
PredictionsSeries,
ProbabilitiesDataFrame,
)
from utils.helpers import get_first_valid_return_index
from tqdm import tqdm
from typing import Optional
from data_loader.types import XDataFrame
from tqdm import tqdm
def walk_forward_inference_batched(
model_name: str,
model_over_time: ModelOverTime,
transformations_over_time: TransformationsOverTime,
X: XDataFrame,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
predictions = pd.Series(index=X.index, dtype='object').rename(model_name)
probabilities = pd.DataFrame(index=X.index, columns=['0', '1'])
inference_from = max(get_first_valid_return_index(model_over_time), get_first_valid_return_index(X.iloc[:,0])) if from_index is None else X.index.to_list().index(from_index)
def walk_forward_inference_batched(
model_name: str,
model_over_time: ModelOverTime,
transformations_over_time: TransformationsOverTime,
X: XDataFrame,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
predictions = pd.Series(index=X.index, dtype="object").rename(model_name)
probabilities = pd.DataFrame(index=X.index, columns=["0", "1"])
inference_from = (
max(
get_first_valid_return_index(model_over_time),
get_first_valid_return_index(X.iloc[:, 0]),
)
if from_index is None
else X.index.to_list().index(from_index)
)
inference_till = X.shape[0]
first_model = model_over_time[inference_from]
if first_model.only_column is not None:
X = X[[column for column in X.columns if first_model.only_column in column]]
if first_model.data_transformation == 'original':
if first_model.data_transformation == "original":
transformations_over_time = []
batch_indices = range(inference_from, inference_till, retrain_every) if inference_till - inference_from > retrain_every else [inference_from]
batched_results = [__inference_from_window(index, index + retrain_every, X, model_over_time, transformations_over_time, expanding_window, window_size) for index in tqdm(batch_indices)]
batch_indices = (
range(inference_from, inference_till, retrain_every)
if inference_till - inference_from > retrain_every
else [inference_from]
)
batched_results = [
__inference_from_window(
index,
index + retrain_every,
X,
model_over_time,
transformations_over_time,
expanding_window,
window_size,
)
for index in tqdm(batch_indices)
]
for batch in batched_results:
for index, prediction, probs in batch:
predictions[X.index[index]] = prediction
@@ -38,12 +66,24 @@ def walk_forward_inference_batched(
return predictions, probabilities
def __inference_from_window(index_start: int, index_end: int, X: XDataFrame, model_over_time: ModelOverTime, transformations_over_time: TransformationsOverTime, expanding_window: bool, window_size: int) -> list[tuple[int, float, pd.Series]]:
def __inference_from_window(
index_start: int,
index_end: int,
X: XDataFrame,
model_over_time: ModelOverTime,
transformations_over_time: TransformationsOverTime,
expanding_window: bool,
window_size: int,
) -> list[tuple[int, float, pd.Series]]:
current_model = model_over_time[X.index[index_start]]
current_transformations = [transformation_over_time[X.index[index_start]] for transformation_over_time in transformations_over_time]
current_transformations = [
transformation_over_time[X.index[index_start]]
for transformation_over_time in transformations_over_time
]
input_data = X.iloc[index_start:index_end]
for transformation in current_transformations:
input_data = transformation.transform(input_data)
@@ -51,6 +91,9 @@ def __inference_from_window(index_start: int, index_end: int, X: XDataFrame, mod
predictions = current_model.predict(input_data)
probs = current_model.predict_proba(input_data)
results = [(index_start + index, predictions[index], probs[index]) for index in range(len(predictions))]
return results
results = [
(index_start + index, predictions[index], probs[index])
for index in range(len(predictions))
]
return results
training/walk_forward/inference_parallel.py
+72 -26
View File
@@ -1,6 +1,11 @@
import pandas as pd
from models.base import Model
from training.types import ModelOverTime, TransformationsOverTime, PredictionsSeries, ProbabilitiesDataFrame
from training.types import (
ModelOverTime,
TransformationsOverTime,
PredictionsSeries,
ProbabilitiesDataFrame,
)
from transformations.base import Transformation
from utils.helpers import get_first_valid_return_index
from tqdm import tqdm
@@ -8,31 +13,54 @@ from typing import Optional
from data_loader.types import XDataFrame
import ray
def walk_forward_inference(
model_name: str,
model_over_time: ModelOverTime,
transformations_over_time: TransformationsOverTime,
X: XDataFrame,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
predictions = pd.Series(index=X.index, dtype='object').rename(model_name)
model_name: str,
model_over_time: ModelOverTime,
transformations_over_time: TransformationsOverTime,
X: XDataFrame,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
) -> tuple[PredictionsSeries, ProbabilitiesDataFrame]:
predictions = pd.Series(index=X.index, dtype="object").rename(model_name)
probabilities = pd.DataFrame(index=X.index)
inference_from = max(get_first_valid_return_index(model_over_time), get_first_valid_return_index(X.iloc[:,0])) if from_index is None else X.index.to_list().index(from_index)
inference_from = (
max(
get_first_valid_return_index(model_over_time),
get_first_valid_return_index(X.iloc[:, 0]),
)
if from_index is None
else X.index.to_list().index(from_index)
)
inference_till = X.shape[0]
first_model = model_over_time[inference_from]
if first_model.only_column is not None:
X = X[[column for column in X.columns if first_model.only_column in column]]
if first_model.data_transformation == 'original':
if first_model.data_transformation == "original":
transformations_over_time = []
batch_size = int((inference_till - inference_from) / 10)
batched_results = ray.get([__inference_from_window.remote(index, index + batch_size, inference_from, retrain_every, X, model_over_time, transformations_over_time, expanding_window, window_size) for index in range(inference_from, inference_till)])
batched_results = ray.get(
[
__inference_from_window.remote(
index,
index + batch_size,
inference_from,
retrain_every,
X,
model_over_time,
transformations_over_time,
expanding_window,
window_size,
)
for index in range(inference_from, inference_till)
]
)
for batch in batched_results:
for index, prediction, probs in batch:
predictions[X.index[index]] = prediction
@@ -40,23 +68,41 @@ def walk_forward_inference(
return predictions, probabilities
@ray.remote
def __inference_from_window(index_start: int, index_end: int, inference_from: int, retrain_every: int, X: XDataFrame, model_over_time: ModelOverTime, transformations_over_time: TransformationsOverTime, expanding_window: bool, window_size: int) -> list[tuple[int, float, pd.Series]]:
def __inference_from_window(
index_start: int,
index_end: int,
inference_from: int,
retrain_every: int,
X: XDataFrame,
model_over_time: ModelOverTime,
transformations_over_time: TransformationsOverTime,
expanding_window: bool,
window_size: int,
) -> list[tuple[int, float, pd.Series]]:
results = []
for index in range(index_start, index_end):
last_model_index = index - ((index - inference_from) % retrain_every)
train_window_start = X.index[inference_from] if expanding_window else X.index[index - window_size - 1]
train_window_start = (
X.index[inference_from]
if expanding_window
else X.index[index - window_size - 1]
)
current_model = model_over_time[X.index[last_model_index]]
current_transformations = [transformation_over_time[X.index[last_model_index]] for transformation_over_time in transformations_over_time]
current_transformations = [
transformation_over_time[X.index[last_model_index]]
for transformation_over_time in transformations_over_time
]
if current_model.predict_window_size == 'window_size':
next_timestep = X.loc[train_window_start:X.index[index]]
else:
if current_model.predict_window_size == "window_size":
next_timestep = X.loc[train_window_start : X.index[index]]
else:
# we need to get a Dataframe out of it, since the transformation step always expects a 2D array, but it's equivalent to X.iloc[index]
next_timestep = X.loc[X.index[index]:X.index[index]]
next_timestep = X.loc[X.index[index] : X.index[index]]
for transformation in current_transformations:
next_timestep = transformation.transform(next_timestep)
@@ -64,5 +110,5 @@ def __inference_from_window(index_start: int, index_end: int, inference_from: in
prediction, probs = current_model.predict(next_timestep)
results.append((index, prediction, probs))
return results
return results
training/walk_forward/process_transformations.py
+43 -20
View File
@@ -8,40 +8,63 @@ from data_loader.types import ForwardReturnSeries, XDataFrame, ySeries
def walk_forward_process_transformations(
X: XDataFrame,
y: ySeries,
forward_returns: ForwardReturnSeries,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
transformations: list[Transformation],
) -> TransformationsOverTime:
transformations_over_time = [pd.Series(index=y.index, dtype='object').rename(t.get_name()) for t in transformations]
X: XDataFrame,
y: ySeries,
forward_returns: ForwardReturnSeries,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
transformations: list[Transformation],
) -> TransformationsOverTime:
transformations_over_time = [
pd.Series(index=y.index, dtype="object").rename(t.get_name())
for t in transformations
]
first_nonzero_return = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y))
train_from = first_nonzero_return + window_size + 1 if from_index is None else X.index.to_list().index(from_index)
first_nonzero_return = max(
get_first_valid_return_index(forward_returns),
get_first_valid_return_index(X.iloc[:, 0]),
get_first_valid_return_index(y),
)
train_from = (
first_nonzero_return + window_size + 1
if from_index is None
else X.index.to_list().index(from_index)
)
train_till = len(y)
iterations_before_retrain = 0
for index in tqdm(range(train_from, train_till)):
train_window_start = X.index[first_nonzero_return] if expanding_window else X.index[index - window_size - 1]
if iterations_before_retrain <= 0 or pd.isna(transformations_over_time[0][index-1]):
for index in tqdm(range(train_from, train_till)):
train_window_start = (
X.index[first_nonzero_return]
if expanding_window
else X.index[index - window_size - 1]
)
if iterations_before_retrain <= 0 or pd.isna(
transformations_over_time[0][index - 1]
):
train_window_end = X.index[index - 1]
X_expanding_window = X[train_window_start:train_window_end]
y_expanding_window = y[train_window_start:train_window_end]
current_transformations = [t.clone() for t in transformations]
for transformation_index, transformation in enumerate(current_transformations):
X_expanding_window = transformation.fit_transform(X_expanding_window, y_expanding_window)
for transformation_index, transformation in enumerate(
current_transformations
):
X_expanding_window = transformation.fit_transform(
X_expanding_window, y_expanding_window
)
iterations_before_retrain = retrain_every
for transformation_index, transformation in enumerate(current_transformations):
transformations_over_time[transformation_index][X.index[index]] = transformation
transformations_over_time[transformation_index][
X.index[index]
] = transformation
iterations_before_retrain -= 1
training/walk_forward/process_transformations_parallel.py
+62 -21
View File
@@ -8,33 +8,72 @@ from data_loader.types import ForwardReturnSeries, XDataFrame, ySeries
import ray
from utils.parallel import parallel_compute_with_bar
def walk_forward_process_transformations(
X: XDataFrame,
y: ySeries,
forward_returns: ForwardReturnSeries,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
transformations: list[Transformation],
) -> TransformationsOverTime:
transformations_over_time = [pd.Series(index=y.index).rename(t.get_name()) for t in transformations]
first_nonzero_return = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y))
train_from = first_nonzero_return + window_size + 1 if from_index is None else X.index.to_list().index(from_index)
def walk_forward_process_transformations(
X: XDataFrame,
y: ySeries,
forward_returns: ForwardReturnSeries,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
transformations: list[Transformation],
) -> TransformationsOverTime:
transformations_over_time = [
pd.Series(index=y.index).rename(t.get_name()) for t in transformations
]
first_nonzero_return = max(
get_first_valid_return_index(forward_returns),
get_first_valid_return_index(X.iloc[:, 0]),
get_first_valid_return_index(y),
)
train_from = (
first_nonzero_return + window_size + 1
if from_index is None
else X.index.to_list().index(from_index)
)
train_till = len(y)
processed_transformations = parallel_compute_with_bar([preprocess_transformations_window.remote(X, y, expanding_window, window_size, transformations, first_nonzero_return, index) for index in range(train_from, train_till, retrain_every)])
processed_transformations = parallel_compute_with_bar(
[
preprocess_transformations_window.remote(
X,
y,
expanding_window,
window_size,
transformations,
first_nonzero_return,
index,
)
for index in range(train_from, train_till, retrain_every)
]
)
for transformation, index_time in processed_transformations:
for transformation_index, transformation in enumerate(transformation):
transformations_over_time[transformation_index][X.index[index_time]] = transformation
transformations_over_time[transformation_index][
X.index[index_time]
] = transformation
return transformations_over_time
@ray.remote
def preprocess_transformations_window(X: XDataFrame, y: ySeries, expanding_window: bool, window_size: int, transformations: list[Transformation], first_nonzero_return: int, index: int) -> tuple[list[Transformation], int]:
train_window_start = X.index[first_nonzero_return] if expanding_window else X.index[index - window_size - 1]
def preprocess_transformations_window(
X: XDataFrame,
y: ySeries,
expanding_window: bool,
window_size: int,
transformations: list[Transformation],
first_nonzero_return: int,
index: int,
) -> tuple[list[Transformation], int]:
train_window_start = (
X.index[first_nonzero_return]
if expanding_window
else X.index[index - window_size - 1]
)
train_window_end = X.index[index - 1]
X_expanding_window = X[train_window_start:train_window_end]
@@ -42,6 +81,8 @@ def preprocess_transformations_window(X: XDataFrame, y: ySeries, expanding_windo
current_transformations = [t.clone() for t in transformations]
for transformation in current_transformations:
X_expanding_window = transformation.fit_transform(X_expanding_window, y_expanding_window)
X_expanding_window = transformation.fit_transform(
X_expanding_window, y_expanding_window
)
return (current_transformations, index)
return (current_transformations, index)
training/walk_forward/train.py
+37 -21
View File
@@ -7,39 +7,55 @@ from typing import Optional
from data_loader.types import ForwardReturnSeries, XDataFrame, ySeries
from copy import deepcopy
def walk_forward_train(
model: Model,
X: XDataFrame,
y: ySeries,
forward_returns: ForwardReturnSeries,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
transformations_over_time: TransformationsOverTime,
) -> ModelOverTime:
models_over_time = pd.Series(index=y.index, dtype='object').rename(model.name)
first_nonzero_return = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y))
train_from = first_nonzero_return + window_size + 1 if from_index is None else X.index.to_list().index(from_index)
def walk_forward_train(
model: Model,
X: XDataFrame,
y: ySeries,
forward_returns: ForwardReturnSeries,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
transformations_over_time: TransformationsOverTime,
) -> ModelOverTime:
models_over_time = pd.Series(index=y.index, dtype="object").rename(model.name)
first_nonzero_return = max(
get_first_valid_return_index(forward_returns),
get_first_valid_return_index(X.iloc[:, 0]),
get_first_valid_return_index(y),
)
train_from = (
first_nonzero_return + window_size + 1
if from_index is None
else X.index.to_list().index(from_index)
)
train_till = len(y)
if model.only_column is not None:
X = X[[column for column in X.columns if model.only_column in column]]
if model.data_transformation == 'original':
if model.data_transformation == "original":
transformations_over_time = []
for index in tqdm(range(train_from, train_till, retrain_every)):
train_window_start = X.index[first_nonzero_return] if expanding_window else X.index[index - window_size - 1]
train_window_start = (
X.index[first_nonzero_return]
if expanding_window
else X.index[index - window_size - 1]
)
train_window_end = X.index[index - 1]
current_transformations = [transformation_over_time[index] for transformation_over_time in transformations_over_time]
current_transformations = [
transformation_over_time[index]
for transformation_over_time in transformations_over_time
]
X_slice = X[train_window_start:train_window_end]
for transformation in current_transformations:
X_slice = transformation.transform(X_slice)
X_slice = X_slice.to_numpy()
y_slice = y[train_window_start:train_window_end].to_numpy()
@@ -48,4 +64,4 @@ def walk_forward_train(
models_over_time[X.index[index]] = current_model
return models_over_time
return models_over_time
training/walk_forward/train_parallel.py
+61 -21
View File
@@ -9,46 +9,86 @@ import ray
from utils.parallel import parallel_compute_with_bar
from copy import deepcopy
def walk_forward_train(
model: Model,
X: XDataFrame,
y: ySeries,
forward_returns: ForwardReturnSeries,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
transformations_over_time: TransformationsOverTime,
) -> ModelOverTime:
model: Model,
X: XDataFrame,
y: ySeries,
forward_returns: ForwardReturnSeries,
expanding_window: bool,
window_size: int,
retrain_every: int,
from_index: Optional[pd.Timestamp],
transformations_over_time: TransformationsOverTime,
) -> ModelOverTime:
models_over_time = pd.Series(index=y.index).rename(model.name)
first_nonzero_return = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(X.iloc[:,0]), get_first_valid_return_index(y))
train_from = first_nonzero_return + window_size + 1 if from_index is None else X.index.to_list().index(from_index)
first_nonzero_return = max(
get_first_valid_return_index(forward_returns),
get_first_valid_return_index(X.iloc[:, 0]),
get_first_valid_return_index(y),
)
train_from = (
first_nonzero_return + window_size + 1
if from_index is None
else X.index.to_list().index(from_index)
)
train_till = len(y)
if model.only_column is not None:
X = X[[column for column in X.columns if model.only_column in column]]
if model.data_transformation == 'original':
if model.data_transformation == "original":
transformations_over_time = []
models = parallel_compute_with_bar([train_on_window.remote(index, first_nonzero_return, window_size, X, y, model, expanding_window, transformations_over_time) for index in tqdm(range(train_from, train_till, retrain_every))])
for index, current_model in models:
models = parallel_compute_with_bar(
[
train_on_window.remote(
index,
first_nonzero_return,
window_size,
X,
y,
model,
expanding_window,
transformations_over_time,
)
for index in tqdm(range(train_from, train_till, retrain_every))
]
)
for index, current_model in models:
models_over_time[X.index[index]] = current_model
return models_over_time
@ray.remote
def train_on_window(index: int, first_nonzero_return: int, window_size: int, X: XDataFrame, y: ySeries, model: Model, expanding_window: bool, transformations_over_time: TransformationsOverTime) -> tuple[int, Model]:
train_window_start = X.index[first_nonzero_return] if expanding_window else X.index[index - window_size - 1]
def train_on_window(
index: int,
first_nonzero_return: int,
window_size: int,
X: XDataFrame,
y: ySeries,
model: Model,
expanding_window: bool,
transformations_over_time: TransformationsOverTime,
) -> tuple[int, Model]:
train_window_start = (
X.index[first_nonzero_return]
if expanding_window
else X.index[index - window_size - 1]
)
train_window_end = X.index[index - 1]
current_transformations = [transformation_over_time[index] for transformation_over_time in transformations_over_time]
current_transformations = [
transformation_over_time[index]
for transformation_over_time in transformations_over_time
]
X_slice = X[train_window_start:train_window_end]
for transformation in current_transformations:
X_slice = transformation.transform(X_slice)
X_slice = X_slice.to_numpy()
y_slice = y[train_window_start:train_window_end].to_numpy()
transformations/base.py
+4 -4
View File
@@ -3,14 +3,16 @@ from typing import Literal, Optional, Union
from abc import ABC, abstractmethod
import pandas as pd
class Transformation(ABC):
class Transformation(ABC):
@abstractmethod
def fit(self, X: pd.DataFrame, y: Optional[pd.Series]) -> None:
raise NotImplementedError
@abstractmethod
def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> pd.DataFrame:
def fit_transform(
self, X: pd.DataFrame, y: Optional[pd.Series] = None
) -> pd.DataFrame:
raise NotImplementedError
@abstractmethod
@@ -24,5 +26,3 @@ class Transformation(ABC):
@abstractmethod
def get_name(self) -> str:
raise NotImplementedError
transformations/pca.py
+13 -11
View File
@@ -5,6 +5,7 @@ from copy import deepcopy
from sklearn.decomposition import PCA
import pandas as pd
class PCATransformation(Transformation):
pca: PCA
@@ -14,16 +15,23 @@ class PCATransformation(Transformation):
self.sliding_window_size = sliding_window_size
def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> None:
self.pca = PCA(n_components = min(int(len(X.columns) * self.ratio_components_to_keep), self.sliding_window_size))
self.pca = PCA(
n_components=min(
int(len(X.columns) * self.ratio_components_to_keep),
self.sliding_window_size,
)
)
self.pca.fit(X, y)
def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> pd.DataFrame:
def fit_transform(
self, X: pd.DataFrame, y: Optional[pd.Series] = None
) -> pd.DataFrame:
self.fit(X, y)
return self.transform(X)
def transform(self, X: pd.DataFrame) -> pd.DataFrame:
X = pd.DataFrame(self.pca.transform(X), index = X.index)
X.columns = ['PCA_' + str(i) for i in range(1, len(X.columns)+1)]
X = pd.DataFrame(self.pca.transform(X), index=X.index)
X.columns = ["PCA_" + str(i) for i in range(1, len(X.columns) + 1)]
return X
def clone(self) -> PCATransformation:
@@ -31,9 +39,3 @@ class PCATransformation(Transformation):
def get_name(self) -> str:
return "PCA"
transformations/rfe.py
+13 -13
View File
@@ -6,37 +6,37 @@ from sklearn.feature_selection import RFE
import pandas as pd
from models.sklearn import SKLearnModel
class RFETransformation(Transformation):
rfe: RFE
n_feature_to_select: int
def __init__(self, n_feature_to_select: int, model: SKLearnModel, step = 0.1):
def __init__(self, n_feature_to_select: int, model: SKLearnModel, step=0.1):
self.n_feature_to_keep = n_feature_to_select
self.model = model
self.rfe = RFE(model, n_features_to_select= n_feature_to_select, step=step)
self.rfe = RFE(model, n_features_to_select=n_feature_to_select, step=step)
def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> None:
if self.rfe is None: return
if self.rfe is None:
return
self.rfe.fit(X, y)
def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> pd.DataFrame:
if self.rfe is None: return X
def fit_transform(
self, X: pd.DataFrame, y: Optional[pd.Series] = None
) -> pd.DataFrame:
if self.rfe is None:
return X
self.fit(X, y)
return self.transform(X)
def transform(self, X: pd.DataFrame) -> pd.DataFrame:
if self.rfe is None: return X
return pd.DataFrame(X[X.columns[self.rfe.support_]], index= X.index)
if self.rfe is None:
return X
return pd.DataFrame(X[X.columns[self.rfe.support_]], index=X.index)
def clone(self) -> RFETransformation:
return deepcopy(self)
def get_name(self) -> str:
return "RFE"
transformations/scaler.py
+7 -6
View File
@@ -2,14 +2,15 @@ from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler
from .sklearn import SKLearnTransformation
from typing import Literal
ScalerTypes = Literal['normalize', 'minmax', 'standardize']
ScalerTypes = Literal["normalize", "minmax", "standardize"]
def get_scaler(type: ScalerTypes) -> SKLearnTransformation:
if type == 'normalize':
if type == "normalize":
return SKLearnTransformation(Normalizer())
elif type == 'minmax':
return SKLearnTransformation(MinMaxScaler(feature_range= (-1, 1)))
elif type == 'standardize':
elif type == "minmax":
return SKLearnTransformation(MinMaxScaler(feature_range=(-1, 1)))
elif type == "standardize":
return SKLearnTransformation(StandardScaler())
else:
raise Exception("Scaler type not supported")
raise Exception("Scaler type not supported")
transformations/sklearn.py
+5 -8
View File
@@ -4,6 +4,7 @@ from typing import Literal, Optional, Union
from sklearn.base import clone, BaseEstimator
import pandas as pd
class SKLearnTransformation(Transformation):
transformer: BaseEstimator
@@ -13,22 +14,18 @@ class SKLearnTransformation(Transformation):
def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> None:
self.transformer.fit(X, y)
def fit_transform(self, X: pd.DataFrame, y: Optional[pd.Series]) -> pd.DataFrame:
self.fit(X, y)
return self.transform(X)
def transform(self, X: pd.DataFrame) -> pd.DataFrame:
return pd.DataFrame(self.transformer.transform(X), index = X.index, columns = X.columns)
return pd.DataFrame(
self.transformer.transform(X), index=X.index, columns=X.columns
)
def clone(self) -> SKLearnTransformation:
return SKLearnTransformation(clone(self.transformer))
def get_name(self) -> str:
return self.transformer.__class__.__name__
utils/evaluate.py
+78 -48
View File
@@ -8,95 +8,125 @@ import numpy as np
from data_loader.types import ForwardReturnSeries, ySeries
from training.types import Stats, WeightsSeries
def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.002) -> pd.Series:
delta_pos = signal.diff(1).abs().fillna(0.)
def backtest(
returns: pd.Series, signal: pd.Series, transaction_cost=0.002
) -> pd.Series:
delta_pos = signal.diff(1).abs().fillna(0.0)
costs = transaction_cost * delta_pos
return (signal * returns) - costs
def __preprocess(forward_returns: ForwardReturnSeries, y_pred: pd.Series, y_true: pd.Series, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame:
y_pred.name = 'y_pred'
forward_returns.name = 'forward_returns'
df = pd.concat([y_pred, forward_returns],axis=1).dropna()
def __preprocess(
forward_returns: ForwardReturnSeries,
y_pred: pd.Series,
y_true: pd.Series,
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
discretize: bool,
) -> pd.DataFrame:
y_pred.name = "y_pred"
forward_returns.name = "forward_returns"
df = pd.concat([y_pred, forward_returns], axis=1).dropna()
discretize_func = get_discretize_function(no_of_classes)
# make sure that we evaluate binary/three-way predictions even if the model is a regression
df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
df['sign_true'] = y_true
df["sign_pred"] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
df["sign_true"] = y_true
df['result'] = backtest(df.forward_returns, df.sign_pred)
df["result"] = backtest(df.forward_returns, df.sign_pred)
return df
def evaluate_predictions(
forward_returns: ForwardReturnSeries,
y_pred: WeightsSeries,
y_true: ySeries,
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
discretize: bool = False,
) -> Stats:
forward_returns: ForwardReturnSeries,
y_pred: WeightsSeries,
y_true: ySeries,
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
discretize: bool = False,
) -> Stats:
# ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
evaluate_from = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(y_pred))
evaluate_from = max(
get_first_valid_return_index(forward_returns),
get_first_valid_return_index(y_pred),
)
forward_returns = pd.Series(forward_returns[evaluate_from:])
y_pred = pd.Series(y_pred[evaluate_from:])
df = __preprocess(forward_returns, y_pred, y_true, no_of_classes, discretize)
scorecard = dict()
def count_non_zero(series: pd.Series) -> int:
return len(series[series != 0])
no_of_samples = count_non_zero(df.y_pred)
scorecard['no_of_samples'] = no_of_samples
sharpe = sharpe_ratio(df.result + 1e-20)
scorecard['sharpe'] = sharpe
benchmark_sharpe = sharpe_ratio(df.forward_returns)
scorecard['benchmark_sharpe'] = benchmark_sharpe
scorecard['prob_sharpe'] = probabilistic_sharpe_ratio(sharpe, benchmark_sharpe, no_of_samples)
scorecard['sortino'] = sortino(df.result)
scorecard['skew'] = skew(df.result)
labels = [1, -1] if no_of_classes == 'two' else [1, -1, 0]
avg_type = 'weighted' if no_of_classes == 'two' else 'macro'
no_of_samples = count_non_zero(df.y_pred)
scorecard["no_of_samples"] = no_of_samples
sharpe = sharpe_ratio(df.result + 1e-20)
scorecard["sharpe"] = sharpe
benchmark_sharpe = sharpe_ratio(df.forward_returns)
scorecard["benchmark_sharpe"] = benchmark_sharpe
scorecard["prob_sharpe"] = probabilistic_sharpe_ratio(
sharpe, benchmark_sharpe, no_of_samples
)
scorecard["sortino"] = sortino(df.result)
scorecard["skew"] = skew(df.result)
labels = [1, -1] if no_of_classes == "two" else [1, -1, 0]
avg_type = "weighted" if no_of_classes == "two" else "macro"
if discretize == True:
scorecard['accuracy'] = accuracy_score(df.sign_true, df.sign_pred) * 100
scorecard['recall'] = recall_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
scorecard['precision'] = precision_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
scorecard['f1_score'] = f1_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
scorecard['edge'] = df.result.mean()
scorecard['noise'] = df.y_pred.diff().abs().mean()
scorecard['edge_to_noise'] = scorecard['edge'] / (scorecard['noise'] + 0.00001)
scorecard["accuracy"] = accuracy_score(df.sign_true, df.sign_pred) * 100
scorecard["recall"] = recall_score(
df.sign_true, df.sign_pred, labels=labels, average=avg_type
)
scorecard["precision"] = precision_score(
df.sign_true, df.sign_pred, labels=labels, average=avg_type
)
scorecard["f1_score"] = f1_score(
df.sign_true, df.sign_pred, labels=labels, average=avg_type
)
scorecard["edge"] = df.result.mean()
scorecard["noise"] = df.y_pred.diff().abs().mean()
scorecard["edge_to_noise"] = scorecard["edge"] / (scorecard["noise"] + 0.00001)
if discretize == True:
for index, row in df.sign_true.value_counts().iteritems():
scorecard['sign_true_ratio_' + str(index)] = row / len(df.sign_true)
scorecard["sign_true_ratio_" + str(index)] = row / len(df.sign_true)
for index, row in df.sign_pred.value_counts().iteritems():
scorecard['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred)
scorecard["sign_pred_ratio_" + str(index)] = row / len(df.sign_pred)
scorecard = {k: round(float(v), 3) for k, v in scorecard.items()}
return scorecard
return scorecard
def __discretize_binary(x):
return 1 if x > 0 else -1
def __discretize_threeway(x):
return 0 if x == 0 else 1 if x > 0 else -1
def __discretize_binary(x): return 1 if x > 0 else -1
def __discretize_threeway(x): return 0 if x == 0 else 1 if x > 0 else -1
def discretize_threeway_threshold(threshold: float) -> Callable:
def discretize(current_value):
lower_threshold = -threshold
upper_threshold = threshold
if np.isnan(current_value):
return np.nan
return np.nan
elif current_value <= lower_threshold:
return -1
elif current_value > lower_threshold and current_value < upper_threshold:
return 0
else:
return 1
return discretize
def get_discretize_function(no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']) -> Callable:
return __discretize_binary if no_of_classes == 'two' else __discretize_threeway
def get_discretize_function(
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"]
) -> Callable:
return __discretize_binary if no_of_classes == "two" else __discretize_threeway
utils/glassnode/blockchain.py
+60 -59
View File
@@ -4,49 +4,50 @@ from .client import GlassnodeClient
class Blockchain:
"""
Blockchain class.
Blockchain class.
Methods
-------
__init__(glassnode_client):
Constructs a Blockchain object.
utxos_total():
Returns the total number of UTXOs in the network.
utxos_created():
Returns the number of created unspent transaction outputs.
utxos_spent():
Returns the number of spent transaction outputs.
utxo_value_created_total():
Returns the total amount of coins in newly created UTXOs.
utxo_value_spent_total():
Returns the total amount of coins in spent transaction outputs.
utxo_value_created_mean():
Returns the mean amount of coins in newly created UTXOs.
utxo_value_spent_mean():
Returns the mean amount of coins in spent transaction outputs.
utxo_value_created_median():
Returns the median amount of coins in newly created UTXOs.
utxo_value_spent_median():
Returns the median amount of coins in spent transaction outputs.
utxos_in_profit():
Returns the number of unspent transaction outputs in profit.
utxos_in_loss():
Returns the number of unspent transaction outputs in loss.
percent_utxos_in_profit():
Returns the percentage of unspent transaction outputs in profit.
block_heights():
Returns the block height.
blocks_mined():
Returns the number of blocks mined.
block_interval_mean():
Returns the mean time (in seconds) between mined blocks.
block_interval_median():
Returns the median time (in seconds) between mined blocks.
block_size_mean():
Returns the mean size of all blocks created within the time period (in bytes).
block_size_total():
Returns the total size of all blocks created within the time period (in bytes).
Methods
-------
__init__(glassnode_client):
Constructs a Blockchain object.
utxos_total():
Returns the total number of UTXOs in the network.
utxos_created():
Returns the number of created unspent transaction outputs.
utxos_spent():
Returns the number of spent transaction outputs.
utxo_value_created_total():
Returns the total amount of coins in newly created UTXOs.
utxo_value_spent_total():
Returns the total amount of coins in spent transaction outputs.
utxo_value_created_mean():
Returns the mean amount of coins in newly created UTXOs.
utxo_value_spent_mean():
Returns the mean amount of coins in spent transaction outputs.
utxo_value_created_median():
Returns the median amount of coins in newly created UTXOs.
utxo_value_spent_median():
Returns the median amount of coins in spent transaction outputs.
utxos_in_profit():
Returns the number of unspent transaction outputs in profit.
utxos_in_loss():
Returns the number of unspent transaction outputs in loss.
percent_utxos_in_profit():
Returns the percentage of unspent transaction outputs in profit.
block_heights():
Returns the block height.
blocks_mined():
Returns the number of blocks mined.
block_interval_mean():
Returns the mean time (in seconds) between mined blocks.
block_interval_median():
Returns the median time (in seconds) between mined blocks.
block_size_mean():
Returns the mean size of all blocks created within the time period (in bytes).
block_size_total():
Returns the total size of all blocks created within the time period (in bytes).
"""
def __init__(self, glassnode_client: GlassnodeClient):
self._gc = glassnode_client
@@ -57,7 +58,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_count'
endpoint = "/v1/metrics/blockchain/utxo_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -70,7 +71,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_created_count'
endpoint = "/v1/metrics/blockchain/utxo_created_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -83,7 +84,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_spent_count'
endpoint = "/v1/metrics/blockchain/utxo_spent_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -96,7 +97,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_created_value_sum'
endpoint = "/v1/metrics/blockchain/utxo_created_value_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -109,7 +110,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_spent_value_sum'
endpoint = "/v1/metrics/blockchain/utxo_spent_value_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -122,7 +123,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_created_value_mean'
endpoint = "/v1/metrics/blockchain/utxo_created_value_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -135,7 +136,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_spent_value_mean'
endpoint = "/v1/metrics/blockchain/utxo_spent_value_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -148,7 +149,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_created_value_median'
endpoint = "/v1/metrics/blockchain/utxo_created_value_median"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -161,7 +162,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_spent_value_median'
endpoint = "/v1/metrics/blockchain/utxo_spent_value_median"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -174,7 +175,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_profit_count'
endpoint = "/v1/metrics/blockchain/utxo_profit_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -187,7 +188,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_loss_count'
endpoint = "/v1/metrics/blockchain/utxo_loss_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -200,7 +201,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/utxo_profit_relative'
endpoint = "/v1/metrics/blockchain/utxo_profit_relative"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -213,7 +214,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/block_height'
endpoint = "/v1/metrics/blockchain/block_height"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -226,7 +227,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/block_count'
endpoint = "/v1/metrics/blockchain/block_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -239,7 +240,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/block_interval_mean'
endpoint = "/v1/metrics/blockchain/block_interval_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -252,7 +253,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/block_interval_median'
endpoint = "/v1/metrics/blockchain/block_interval_median"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -265,7 +266,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/block_size_mean'
endpoint = "/v1/metrics/blockchain/block_size_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -278,7 +279,7 @@ class Blockchain:
:return: DataFrame
"""
endpoint = '/v1/metrics/blockchain/block_size_sum'
endpoint = "/v1/metrics/blockchain/block_size_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
utils/glassnode/client.py
+19 -19
View File
@@ -6,13 +6,13 @@ from .endpoints import Endpoints
class GlassnodeClient:
def __init__(
self,
api_key=None,
asset='BTC',
resolution='24h',
currency='native',
since=None,
until=None
self,
api_key=None,
asset="BTC",
resolution="24h",
currency="native",
since=None,
until=None,
):
"""
Glassnode API client.
@@ -25,11 +25,11 @@ class GlassnodeClient:
"""
if api_key:
self._api_key = api_key
elif 'GLASSNODE_API_KEY' in os.environ:
self._api_key = os.environ.get('GLASSNODE_API_KEY')
elif "GLASSNODE_API_KEY" in os.environ:
self._api_key = os.environ.get("GLASSNODE_API_KEY")
else:
# API key is required for every endpoint!
print(f'\033[91m ERROR: Glassnode API key required!\033[0m')
print(f"\033[91m ERROR: Glassnode API key required!\033[0m")
sys.exit()
self.endpoints = Endpoints()
@@ -53,27 +53,27 @@ class GlassnodeClient:
def __prepare_request_params(self, params):
p = dict()
p['api_key'] = self._api_key
p['a'] = self._asset
p['i'] = self._resolution
p["api_key"] = self._api_key
p["a"] = self._asset
p["i"] = self._resolution
if self._since is not None:
try:
p['s'] = unix_timestamp(self._since)
p["s"] = unix_timestamp(self._since)
except Exception as e:
print(e)
if self._until is not None:
try:
p['u'] = unix_timestamp(self._until)
p["u"] = unix_timestamp(self._until)
except Exception as e:
print(e)
# Set domain specific query parameters if available
if params:
if 'e' in p:
p['e'] = params['e']
if 'm' in p:
p['miner'] = params['m']
if "e" in p:
p["e"] = params["e"]
if "m" in p:
p["miner"] = params["m"]
return p
utils/glassnode/derivatives.py
+65 -64
View File
@@ -4,45 +4,46 @@ from .client import GlassnodeClient
class Derivatives:
"""
Derivatives class.
Derivatives class.
Methods
-------
__init__(glassnode_client):
Constructs a Derivatives object.
futures_perpetual_funding_rate([exchange]):
Returns the average funding rate (in %) set by exchanges for perpetual futures contracts.
futures_perpetual_funding_rate_all():
Returns the average funding rate (in %) set by exchanges for perpetual futures contracts.
futures_volume([exchange]):
Returns the total volume traded in futures contracts in the last 24 hours.
futures_volume_latest_24h():
Returns the total volume traded in futures contracts per exchange over the last 24 hours.
futures_volume_stacked():
Returns the total volume traded in futures contracts in the last 24 hours.
futures_volume_perpetual([exchange]):
Returns The total volume traded in perpetual futures contracts in the last 24 hours.
futures_volume_perpetual_stacked():
Returns the total volume traded in perpetual futures contracts in the last 24 hours.
futures_open_interest([exchange]):
Returns the total amount of funds allocated in open futures contracts.
futures_open_interest_current():
Returns the current amount of allocated funds in futures contracts per exchange.
futures_open_interest_perpetual([exchange]):
Returns the total amount of funds allocated in open perpetual futures contracts.
futures_open_interest_perpetual_stacked():
Returns the total amount of funds allocated in open perpetual futures contracts.
futures_open_interest_stacked():
Returns the total amount of funds allocated in open futures contracts.
futures_long_liquidations([exchange]):
Returns the sum liquidated volume from long positions in futures contracts.
futures_long_liquidations_mean([exchange]):
Returns the mean liquidated volume from long positions in futures contracts.
futures_short_liquidations([exchange]):
Returns the sum liquidated volume from short positions in futures contracts.
futures_short_liquidations_mean([exchange]):
Returns the mean liquidated volume from short positions in futures contracts.
Methods
-------
__init__(glassnode_client):
Constructs a Derivatives object.
futures_perpetual_funding_rate([exchange]):
Returns the average funding rate (in %) set by exchanges for perpetual futures contracts.
futures_perpetual_funding_rate_all():
Returns the average funding rate (in %) set by exchanges for perpetual futures contracts.
futures_volume([exchange]):
Returns the total volume traded in futures contracts in the last 24 hours.
futures_volume_latest_24h():
Returns the total volume traded in futures contracts per exchange over the last 24 hours.
futures_volume_stacked():
Returns the total volume traded in futures contracts in the last 24 hours.
futures_volume_perpetual([exchange]):
Returns The total volume traded in perpetual futures contracts in the last 24 hours.
futures_volume_perpetual_stacked():
Returns the total volume traded in perpetual futures contracts in the last 24 hours.
futures_open_interest([exchange]):
Returns the total amount of funds allocated in open futures contracts.
futures_open_interest_current():
Returns the current amount of allocated funds in futures contracts per exchange.
futures_open_interest_perpetual([exchange]):
Returns the total amount of funds allocated in open perpetual futures contracts.
futures_open_interest_perpetual_stacked():
Returns the total amount of funds allocated in open perpetual futures contracts.
futures_open_interest_stacked():
Returns the total amount of funds allocated in open futures contracts.
futures_long_liquidations([exchange]):
Returns the sum liquidated volume from long positions in futures contracts.
futures_long_liquidations_mean([exchange]):
Returns the mean liquidated volume from long positions in futures contracts.
futures_short_liquidations([exchange]):
Returns the sum liquidated volume from short positions in futures contracts.
futures_short_liquidations_mean([exchange]):
Returns the mean liquidated volume from short positions in futures contracts.
"""
def __init__(self, glassnode_client: GlassnodeClient):
self._gc = glassnode_client
@@ -51,11 +52,11 @@ class Derivatives:
The average funding rate (in %) set by exchanges for perpetual futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesFundingRatePerpetual>`_
"""
endpoint = '/v1/metrics/derivatives/futures_funding_rate_perpetual'
endpoint = "/v1/metrics/derivatives/futures_funding_rate_perpetual"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
@dataframe_with_inner_object
def futures_perpetual_funding_rate_all(self) -> pd.DataFrame:
@@ -63,7 +64,7 @@ class Derivatives:
The average funding rate (in %) set by exchanges for perpetual futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesFundingRatePerpetualAll>`_
"""
endpoint = '/v1/metrics/derivatives/futures_funding_rate_perpetual_all'
endpoint = "/v1/metrics/derivatives/futures_funding_rate_perpetual_all"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -74,11 +75,11 @@ class Derivatives:
The total volume traded in futures contracts in the last 24 hours.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesVolumeDailySum>`_
"""
url = '/v1/metrics/derivatives/futures_volume_daily_sum'
url = "/v1/metrics/derivatives/futures_volume_daily_sum"
if not is_supported_by_endpoint(self._gc, url):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(url, {'e': exchange}))
return response_to_dataframe(self._gc.get(url, {"e": exchange}))
# TODO: Unpack inner object from response
def futures_volume_latest_24h(self) -> pd.DataFrame:
@@ -87,7 +88,7 @@ class Derivatives:
Values are updated every 10 min.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesVolumeDailyLatest>`_
"""
endpoint = '/v1/metrics/derivatives/futures_volume_daily_latest'
endpoint = "/v1/metrics/derivatives/futures_volume_daily_latest"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -99,7 +100,7 @@ class Derivatives:
The total volume traded in futures contracts in the last 24 hours.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesVolumeDailySumAll>`_
"""
endpoint = '/v1/metrics/derivatives/futures_volume_daily_sum_all'
endpoint = "/v1/metrics/derivatives/futures_volume_daily_sum_all"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -110,11 +111,11 @@ class Derivatives:
The total volume traded in perpetual (non-expiring) futures contracts in the last 24 hours.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesVolumeDailyPerpetualSum>`_
"""
endpoint = '/v1/metrics/derivatives/futures_volume_daily_perpetual_sum'
endpoint = "/v1/metrics/derivatives/futures_volume_daily_perpetual_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
@dataframe_with_inner_object
def futures_volume_perpetual_stacked(self) -> pd.DataFrame:
@@ -122,7 +123,7 @@ class Derivatives:
The total volume traded in perpetual (non-expiring) futures contracts in the last 24 hours.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesVolumeDailyPerpetualSumAll>`_
"""
endpoint = '/v1/metrics/derivatives/futures_volume_daily_perpetual_sum_all'
endpoint = "/v1/metrics/derivatives/futures_volume_daily_perpetual_sum_all"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -133,11 +134,11 @@ class Derivatives:
The total amount of funds allocated in open futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesOpenInterestSum>`_
"""
endpoint = '/v1/metrics/derivatives/futures_open_interest_sum'
endpoint = "/v1/metrics/derivatives/futures_open_interest_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
# TODO: Unpack inner object from response
def futures_open_interest_current(self) -> pd.DataFrame:
@@ -145,7 +146,7 @@ class Derivatives:
The current amount of allocated funds in futures contracts per exchange.Values are updated every 10 min.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesOpenInterestLatest>`_
"""
endpoint = '/v1/metrics/derivatives/futures_open_interest_latest'
endpoint = "/v1/metrics/derivatives/futures_open_interest_latest"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -156,11 +157,11 @@ class Derivatives:
The total amount of funds allocated in open perpetual (non-expiring) futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesOpenInterestPerpetualSum>`_
"""
endpoint = '/v1/metrics/derivatives/futures_open_interest_perpetual_sum'
endpoint = "/v1/metrics/derivatives/futures_open_interest_perpetual_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
@dataframe_with_inner_object
def futures_open_interest_perpetual_stacked(self) -> pd.DataFrame:
@@ -168,7 +169,7 @@ class Derivatives:
The total amount of funds allocated in open perpetual (non-expiring) futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesOpenInterestPerpetualSumAll>`_
"""
endpoint = '/v1/metrics/derivatives/futures_open_interest_perpetual_sum_all'
endpoint = "/v1/metrics/derivatives/futures_open_interest_perpetual_sum_all"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -180,7 +181,7 @@ class Derivatives:
The total amount of funds allocated in open futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesOpenInterestSumAll>`_
"""
endpoint = '/v1/metrics/derivatives/futures_open_interest_sum_all'
endpoint = "/v1/metrics/derivatives/futures_open_interest_sum_all"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -191,49 +192,49 @@ class Derivatives:
The sum liquidated volume from long positions in futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesLiquidatedVolumeLongSum>`_
"""
endpoint = '/v1/metrics/derivatives/futures_liquidated_volume_long_sum'
endpoint = "/v1/metrics/derivatives/futures_liquidated_volume_long_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
def futures_long_liquidations_mean(self, exchange: str = None) -> pd.DataFrame:
"""
The mean liquidated volume from long positions in futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesLiquidatedVolumeLongMean>`_
"""
endpoint = '/v1/metrics/derivatives/futures_liquidated_volume_long_mean'
endpoint = "/v1/metrics/derivatives/futures_liquidated_volume_long_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
def futures_short_liquidations(self, exchange: str = None) -> pd.DataFrame:
"""
The sum liquidated volume from short positions in futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesLiquidatedVolumeShortSum>`_
"""
endpoint = '/v1/metrics/derivatives/futures_liquidated_volume_short_sum'
endpoint = "/v1/metrics/derivatives/futures_liquidated_volume_short_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
def futures_short_liquidations_mean(self, exchange: str = None) -> pd.DataFrame:
"""
The mean liquidated volume from short positions in futures contracts.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=derivatives.FuturesLiquidatedVolumeShortMean>`_
"""
endpoint = '/v1/metrics/derivatives/futures_liquidated_volume_short_mean'
endpoint = "/v1/metrics/derivatives/futures_liquidated_volume_short_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
def futures_estimated_leverage_ratio(self, exchange: str = None) -> pd.DataFrame:
endpoint = '/v1/metrics/derivatives/futures_estimated_leverage_ratio'
endpoint = "/v1/metrics/derivatives/futures_estimated_leverage_ratio"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
utils/glassnode/distribution.py
+39 -38
View File
@@ -3,33 +3,34 @@ from .utils import *
class Distribution:
"""
Distribution class.
Distribution class.
Methods
-------
__init__(glassnode_client):
Constructs a Distribution object.
exchange_balance_total(exchange):
Returns the total amount of coins held on exchange addresses.
exchange_balance_percent(exchange):
Returns the percent supply held on exchange addresses.
exchange_balance_stacked():
Returns the total amount of coins held on exchange addresses.
miner_balance():
Returns the total supply held in miner addresses.
miner_balance_stacked():
Returns the total supply held in miner addresses.
balance_miners_change():
Returns 30d change of the supply held in miner addresses.
supply_top_one_pct_addresses():
Returns the percentage of supply held by the top 1% addresses.
gini_coefficient():
Returns gini coefficient data.
herfindahl_index():
Returns herfindahl index data.
supply_in_smart_contracts():
Returns percent of total supply that is held in smart contracts.
Methods
-------
__init__(glassnode_client):
Constructs a Distribution object.
exchange_balance_total(exchange):
Returns the total amount of coins held on exchange addresses.
exchange_balance_percent(exchange):
Returns the percent supply held on exchange addresses.
exchange_balance_stacked():
Returns the total amount of coins held on exchange addresses.
miner_balance():
Returns the total supply held in miner addresses.
miner_balance_stacked():
Returns the total supply held in miner addresses.
balance_miners_change():
Returns 30d change of the supply held in miner addresses.
supply_top_one_pct_addresses():
Returns the percentage of supply held by the top 1% addresses.
gini_coefficient():
Returns gini coefficient data.
herfindahl_index():
Returns herfindahl index data.
supply_in_smart_contracts():
Returns percent of total supply that is held in smart contracts.
"""
def __init__(self, glassnode_client):
self._gc = glassnode_client
@@ -41,11 +42,11 @@ class Distribution:
:return: A DataFrame with exchange balance data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/balance_exchanges'
endpoint = "/v1/metrics/distribution/balance_exchanges"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
def exchange_balance_percent(self, exchange=None) -> pd.DataFrame:
"""
@@ -55,11 +56,11 @@ class Distribution:
:return: A DataFrame with exchange balance data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/balance_exchanges_relative'
endpoint = "/v1/metrics/distribution/balance_exchanges_relative"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'e': exchange}))
return response_to_dataframe(self._gc.get(endpoint, {"e": exchange}))
def exchange_balance_stacked(self) -> pd.DataFrame:
"""
@@ -69,7 +70,7 @@ class Distribution:
:return: A DataFrame with stacked exchange balance data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/balance_exchanges_all'
endpoint = "/v1/metrics/distribution/balance_exchanges_all"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -83,7 +84,7 @@ class Distribution:
:return: A DataFrame miner balance data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/balance_miners_sum'
endpoint = "/v1/metrics/distribution/balance_miners_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -97,7 +98,7 @@ class Distribution:
:return: A DataFrame with stacked miner balance data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/balance_miners_all'
endpoint = "/v1/metrics/distribution/balance_miners_all"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -105,13 +106,13 @@ class Distribution:
def balance_miners_change(self) -> pd.DataFrame:
"""
The 30d change of the supply held in miner addresses.
The 30d change of the supply held in miner addresses.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=distribution.BalanceMinersChange>`_
:return: A DataFrame with 30d change of the supply held in miner addresses.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/balance_miners_change'
endpoint = "/v1/metrics/distribution/balance_miners_change"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -125,7 +126,7 @@ class Distribution:
:return: A DataFrame with top 1% supply data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/balance_1pct_holders'
endpoint = "/v1/metrics/distribution/balance_1pct_holders"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -139,7 +140,7 @@ class Distribution:
:return: A DataFrame Gini Coefficient data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/gini'
endpoint = "/v1/metrics/distribution/gini"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -153,7 +154,7 @@ class Distribution:
:return: A DataFrame Herfindahl index data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/herfindahl'
endpoint = "/v1/metrics/distribution/herfindahl"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -167,7 +168,7 @@ class Distribution:
:return: A DataFrame smart contracts supply data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/distribution/supply_contracts'
endpoint = "/v1/metrics/distribution/supply_contracts"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
utils/glassnode/endpoints.py
+8 -6
View File
@@ -3,11 +3,11 @@ from .utils import fetch
def create_endpoints_dict(endpoints):
return {
endpoint['path']: {
'assets': {asset['symbol']: asset['tags'] for asset in endpoint['assets']},
'currencies': endpoint['currencies'],
'resolutions': endpoint['resolutions'],
'formats': endpoint['formats']
endpoint["path"]: {
"assets": {asset["symbol"]: asset["tags"] for asset in endpoint["assets"]},
"currencies": endpoint["currencies"],
"resolutions": endpoint["resolutions"],
"formats": endpoint["formats"],
}
for endpoint in endpoints
}
@@ -37,7 +37,9 @@ class Endpoints(metaclass=MetaEndpoints):
@endpoints.setter
def endpoints(self, api_key):
self._endpoints = create_endpoints_dict(fetch('/v2/metrics/endpoints', {'api_key': api_key}))
self._endpoints = create_endpoints_dict(
fetch("/v2/metrics/endpoints", {"api_key": api_key})
)
def query(self, path):
return self._endpoints[path]
utils/glassnode/entities.py
+61 -60
View File
@@ -4,49 +4,50 @@ from .client import GlassnodeClient
class Entities:
"""
Entities class.
Entities class.
Methods
-------
__init__(glassnode_client):
Constructs a Entities object.
sending_entities():
Returns the number of unique entities that were active as a sender.
receiving_entities():
Returns the number of unique entities that were active as a receiver.
active_entities():
Returns the number of unique entities that were active either as a sender or receiver.
new_entities():
Returns The number of unique entities that appeared for the first time in a transaction.
entities_net_growth():
Returns the net growth of unique entities in the network.
number_of_whales():
Returns the number of unique entities holding at least 1k coins.
supply_balance_less_0001():
Returns the total circulating supply held by entities with a balance lower than 0.001 coins.
supply_balance_0001_001():
Returns the total circulating supply held by entities with a balance between 0.001 and 0.01 coins.
supply_balance_001_01():
Returns the total circulating supply held by entities with a balance between 0.01 and 0.1 coins.
supply_balance_01_1():
Returns the total circulating supply held by entities with a balance between 0.1 and 1 coins.
supply_balance_1_10():
Returns the total circulating supply held by entities with a balance between 1 and 10 coins.
supply_balance_10_100():
Returns the total circulating supply held by entities with a balance between 10 and 100 coins.
supply_balance_100_1k():
Returns the total circulating supply held by entities with a balance between 100 and 1,000 coins.
supply_balance_1k_10k():
Returns the total circulating supply held by entities with a balance between 1,000 and 10,000 coins.
supply_balance_10k_100k():
Returns the total circulating supply held by entities with a balance between 10,000 and 100,000 coins.
supply_balance_more_100k():
Returns the total circulating supply held by entities with a balance of at least 100,000 coins.
entities_supply_distribution():
Returns relative distribution of the circulating supply held by entities with specific balance bands.
percent_entities_in_profit():
Returns the percentage of entities in the network that are currently in profit.
Methods
-------
__init__(glassnode_client):
Constructs a Entities object.
sending_entities():
Returns the number of unique entities that were active as a sender.
receiving_entities():
Returns the number of unique entities that were active as a receiver.
active_entities():
Returns the number of unique entities that were active either as a sender or receiver.
new_entities():
Returns The number of unique entities that appeared for the first time in a transaction.
entities_net_growth():
Returns the net growth of unique entities in the network.
number_of_whales():
Returns the number of unique entities holding at least 1k coins.
supply_balance_less_0001():
Returns the total circulating supply held by entities with a balance lower than 0.001 coins.
supply_balance_0001_001():
Returns the total circulating supply held by entities with a balance between 0.001 and 0.01 coins.
supply_balance_001_01():
Returns the total circulating supply held by entities with a balance between 0.01 and 0.1 coins.
supply_balance_01_1():
Returns the total circulating supply held by entities with a balance between 0.1 and 1 coins.
supply_balance_1_10():
Returns the total circulating supply held by entities with a balance between 1 and 10 coins.
supply_balance_10_100():
Returns the total circulating supply held by entities with a balance between 10 and 100 coins.
supply_balance_100_1k():
Returns the total circulating supply held by entities with a balance between 100 and 1,000 coins.
supply_balance_1k_10k():
Returns the total circulating supply held by entities with a balance between 1,000 and 10,000 coins.
supply_balance_10k_100k():
Returns the total circulating supply held by entities with a balance between 10,000 and 100,000 coins.
supply_balance_more_100k():
Returns the total circulating supply held by entities with a balance of at least 100,000 coins.
entities_supply_distribution():
Returns relative distribution of the circulating supply held by entities with specific balance bands.
percent_entities_in_profit():
Returns the percentage of entities in the network that are currently in profit.
"""
def __init__(self, glassnode_client: GlassnodeClient):
self._gc = glassnode_client
@@ -55,7 +56,7 @@ class Entities:
The number of unique entities that were active as a sender.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SendingCount>`_
"""
endpoint = '/v1/metrics/entities/sending_count'
endpoint = "/v1/metrics/entities/sending_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -66,7 +67,7 @@ class Entities:
The number of unique entities that were active as a receiver.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.ReceivingCount>`_
"""
endpoint = '/v1/metrics/entities/receiving_count'
endpoint = "/v1/metrics/entities/receiving_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -77,7 +78,7 @@ class Entities:
The number of unique entities that were active either as a sender or receiver.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.ActiveCount>`_
"""
endpoint = '/v1/metrics/entities/active_count'
endpoint = "/v1/metrics/entities/active_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -89,7 +90,7 @@ class Entities:
in a transaction of the native coin in the network.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.NewCount>`_
"""
endpoint = '/v1/metrics/entities/new_count'
endpoint = "/v1/metrics/entities/new_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -100,7 +101,7 @@ class Entities:
The net growth of unique entities in the network.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.NetGrowthCount>`_
"""
endpoint = '/v1/metrics/entities/net_growth_count'
endpoint = "/v1/metrics/entities/net_growth_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -111,7 +112,7 @@ class Entities:
The number of unique entities holding at least 1k coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.Min1KCount>`_
"""
endpoint = '/v1/metrics/entities/min_1k_count'
endpoint = "/v1/metrics/entities/min_1k_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -122,7 +123,7 @@ class Entities:
The total circulating supply held by entities with a balance lower than 0.001 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalanceLess0001>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_less_0001'
endpoint = "/v1/metrics/entities/supply_balance_less_0001"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -133,7 +134,7 @@ class Entities:
The total circulating supply held by entities with a balance between 0.001 and 0.01 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalance0001001>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_0001_001'
endpoint = "/v1/metrics/entities/supply_balance_0001_001"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -144,7 +145,7 @@ class Entities:
The total circulating supply held by entities with a balance between 0.01 and 0.1 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalance00101>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_001_01'
endpoint = "/v1/metrics/entities/supply_balance_001_01"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -155,7 +156,7 @@ class Entities:
The total circulating supply held by entities with a balance between 0.1 and 1 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalance011>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_01_1'
endpoint = "/v1/metrics/entities/supply_balance_01_1"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -166,7 +167,7 @@ class Entities:
The total circulating supply held by entities with a balance between 1 and 10 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalance110>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_1_10'
endpoint = "/v1/metrics/entities/supply_balance_1_10"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -177,7 +178,7 @@ class Entities:
The total circulating supply held by entities with a balance between 10 and 100 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalance10100>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_10_100'
endpoint = "/v1/metrics/entities/supply_balance_10_100"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -188,7 +189,7 @@ class Entities:
The total circulating supply held by entities with a balance between 100 and 1,000 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalance1001K>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_100_1k'
endpoint = "/v1/metrics/entities/supply_balance_100_1k"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -199,7 +200,7 @@ class Entities:
The total circulating supply held by entities with a balance between 1,000 and 10,000 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalance1K10K>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_1k_10k'
endpoint = "/v1/metrics/entities/supply_balance_1k_10k"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -210,7 +211,7 @@ class Entities:
The total circulating supply held by entities with a balance between 10,000 and 100,000 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalance10K100K>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_10k_100k'
endpoint = "/v1/metrics/entities/supply_balance_10k_100k"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -221,7 +222,7 @@ class Entities:
The total circulating supply held by entities with a balance of at least 100,000 coins.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyBalanceMore100K>`_
"""
endpoint = '/v1/metrics/entities/supply_balance_more_100k'
endpoint = "/v1/metrics/entities/supply_balance_more_100k"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -233,7 +234,7 @@ class Entities:
Relative distribution of the circulating supply held by entities with specific balance bands.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.SupplyDistributionRelative>`_
"""
endpoint = '/v1/metrics/entities/supply_distribution_relative'
endpoint = "/v1/metrics/entities/supply_distribution_relative"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -244,8 +245,8 @@ class Entities:
The percentage of entities in the network that are currently in profit.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=entities.ProfitRelative>`_
"""
endpoint = '/v1/metrics/entities/profit_relative'
endpoint = "/v1/metrics/entities/profit_relative"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint))
return response_to_dataframe(self._gc.get(endpoint))
utils/glassnode/eth2.py
+27 -26
View File
@@ -3,27 +3,28 @@ from .utils import *
class ETH2:
"""
ETH 2.0 class.
ETH 2.0 class.
Methods
-------
__init__(glassnode_client):
Constructs an ETH2 object.
new_deposits():
Returns the number transactions depositing 32 ETH to the ETH2 deposit contract.
new_value_staked():
Returns the amount of ETH transferred to the ETH2 deposit contract.
new_validators():
Returns the number of new validators depositing 32 ETH to the ETH2 deposit contract.
total_number_of_deposits():
Returns the total number of transactions to the ETH2 deposit contract.
total_value_staked():
Returns the amount of ETH deposited to the ETH2 deposit contract.
total_number_of_validators():
Returns the total number of unique validators.
phase_zero_staking_goal():
Returns the percentage of the Phase 0 staking goal.
Methods
-------
__init__(glassnode_client):
Constructs an ETH2 object.
new_deposits():
Returns the number transactions depositing 32 ETH to the ETH2 deposit contract.
new_value_staked():
Returns the amount of ETH transferred to the ETH2 deposit contract.
new_validators():
Returns the number of new validators depositing 32 ETH to the ETH2 deposit contract.
total_number_of_deposits():
Returns the total number of transactions to the ETH2 deposit contract.
total_value_staked():
Returns the amount of ETH deposited to the ETH2 deposit contract.
total_number_of_validators():
Returns the total number of unique validators.
phase_zero_staking_goal():
Returns the percentage of the Phase 0 staking goal.
"""
def __init__(self, glassnode_client):
self._gc = glassnode_client
@@ -35,7 +36,7 @@ class ETH2:
:return: A DataFrame with ETH2 deposit data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/eth2/staking_deposits_count'
endpoint = "/v1/metrics/eth2/staking_deposits_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -49,7 +50,7 @@ class ETH2:
:return: A DataFrame with staked value data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/eth2/staking_volume_sum'
endpoint = "/v1/metrics/eth2/staking_volume_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -63,7 +64,7 @@ class ETH2:
:return: A DataFrame with new validators data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/eth2/staking_validators_count'
endpoint = "/v1/metrics/eth2/staking_validators_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -77,7 +78,7 @@ class ETH2:
:return: A DataFrame with ETH2 deposit data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/eth2/staking_total_deposits_count'
endpoint = "/v1/metrics/eth2/staking_total_deposits_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -92,7 +93,7 @@ class ETH2:
:return: A DataFrame with ETH2 deposit data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/eth2/staking_total_volume_sum'
endpoint = "/v1/metrics/eth2/staking_total_volume_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -106,7 +107,7 @@ class ETH2:
:return: A DataFrame with ETH2 deposit data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/eth2/staking_total_validators_count'
endpoint = "/v1/metrics/eth2/staking_total_validators_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -120,7 +121,7 @@ class ETH2:
:return: A DataFrame with staking goal data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/eth2/staking_phase_0_goal_percent'
endpoint = "/v1/metrics/eth2/staking_phase_0_goal_percent"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
utils/glassnode/fees.py
+48 -47
View File
@@ -4,41 +4,42 @@ from .client import GlassnodeClient
class Fees:
"""
Fees class.
Fees class.
Methods
-------
__init__(glassnode_client):
Constructs a Mining object.
fee_ratio_multiple():
Returns the Fee Ratio Multiple (FRM).
fees_total():
Returns the total amount of fees paid to miners.
fees_mean():
Returns the mean fee per transaction.
fees_median():
Returns the median fee per transaction.
gas_used_total():
Returns the total amount of gas used in all transactions.
gas_used_mean():
Returns the mean amount of gas used per transaction.
gas_used_median():
Returns the median amount of gas used per transaction.
gas_price_mean():
Returns the mean gas price paid per transaction.
gas_price_median():
Returns the median gas price paid per transaction.
transaction_gas_limit_mean():
Returns the mean gas limit per transaction.
transaction_gas_limit_median():
Returns the median gas limit per transaction.
exchange_fees_total():
Returns the total amount of fees paid in transactions related to on-chain exchange activity.
exchange_fees_mean():
Returns the mean amount of fees paid in transactions related to on-chain exchange activity.
exchange_fees_dominance():
Returns the exchange fee dominance.
Methods
-------
__init__(glassnode_client):
Constructs a Mining object.
fee_ratio_multiple():
Returns the Fee Ratio Multiple (FRM).
fees_total():
Returns the total amount of fees paid to miners.
fees_mean():
Returns the mean fee per transaction.
fees_median():
Returns the median fee per transaction.
gas_used_total():
Returns the total amount of gas used in all transactions.
gas_used_mean():
Returns the mean amount of gas used per transaction.
gas_used_median():
Returns the median amount of gas used per transaction.
gas_price_mean():
Returns the mean gas price paid per transaction.
gas_price_median():
Returns the median gas price paid per transaction.
transaction_gas_limit_mean():
Returns the mean gas limit per transaction.
transaction_gas_limit_median():
Returns the median gas limit per transaction.
exchange_fees_total():
Returns the total amount of fees paid in transactions related to on-chain exchange activity.
exchange_fees_mean():
Returns the mean amount of fees paid in transactions related to on-chain exchange activity.
exchange_fees_dominance():
Returns the exchange fee dominance.
"""
def __init__(self, glassnode_client: GlassnodeClient):
self._gc = glassnode_client
@@ -48,7 +49,7 @@ class Fees:
and gives an assessment how secure a chain is once block rewards disappear.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=fees.FeeRatioMultiple>`_
"""
endpoint = '/v1/metrics/fees/fee_ratio_multiple'
endpoint = "/v1/metrics/fees/fee_ratio_multiple"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -59,7 +60,7 @@ class Fees:
The total amount of fees paid to miners. Issued (minted) coins are not included.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=fees.VolumeSum>`_
"""
endpoint = '/v1/metrics/fees/volume_sum'
endpoint = "/v1/metrics/fees/volume_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -70,7 +71,7 @@ class Fees:
The mean fee per transaction. Issued (minted) coins are not included.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=fees.VolumeMean>`_
"""
endpoint = '/v1/metrics/fees/volume_mean'
endpoint = "/v1/metrics/fees/volume_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -81,7 +82,7 @@ class Fees:
The median fee per transaction. Issued (minted) coins are not included.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=fees.VolumeMedian>`_
"""
endpoint = '/v1/metrics/fees/volume_median'
endpoint = "/v1/metrics/fees/volume_median"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -92,7 +93,7 @@ class Fees:
The total amount of gas used in all transactions.
`View in Studio <https://studio.glassnode.com/metrics?a=ETH&m=fees.GasUsedSum>`_
"""
endpoint = '/v1/metrics/fees/gas_used_sum'
endpoint = "/v1/metrics/fees/gas_used_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -103,7 +104,7 @@ class Fees:
The mean amount of gas used per transaction.
`View in Studio <https://studio.glassnode.com/metrics?a=ETH&m=fees.GasUsedMean>`_
"""
endpoint = '/v1/metrics/fees/gas_used_mean'
endpoint = "/v1/metrics/fees/gas_used_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -114,7 +115,7 @@ class Fees:
The median amount of gas used per transaction.
`View in Studio <https://studio.glassnode.com/metrics?a=ETH&m=fees.GasUsedMedian>`_
"""
endpoint = '/v1/metrics/fees/gas_used_median'
endpoint = "/v1/metrics/fees/gas_used_median"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -125,7 +126,7 @@ class Fees:
The mean gas price paid per transaction.
`View in Studio <https://studio.glassnode.com/metrics?a=ETH&m=fees.GasPriceMean>`_
"""
endpoint = '/v1/metrics/fees/gas_price_mean'
endpoint = "/v1/metrics/fees/gas_price_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -136,7 +137,7 @@ class Fees:
The median gas price paid per transaction.
`View in Studio <https://studio.glassnode.com/metrics?a=ETH&m=fees.GasPriceMedian>`_
"""
endpoint = '/v1/metrics/fees/gas_price_median'
endpoint = "/v1/metrics/fees/gas_price_median"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -147,7 +148,7 @@ class Fees:
The mean gas limit per transaction.
`View in Studio <https://studio.glassnode.com/metrics?a=ETH&m=fees.GasLimitTxMean>`_
"""
endpoint = '/v1/metrics/fees/gas_limit_tx_mean'
endpoint = "/v1/metrics/fees/gas_limit_tx_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -158,7 +159,7 @@ class Fees:
The median gas limit per transaction.
`View in Studio <https://studio.glassnode.com/metrics?a=ETH&m=fees.GasLimitTxMedian>`_
"""
endpoint = '/v1/metrics/fees/gas_limit_tx_median'
endpoint = "/v1/metrics/fees/gas_limit_tx_median"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -170,7 +171,7 @@ class Fees:
The total amount of fees paid in transactions related to on-chain exchange activity.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=fees.ExchangesSum>`_
"""
endpoint = '/v1/metrics/fees/exchanges_sum'
endpoint = "/v1/metrics/fees/exchanges_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -182,7 +183,7 @@ class Fees:
The mean amount of fees paid in transactions related to on-chain exchange activity.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=fees.ExchangesMean>`_
"""
endpoint = '/v1/metrics/fees/exchanges_mean'
endpoint = "/v1/metrics/fees/exchanges_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -195,7 +196,7 @@ class Fees:
paid in transactions related to on-chain exchange activity.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=fees.ExchangesRelative>`_
"""
endpoint = '/v1/metrics/fees/exchanges_relative'
endpoint = "/v1/metrics/fees/exchanges_relative"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
utils/glassnode/indicators.py
+40 -42
View File
@@ -11,7 +11,7 @@ class Indicators:
The Realized HODL Ratio is a market indicator that uses a ratio of the Realized Cap HODL Waves.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.RhodlRatio>`_
"""
endpoint = '/v1/metrics/indicators/rhodl_ratio'
endpoint = "/v1/metrics/indicators/rhodl_ratio"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -23,7 +23,7 @@ class Indicators:
the market age (in days). Historically, CVDD has been an accurate indicator for global Bitcoin market bottoms.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Cvdd>`_
"""
endpoint = '/v1/metrics/indicators/cvdd'
endpoint = "/v1/metrics/indicators/cvdd"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -37,7 +37,7 @@ class Indicators:
the worst of the miner capitulation is over when the 30d MA of the hash rate crosses above the 60d MA.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.HashRibbon>`_
"""
endpoint = '/v1/metrics/indicators/hash_ribbon'
endpoint = "/v1/metrics/indicators/hash_ribbon"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -50,7 +50,7 @@ class Indicators:
of the Bitcoin mining difficulty to create the ribbon.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.DifficultyRibbon>`_
"""
endpoint = '/v1/metrics/indicators/difficulty_ribbon'
endpoint = "/v1/metrics/indicators/difficulty_ribbon"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -62,7 +62,7 @@ class Indicators:
standard deviation to quantify compression of the Difficulty Ribbon.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.DifficultyRibbonCompression>`_
"""
endpoint = '/v1/metrics/indicators/difficulty_ribbon_compression'
endpoint = "/v1/metrics/indicators/difficulty_ribbon_compression"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -74,7 +74,7 @@ class Indicators:
the market cap by the transferred on-chain volume measured in USD.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Nvt>`_
"""
endpoint = '/v1/metrics/indicators/nvt'
endpoint = "/v1/metrics/indicators/nvt"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -85,7 +85,7 @@ class Indicators:
The NVT Signal (NVTS) is a modified version of the original NVT Ratio.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Nvts>`_
"""
endpoint = '/v1/metrics/indicators/nvts'
endpoint = "/v1/metrics/indicators/nvts"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -97,7 +97,7 @@ class Indicators:
by dividing the on-chain transaction volume (in USD) by the market cap, i.e. the inverse of the NVT ratio.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Velocity>`_
"""
endpoint = '/v1/metrics/indicators/velocity'
endpoint = "/v1/metrics/indicators/velocity"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -108,7 +108,7 @@ class Indicators:
Adjusted Coin Days Destroyed simply divides CDD by the circulating supply.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.CddSupplyAdjusted>`_
"""
endpoint = '/v1/metrics/indicators/cdd_supply_adjusted'
endpoint = "/v1/metrics/indicators/cdd_supply_adjusted"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -119,7 +119,7 @@ class Indicators:
Binary Coin Days Destroyed is computed by thresholding Adjusted CDD by its average over time.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.CddSupplyAdjustedBinary>`_
"""
endpoint = '/v1/metrics/indicators/cdd_supply_adjusted_binary'
endpoint = "/v1/metrics/indicators/cdd_supply_adjusted_binary"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -131,7 +131,7 @@ class Indicators:
and is defined as the ratio of coin days destroyed and total transfer volume.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.AverageDormancySupplyAdjusted>`_
"""
endpoint = '/v1/metrics/indicators/average_dormancy_supply_adjusted'
endpoint = "/v1/metrics/indicators/average_dormancy_supply_adjusted"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -144,7 +144,7 @@ class Indicators:
0 and the current ATH in 100 equally-spaced partitions.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.SpentOutputPriceDistributionAth>`_
"""
endpoint = '/v1/metrics/indicators/spent_output_price_distribution_ath'
endpoint = "/v1/metrics/indicators/spent_output_price_distribution_ath"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -157,7 +157,7 @@ class Indicators:
and creating 50 equally-spaced bucket each above and below the current price in steps of +/- 2%.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.SpentOutputPriceDistributionPercent>`_
"""
endpoint = '/v1/metrics/indicators/spent_output_price_distribution_percent'
endpoint = "/v1/metrics/indicators/spent_output_price_distribution_percent"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -169,7 +169,7 @@ class Indicators:
by the 365-day moving average of daily issuance value.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.PuellMultiple>`_
"""
endpoint = '/v1/metrics/indicators/puell_multiple'
endpoint = "/v1/metrics/indicators/puell_multiple"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -180,7 +180,7 @@ class Indicators:
Adjusted SOPR is SOPR ignoring all outputs with a lifespan of less than 1 hour.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.SoprAdjusted>`_
"""
endpoint = '/v1/metrics/indicators/sopr_adjusted'
endpoint = "/v1/metrics/indicators/sopr_adjusted"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -192,7 +192,7 @@ class Indicators:
When confidence is low and price is high then risk/reward is unattractive at that time (Reserve Risk is high).
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.ReserveRisk>`_
"""
endpoint = '/v1/metrics/indicators/reserve_risk'
endpoint = "/v1/metrics/indicators/reserve_risk"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -204,7 +204,7 @@ class Indicators:
younger than 155 days and serves as an indicator to assess the behaviour of short term investors.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.SoprLess155>`_
"""
endpoint = '/v1/metrics/indicators/sopr_less_155'
endpoint = "/v1/metrics/indicators/sopr_less_155"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -216,7 +216,7 @@ class Indicators:
of at least 155 days and serves as an indicator to assess the behaviour of long term investors.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.SoprMore155>`_
"""
endpoint = '/v1/metrics/indicators/sopr_more_155'
endpoint = "/v1/metrics/indicators/sopr_more_155"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -227,7 +227,7 @@ class Indicators:
HODLer Net Position Change shows the monthly position change of long term investors.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.HodlerNetPositionChange>`_
"""
endpoint = '/v1/metrics/indicators/hodler_net_position_change'
endpoint = "/v1/metrics/indicators/hodler_net_position_change"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -238,7 +238,7 @@ class Indicators:
Lost or HODLed Bitcoins indicates moves of large and old stashes.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.HodledLostCoins>`_
"""
endpoint = '/v1/metrics/indicators/hodled_lost_coins'
endpoint = "/v1/metrics/indicators/hodled_lost_coins"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -250,7 +250,7 @@ class Indicators:
divided by the value at creation (USD) of a spent output. Or simply: price sold / price paid.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Sopr>`_
"""
endpoint = '/v1/metrics/indicators/sopr'
endpoint = "/v1/metrics/indicators/sopr"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -262,7 +262,7 @@ class Indicators:
in a transaction and multiplying it by the number of days it has been since those coins were last spent.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Cdd>`_
"""
endpoint = '/v1/metrics/indicators/cdd'
endpoint = "/v1/metrics/indicators/cdd"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -274,7 +274,7 @@ class Indicators:
Outputs with a lifespan of less than 1h are discarded.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Asol>`_
"""
endpoint = '/v1/metrics/indicators/asol'
endpoint = "/v1/metrics/indicators/asol"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -286,7 +286,7 @@ class Indicators:
Outputs with a lifespan of less than 1h are discarded.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Msol>`_
"""
endpoint = '/v1/metrics/indicators/msol'
endpoint = "/v1/metrics/indicators/msol"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -298,7 +298,7 @@ class Indicators:
and is defined as the ratio of coin days destroyed and total transfer volume.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.AverageDormancy>`_
"""
endpoint = '/v1/metrics/indicators/average_dormancy'
endpoint = "/v1/metrics/indicators/average_dormancy"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -310,7 +310,7 @@ class Indicators:
Liveliness increases as long term holder liquidate positions and decreases while they accumulate to HODL.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Liveliness>`_
"""
endpoint = '/v1/metrics/indicators/liveliness'
endpoint = "/v1/metrics/indicators/liveliness"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -322,7 +322,7 @@ class Indicators:
whose price at realisation time was lower than the current price normalised by the market cap.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.UnrealizedProfit>`_
"""
endpoint = '/v1/metrics/indicators/unrealized_profit'
endpoint = "/v1/metrics/indicators/unrealized_profit"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -334,7 +334,7 @@ class Indicators:
whose price at realisation time was higher than the current price normalised by the market cap.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.UnrealizedLoss>`_
"""
endpoint = '/v1/metrics/indicators/unrealized_loss'
endpoint = "/v1/metrics/indicators/unrealized_loss"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -345,7 +345,7 @@ class Indicators:
Net Unrealized Profit/Loss (NUPL) is the difference between Relative Unrealized Profit/Loss.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.NetUnrealizedProfitLoss>`_
"""
endpoint = '/v1/metrics/indicators/net_unrealized_profit_loss'
endpoint = "/v1/metrics/indicators/net_unrealized_profit_loss"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -357,7 +357,7 @@ class Indicators:
younger than 155 days and serves as an indicator to assess the behaviour of short term investors.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.NuplLess155>`_
"""
endpoint = '/v1/metrics/indicators/nupl_less_155'
endpoint = "/v1/metrics/indicators/nupl_less_155"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -369,7 +369,7 @@ class Indicators:
with a lifespan of at least 155 days and serves as an indicator to assess the behaviour of long term investors.
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.NuplMore155>`_
"""
endpoint = '/v1/metrics/indicators/nupl_more_155'
endpoint = "/v1/metrics/indicators/nupl_more_155"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -378,28 +378,26 @@ class Indicators:
@dataframe_with_inner_object
def ssr(self) -> pd.DataFrame:
"""
The Stablecoin Supply Ratio (SSR) is the ratio between Bitcoin supply and the supply of stablecoins denoted
in BTC, or: Bitcoin Marketcap / Stablecoin Marketcap. We use the following stablecoins for the supply: USDT,
TUSD, USDC, PAX, GUSD, DAI, SAI, and BUSD. When the SSR is low, the current stablecoin supply has more "buying power"
to purchase BTC. It serves as a proxy for the supply/demand mechanics between BTC and USD. For more information see
this article (https://medium.com/@glassnode/stablecoins-buying-power-over-bitcoin-3475c0d8779d).
The Stablecoin Supply Ratio (SSR) is the ratio between Bitcoin supply and the supply of stablecoins denoted
in BTC, or: Bitcoin Marketcap / Stablecoin Marketcap. We use the following stablecoins for the supply: USDT,
TUSD, USDC, PAX, GUSD, DAI, SAI, and BUSD. When the SSR is low, the current stablecoin supply has more "buying power"
to purchase BTC. It serves as a proxy for the supply/demand mechanics between BTC and USD. For more information see
this article (https://medium.com/@glassnode/stablecoins-buying-power-over-bitcoin-3475c0d8779d).
`View in Studio <https://studio.glassnode.com/metrics?a=BTC&m=indicators.Ssr>`_
"""
endpoint = '/v1/metrics/indicators/ssr'
endpoint = "/v1/metrics/indicators/ssr"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint))
def bvin(self):
'''
"""
The Bitcoin Volatility Index (BVIN) is an implied volatility index that also represents the fair value of a bitcoin variance swap. The index is calculated by CryptoCompare using options data from Deribit and has been developed in collaboration with Carol Alexander and Arben Imeraj at the University of Sussex Business School. The index is suitable for use as a settlement price for bitcoin volatility futures. For more information on the methodology please see Alexander and Imeraj (2020).
'''
endpoint = '/v1/metrics/indicators/bvin'
"""
endpoint = "/v1/metrics/indicators/bvin"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint))
utils/glassnode/market.py
+36 -35
View File
@@ -3,33 +3,34 @@ from .utils import *
class Market:
"""
Market class.
Market class.
Methods
-------
__init__(glassnode_client):
Constructs a Market object.
price():
Returns the asset's price in USD.
price_ohlc():
Returns OHLC candlestick data.
price_drawdown_from_ath():
Returns the percent drawdown from previous all-time high.
marketcap():
Returns the market capitalization of the asset.
mvrv_ratio():
Returns MVRV ratio.
realized_cap():
Returns realized cap data.
mvrv_z_score():
Returns MVRV Z-Score.
sth_mvrv():
Returns Short Term Holder MVRV data.
lth_mvrv():
Returns Long Term Holder MVRV data.
realized_price():
Returns realized price data.
Methods
-------
__init__(glassnode_client):
Constructs a Market object.
price():
Returns the asset's price in USD.
price_ohlc():
Returns OHLC candlestick data.
price_drawdown_from_ath():
Returns the percent drawdown from previous all-time high.
marketcap():
Returns the market capitalization of the asset.
mvrv_ratio():
Returns MVRV ratio.
realized_cap():
Returns realized cap data.
mvrv_z_score():
Returns MVRV Z-Score.
sth_mvrv():
Returns Short Term Holder MVRV data.
lth_mvrv():
Returns Long Term Holder MVRV data.
realized_price():
Returns realized price data.
"""
def __init__(self, glassnode_client):
self._gc = glassnode_client
@@ -41,7 +42,7 @@ class Market:
:return: A DataFrame containing the asset's price data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/market/price_usd'
endpoint = "/v1/metrics/market/price_usd"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -56,7 +57,7 @@ class Market:
:return: A DataFrame containing OHLC candlestick data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/market/price_usd_ohlc'
endpoint = "/v1/metrics/market/price_usd_ohlc"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -70,7 +71,7 @@ class Market:
:return: A DataFrame containing the percent drawdown data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/market/price_drawdown_relative'
endpoint = "/v1/metrics/market/price_drawdown_relative"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -84,7 +85,7 @@ class Market:
:return: DataFrame
"""
endpoint = '/v1/metrics/market/marketcap_usd'
endpoint = "/v1/metrics/market/marketcap_usd"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -98,7 +99,7 @@ class Market:
:return: DataFrame
"""
endpoint = '/v1/metrics/market/mvrv'
endpoint = "/v1/metrics/market/mvrv"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -112,7 +113,7 @@ class Market:
:return: DataFrame
"""
endpoint = '/v1/metrics/market/marketcap_realized_usd'
endpoint = "/v1/metrics/market/marketcap_realized_usd"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -125,7 +126,7 @@ class Market:
:return: DataFrame
"""
endpoint = '/v1/metrics/market/mvrv_z_score'
endpoint = "/v1/metrics/market/mvrv_z_score"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -139,7 +140,7 @@ class Market:
:return: DataFrame
"""
endpoint = '/v1/metrics/market/mvrv_less_155'
endpoint = "/v1/metrics/market/mvrv_less_155"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -153,7 +154,7 @@ class Market:
:return: DataFrame
"""
endpoint = '/v1/metrics/market/mvrv_more_155'
endpoint = "/v1/metrics/market/mvrv_more_155"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -166,7 +167,7 @@ class Market:
:return: DataFrame
"""
endpoint = '/v1/metrics/market/price_realized_usd'
endpoint = "/v1/metrics/market/price_realized_usd"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
utils/glassnode/mining.py
+40 -39
View File
@@ -3,33 +3,34 @@ from .utils import *
class Mining:
"""
Mining class.
Mining class.
Methods
-------
__init__(glassnode_client):
Constructs a Mining object.
difficulty():
Returns difficulty to mine a block.
hash_rate():
Returns hash rate.
miner_revenue_total():
Returns the total miner revenue.
miner_revenue_fees():
Returns the percentage of miner revenue derived from fees.
miner_revenue_block_rewards():
Returns the total amount of newly minted coins.
miner_outflow_multiple():
Returns the miner outflow multiple.
thermocap():
Returns Thermocap data.
market_cap_to_thermocap_ratio():
Returns the Marketcap to Thermocap Ratio.
miner_unspent_supply():
Returns unspent miner supply.
miner_names():
Returns miner names for a mining endpoint.
Methods
-------
__init__(glassnode_client):
Constructs a Mining object.
difficulty():
Returns difficulty to mine a block.
hash_rate():
Returns hash rate.
miner_revenue_total():
Returns the total miner revenue.
miner_revenue_fees():
Returns the percentage of miner revenue derived from fees.
miner_revenue_block_rewards():
Returns the total amount of newly minted coins.
miner_outflow_multiple():
Returns the miner outflow multiple.
thermocap():
Returns Thermocap data.
market_cap_to_thermocap_ratio():
Returns the Marketcap to Thermocap Ratio.
miner_unspent_supply():
Returns unspent miner supply.
miner_names():
Returns miner names for a mining endpoint.
"""
def __init__(self, glassnode_client):
self._gc = glassnode_client
@@ -41,7 +42,7 @@ class Mining:
:return: A DataFrame with the latest difficulty data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/difficulty_latest'
endpoint = "/v1/metrics/mining/difficulty_latest"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -55,7 +56,7 @@ class Mining:
:return: A DataFrame with hash rate data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/hash_rate_mean'
endpoint = "/v1/metrics/mining/hash_rate_mean"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -69,11 +70,11 @@ class Mining:
:return: A DataFrame with total revenue data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/revenue_sum'
endpoint = "/v1/metrics/mining/revenue_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'m': miner}))
return response_to_dataframe(self._gc.get(endpoint, {"m": miner}))
def miner_revenue_fees(self) -> pd.DataFrame:
"""
@@ -83,7 +84,7 @@ class Mining:
:return: A DataFrame with revenue fees data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/revenue_from_fees'
endpoint = "/v1/metrics/mining/revenue_from_fees"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -97,11 +98,11 @@ class Mining:
:return: A DataFrame with revenue block rewards data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/volume_mined_sum'
endpoint = "/v1/metrics/mining/volume_mined_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'m': miner}))
return response_to_dataframe(self._gc.get(endpoint, {"m": miner}))
def miner_outflow_multiple(self, miner=None) -> pd.DataFrame:
"""
@@ -112,11 +113,11 @@ class Mining:
:return: A DataFrame MOM data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/miners_outflow_multiple'
endpoint = "/v1/metrics/mining/miners_outflow_multiple"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint, {'m': miner}))
return response_to_dataframe(self._gc.get(endpoint, {"m": miner}))
def thermocap(self) -> pd.DataFrame:
"""
@@ -127,7 +128,7 @@ class Mining:
:return: A DataFrame with thermocap data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/thermocap'
endpoint = "/v1/metrics/mining/thermocap"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -142,7 +143,7 @@ class Mining:
:return: A DataFrame with M/T ratio data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/marketcap_thermocap_ratio'
endpoint = "/v1/metrics/mining/marketcap_thermocap_ratio"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -156,13 +157,13 @@ class Mining:
:return: A DataFrame with unspent miner supply data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/mining/miners_unspent_supply'
endpoint = "/v1/metrics/mining/miners_unspent_supply"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
return response_to_dataframe(self._gc.get(endpoint))
def miner_names(self, endpoint='revenue_sum') -> list:
def miner_names(self, endpoint="revenue_sum") -> list:
"""
Returns a list of miner names for a mining endpoint.
@@ -170,5 +171,5 @@ class Mining:
:return: A List with miner names.
:rtype: List
"""
miners = self._gc.get(f'/v1/metrics/mining/{endpoint}/miners')
miners = self._gc.get(f"/v1/metrics/mining/{endpoint}/miners")
return miners[self._gc.asset]
utils/glassnode/protocols.py
+16 -15
View File
@@ -3,20 +3,21 @@ from .utils import *
class Protocols:
"""
Protocols class.
Protocols class.
Methods
-------
__init__(glassnode_client):
Constructs a Protocols object.
uniswap_transactions():
Returns the total number of transactions
that contains an interaction within Uniswap contracts.
uniswap_liquidity():
Returns the current liquidity on Uniswap.
uniswap_volume():
Returns the total volume traded on Uniswap.
Methods
-------
__init__(glassnode_client):
Constructs a Protocols object.
uniswap_transactions():
Returns the total number of transactions
that contains an interaction within Uniswap contracts.
uniswap_liquidity():
Returns the current liquidity on Uniswap.
uniswap_volume():
Returns the total volume traded on Uniswap.
"""
def __init__(self, glassnode_client):
self._gc = glassnode_client
self._endpoints = self._gc.endpoints
@@ -30,7 +31,7 @@ class Protocols:
:return: A DataFrame containing Uniswap transactions data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/protocols/uniswap_transaction_count'
endpoint = "/v1/metrics/protocols/uniswap_transaction_count"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -44,7 +45,7 @@ class Protocols:
:return: A DataFrame containing Uniswap liquidity data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/protocols/uniswap_liquidity_latest'
endpoint = "/v1/metrics/protocols/uniswap_liquidity_latest"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -58,7 +59,7 @@ class Protocols:
:return: A DataFrame containing Uniswap volume data.
:rtype: DataFrame
"""
endpoint = '/v1/metrics/protocols/uniswap_volume_sum'
endpoint = "/v1/metrics/protocols/uniswap_volume_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
utils/glassnode/supply.py
+114 -113
View File
@@ -4,85 +4,86 @@ from .client import GlassnodeClient
class Supply:
"""
Supply class.
Supply class.
Methods
-------
__init__(glassnode_client):
Constructs a Supply object.
liquid_illiquid_supply():
Returns the total supply held by illiquid, liquid, and highly liquid entities.
liquid_supply_change():
Returns the monthly (30d) net change of supply held by liquid and highly liquid entities.
illiquid_supply_change():
Returns the monthly (30d) net change of supply held by illiquid entities.
circulating_supply():
Returns the total amount of all coins ever created/issued.
issuance():
Returns the total amount of new coins added to the current supply.
inflation_rate():
Returns the yearly inflation rate.
supply_last_active_less_24h():
Returns the amount of circulating supply last moved in the last 24 hours.
supply_last_active_1d_1w():
Returns the amount of circulating supply last moved between 1 day and 1 week ago.
supply_last_active_1w_1m():
Returns the amount of circulating supply last moved between 1 week and 1 month ago.
supply_last_active_1m_3m():
Returns the amount of circulating supply last moved between 1 month and 3 months ago.
supply_last_active_3m_6m():
Returns the amount of circulating supply last moved between 3 months and 6 months ago.
supply_last_active_6m_12m():
Returns the amount of circulating supply last moved between 6 months and 12 months ago.
supply_last_active_1y_2y():
Returns the amount of circulating supply last moved between 1 year and 6 years ago.
supply_last_active_2y_3y():
Returns the amount of circulating supply last moved between 2 years and 3 years ago.
supply_last_active_3y_5y():
Returns the amount of circulating supply last moved between 3 years and 5 years ago.
supply_last_active_5y_7y():
Returns the amount of circulating supply last moved between 5 years and 7 years ago.
supply_last_active_7y_10y():
Returns the amount of circulating supply last moved between 7 years and 10 years ago.
supply_last_active_more_10y():
Returns the amount of circulating supply last moved more than 10 years ago.
hodl_waves():
Returns a bundle of all active supply age bands, aka HODL waves.
supply_last_active_more_1y_ago():
Returns the percent of circulating supply that has not moved in at least 1 year.
supply_last_active_more_2y_ago():
Returns the percent of circulating supply that has not moved in at least 2 years.
supply_last_active_more_3y_ago():
Returns the percent of circulating supply that has not moved in at least 3 years.
supply_last_active_more_5y_ago():
Returns the percent of circulating supply that has not moved in at least 5 years.
realized_cap_hodl_waves():
Returns HODL waves weighted by Realized Price.
adjusted_supply():
Returns the circulating supply adjusted by accounting for lost coins.
supply_in_profit():
Returns the circulating supply in profit.
supply_in_loss():
Returns the circulating supply in loss.
supply_in_profit_relative():
Returns the percentage of circulating supply in profit.
short_term_holder_supply():
Returns the total amount of circulating supply held by short-term holders.
long_term_holder_supply():
Returns the total amount of circulating supply held by long-term holders.
short_term_holder_supply_in_loss():
Returns the total amount of circulating supply that is currently at loss and held by short-term holders.
long_term_holder_supply_in_loss():
Returns the total amount of circulating supply that is currently at loss and held by long-term holders.
short_term_holder_supply_in_profit():
Returns the total amount of circulating supply that is currently in profit and held by short-term holders.
long_term_holder_supply_in_profit():
Returns the total amount of circulating supply that is currently in profit and held by long-term holders.
relative_long_short_term_holder_supply():
Returns the relative amount of circulating supply of held by long- and short-term holders in profit/loss.
long_term_holder_position_change():
Returns the monthly net position change of long-term holders.
Methods
-------
__init__(glassnode_client):
Constructs a Supply object.
liquid_illiquid_supply():
Returns the total supply held by illiquid, liquid, and highly liquid entities.
liquid_supply_change():
Returns the monthly (30d) net change of supply held by liquid and highly liquid entities.
illiquid_supply_change():
Returns the monthly (30d) net change of supply held by illiquid entities.
circulating_supply():
Returns the total amount of all coins ever created/issued.
issuance():
Returns the total amount of new coins added to the current supply.
inflation_rate():
Returns the yearly inflation rate.
supply_last_active_less_24h():
Returns the amount of circulating supply last moved in the last 24 hours.
supply_last_active_1d_1w():
Returns the amount of circulating supply last moved between 1 day and 1 week ago.
supply_last_active_1w_1m():
Returns the amount of circulating supply last moved between 1 week and 1 month ago.
supply_last_active_1m_3m():
Returns the amount of circulating supply last moved between 1 month and 3 months ago.
supply_last_active_3m_6m():
Returns the amount of circulating supply last moved between 3 months and 6 months ago.
supply_last_active_6m_12m():
Returns the amount of circulating supply last moved between 6 months and 12 months ago.
supply_last_active_1y_2y():
Returns the amount of circulating supply last moved between 1 year and 6 years ago.
supply_last_active_2y_3y():
Returns the amount of circulating supply last moved between 2 years and 3 years ago.
supply_last_active_3y_5y():
Returns the amount of circulating supply last moved between 3 years and 5 years ago.
supply_last_active_5y_7y():
Returns the amount of circulating supply last moved between 5 years and 7 years ago.
supply_last_active_7y_10y():
Returns the amount of circulating supply last moved between 7 years and 10 years ago.
supply_last_active_more_10y():
Returns the amount of circulating supply last moved more than 10 years ago.
hodl_waves():
Returns a bundle of all active supply age bands, aka HODL waves.
supply_last_active_more_1y_ago():
Returns the percent of circulating supply that has not moved in at least 1 year.
supply_last_active_more_2y_ago():
Returns the percent of circulating supply that has not moved in at least 2 years.
supply_last_active_more_3y_ago():
Returns the percent of circulating supply that has not moved in at least 3 years.
supply_last_active_more_5y_ago():
Returns the percent of circulating supply that has not moved in at least 5 years.
realized_cap_hodl_waves():
Returns HODL waves weighted by Realized Price.
adjusted_supply():
Returns the circulating supply adjusted by accounting for lost coins.
supply_in_profit():
Returns the circulating supply in profit.
supply_in_loss():
Returns the circulating supply in loss.
supply_in_profit_relative():
Returns the percentage of circulating supply in profit.
short_term_holder_supply():
Returns the total amount of circulating supply held by short-term holders.
long_term_holder_supply():
Returns the total amount of circulating supply held by long-term holders.
short_term_holder_supply_in_loss():
Returns the total amount of circulating supply that is currently at loss and held by short-term holders.
long_term_holder_supply_in_loss():
Returns the total amount of circulating supply that is currently at loss and held by long-term holders.
short_term_holder_supply_in_profit():
Returns the total amount of circulating supply that is currently in profit and held by short-term holders.
long_term_holder_supply_in_profit():
Returns the total amount of circulating supply that is currently in profit and held by long-term holders.
relative_long_short_term_holder_supply():
Returns the relative amount of circulating supply of held by long- and short-term holders in profit/loss.
long_term_holder_position_change():
Returns the monthly net position change of long-term holders.
"""
def __init__(self, glassnode_client: GlassnodeClient):
self._gc = glassnode_client
@@ -93,7 +94,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/liquid_illiquid_sum'
endpoint = "/v1/metrics/supply/liquid_illiquid_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -106,7 +107,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/liquid_change'
endpoint = "/v1/metrics/supply/liquid_change"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -119,7 +120,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/illiquid_change'
endpoint = "/v1/metrics/supply/illiquid_change"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -132,7 +133,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/current'
endpoint = "/v1/metrics/supply/current"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -146,7 +147,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/issued'
endpoint = "/v1/metrics/supply/issued"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -160,7 +161,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/inflation_rate'
endpoint = "/v1/metrics/supply/inflation_rate"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -173,7 +174,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_24h'
endpoint = "/v1/metrics/supply/active_24h"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -186,7 +187,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_1d_1w'
endpoint = "/v1/metrics/supply/active_1d_1w"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -199,7 +200,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_1w_1m'
endpoint = "/v1/metrics/supply/active_1w_1m"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -212,7 +213,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_1m_3m'
endpoint = "/v1/metrics/supply/active_1m_3m"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -225,7 +226,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_3m_6m'
endpoint = "/v1/metrics/supply/active_3m_6m"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -238,7 +239,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_6m_12m'
endpoint = "/v1/metrics/supply/active_6m_12m"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -251,7 +252,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_1y_2y'
endpoint = "/v1/metrics/supply/active_1y_2y"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -264,7 +265,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_2y_3y'
endpoint = "/v1/metrics/supply/active_2y_3y"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -277,7 +278,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_3y_5y'
endpoint = "/v1/metrics/supply/active_3y_5y"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -290,7 +291,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_5y_7y'
endpoint = "/v1/metrics/supply/active_5y_7y"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -303,7 +304,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_7y_10y'
endpoint = "/v1/metrics/supply/active_7y_10y"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -316,7 +317,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_more_10y'
endpoint = "/v1/metrics/supply/active_more_10y"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -329,7 +330,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/hodl_waves'
endpoint = "/v1/metrics/supply/hodl_waves"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -342,7 +343,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_more_1y_percent'
endpoint = "/v1/metrics/supply/active_more_1y_percent"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -355,7 +356,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_more_2y_percent'
endpoint = "/v1/metrics/supply/active_more_2y_percent"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -368,7 +369,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_more_3y_percent'
endpoint = "/v1/metrics/supply/active_more_3y_percent"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -381,7 +382,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/active_more_5y_percent'
endpoint = "/v1/metrics/supply/active_more_5y_percent"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -394,7 +395,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/rcap_hodl_waves'
endpoint = "/v1/metrics/supply/rcap_hodl_waves"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -407,7 +408,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/current_adjusted'
endpoint = "/v1/metrics/supply/current_adjusted"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -421,7 +422,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/profit_sum'
endpoint = "/v1/metrics/supply/profit_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -435,7 +436,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/loss_sum'
endpoint = "/v1/metrics/supply/loss_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -448,7 +449,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/profit_relative'
endpoint = "/v1/metrics/supply/profit_relative"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -461,7 +462,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/sth_sum'
endpoint = "/v1/metrics/supply/sth_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -474,7 +475,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/lth_sum'
endpoint = "/v1/metrics/supply/lth_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -487,7 +488,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/sth_loss_sum'
endpoint = "/v1/metrics/supply/sth_loss_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -500,7 +501,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/lth_loss_sum'
endpoint = "/v1/metrics/supply/lth_loss_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -513,7 +514,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/sth_profit_sum'
endpoint = "/v1/metrics/supply/sth_profit_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -526,7 +527,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/lth_profit_sum'
endpoint = "/v1/metrics/supply/lth_profit_sum"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -539,7 +540,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/lth_sth_profit_loss_relative'
endpoint = "/v1/metrics/supply/lth_sth_profit_loss_relative"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
@@ -553,7 +554,7 @@ class Supply:
:rtype: DataFrame
"""
endpoint = '/v1/metrics/supply/lth_net_change'
endpoint = "/v1/metrics/supply/lth_net_change"
if not is_supported_by_endpoint(self._gc, endpoint):
return pd.DataFrame()
utils/glassnode/utils.py
+9 -8
View File
@@ -18,11 +18,11 @@ def unix_timestamp(date_str):
def is_supported_by_endpoint(glassnode_client, url):
path = glassnode_client.endpoints.query(url)
if glassnode_client.asset not in path['assets']:
print(f'{url} metric is not available for {glassnode_client.asset}')
if glassnode_client.asset not in path["assets"]:
print(f"{url} metric is not available for {glassnode_client.asset}")
return False
if glassnode_client.resolution not in path['resolutions']:
print(f'{url} metric is not available for {glassnode_client.resolution}')
if glassnode_client.resolution not in path["resolutions"]:
print(f"{url} metric is not available for {glassnode_client.resolution}")
return False
return True
@@ -36,8 +36,8 @@ def response_to_dataframe(response):
"""
try:
df = pd.DataFrame(response)
df.set_index('t', inplace=True)
df.index = pd.to_datetime(df.index, unit='s')
df.set_index("t", inplace=True)
df.index = pd.to_datetime(df.index, unit="s")
df.index.name = None
df.sort_index(ascending=False, inplace=True)
return df
@@ -48,7 +48,8 @@ def response_to_dataframe(response):
def dataframe_with_inner_object(func):
def wrapper(*args, **kwargs):
df = func(*args, **kwargs)
return pd.concat([df.drop(['o'], axis=1), df['o'].apply(pd.Series)], axis=1)
return pd.concat([df.drop(["o"], axis=1), df["o"].apply(pd.Series)], axis=1)
return wrapper
@@ -60,7 +61,7 @@ def fetch(endpoint, params=None):
:param endpoint: Endpoint url corresponding to some metric (ex. '/v1/metrics/market/price_usd')
:return: DataFrame of {'t' : datetime, 'v' : 'metric-value'} pairs
"""
r = requests.get(f'https://api.glassnode.com{endpoint}', params=params, stream=True)
r = requests.get(f"https://api.glassnode.com{endpoint}", params=params, stream=True)
try:
r.raise_for_status()
except requests.exceptions.HTTPError as e:
utils/hashing.py
+3 -1
View File
@@ -1,10 +1,12 @@
from hashlib import sha256
import pandas as pd
def hash_df(df: pd.DataFrame) -> str:
s = str(df.columns) + str(df.index) + str(df.values)
return sha256(s.encode()).hexdigest()
def hash_series(df: pd.Series) -> str:
s = str(df.name) + str(df.index) + str(df.values)
return sha256(s.encode()).hexdigest()
return sha256(s.encode()).hexdigest()
utils/helpers.py
+33 -11
View File
@@ -5,11 +5,19 @@ import string
import random
from itertools import dropwhile
def get_files_from_dir(path: str) -> list[str]:
return [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
return [
f
for f in os.listdir(path)
if os.path.isfile(os.path.join(path, f)) and not f.startswith(".")
]
def get_first_valid_return_index(series: pd.Series) -> int:
double_nested_results = np.where(np.logical_and(series != 0, np.logical_not(pd.isna(series))))
double_nested_results = np.where(
np.logical_and(series != 0, np.logical_not(pd.isna(series)))
)
if len(double_nested_results) == 0:
return 0
nested_result = double_nested_results[0]
@@ -17,42 +25,56 @@ def get_first_valid_return_index(series: pd.Series) -> int:
return 0
return nested_result[0]
def get_last_non_na_index(series: pd.Series, index: int) -> int:
return next(dropwhile(lambda x: pd.isna(x[1]), enumerate(reversed(series[:index+1]))))[0]
return next(
dropwhile(lambda x: pd.isna(x[1]), enumerate(reversed(series[: index + 1])))
)[0]
def flatten(list_of_lists: list) -> list:
return [item for sublist in list_of_lists for item in sublist]
def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.Series:
if df.shape[1] == 0:
return df
mean_df = df.iloc[:,0]
mean_df = df.iloc[:, 0]
weights = df.loc[weights_source]
for i, row in df.iterrows():
if i == weights_source: continue
if i == weights_source:
continue
mean_df.loc[i] = (row * weights).sum() / df.loc[weights_source].sum()
return mean_df
def drop_columns_if_exist(df: pd.DataFrame, columns: list) -> pd.DataFrame:
for column in columns:
if column in df.columns:
df = df.drop(column, axis=1)
return df
def random_string(n: int) -> str:
return ''.join(random.choices(string.ascii_uppercase + string.digits, k=n))
return "".join(random.choices(string.ascii_uppercase + string.digits, k=n))
def equal_except_nan(row: pd.Series):
if np.isnan(row.iloc[0]) or np.isnan(row.iloc[1]):
return np.nan
if row.iloc[0] == row.iloc[1]:
return 1.
return 1.0
else:
return 0.
return 0.0
def drop_until_first_valid_index(df: pd.DataFrame, series: pd.Series) -> tuple[pd.DataFrame, pd.Series]:
first_valid_index = max(get_first_valid_return_index(df.iloc[:,0]), get_first_valid_return_index(series))
return df.iloc[first_valid_index:], series.iloc[first_valid_index:]
def drop_until_first_valid_index(
df: pd.DataFrame, series: pd.Series
) -> tuple[pd.DataFrame, pd.Series]:
first_valid_index = max(
get_first_valid_return_index(df.iloc[:, 0]),
get_first_valid_return_index(series),
)
return df.iloc[first_valid_index:], series.iloc[first_valid_index:]
utils/metrics.py
+129 -61
View File
@@ -3,6 +3,7 @@ import pandas as pd
import scipy.stats as ss
import numpy as np
def timing_of_flattening_and_flips(target_positions: pd.Series) -> pd.DatetimeIndex:
"""
Advances in Financial Machine Learning, Snippet 14.1, page 197
@@ -15,8 +16,12 @@ def timing_of_flattening_and_flips(target_positions: pd.Series) -> pd.DatetimeIn
:return: (pd.DatetimeIndex) Timestamps of trades flattening, flipping and last bet
"""
empty_positions = target_positions[(target_positions == 0)].index # Empty positions index
previous_positions = target_positions.shift(1) # Timestamps pointing at previous positions
empty_positions = target_positions[
(target_positions == 0)
].index # Empty positions index
previous_positions = target_positions.shift(
1
) # Timestamps pointing at previous positions
# Index of positions where previous one wasn't empty
previous_positions = previous_positions[(previous_positions != 0)].index
@@ -30,8 +35,12 @@ def timing_of_flattening_and_flips(target_positions: pd.Series) -> pd.DatetimeIn
# FLIPS - if current position has another direction compared to the next
flips = multiplied_posions[(multiplied_posions < 0)].index
flips_and_flattenings = flattening.union(flips).sort_values()
if target_positions.index[-1] not in flips_and_flattenings: # Appending with last bet
flips_and_flattenings = flips_and_flattenings.append(target_positions.index[-1:])
if (
target_positions.index[-1] not in flips_and_flattenings
): # Appending with last bet
flips_and_flattenings = flips_and_flattenings.append(
target_positions.index[-1:]
)
return flips_and_flattenings
@@ -50,19 +59,23 @@ def average_holding_period(target_positions: pd.Series) -> float:
:return: (float) Estimated average holding period, NaN if zero or unpredicted
"""
holding_period = pd.DataFrame(columns=['holding_time', 'weight'])
holding_period = pd.DataFrame(columns=["holding_time", "weight"])
entry_time = 0
position_difference = target_positions.diff()
# Time elapsed from the starting time for each position
time_difference = (target_positions.index - target_positions.index[0]) / np.timedelta64(1, 'D')
time_difference = (
target_positions.index - target_positions.index[0]
) / np.timedelta64(1, "D")
for i in range(1, target_positions.size):
# Increased or unchanged position
if float(position_difference.iloc[i] * target_positions.iloc[i - 1]) >= 0:
if float(target_positions.iloc[i]) != 0: # And not an empty position
entry_time = (entry_time * target_positions.iloc[i - 1] +
time_difference[i] * position_difference.iloc[i]) / target_positions.iloc[i]
entry_time = (
entry_time * target_positions.iloc[i - 1]
+ time_difference[i] * position_difference.iloc[i]
) / target_positions.iloc[i]
# Decreased
if float(position_difference.iloc[i] * target_positions.iloc[i - 1]) < 0:
@@ -71,19 +84,25 @@ def average_holding_period(target_positions: pd.Series) -> float:
# Flip of a position
if float(target_positions.iloc[i] * target_positions.iloc[i - 1]) < 0:
weight = abs(target_positions.iloc[i - 1])
holding_period.loc[target_positions.index[i], ['holding_time', 'weight']] = (hold_time, weight)
holding_period.loc[
target_positions.index[i], ["holding_time", "weight"]
] = (hold_time, weight)
entry_time = time_difference[i] # Reset entry time
# Only a part of position is closed
else:
weight = abs(position_difference.iloc[i])
holding_period.loc[target_positions.index[i], ['holding_time', 'weight']] = (hold_time, weight)
holding_period.loc[
target_positions.index[i], ["holding_time", "weight"]
] = (hold_time, weight)
if float(holding_period['weight'].sum()) > 0: # If there were closed trades at all
avg_holding_period = float((holding_period['holding_time'] * \
holding_period['weight']).sum() / holding_period['weight'].sum())
if float(holding_period["weight"].sum()) > 0: # If there were closed trades at all
avg_holding_period = float(
(holding_period["holding_time"] * holding_period["weight"]).sum()
/ holding_period["weight"].sum()
)
else:
avg_holding_period = float('nan')
avg_holding_period = float("nan")
return avg_holding_period
@@ -99,15 +118,15 @@ def bets_concentration(returns: pd.Series) -> float:
"""
if returns.size <= 2:
return float('nan') # If less than 3 bets
return float("nan") # If less than 3 bets
weights = returns / returns.sum() # Weights of each bet
hhi = (weights ** 2).sum() # Herfindahl-Hirschman Index for weights
hhi = (weights**2).sum() # Herfindahl-Hirschman Index for weights
hhi = float((hhi - returns.size ** (-1)) / (1 - returns.size ** (-1)))
return hhi
def all_bets_concentration(returns: pd.Series, frequency: str = 'M') -> tuple:
def all_bets_concentration(returns: pd.Series, frequency: str = "M") -> tuple:
"""
Advances in Financial Machine Learning, Snippet 14.3, page 201
Given a pd.Series of returns, derives concentration of positive returns, negative returns
@@ -131,7 +150,9 @@ def all_bets_concentration(returns: pd.Series, frequency: str = 'M') -> tuple:
negative_concentration = bets_concentration(returns[returns < 0])
# Concentration of bets/time period (month by default)
time_concentration = bets_concentration(returns.groupby(pd.Grouper(freq=frequency)).count())
time_concentration = bets_concentration(
returns.groupby(pd.Grouper(freq=frequency)).count()
)
return (positive_concentration, negative_concentration, time_concentration)
@@ -157,35 +178,42 @@ def drawdown_and_time_under_water(returns: pd.Series, dollars: bool = False) ->
:return: (tuple of pd.Series) Series of drawdowns and time under water
"""
frame = returns.to_frame('pnl')
frame['hwm'] = returns.expanding().max() # Adding high watermarks as column
frame = returns.to_frame("pnl")
frame["hwm"] = returns.expanding().max() # Adding high watermarks as column
# Grouped as min returns by high watermarks
high_watermarks = frame.groupby('hwm').min().reset_index()
high_watermarks.columns = ['hwm', 'min']
high_watermarks = frame.groupby("hwm").min().reset_index()
high_watermarks.columns = ["hwm", "min"]
# Time high watermark occurred
high_watermarks.index = frame['hwm'].drop_duplicates(keep='first').index
high_watermarks.index = frame["hwm"].drop_duplicates(keep="first").index
# Picking ones that had a drawdown after high watermark
high_watermarks = high_watermarks[high_watermarks['hwm'] > high_watermarks['min']]
high_watermarks = high_watermarks[high_watermarks["hwm"] > high_watermarks["min"]]
if dollars:
drawdown = high_watermarks['hwm'] - high_watermarks['min']
drawdown = high_watermarks["hwm"] - high_watermarks["min"]
else:
drawdown = 1 - high_watermarks['min'] / high_watermarks['hwm']
drawdown = 1 - high_watermarks["min"] / high_watermarks["hwm"]
time_under_water = ((high_watermarks.index[1:] - high_watermarks.index[:-1]) / np.timedelta64(1, 'Y')).values
time_under_water = (
(high_watermarks.index[1:] - high_watermarks.index[:-1])
/ np.timedelta64(1, "Y")
).values
# Adding also period from last High watermark to last return observed.
time_under_water = np.append(time_under_water,
(returns.index[-1] - high_watermarks.index[-1]) / np.timedelta64(1, 'Y'))
time_under_water = np.append(
time_under_water,
(returns.index[-1] - high_watermarks.index[-1]) / np.timedelta64(1, "Y"),
)
time_under_water = pd.Series(time_under_water, index=high_watermarks.index)
return drawdown, time_under_water
def sharpe_ratio(returns: pd.Series, entries_per_year: int = 252, risk_free_rate: float = 0) -> float:
def sharpe_ratio(
returns: pd.Series, entries_per_year: int = 252, risk_free_rate: float = 0
) -> float:
"""
Calculates annualized Sharpe ratio for pd.Series of normal or log returns.
Risk_free_rate should be given for the same period the returns are given.
@@ -197,12 +225,18 @@ def sharpe_ratio(returns: pd.Series, entries_per_year: int = 252, risk_free_rate
:return: (float) Annualized Sharpe ratio
"""
sharpe_r = (returns.mean() - risk_free_rate) / returns.std() * (entries_per_year) ** (1 / 2)
sharpe_r = (
(returns.mean() - risk_free_rate)
/ returns.std()
* (entries_per_year) ** (1 / 2)
)
return sharpe_r
def information_ratio(returns: pd.Series, benchmark: float = 0, entries_per_year: int = 252) -> float:
def information_ratio(
returns: pd.Series, benchmark: float = 0, entries_per_year: int = 252
) -> float:
"""
Calculates annualized information ratio for pd.Series of normal or log returns.
Benchmark should be provided as a return for the same time period as that between
@@ -223,8 +257,13 @@ def information_ratio(returns: pd.Series, benchmark: float = 0, entries_per_year
return information_r
def probabilistic_sharpe_ratio(observed_sr: float, benchmark_sr: float, number_of_returns: int,
skewness_of_returns: float = 0, kurtosis_of_returns: float = 3) -> float:
def probabilistic_sharpe_ratio(
observed_sr: float,
benchmark_sr: float,
number_of_returns: int,
skewness_of_returns: float = 0,
kurtosis_of_returns: float = 3,
) -> float:
"""
Calculates the probabilistic Sharpe ratio (PSR) that provides an adjusted estimate of SR,
by removing the inflationary effect caused by short series with skewed and/or
@@ -241,30 +280,47 @@ def probabilistic_sharpe_ratio(observed_sr: float, benchmark_sr: float, number_o
:return: (float) Probabilistic Sharpe ratio
"""
test_value = ((observed_sr - benchmark_sr) * np.sqrt(number_of_returns - 1)) / \
((1 - skewness_of_returns * observed_sr + (kurtosis_of_returns - 1) / \
4 * observed_sr ** 2)**(1 / 2))
test_value = ((observed_sr - benchmark_sr) * np.sqrt(number_of_returns - 1)) / (
(
1
- skewness_of_returns * observed_sr
+ (kurtosis_of_returns - 1) / 4 * observed_sr**2
)
** (1 / 2)
)
if np.isnan(test_value):
warnings.warn('Test value is nan. Please check the input values.', UserWarning)
warnings.warn("Test value is nan. Please check the input values.", UserWarning)
return test_value
if isinstance(test_value, complex):
warnings.warn('Output is a complex number. You may want to check the input skewness (too high), '
'kurtosis (too low), or observed_sr values.', UserWarning)
warnings.warn(
"Output is a complex number. You may want to check the input skewness (too high), "
"kurtosis (too low), or observed_sr values.",
UserWarning,
)
if np.isinf(test_value):
warnings.warn('Test value is infinite. You may want to check the input skewness, '
'kurtosis, or observed_sr values.', UserWarning)
warnings.warn(
"Test value is infinite. You may want to check the input skewness, "
"kurtosis, or observed_sr values.",
UserWarning,
)
probab_sr = ss.norm.cdf(test_value)
return probab_sr
def deflated_sharpe_ratio(observed_sr: float, sr_estimates: list, number_of_returns: int,
skewness_of_returns: float = 0, kurtosis_of_returns: float = 3,
estimates_param: bool = False, benchmark_out: bool = False) -> float:
def deflated_sharpe_ratio(
observed_sr: float,
sr_estimates: list,
number_of_returns: int,
skewness_of_returns: float = 0,
kurtosis_of_returns: float = 3,
estimates_param: bool = False,
benchmark_out: bool = False,
) -> float:
"""
Calculates the deflated Sharpe ratio (DSR) - a PSR where the rejection threshold is
adjusted to reflect the multiplicity of trials. DSR is estimated as PSR[SR∗], where
@@ -290,18 +346,25 @@ def deflated_sharpe_ratio(observed_sr: float, sr_estimates: list, number_of_retu
# Calculating benchmark_SR from the parameters of estimates
if estimates_param:
benchmark_sr = sr_estimates[0] * \
((1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / sr_estimates[1]) +
np.euler_gamma * ss.norm.ppf(1 - 1 / sr_estimates[1] * np.e ** (-1)))
benchmark_sr = sr_estimates[0] * (
(1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / sr_estimates[1])
+ np.euler_gamma * ss.norm.ppf(1 - 1 / sr_estimates[1] * np.e ** (-1))
)
# Calculating benchmark_SR from a list of estimates
else:
benchmark_sr = np.array(sr_estimates).std() * \
((1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / len(sr_estimates)) +
np.euler_gamma * ss.norm.ppf(1 - 1 / len(sr_estimates) * np.e ** (-1)))
benchmark_sr = np.array(sr_estimates).std() * (
(1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / len(sr_estimates))
+ np.euler_gamma * ss.norm.ppf(1 - 1 / len(sr_estimates) * np.e ** (-1))
)
deflated_sr = probabilistic_sharpe_ratio(observed_sr, benchmark_sr, number_of_returns,
skewness_of_returns, kurtosis_of_returns)
deflated_sr = probabilistic_sharpe_ratio(
observed_sr,
benchmark_sr,
number_of_returns,
skewness_of_returns,
kurtosis_of_returns,
)
if benchmark_out:
return benchmark_sr
@@ -309,10 +372,13 @@ def deflated_sharpe_ratio(observed_sr: float, sr_estimates: list, number_of_retu
return deflated_sr
def minimum_track_record_length(observed_sr: float, benchmark_sr: float,
skewness_of_returns: float = 0,
kurtosis_of_returns: float = 3,
alpha: float = 0.05) -> float:
def minimum_track_record_length(
observed_sr: float,
benchmark_sr: float,
skewness_of_returns: float = 0,
kurtosis_of_returns: float = 3,
alpha: float = 0.05,
) -> float:
"""
Calculates the minimum track record length (MinTRL) - "How long should a track
record be in order to have statistical confidence that its Sharpe ratio is above
@@ -329,8 +395,10 @@ def minimum_track_record_length(observed_sr: float, benchmark_sr: float,
:return: (float) Minimum number of track records
"""
track_rec_length = 1 + (1 - skewness_of_returns * observed_sr +
(kurtosis_of_returns - 1) / 4 * observed_sr ** 2) * \
(ss.norm.ppf(1 - alpha) / (observed_sr - benchmark_sr)) ** (2)
track_rec_length = 1 + (
1
- skewness_of_returns * observed_sr
+ (kurtosis_of_returns - 1) / 4 * observed_sr**2
) * (ss.norm.ppf(1 - alpha) / (observed_sr - benchmark_sr)) ** (2)
return track_rec_length
return track_rec_length
utils/parallel.py
+2 -2
View File
@@ -1,8 +1,8 @@
from tqdm import tqdm
import ray
def parallel_compute_with_bar(computations) -> list:
def parallel_compute_with_bar(computations) -> list:
def to_iterator(obj_ids):
while obj_ids:
done, obj_ids = ray.wait(obj_ids)
@@ -12,4 +12,4 @@ def parallel_compute_with_bar(computations) -> list:
for x in tqdm(to_iterator(computations), total=len(computations)):
ret.append(x)
return ret
return ret
utils/resample.py
+6 -5
View File
@@ -1,9 +1,10 @@
import pandas as pd
def resample_ohlc(df, period):
output = pd.DataFrame()
output['open'] = df.open.resample(period).first()
output['high'] = df.high.resample(period).max()
output['low'] = df.low.resample(period).min()
output['close'] = df.close.resample(period).last()
return output
output["open"] = df.open.resample(period).first()
output["high"] = df.high.resample(period).max()
output["low"] = df.low.resample(period).min()
output["close"] = df.close.resample(period).last()
return output
utils/sktime.py
+7 -6
View File
@@ -1,17 +1,18 @@
import pandas as pd
import numpy as np
def from_df_to_sktime_data(x: pd.DataFrame) -> pd.DataFrame:
instance_list = []
instance_list.append([])
x_data = pd.DataFrame(dtype=np.float32)
for i in range(len(x.index)):
instance_list[0].append(pd.Series(x.iloc[i]))
# only dim_0 for univariate time series
for dim in range(len(instance_list)):
x_data['dim_' + str(dim)] = instance_list[dim]
return x_data
x_data["dim_" + str(dim)] = instance_list[dim]
return x_data
utils/technical_indicators.py
+24 -10
View File
@@ -1,29 +1,43 @@
import pandas as pd
import numpy as np
def STOK(close, low, high, n):
STOK = ((close - low.rolling(n).min()) / (high.rolling(n).max() - low.rolling(n).min())) * 100
STOK = (
(close - low.rolling(n).min()) / (high.rolling(n).max() - low.rolling(n).min())
) * 100
return STOK
def STOD(close, low, high, n):
STOK = ((close - low.rolling(n).min()) / (high.rolling(n).max() - low.rolling(n).min())) * 100
STOK = (
(close - low.rolling(n).min()) / (high.rolling(n).max() - low.rolling(n).min())
) * 100
STOD = STOK.rolling(3).mean()
return STOD
def RSI(series, period):
delta = series.diff().dropna()
u=delta*0
u = delta * 0
d = u.copy()
u[delta > 0] = delta[delta > 0]
d[delta < 0] = -delta[delta < 0]
u[u.index[period-1]] = np.mean( u[:period] ) #first value is sum of avg gains u = u.drop(u.index[:(period-1)])
d[d.index[period-1]] = np.mean( d[:period] ) #first value is sum of avg losses d = d.drop(d.index[:(period-1)])
rs = u.ewm(com=period-1, adjust=False).mean() / \
d.ewm(com=period-1, adjust=False).mean()
return 100-100/(1+rs)
u[u.index[period - 1]] = np.mean(
u[:period]
) # first value is sum of avg gains u = u.drop(u.index[:(period-1)])
d[d.index[period - 1]] = np.mean(
d[:period]
) # first value is sum of avg losses d = d.drop(d.index[:(period-1)])
rs = (
u.ewm(com=period - 1, adjust=False).mean()
/ d.ewm(com=period - 1, adjust=False).mean()
)
return 100 - 100 / (1 + rs)
def ROC(df, n):
M = df.diff(n - 1)
N = df.shift(n - 1)
ROC = pd.Series(((M / N) * 100), name = 'ROC_' + str(n))
return ROC
ROC = pd.Series(((M / N) * 100), name="ROC_" + str(n))
return ROC

Some files were not shown because too many files have changed in this diff Show More