mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-27 18:57:55 +00:00
feat(Data): added daily_glassnode DataCollection (#99)
This commit is contained in:
Mark Aron Szulyovszky
committed by
GitHub
GitHub
parent
442915f847
commit
867269df2b
+37
-32
@@ -1,43 +1,48 @@
|
||||
from utils.types import Path, FileName, DataSource, DataCollection
|
||||
from utils.helpers import flatten
|
||||
|
||||
daily_etf = ["GLD", "IEF", "QQQ", "SPY", "TLT"]
|
||||
daily_etf = list(zip(["data/daily_etf"] * len(daily_etf), daily_etf))
|
||||
def transform_to_data_collection(path: str, file_names: list[str]) -> DataCollection:
|
||||
return list(zip([path] * len(file_names), file_names))
|
||||
|
||||
daily_crypto = ["ADA_USD",
|
||||
"BCH_USD",
|
||||
"BNB_USD",
|
||||
"BTC_USD",
|
||||
"DOT_USD",
|
||||
"ETC_USD",
|
||||
"ETH_USD",
|
||||
"FIL_USD",
|
||||
"LTC_USD",
|
||||
"SOL_USD",
|
||||
"THETA_USD",
|
||||
"TRX_USD",
|
||||
"UNI_USD",
|
||||
"XLM_USD",
|
||||
"XRP_USD",
|
||||
"XTZ_USD"]
|
||||
daily_crypto = list(zip(["data/daily_crypto"] * len(daily_crypto), daily_crypto))
|
||||
__daily_etf = ["GLD", "IEF", "QQQ", "SPY", "TLT"]
|
||||
|
||||
__daily_crypto = ["ADA_USD", "BCH_USD", "BNB_USD", "BTC_USD", "DOT_USD", "ETC_USD", "ETH_USD", "FIL_USD", "LTC_USD", "SOL_USD", "THETA_USD", "TRX_USD", "UNI_USD", "XLM_USD", "XRP_USD", "XTZ_USD"]
|
||||
|
||||
hourly_crypto = ["BTC_USD", "DASH_USD", "ETC_USD", "ETH_USD", "LTC_USD", "TRX_USD", "XLM_USD", "XMR_USD", "XRP_USD"]
|
||||
hourly_crypto = list(zip(["data/hourly_crypto"] * len(hourly_crypto), hourly_crypto))
|
||||
__hourly_crypto = ["BTC_USD", "DASH_USD", "ETC_USD", "ETH_USD", "LTC_USD", "TRX_USD", "XLM_USD", "XMR_USD", "XRP_USD"]
|
||||
|
||||
__daily_glassnode = ['rhodl_ratio',
|
||||
# 'cvdd',
|
||||
'nvt_ratio',
|
||||
'nvt_signal',
|
||||
'velocity',
|
||||
'supply_adjusted_cdd',
|
||||
'binary_cdd',
|
||||
'supply_adjusted_dormancy',
|
||||
'puell_multiple',
|
||||
'asopr',
|
||||
'reserve_risk',
|
||||
'sopr',
|
||||
'cdd',
|
||||
'asol',
|
||||
'msol',
|
||||
'dormancy',
|
||||
'liveliness',
|
||||
'relative_unrealized_profit',
|
||||
'relative_unrealized_loss',
|
||||
'nupl',
|
||||
'sth_nupl',
|
||||
'lth_nupl',
|
||||
'ssr',
|
||||
'bvin',
|
||||
# 'hash_rate'
|
||||
]
|
||||
|
||||
|
||||
data_collections = dict(
|
||||
daily_crypto = daily_crypto,
|
||||
daily_etf = daily_etf,
|
||||
hourly_crypto = hourly_crypto
|
||||
daily_only_btc = transform_to_data_collection("data/daily_crypto", ['BTC_USD']),
|
||||
daily_crypto = transform_to_data_collection("data/daily_crypto", __daily_crypto),
|
||||
daily_etf = transform_to_data_collection("data/daily_etf", __daily_etf),
|
||||
hourly_crypto = transform_to_data_collection("data/hourly_crypto", __hourly_crypto),
|
||||
daily_glassnode =transform_to_data_collection("data/daily_glassnode", __daily_glassnode),
|
||||
)
|
||||
|
||||
def preprocess_data_collections_config(data_dict: dict) -> dict:
|
||||
data_dict = data_dict.copy()
|
||||
keys = ['assets', 'other_assets']
|
||||
# keys = ['assets', 'other_assets', 'exogenous_data']
|
||||
for key in keys:
|
||||
preset_names = data_dict[key]
|
||||
data_dict[key] = flatten([data_collections[preset_name] for preset_name in preset_names])
|
||||
return data_dict
|
||||
|
||||
+22
-13
@@ -1,7 +1,7 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from utils.types import DataSource, FeatureExtractor
|
||||
from utils.helpers import deduplicate_indexes
|
||||
from utils.helpers import deduplicate_indexes, drop_columns_if_exist
|
||||
from data_loader.collections import DataCollection
|
||||
from typing import Literal
|
||||
import ray
|
||||
@@ -9,13 +9,14 @@ import os
|
||||
|
||||
def load_data(assets: DataCollection,
|
||||
other_assets: DataCollection,
|
||||
# exogenous_data: DataCollection,
|
||||
target_asset: str,
|
||||
exogenous_data: DataCollection,
|
||||
target_asset: DataSource,
|
||||
load_non_target_asset: bool,
|
||||
log_returns: bool,
|
||||
forecasting_horizon: int,
|
||||
own_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
other_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
exogenous_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
index_column: Literal['date', 'int'],
|
||||
method: Literal['regression', 'classification'],
|
||||
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
|
||||
@@ -29,25 +30,36 @@ def load_data(assets: DataCollection,
|
||||
- Series `forward_returns` with the target asset returns shifted by 1 day
|
||||
"""
|
||||
|
||||
target_file = [f for f in assets if f[1].startswith(target_asset)]
|
||||
other_files = [f for f in assets if load_non_target_asset == True and f[1].startswith(target_asset) == False]
|
||||
target_file = [f for f in assets if f[1].startswith(target_asset[1])]
|
||||
assert len(target_file) == 1, "There should be exactly one target file"
|
||||
other_files = [f for f in assets if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False]
|
||||
files = target_file + other_files + other_assets
|
||||
def is_target_asset(target_asset: str, file: str): return file.split('.')[0].startswith(target_asset)
|
||||
futures = [__load_df.remote(
|
||||
asset_futures = [__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns='log_returns' if log_returns else 'returns',
|
||||
feature_extractors=own_features if is_target_asset(target_asset[1], data_source[1]) else other_features,
|
||||
narrow_format=narrow_format,
|
||||
) for data_source in files]
|
||||
dfs = ray.get(futures)
|
||||
asset_dfs = ray.get(asset_futures)
|
||||
|
||||
exogenous_futures = [__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns='returns',
|
||||
feature_extractors=exogenous_features,
|
||||
narrow_format=narrow_format,
|
||||
) for data_source in exogenous_data]
|
||||
exogenous_dfs = ray.get(exogenous_futures)
|
||||
|
||||
dfs = asset_dfs + exogenous_dfs
|
||||
dfs = [deduplicate_indexes(df) for df in dfs]
|
||||
longest_df = max(dfs, key=lambda df: df.shape[0])
|
||||
if narrow_format:
|
||||
dfs = pd.concat(dfs, axis=0).fillna(0.)
|
||||
dfs = pd.concat([df.sort_index().reindex(longest_df.index) for df in dfs], axis=0).fillna(0.)
|
||||
else:
|
||||
dfs = pd.concat([df.reindex(longest_df.index) for df in dfs], axis=1).fillna(0.)
|
||||
dfs = pd.concat([df.sort_index().reindex(longest_df.index) for df in dfs], axis=1).fillna(0.)
|
||||
|
||||
dfs.index = pd.DatetimeIndex(dfs.index)
|
||||
|
||||
@@ -93,10 +105,7 @@ def __load_df(data_source: DataSource,
|
||||
df = __apply_feature_extractors(df, log_returns=True if returns == 'log_returns' else False, feature_extractors = feature_extractors)
|
||||
|
||||
df = df.replace([np.inf, -np.inf], 0.)
|
||||
df = df.drop(columns=['open', 'high', 'low', 'close'])
|
||||
# we're not ready for this just yet
|
||||
if 'volume' in df.columns:
|
||||
df = df.drop(columns=['volume'])
|
||||
df = drop_columns_if_exist(df, ['open', 'high', 'low', 'close', 'volume'])
|
||||
|
||||
if narrow_format:
|
||||
df["ticker"] = np.repeat(prefix, df.shape[0])
|
||||
|
||||
Reference in New Issue
Block a user