chore(Linter): reformatted code with black (#211)

* chore(Linter): reformatted code with black

* Create black.yaml

data_loader/collections.py

@@ -1,51 +1,83 @@
 from .types import Path, FileName, DataSource, DataCollection
 from utils.helpers import flatten
 
+
 def transform_to_data_collection(path: str, file_names: list[str]) -> DataCollection:
     return list(zip([path] * len(file_names), file_names))
 
+
 __daily_etf = ["GLD", "IEF", "QQQ", "SPY", "TLT"]
 
-__daily_crypto = ["ADA_USD", "BCH_USD", "BNB_USD", "BTC_USD", "DOT_USD", "ETC_USD", "ETH_USD", "FIL_USD", "LTC_USD", "SOL_USD", "THETA_USD", "TRX_USD", "UNI_USD", "XLM_USD", "XRP_USD", "XTZ_USD"]
+__daily_crypto = [
+    "ADA_USD",
+    "BCH_USD",
+    "BNB_USD",
+    "BTC_USD",
+    "DOT_USD",
+    "ETC_USD",
+    "ETH_USD",
+    "FIL_USD",
+    "LTC_USD",
+    "SOL_USD",
+    "THETA_USD",
+    "TRX_USD",
+    "UNI_USD",
+    "XLM_USD",
+    "XRP_USD",
+    "XTZ_USD",
+]
 
 __daily_crypto_lightweight = ["ADA_USD", "BCH_USD"]
 
-__5min_crypto = ["BTC_USD", "DASH_USD", "ETC_USD", "ETH_USD", "LTC_USD", "TRX_USD", "XLM_USD", "XMR_USD"]
+__5min_crypto = [
+    "BTC_USD",
+    "DASH_USD",
+    "ETC_USD",
+    "ETH_USD",
+    "LTC_USD",
+    "TRX_USD",
+    "XLM_USD",
+    "XMR_USD",
+]
 
-__daily_glassnode = ['rhodl_ratio',
+__daily_glassnode = [
+    "rhodl_ratio",
     # 'cvdd',
-    'nvt_ratio',
-    'nvt_signal',
-    'velocity',
-    'supply_adjusted_cdd',
-    'binary_cdd',
-    'supply_adjusted_dormancy',
-    'puell_multiple',
-    'asopr',
-    'reserve_risk',
-    'sopr',
-    'cdd',
-    'asol',
-    'msol',
-    'dormancy',
-    'liveliness',
-    'relative_unrealized_profit',
-    'relative_unrealized_loss',
-    'nupl',
-    'sth_nupl',
-    'lth_nupl',
-    'ssr',
-    'bvin',
+    "nvt_ratio",
+    "nvt_signal",
+    "velocity",
+    "supply_adjusted_cdd",
+    "binary_cdd",
+    "supply_adjusted_dormancy",
+    "puell_multiple",
+    "asopr",
+    "reserve_risk",
+    "sopr",
+    "cdd",
+    "asol",
+    "msol",
+    "dormancy",
+    "liveliness",
+    "relative_unrealized_profit",
+    "relative_unrealized_loss",
+    "nupl",
+    "sth_nupl",
+    "lth_nupl",
+    "ssr",
+    "bvin",
     # 'hash_rate'
 ]
 
 
 data_collections = dict(
-    daily_only_btc = transform_to_data_collection("data/daily_crypto", ['BTC_USD']),
-    daily_crypto = transform_to_data_collection("data/daily_crypto", __daily_crypto),
-    daily_crypto_lightweight = transform_to_data_collection("data/daily_crypto", __daily_crypto_lightweight),
-    daily_etf = transform_to_data_collection("data/daily_etf", __daily_etf),
-    fivemin_crypto = transform_to_data_collection("data/5min_crypto", __5min_crypto),
-    daily_glassnode =transform_to_data_collection("data/daily_glassnode",  __daily_glassnode),
+    daily_only_btc=transform_to_data_collection("data/daily_crypto", ["BTC_USD"]),
+    daily_crypto=transform_to_data_collection("data/daily_crypto", __daily_crypto),
+    daily_crypto_lightweight=transform_to_data_collection(
+        "data/daily_crypto", __daily_crypto_lightweight
+    ),
+    daily_etf=transform_to_data_collection("data/daily_etf", __daily_etf),
+    fivemin_crypto=transform_to_data_collection("data/5min_crypto", __5min_crypto),
+    daily_glassnode=transform_to_data_collection(
+        "data/daily_glassnode", __daily_glassnode
+    ),
 )
-

data_loader/get_prices.py

@@ -3,45 +3,76 @@ import requests
 import pandas as pd
 
 # %%
-def get_crypto_price_crypto_compare(symbol: str, exchange: str, days: int) -> pd.DataFrame:
-    api_url = f'https://min-api.cryptocompare.com/data/v2/histoday?fsym={symbol}&tsym={exchange}&limit={days}&api_key={CC_API_KEY}'
+def get_crypto_price_crypto_compare(
+    symbol: str, exchange: str, days: int
+) -> pd.DataFrame:
+    api_url = f"https://min-api.cryptocompare.com/data/v2/histoday?fsym={symbol}&tsym={exchange}&limit={days}&api_key={CC_API_KEY}"
     raw = requests.get(api_url).json()
-    df = pd.DataFrame(raw['Data']['Data'])[['time', 'high', 'low', 'open', 'close']].set_index('time')
-    df.index = pd.to_datetime(df.index, unit = 's')
-    df.sort_index(inplace = True, ascending= True)
+    df = pd.DataFrame(raw["Data"]["Data"])[
+        ["time", "high", "low", "open", "close"]
+    ].set_index("time")
+    df.index = pd.to_datetime(df.index, unit="s")
+    df.sort_index(inplace=True, ascending=True)
     return df
 
-ada = get_crypto_price_crypto_compare('ADA', 'USD', 1500)
+
+ada = get_crypto_price_crypto_compare("ADA", "USD", 1500)
 ada
 
 
-
-def get_crypto_price_av(symbol: str, exchange: str, start_date = None) -> pd.DataFrame:
-    api_url = f'https://www.alphavantage.co/query?function=DIGITAL_CURRENCY_DAILY&symbol={symbol}&market={exchange}&apikey={AV_API_KEY}'
+def get_crypto_price_av(symbol: str, exchange: str, start_date=None) -> pd.DataFrame:
+    api_url = f"https://www.alphavantage.co/query?function=DIGITAL_CURRENCY_DAILY&symbol={symbol}&market={exchange}&apikey={AV_API_KEY}"
     raw_df = requests.get(api_url).json()
-    df = pd.DataFrame(raw_df['Time Series (Digital Currency Daily)']).T
-    df = df.rename(columns = {'1a. open (USD)': 'open', '2a. high (USD)': 'high', '3a. low (USD)': 'low', '4a. close (USD)': 'close', '5. volume': 'volume'})
+    df = pd.DataFrame(raw_df["Time Series (Digital Currency Daily)"]).T
+    df = df.rename(
+        columns={
+            "1a. open (USD)": "open",
+            "2a. high (USD)": "high",
+            "3a. low (USD)": "low",
+            "4a. close (USD)": "close",
+            "5. volume": "volume",
+        }
+    )
     for i in df.columns:
         df[i] = df[i].astype(float)
     df.index = pd.to_datetime(df.index)
-    df = df.iloc[::-1].drop(['1b. open (USD)', '2b. high (USD)', '3b. low (USD)', '4b. close (USD)', '6. market cap (USD)'], axis = 1)
+    df = df.iloc[::-1].drop(
+        [
+            "1b. open (USD)",
+            "2b. high (USD)",
+            "3b. low (USD)",
+            "4b. close (USD)",
+            "6. market cap (USD)",
+        ],
+        axis=1,
+    )
     if start_date:
         df = df[df.index >= start_date]
-    df.sort_index(inplace = True, ascending= True)
+    df.sort_index(inplace=True, ascending=True)
     return df
 
+
 def get_stock_price_av(symbol: str, start_date: str = None) -> pd.DataFrame:
-    api_url = f'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&outputsize=full&apikey={AV_API_KEY}'
+    api_url = f"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&outputsize=full&apikey={AV_API_KEY}"
     raw_df = requests.get(api_url).json()
-    df = pd.DataFrame(raw_df['Time Series (Daily)']).T
-    df = df.rename(columns = {'1. open': 'open', '2. high': 'high', '3. low': 'low', '5. adjusted close': 'close', '6. volume': 'volume'})
+    df = pd.DataFrame(raw_df["Time Series (Daily)"]).T
+    df = df.rename(
+        columns={
+            "1. open": "open",
+            "2. high": "high",
+            "3. low": "low",
+            "5. adjusted close": "close",
+            "6. volume": "volume",
+        }
+    )
     for i in df.columns:
         df[i] = df[i].astype(float)
     df.index = pd.to_datetime(df.index)
-    df = df.iloc[::-1].drop(['4. close', '7. dividend amount', '8. split coefficient'], axis = 1)
+    df = df.iloc[::-1].drop(
+        ["4. close", "7. dividend amount", "8. split coefficient"], axis=1
+    )
     if start_date:
         df = df[df.index >= start_date]
-    df.sort_index(inplace = True, ascending= True)
-    df = df.rename_axis('time')
+    df.sort_index(inplace=True, ascending=True)
+    df = df.rename_axis("time")
     return df
-

data_loader/load.py

@@ -10,8 +10,8 @@ import os
 from config.hashing import hash_data_config
 from .types import XDataFrame, ReturnSeries, ForwardReturnSeries
 from diskcache import Cache
-cache = Cache(".cachedir/data")
 
+cache = Cache(".cachedir/data")
 
 
 def load_data(**kwargs) -> tuple[XDataFrame, ReturnSeries]:
@@ -23,15 +23,17 @@ def load_data(**kwargs) -> tuple[XDataFrame, ReturnSeries]:
         cache[hashed] = return_value
         return return_value
 
-def __load_data(assets: DataCollection,
-            other_assets: DataCollection,
-            exogenous_data: DataCollection,
-            target_asset: DataSource,
-            load_non_target_asset: bool,
-            own_features: list[tuple[str, FeatureExtractor, list[int]]],
-            other_features: list[tuple[str, FeatureExtractor, list[int]]],
-            exogenous_features: list[tuple[str, FeatureExtractor, list[int]]],
-        ) -> tuple[XDataFrame, ReturnSeries]:
+
+def __load_data(
+    assets: DataCollection,
+    other_assets: DataCollection,
+    exogenous_data: DataCollection,
+    target_asset: DataSource,
+    load_non_target_asset: bool,
+    own_features: list[tuple[str, FeatureExtractor, list[int]]],
+    other_features: list[tuple[str, FeatureExtractor, list[int]]],
+    exogenous_features: list[tuple[str, FeatureExtractor, list[int]]],
+) -> tuple[XDataFrame, ReturnSeries]:
     """
     Loads asset data from the specified path.
     Returns:
@@ -42,77 +44,96 @@ def __load_data(assets: DataCollection,
 
     target_file = [f for f in assets if f[1].startswith(target_asset[1])]
     assert len(target_file) == 1, "There should be exactly one target file"
-    other_files = [f for f in assets if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False]
+    other_files = [
+        f
+        for f in assets
+        if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False
+    ]
     files = other_files + other_assets
-    
-    target_asset_future = [__load_df.remote(
-        data_source=data_source,
-        prefix=data_source[1],
-        returns='log_returns',
-        feature_extractors=own_features,
-    ) for data_source in target_file]
+
+    target_asset_future = [
+        __load_df.remote(
+            data_source=data_source,
+            prefix=data_source[1],
+            returns="log_returns",
+            feature_extractors=own_features,
+        )
+        for data_source in target_file
+    ]
     target_asset_df = ray.get(target_asset_future)
-    
+
     target_asset_only_returns_future = __load_df.remote(
         data_source=target_file[0],
         prefix=target_file[0][1],
-        returns='returns',
+        returns="returns",
         feature_extractors=[],
     )
     df_target_asset_only_returns = ray.get(target_asset_only_returns_future)
 
-    asset_futures = [__load_df.remote(
-        data_source=data_source,
-        prefix=data_source[1],
-        returns='log_returns',
-        feature_extractors=other_features,
-    ) for data_source in files]
+    asset_futures = [
+        __load_df.remote(
+            data_source=data_source,
+            prefix=data_source[1],
+            returns="log_returns",
+            feature_extractors=other_features,
+        )
+        for data_source in files
+    ]
     asset_dfs = ray.get(asset_futures)
 
-    exogenous_futures = [__load_df.remote(
-        data_source=data_source,
-        prefix=data_source[1],
-        returns='none',
-        feature_extractors=exogenous_features,
-    ) for data_source in exogenous_data]
+    exogenous_futures = [
+        __load_df.remote(
+            data_source=data_source,
+            prefix=data_source[1],
+            returns="none",
+            feature_extractors=exogenous_features,
+        )
+        for data_source in exogenous_data
+    ]
     exogenous_dfs = ray.get(exogenous_futures)
 
     X = target_asset_df + asset_dfs + exogenous_dfs
-    X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.)
+    X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.0)
 
     X.index = pd.DatetimeIndex(X.index)
-    
-    ## Create target 
-    returns = df_target_asset_only_returns[target_asset[1] + '_returns']
+
+    ## Create target
+    returns = df_target_asset_only_returns[target_asset[1] + "_returns"]
     returns.index = pd.DatetimeIndex(X.index)
-    
-    return X, returns 
+
+    return X, returns
 
 
 @ray.remote
-def __load_df(data_source: DataSource,
-            prefix: str,
-            returns: Literal['none', 'price', 'returns', 'log_returns'],
-            feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]) -> pd.DataFrame:
-    df = pd.read_csv(os.path.join(data_source[0], data_source[1] + '.csv'), header=0, index_col=0).fillna(0)
+def __load_df(
+    data_source: DataSource,
+    prefix: str,
+    returns: Literal["none", "price", "returns", "log_returns"],
+    feature_extractors: list[tuple[str, FeatureExtractor, list[int]]],
+) -> pd.DataFrame:
+    df = pd.read_csv(
+        os.path.join(data_source[0], data_source[1] + ".csv"), header=0, index_col=0
+    ).fillna(0)
 
-    if returns == 'log_returns':
-        df['returns'] = np.log(df['close']).diff(1)
-    elif returns == 'price':
-        df['returns'] = df['close']
-    elif returns == 'returns':
-        df['returns'] = df['close'].pct_change()
+    if returns == "log_returns":
+        df["returns"] = np.log(df["close"]).diff(1)
+    elif returns == "price":
+        df["returns"] = df["close"]
+    elif returns == "returns":
+        df["returns"] = df["close"].pct_change()
 
-    df = __apply_feature_extractors(df, feature_extractors = feature_extractors)
+    df = __apply_feature_extractors(df, feature_extractors=feature_extractors)
 
-    df = df.replace([np.inf, -np.inf], 0.)
-    df = drop_columns_if_exist(df, ['open', 'high', 'low', 'close', 'volume'])
-    
-    df.columns = [prefix + "_" + c if 'date' not in c else c for c in df.columns]
+    df = df.replace([np.inf, -np.inf], 0.0)
+    df = drop_columns_if_exist(df, ["open", "high", "low", "close", "volume"])
+
+    df.columns = [prefix + "_" + c if "date" not in c else c for c in df.columns]
     return df
 
 
-def __apply_feature_extractors(df: pd.DataFrame, feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]) -> pd.DataFrame:
+def __apply_feature_extractors(
+    df: pd.DataFrame, feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]
+) -> pd.DataFrame:
 
     for name, extractor, periods in feature_extractors:
         for period in periods:
@@ -120,20 +141,27 @@ def __apply_feature_extractors(df: pd.DataFrame, feature_extractors: list[tuple[
             if type(features) == pd.DataFrame:
                 df = pd.concat([df, features], axis=1)
             elif type(features) == pd.Series:
-                df[name + '_' + str(period)] = features
+                df[name + "_" + str(period)] = features
             else:
-                assert False, "Feature extractor must return a pd.DataFrame or pd.Series"
+                assert (
+                    False
+                ), "Feature extractor must return a pd.DataFrame or pd.Series"
     return df
 
 
-def load_only_returns(assets: DataCollection, returns: Literal['price', 'returns']) -> pd.DataFrame:
+def load_only_returns(
+    assets: DataCollection, returns: Literal["price", "returns"]
+) -> pd.DataFrame:
 
-    assets_future = [__load_df.remote(
-        data_source=data_source,
-        prefix=data_source[1],
-        returns=returns,
-        feature_extractors=[],
-    ) for data_source in assets]
+    assets_future = [
+        __load_df.remote(
+            data_source=data_source,
+            prefix=data_source[1],
+            returns=returns,
+            feature_extractors=[],
+        )
+        for data_source in assets
+    ]
     dfs = ray.get(assets_future)
 
     dfs = pd.concat(dfs, axis=1)

data_loader/process.py

@@ -4,16 +4,21 @@ import warnings
 from utils.helpers import get_first_valid_return_index
 from data_loader.types import XDataFrame
 
+
 def check_data(X: XDataFrame, config: Config) -> bool:
-    """ Returns True if data is valid, else returns False."""
-    
+    """Returns True if data is valid, else returns False."""
+
     if has_enough_samples_to_train(X, config) == False:
         warnings.warn("Not enough samples to train")
         return False
-    
+
     return True
 
+
 def has_enough_samples_to_train(X: XDataFrame, config: Config) -> bool:
-    first_valid_index = get_first_valid_return_index(X.iloc[:,0])
+    first_valid_index = get_first_valid_return_index(X.iloc[:, 0])
     samples_to_train = len(X) - first_valid_index
-    return samples_to_train > config.sliding_window_size_base + config.sliding_window_size_meta + 100
+    return (
+        samples_to_train
+        > config.sliding_window_size_base + config.sliding_window_size_meta + 100
+    )

data_loader/types.py

@@ -8,4 +8,4 @@ DataCollection = list[DataSource]
 ReturnSeries = pd.Series
 ForwardReturnSeries = pd.Series
 XDataFrame = pd.DataFrame
-ySeries = pd.Series
+ySeries = pd.Series

data_loader/utils.py

@@ -1,3 +1,5 @@
 import pandas as pd
 
-def deduplicate_indexes(df: pd.DataFrame) -> pd.DataFrame: return df[~df.index.duplicated(keep='last')]
+
+def deduplicate_indexes(df: pd.DataFrame) -> pd.DataFrame:
+    return df[~df.index.duplicated(keep="last")]