feat(Data): added script to download data from binance (#224)

* feat(Data): added script to download data from binance

* feat(Data): saving unified parquet file/loading

* fix(Config): tweak the cusum filter's threshold

* fix(Dependencies): added binance_historical_data

feature_extractors/feature_extractor_presets.py

@@ -15,7 +15,6 @@ from .feature_extractors import (
 )
 from .fractional_differentiation import (
     feature_fractional_differentiation,
-    feature_fractional_differentiation_log,
 )
 
 __presets = dict(
@@ -40,7 +39,6 @@ __presets = dict(
     stod=[("stod", feature_STOD, [100, 300, 2000])],
     stok=[("stok", feature_STOK, [100, 300, 2000])],
     fracdiff=[("fracdiff", feature_fractional_differentiation, [100, 300])],
-    fracdiff_log=[("fracdiff_log", feature_fractional_differentiation_log, [100, 300])],
     z_score=[("z_score", feature_expanding_zscore, [100])],
 )
 

feature_extractors/feature_extractors.py

@@ -1,7 +1,6 @@
 import pandas as pd
 import numpy as np
 from feature_extractors.utils import get_close_low_high
-from feature_extractors.utils import apply_log_if_necessary_series
 
 
 def feature_debug_future_lookahead(df: pd.DataFrame, period: int) -> pd.Series:
@@ -51,7 +50,7 @@ def feature_STOK(df: pd.DataFrame, period: int) -> pd.Series:
         (close - low.rolling(period).min())
         / (high.rolling(period).max() - low.rolling(period).min())
     ) * 100
-    return apply_log_if_necessary_series(STOK, "stok")
+    return STOK
 
 
 def feature_STOD(df: pd.DataFrame, period: int) -> pd.Series:
@@ -76,7 +75,7 @@ def feature_RSI(df: pd.DataFrame, period: int) -> pd.Series:
         u.ewm(com=period - 1, adjust=False).mean()
         / d.ewm(com=period - 1, adjust=False).mean()
     )
-    return apply_log_if_necessary_series(100 - 100 / (1 + rs), "rsi")
+    return 100 - 100 / (1 + rs)
 
 
 def feature_ROC(df: pd.DataFrame, period: int) -> pd.Series:
@@ -84,4 +83,4 @@ def feature_ROC(df: pd.DataFrame, period: int) -> pd.Series:
     M = returns.diff(period - 1)
     N = returns.shift(period - 1)
     roc = pd.Series(((M / N) * 100), name="ROC_" + str(period))
-    return apply_log_if_necessary_series(roc, "roc")
+    return roc

feature_extractors/fractional_differentiation.py

@@ -1,7 +1,6 @@
 from fracdiff.sklearn import FracdiffStat
 import pandas as pd
 import numpy as np
-from feature_extractors.utils import apply_log_if_necessary_series
 
 
 def feature_fractional_differentiation(df: pd.DataFrame, period: int) -> pd.Series:
@@ -9,8 +8,3 @@ def feature_fractional_differentiation(df: pd.DataFrame, period: int) -> pd.Seri
     input_series = df["close"].to_numpy().reshape(-1, 1)
     result = frac_diff.fit_transform(input_series)
     return pd.Series(result.squeeze(), index=df.index)
-
-
-def feature_fractional_differentiation_log(df: pd.DataFrame, period: int) -> pd.Series:
-    series = feature_fractional_differentiation(df, period, is_log_return)
-    return apply_log_if_necessary_series(series, "fracdiff")

feature_extractors/utils.py

@@ -8,19 +8,3 @@ def get_close_low_high(df: pd.DataFrame) -> tuple[pd.Series, pd.Series, pd.Serie
     low = df["low"]
     high = df["high"]
     return close, low, high
-
-
-def apply_log_if_necessary_series(series: pd.Series, name: str) -> pd.Series:
-    values = series.to_numpy()
-    no_of_unique_values = np.unique(values)
-    if len(no_of_unique_values) < 4:
-        return series
-    is_normal = shapiro(values).pvalue > 0.05
-    if not is_normal:
-        # print("Applying log to column: " + column)
-        min_value = np.min(series)
-        series = (series + min_value).apply(lambda x: np.log(x))
-        is_normal_after_log = shapiro(series).pvalue > 0.05
-        if not is_normal_after_log:
-            print("Failed to normalize column: ", name)
-    return series