feat(DataLoader): caching MVP, added ability to use standard scaling for exogenous data, scaling is now also done before feature selection (#105)

* fix(FeatureExtractor): apply log to transform some series to normality

* feat(DataLoader): add ability of not returning returns when they're not needed (exogenous data), applied log to certain features

* feat(FeatureExtractors): added standard scaling for exogenous data

* feat(FeatureSelection): scale data with the passed in scaler before doing feature-selection

* fix(Config): sweep config

* feat(Models): output probability, store it

* feat(Core): added caching to select_features() and load_data()

* fix(Dependencies): added diskcache

* fix(Training): error when creating results DF

* feat(Models): added xgboost, fixed tests

* refactor(Cache): moved hashing to a separate function, created wrapper functions to separate business logic and caching

* fix(Tests): new syntax

* fix(Model): XGboost can't handle -1 class, so we'll use the deprecated label_encoder fornow

* fix(Model): XGBoost config

* feat(Cache): add run_clear_cache script

* fix(Pipeline) accidentally re-instatiating all_predictions for each asset

feature_extractors/feature_extractor_presets.py

@@ -1,7 +1,7 @@
-from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
+from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_standard_scaling, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
 from utils.types import FeatureExtractorConfig
 from utils.helpers import flatten
-from feature_extractors.fractional_differentiation import feature_fractional_differentiation
+from feature_extractors.fractional_differentiation import feature_fractional_differentiation, feature_fractional_differentiation_log
 
 __presets = dict(
     debug_future_lookahead = [('debug_future', feature_debug_future_lookahead, [1])],
@@ -24,6 +24,8 @@ __presets = dict(
     stod = [('stod', feature_STOD, [10, 30, 200])],
     stok = [('stok', feature_STOK, [10, 30, 200])],
     fracdiff = [('fracdiff', feature_fractional_differentiation, [10, 30])],
+    fracdiff_log = [('fracdiff_log', feature_fractional_differentiation_log, [10, 30])],
+    standard_scaling = [('standard_scaling', feature_standard_scaling, [0])],
 )
 
 presets = __presets | dict(

feature_extractors/feature_extractors.py

@@ -1,6 +1,8 @@
 import pandas as pd
 import numpy as np
 from feature_extractors.utils import get_close_low_high
+from feature_extractors.utils import apply_log_if_necessary_series
+from sklearn.preprocessing import StandardScaler
 
 def feature_debug_future_lookahead(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
     return df['returns'].shift(-period)
@@ -9,6 +11,10 @@ def feature_lag(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series
     assert period > 0
     return df['returns'].shift(period)
 
+def feature_standard_scaling(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
+    scaler = StandardScaler()
+    return pd.Series(scaler.fit_transform(df['close'].to_numpy().reshape(-1, 1)).squeeze(), index = df.index)
+
 def feature_day_of_week(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.DataFrame:
     return pd.get_dummies(pd.DatetimeIndex(df.index).dayofweek, drop_first=True, prefix="date_day_week").set_index(df.index)
 
@@ -31,7 +37,7 @@ def feature_STOK(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Serie
     close, low, high = get_close_low_high(df)
 
     STOK = ((close - low.rolling(period).min()) / (high.rolling(period).max() - low.rolling(period).min())) * 100
-    return STOK
+    return apply_log_if_necessary_series(STOK, "stok")
 
 def feature_STOD(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
     stok = feature_STOK(df, period, is_log_return)
@@ -48,10 +54,11 @@ def feature_RSI(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series
     d[d.index[period-1]] = np.mean( d[:period] ) #first value is sum of avg losses d = d.drop(d.index[:(period-1)])
     rs = u.ewm(com=period-1, adjust=False).mean() / \
     d.ewm(com=period-1, adjust=False).mean()
-    return 100-100/(1+rs)
+    return apply_log_if_necessary_series(100-100/(1+rs), "rsi")
 
 def feature_ROC(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
     returns = df['returns']
     M = returns.diff(period - 1)
     N = returns.shift(period - 1)
-    return pd.Series(((M / N) * 100), name = 'ROC_' + str(period))
+    roc = pd.Series(((M / N) * 100), name = 'ROC_' + str(period))
+    return apply_log_if_necessary_series(roc, "roc")

feature_extractors/fractional_differentiation.py

@@ -1,9 +1,15 @@
 from fracdiff.sklearn import FracdiffStat
 import pandas as pd
 import numpy as np
+from feature_extractors.utils import apply_log_if_necessary_series
 
 def feature_fractional_differentiation(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
     frac_diff = FracdiffStat(window = period)
     input_series = df["close"].to_numpy().reshape(-1, 1)
     result = frac_diff.fit_transform(input_series)
-    return pd.Series(result.squeeze(), index = df.index)
+    return pd.Series(result.squeeze(), index = df.index)
+
+def feature_fractional_differentiation_log(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
+    series = feature_fractional_differentiation(df, period, is_log_return)
+    return apply_log_if_necessary_series(series, "fracdiff")
+    

feature_extractors/utils.py

@@ -1,7 +1,25 @@
 import pandas as pd
+import numpy as np
+from scipy.stats import shapiro
 
 def get_close_low_high(df: pd.DataFrame) -> tuple[pd.Series, pd.Series, pd.Series]:
     close = df['close']
     low = df['low']
     high = df['high']
-    return close, low, high
+    return close, low, high
+
+
+def apply_log_if_necessary_series(series: pd.Series, name: str) -> pd.Series:
+    values = series.to_numpy()
+    no_of_unique_values = np.unique(values)
+    if len(no_of_unique_values) < 4:
+        return series
+    is_normal = shapiro(values).pvalue > 0.05
+    if not is_normal:
+        # print("Applying log to column: " + column)
+        min_value = np.min(series)
+        series = (series + min_value).apply(lambda x: np.log(x))
+        is_normal_after_log = shapiro(series).pvalue > 0.05
+        if not is_normal_after_log:
+            print("Failed to normalize column: ", name)
+    return series