feat(DataLoader): caching MVP, added ability to use standard scaling for exogenous data, scaling is now also done before feature selection (#105)

* fix(FeatureExtractor): apply log to transform some series to normality

* feat(DataLoader): add ability of not returning returns when they're not needed (exogenous data), applied log to certain features

* feat(FeatureExtractors): added standard scaling for exogenous data

* feat(FeatureSelection): scale data with the passed in scaler before doing feature-selection

* fix(Config): sweep config

* feat(Models): output probability, store it

* feat(Core): added caching to select_features() and load_data()

* fix(Dependencies): added diskcache

* fix(Training): error when creating results DF

* feat(Models): added xgboost, fixed tests

* refactor(Cache): moved hashing to a separate function, created wrapper functions to separate business logic and caching

* fix(Tests): new syntax

* fix(Model): XGboost can't handle -1 class, so we'll use the deprecated label_encoder fornow

* fix(Model): XGBoost config

* feat(Cache): add run_clear_cache script

* fix(Pipeline) accidentally re-instatiating all_predictions for each asset

utils/hashing.py

@@ -0,0 +1,10 @@
+from hashlib import sha256
+import pandas as pd
+
+def hash_df(df: pd.DataFrame) -> str:
+    s = str(df.columns) + str(df.index) + str(df.values)
+    return sha256(s.encode()).hexdigest()
+
+def hash_series(df: pd.Series) -> str:
+    s = str(df.name) + str(df.index) + str(df.values)
+    return sha256(s.encode()).hexdigest()

utils/normalize.py

@@ -1,6 +0,0 @@
-import pandas as pd
-
-def normalize(data: pd.DataFrame) -> pd.DataFrame:
-    data_mean = data.mean(axis=0)
-    data_std = data.std(axis=0)
-    return ((data - data_mean) / data_std).fillna(0.)

utils/scaler.py

@@ -0,0 +1,13 @@
+from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler
+from typing import Optional, Union
+from utils.types import ScalerTypes
+
+def get_scaler(type: ScalerTypes) -> Optional[Union[MinMaxScaler, Normalizer, StandardScaler]]:
+    if type == 'normalize':
+        return Normalizer()
+    elif type == 'minmax':
+        return MinMaxScaler(feature_range= (-1, 1))
+    elif type == 'standardize':
+        return StandardScaler()
+    else:
+        return None

utils/types.py

@@ -1,4 +1,4 @@
-from typing import Callable, Union
+from typing import Callable, Union, Literal
 import pandas as pd
 
 Period = int
@@ -9,4 +9,6 @@ FeatureExtractorConfig = tuple[Name, FeatureExtractor, list[Period]]
 Path = str
 FileName = str
 DataSource = list[tuple[Path, FileName]]
-DataCollection = list[DataSource]
+DataCollection = list[DataSource]
+
+ScalerTypes = Literal['normalize', 'minmax', 'standardize', 'none']