mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-28 03:08:01 +00:00
Mark Aron Szulyovszky
1cd0119589
* fix(FeatureExtractor): apply log to transform some series to normality * feat(DataLoader): add ability of not returning returns when they're not needed (exogenous data), applied log to certain features * feat(FeatureExtractors): added standard scaling for exogenous data * feat(FeatureSelection): scale data with the passed in scaler before doing feature-selection * fix(Config): sweep config * feat(Models): output probability, store it * feat(Core): added caching to select_features() and load_data() * fix(Dependencies): added diskcache * fix(Training): error when creating results DF * feat(Models): added xgboost, fixed tests * refactor(Cache): moved hashing to a separate function, created wrapper functions to separate business logic and caching * fix(Tests): new syntax * fix(Model): XGboost can't handle -1 class, so we'll use the deprecated label_encoder fornow * fix(Model): XGBoost config * feat(Cache): add run_clear_cache script * fix(Pipeline) accidentally re-instatiating all_predictions for each asset
26 lines
855 B
Python
26 lines
855 B
Python
import pandas as pd
|
|
import numpy as np
|
|
from scipy.stats import shapiro
|
|
|
|
def get_close_low_high(df: pd.DataFrame) -> tuple[pd.Series, pd.Series, pd.Series]:
|
|
close = df['close']
|
|
low = df['low']
|
|
high = df['high']
|
|
return close, low, high
|
|
|
|
|
|
def apply_log_if_necessary_series(series: pd.Series, name: str) -> pd.Series:
|
|
values = series.to_numpy()
|
|
no_of_unique_values = np.unique(values)
|
|
if len(no_of_unique_values) < 4:
|
|
return series
|
|
is_normal = shapiro(values).pvalue > 0.05
|
|
if not is_normal:
|
|
# print("Applying log to column: " + column)
|
|
min_value = np.min(series)
|
|
series = (series + min_value).apply(lambda x: np.log(x))
|
|
is_normal_after_log = shapiro(series).pvalue > 0.05
|
|
if not is_normal_after_log:
|
|
print("Failed to normalize column: ", name)
|
|
return series
|