gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-16 20:38:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Transformations): replaced feature selection pre-processing step with online version (with cache) (#170)

Browse Source 
* feat(Transformations): removed feature-selection pre-processing step completely

* fix(Core): removed unnecessary `original_X`

* fix(Transformations): use the X_expanding_window to transform subsequent data

* fix(RFE): should check for model correctly

* fix(Config): only re-train the model every 40 timestamp

* fix(MetaLabeling): pass in the correct X to meta-labeling step

* fix(Transformation): PCA should at least keep as many features as sliding_window_size

* feat(Transformations): cache transformations across the same asset

* fix(Tests): missing preloaded_transformations arg

* chore(Config): got rid of unnecessary 'classification_models' and 'regression_models' dictionary keys
This commit is contained in:
Mark Aron Szulyovszky
2022-01-17 11:43:51 +01:00
committed by GitHub
parent 31dc847be1
commit 6b26643ece
27 changed files with 240 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
training/walk_forward.py
+8 -6
View File
@@ -3,10 +3,8 @@ from models.base import Model
import numpy as np
from utils.helpers import get_first_valid_return_index
from tqdm import tqdm
from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler
from typing import Union
from sklearn.base import clone
from transformations.base import Transformation
from typing import Optional
def walk_forward_train(
model_name: str,
@@ -18,6 +16,7 @@ def walk_forward_train(
window_size: int,
retrain_every: int,
transformations: list[Transformation],
preloaded_transformations: Optional[list[pd.Series]],
) -> tuple[pd.Series, list[pd.Series]]:
assert len(X) == len(y)
models_over_time = pd.Series(index=y.index).rename(model_name)
@@ -44,9 +43,12 @@ def walk_forward_train(
X_expanding_window = X[first_nonzero_return:train_window_end]
y_expanding_window = y[first_nonzero_return:train_window_end]
current_transformations = [t.clone() for t in transformations]
for transformation_index, transformation in enumerate(current_transformations):
transformation.fit_transform(X_expanding_window, y_expanding_window)
if preloaded_transformations is not None and len(transformations) > 0:
current_transformations = [transformation_over_time[index] for transformation_over_time in preloaded_transformations]
else:
current_transformations = [t.clone() for t in transformations]
for transformation_index, transformation in enumerate(current_transformations):
X_expanding_window = transformation.fit_transform(X_expanding_window, y_expanding_window)
X_slice = X[train_window_start:train_window_end]