gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-18 21:38:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Transformations): replaced feature selection pre-processing step with online version (with cache) (#170)

Browse Source 
* feat(Transformations): removed feature-selection pre-processing step completely

* fix(Core): removed unnecessary `original_X`

* fix(Transformations): use the X_expanding_window to transform subsequent data

* fix(RFE): should check for model correctly

* fix(Config): only re-train the model every 40 timestamp

* fix(MetaLabeling): pass in the correct X to meta-labeling step

* fix(Transformation): PCA should at least keep as many features as sliding_window_size

* feat(Transformations): cache transformations across the same asset

* fix(Tests): missing preloaded_transformations arg

* chore(Config): got rid of unnecessary 'classification_models' and 'regression_models' dictionary keys
This commit is contained in:
Mark Aron Szulyovszky
2022-01-17 11:43:51 +01:00
committed by GitHub
parent 31dc847be1
commit 6b26643ece
27 changed files with 240 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
training/meta_labeling.py
+3 -12
View File
@@ -1,9 +1,8 @@
from utils.evaluate import discretize_threeway_threshold, evaluate_predictions
from utils.helpers import random_string, equal_except_nan, drop_until_first_valid_index
from utils.helpers import equal_except_nan
from training.primary_model import train_primary_model
from feature_selection.feature_selection import select_features
import pandas as pd
from models.model_map import get_model_map
from models.model_map import default_feature_selector_classification, default_feature_selector_regression
from models.base import Model
from reporting.types import Reporting
from typing import Union
@@ -23,22 +22,14 @@ def train_meta_labeling_model(
preloaded_models: Union[list[Reporting.Single_Model], None] = None
) -> tuple[pd.Series, pd.Series, pd.DataFrame, list[Reporting.Single_Model]]:
_, _, _, default_feature_selector_regression, default_feature_selector_classification = get_model_map(model_config)
discretize = discretize_threeway_threshold(0.33)
discretized_predictions = input_predictions.apply(discretize)
meta_y: pd.Series = pd.concat([discretized_predictions, y], axis=1).apply(equal_except_nan, axis = 1)
print("Feature Selection started")
backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
meta_feature_selection_input_X, meta_feature_selection_input_y = drop_until_first_valid_index(X, meta_y)
feature_selection_output = select_features(X = meta_feature_selection_input_X, y = meta_feature_selection_input_y, model = models[0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model, scaling = training_config['scaler'])
meta_selected_features_X = X[feature_selection_output.columns]
meta_X = pd.concat([meta_selected_features_X, input_predictions, discretized_predictions], axis = 1)
meta_X = pd.concat([X, input_predictions, discretized_predictions], axis = 1)
_, meta_preds, meta_probabilities, all_models_single_asset = train_primary_model(
ticker_to_predict = "prediction_correct",
original_X = meta_X,
X = meta_X,
y = meta_y,
target_returns = target_returns,