mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-28 19:27:47 +00:00
Mark Aron Szulyovszky
9488e92597
* feature(MetaLabeling): added hacky prototype * fix(MetaLabeling): drop index until first valid X & y * fix(MetaLabeling): transform both X & y before feature selection * fix(MetaLabeling): got feature selection to work * fix(MetaLabeling): correct values for meta_y * feat(MetaLabeling): created predictions multiplied by bet sizes * feat(Pipeline): print out averaged result * fix(Evaluation): correctly deal with non-discretized data * fix(Pipeline): use the right column names * refactor(Pipeline): move out meta-labeling * refactor(Pipeline): complete refactoring * feat(CI): post results to PR * fix(Pipeline): use the correct filename * chore(Config): removed now redundant feature_selection flag * feat(Models): added SVC * fix(Pipeline): accidentally switched two return values * feat(Sweep): prepared sweep_meta.yaml, moved report_results() into a separate file * fix(Pipeline): wrong function name * fix(Sweep): yaml + run_sweep * fix(Sweep): typo in name * fix(Reporting): only save averaged results * feat(MetaLabeling): use optional meta-labeling step for every lvl1 models, before averaging * feat(Reporting): print out sharpe improvement in meta-labeling step * fix(Sweep): adjusted config, defaulted to good defaults * fix(Sweep): adjusted sweep
105 lines
2.6 KiB
Python
105 lines
2.6 KiB
Python
|
|
import numpy as np
|
|
import pandas as pd
|
|
from training.walk_forward import walk_forward_train_test
|
|
from models.base import Model
|
|
from utils.evaluate import evaluate_predictions
|
|
|
|
no_of_rows = 100
|
|
|
|
def __generate_even_odd_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]:
|
|
''' Test data, where X[n][any_column] == 1 if n is even, else 0
|
|
'''
|
|
|
|
no_columns = 6
|
|
X = [[-1 if row % 2 == 0 else 1] * no_columns for row in range(no_of_rows)]
|
|
assert X[0][0] == -1
|
|
assert X[1][0] == 1
|
|
assert X[2][0] == -1
|
|
assert X[3][0] == 1
|
|
X = pd.DataFrame(X)
|
|
|
|
y = [-1 if (row+1) % 2 == 0 else 1 for row in range(no_of_rows)]
|
|
assert y[0] == 1
|
|
assert y[1] == -1
|
|
assert y[2] == 1
|
|
assert y[3] == -1
|
|
|
|
y = pd.Series(y)
|
|
|
|
return X, y
|
|
|
|
class EvenOddStubModel(Model):
|
|
'''
|
|
A deteministic model that can predict the future with 100% accuracy
|
|
It verifies that the X[n][any_column] == 1 if n is even,
|
|
'''
|
|
|
|
data_scaling = "unscaled"
|
|
only_column = None
|
|
predict_window_size = 'single_timestamp'
|
|
|
|
def __init__(self, window_length) -> None:
|
|
super().__init__()
|
|
self.window_length = window_length
|
|
|
|
def fit(self, X, y):
|
|
assert len(X) == self.window_length
|
|
for i in range(len(X)):
|
|
assert y[i] == -1 if X[i][0] == 1 else 1
|
|
|
|
def predict(self, X):
|
|
return (-1 if X[0][0] == 1 else 1, np.array([]))
|
|
|
|
def clone(self):
|
|
return self
|
|
|
|
def get_name(self) -> str:
|
|
return 'test'
|
|
|
|
def initialize_network(self, input_dim: int, output_dim: int):
|
|
pass
|
|
|
|
|
|
def test_evaluation():
|
|
X, y = __generate_even_odd_test_data(no_of_rows)
|
|
|
|
window_length = 10
|
|
|
|
model = EvenOddStubModel(window_length = window_length)
|
|
scaler = None
|
|
|
|
models, predictions, probs = walk_forward_train_test(
|
|
model_name='test',
|
|
model=model,
|
|
X=X,
|
|
y=y,
|
|
target_returns=y,
|
|
expanding_window=False,
|
|
window_size=window_length,
|
|
retrain_every=10,
|
|
scaler=scaler
|
|
)
|
|
|
|
# verify if predictions are the same as y
|
|
for i in range(window_length+2, no_of_rows):
|
|
assert predictions[i] == y[i]
|
|
|
|
fake_target_returns = y * 0.1
|
|
processed_predictions_to_match_returns = predictions * 0.1
|
|
|
|
result = evaluate_predictions(
|
|
model_name='test',
|
|
target_returns=fake_target_returns,
|
|
y_pred=processed_predictions_to_match_returns,
|
|
y_true=y,
|
|
method='classification',
|
|
no_of_classes='two',
|
|
discretize=True
|
|
)
|
|
|
|
assert result['accuracy'] == 100.0
|
|
|
|
|
|
|