fix(WalkForward): major bug where we passed in "window of windows of data" is resolved, refactored walk_forward_train_test() and load_data()

This commit is contained in:
Mark Aron Szulyovszky
2021-12-14 21:07:03 +01:00
parent d047b7417e
commit 1ef314c034
7 changed files with 131 additions and 302 deletions
+14
View File
@@ -0,0 +1,14 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Run pipeline",
"type": "python",
"request": "launch",
"module": "model_walk_forward"
}
]
}
+27 -15
View File
@@ -4,33 +4,34 @@ import os
import numpy as np
from pandas.core.frame import DataFrame
from utils.technical_indicators import ROC, RSI, STOK, STOD
from typing import List, Literal
from typing import Literal
#%%
def load_files(path: str,
own_asset: str,
own_asset_lags: List[int],
def load_data(path: str,
target_asset: str,
target_asset_lags: list[int],
load_other_assets: bool,
other_asset_lags: List[int],
other_asset_lags: list[int],
log_returns: bool,
add_date_features: bool,
own_technical_features: Literal['none', 'level1', 'level2'],
other_technical_features: Literal['none', 'level1', 'level2'],
exogenous_features: Literal['none', 'level1'],
index_column: Literal['date', 'int'],
method: Literal['regression', 'classification'],
narrow_format: bool = False,
) -> pd.DataFrame:
) -> tuple[pd.DataFrame, pd.Series]:
files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(own_asset))]
def is_own_asset(own_asset: str, file: str): return file.split('.')[0].startswith(own_asset)
files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(target_asset))]
def is_target_asset(target_asset: str, file: str): return file.split('.')[0].startswith(target_asset)
dfs = [__load_df(
path=os.path.join(path,f),
prefix=f.split('.')[0],
log_returns=log_returns,
technical_features=own_technical_features if is_own_asset(own_asset, f) else other_technical_features,
lags= own_asset_lags if is_own_asset(own_asset, f) else other_asset_lags,
technical_features=own_technical_features if is_target_asset(target_asset, f) else other_technical_features,
lags= target_asset_lags if is_target_asset(target_asset, f) else other_asset_lags,
narrow_format=narrow_format,
) for f in files]
if narrow_format:
@@ -49,11 +50,22 @@ def load_files(path: str,
dfs.reset_index(drop=True, inplace=True)
if narrow_format:
return dfs
else:
return dfs.drop(index=dfs.index[0], axis=0)
dfs = dfs.drop(index=dfs.index[0], axis=0)
def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Literal['none', 'level1', 'level2'], lags: List[int], narrow_format: bool = False) -> pd.DataFrame:
## Create target
target_col = 'target'
returns_col = target_asset + '_returns'
if method == 'regression':
dfs = create_target_cum_forward_returns(dfs, returns_col, 1)
elif method == 'classification':
dfs = create_target_classes(dfs, returns_col, 1, 'two')
X = dfs.drop(columns=[target_col])
y = dfs[target_col]
return X, y
def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Literal['none', 'level1', 'level2'], lags: list[int], narrow_format: bool = False) -> pd.DataFrame:
df = pd.read_csv(path, header=0, index_col=0).fillna(0)
if log_returns:
@@ -84,7 +96,7 @@ def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_fe
df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5)
df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5)
df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5)
df['vol_60'] = df['returns'].rolling(30).std()*(252**0.5)
df['vol_60'] = df['returns'].rolling(60).std()*(252**0.5)
# momentum (10, 20, 30, 60, 90 days)
if log_returns:
+38 -86
View File
@@ -3,7 +3,7 @@ from typing import Literal
from sklearnex import patch_sklearn
patch_sklearn()
from load_data import create_target_cum_forward_returns, load_files, create_target_classes
from load_data import create_target_cum_forward_returns, load_data, create_target_classes
from sktime.forecasting.model_selection import temporal_train_test_split
from utils.evaluate import evaluate_predictions_regression, evaluate_predictions_classification
@@ -21,79 +21,42 @@ from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTree
from sklearn.metrics import r2_score, mean_absolute_error, confusion_matrix, classification_report, accuracy_score
from sklearn.preprocessing import MinMaxScaler
from utils.sliding_window import sliding_window_and_flatten
def walk_forward_train_test(
model_name: str,
create_model,
X: pd.DataFrame,
y: pd.Series,
window_size: int,
retrain_every: int
):
print("Training: ", model_name)
predictions = [None] * (len(y)-1)
models = [None] * len(predictions)
train_from = window_size+1
train_till = len(y)-2
iterations_since_retrain = 0
for i in range(train_from, train_till):
# if i % 20 == 0: print('Fold: ', i)
iterations_since_retrain += 1
window_start = i - window_size
window_end = i
X_train_slice = X[window_start:window_end]
y_train_slice = y[window_start:window_end]
if iterations_since_retrain >= retrain_every or models[i-1] is None:
model = create_model()
model.fit(X_train_slice, y_train_slice)
iterations_since_retrain = 0
else:
model = models[i-1]
models[window_end] = model
predictions[window_end+1] = model.predict(X[window_end+1].reshape(1, -1)).item()
return models, predictions
from utils.walk_forward import walk_forward_train_test
regression_models = [
('LR', lambda: LinearRegression(n_jobs=-1)),
('BayesianRidge', lambda: BayesianRidge()),
('KNN', lambda: KNeighborsRegressor(n_neighbors=15)),
('MLP', lambda: MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
('AB', lambda: AdaBoostRegressor()),
('LR', LinearRegression(n_jobs=-1)),
('BayesianRidge', BayesianRidge()),
('KNN', KNeighborsRegressor(n_neighbors=15)),
# ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
('AB', AdaBoostRegressor()),
# ('RF', lambda: RandomForestRegressor(n_jobs=-1)),
('SVR', lambda: SVR(kernel='rbf', C=1e3, gamma=0.1))
('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1))
]
classification_models = [
('LR', lambda: LogisticRegression(n_jobs=-1)),
('LDA', lambda: LinearDiscriminantAnalysis()),
('KNN', lambda: KNeighborsClassifier()),
('CART', lambda: DecisionTreeClassifier()),
('NB', lambda: GaussianNB()),
('AB', lambda: AdaBoostClassifier()),
('RF', lambda: RandomForestClassifier(n_jobs=-1))
('LR', LogisticRegression(n_jobs=-1)),
('LDA', LinearDiscriminantAnalysis()),
('KNN', KNeighborsClassifier()),
('CART', DecisionTreeClassifier()),
('NB', GaussianNB()),
('AB', AdaBoostClassifier()),
('RF', RandomForestClassifier(n_jobs=-1))
]
def run_whole_pipeline(
ticker_to_predict: str,
models,
method: Literal['regression', 'classification'],
sliding_window_size: int,
retrain_every: int,
):
ticker_to_predict: str,
models,
method: Literal['regression', 'classification'],
sliding_window_size: int,
retrain_every: int,
scaling: bool,
):
print('Predicting: ', ticker_to_predict)
data = load_files(path='data/',
own_asset=ticker_to_predict,
own_asset_lags=[1,2,3,4,5,6,8,10,15],
X, y = load_data(path='data/',
target_asset=ticker_to_predict,
target_asset_lags=[1,2,3,4,5,6,8,10,15],
load_other_assets=False,
other_asset_lags=[],
log_returns=True,
@@ -101,34 +64,21 @@ def run_whole_pipeline(
own_technical_features='level2',
other_technical_features='none',
exogenous_features='none',
index_column='int'
index_column='int',
method=method,
)
target_col = 'target'
returns_col = ticker_to_predict + '_returns'
if method == 'regression':
data = create_target_cum_forward_returns(data, returns_col, 1)
elif method == 'classification':
data = create_target_classes(data, returns_col, 1, 'two')
X = data.drop(columns=[target_col])
y = data[target_col]
if scaling:
# TODO: should move scaling to an expanding window compomenent
feature_scaler = MinMaxScaler(feature_range= (-1, 1))
X = pd.DataFrame(feature_scaler.fit_transform(X), columns=X.columns, index=X.index)
# TODO: should scale y as well probably
# TODO: should move scaling to an expanding window compomenent
feature_scaler = MinMaxScaler(feature_range= (-1, 1))
X = feature_scaler.fit_transform(X)
# TODO: should scale y as well probably
X = sliding_window_and_flatten(X, sliding_window_size)
y = y[sliding_window_size-1:]
for model_name, create_model in models:
for model_name, model in models:
model_over_time, preds = walk_forward_train_test(
model_name=model_name,
create_model = create_model,
model = model,
X = X,
y = y,
window_size = sliding_window_size,
@@ -146,12 +96,14 @@ run_whole_pipeline(
models = regression_models,
method = 'regression',
sliding_window_size = 120,
retrain_every = 50
retrain_every = 50,
scaling=False
)
run_whole_pipeline(
ticker_to_predict = ticker_to_predict,
models = classification_models,
method = 'classification',
sliding_window_size = 120,
retrain_every = 50
retrain_every = 50,
scaling=False
)
+1 -1
View File
@@ -47,7 +47,7 @@ def evaluate_predictions_regression(model_name: str, y_true, y_pred, sliding_win
def evaluate_predictions_classification(model_name: str, y, preds, sliding_window_size: int):
print("Model: ", model_name)
evaluate_from = sliding_window_size+1
y = pd.Series(y[evaluate_from:-1])
y = pd.Series(y[evaluate_from:])
preds = pd.Series(preds[evaluate_from:])
print(accuracy_score(y, preds))
print(confusion_matrix(y, preds))
-200
View File
@@ -1,200 +0,0 @@
import numpy as np
from numpy.lib.stride_tricks import as_strided
from numpy.core.numeric import normalize_axis_tuple
def sliding_window_and_flatten(x: np.array, window_size: int) -> np.array:
n_features = x.shape[1]
x = np.squeeze(__numpy_sliding_window_view(x, (window_size, n_features)), axis = 1)
return x.reshape(x.shape[0], -1)
def __numpy_sliding_window_view(x, window_shape, axis=None, *,
subok=False, writeable=False):
"""
Create a sliding window view into the array with the given window shape.
Also known as rolling or moving window, the window slides across all
dimensions of the array and extracts subsets of the array at all window
positions.
.. versionadded:: 1.20.0
Parameters
----------
x : array_like
Array to create the sliding window view from.
window_shape : int or tuple of int
Size of window over each axis that takes part in the sliding window.
If `axis` is not present, must have same length as the number of input
array dimensions. Single integers `i` are treated as if they were the
tuple `(i,)`.
axis : int or tuple of int, optional
Axis or axes along which the sliding window is applied.
By default, the sliding window is applied to all axes and
`window_shape[i]` will refer to axis `i` of `x`.
If `axis` is given as a `tuple of int`, `window_shape[i]` will refer to
the axis `axis[i]` of `x`.
Single integers `i` are treated as if they were the tuple `(i,)`.
subok : bool, optional
If True, sub-classes will be passed-through, otherwise the returned
array will be forced to be a base-class array (default).
writeable : bool, optional
When true, allow writing to the returned view. The default is false,
as this should be used with caution: the returned view contains the
same memory location multiple times, so writing to one location will
cause others to change.
Returns
-------
view : ndarray
Sliding window view of the array. The sliding window dimensions are
inserted at the end, and the original dimensions are trimmed as
required by the size of the sliding window.
That is, ``view.shape = x_shape_trimmed + window_shape``, where
``x_shape_trimmed`` is ``x.shape`` with every entry reduced by one less
than the corresponding window size.
See Also
--------
lib.stride_tricks.as_strided: A lower-level and less safe routine for
creating arbitrary views from custom shape and strides.
broadcast_to: broadcast an array to a given shape.
Notes
-----
For many applications using a sliding window view can be convenient, but
potentially very slow. Often specialized solutions exist, for example:
- `scipy.signal.fftconvolve`
- filtering functions in `scipy.ndimage`
- moving window functions provided by
`bottleneck <https://github.com/pydata/bottleneck>`_.
As a rough estimate, a sliding window approach with an input size of `N`
and a window size of `W` will scale as `O(N*W)` where frequently a special
algorithm can achieve `O(N)`. That means that the sliding window variant
for a window size of 100 can be a 100 times slower than a more specialized
version.
Nevertheless, for small window sizes, when no custom algorithm exists, or
as a prototyping and developing tool, this function can be a good solution.
Examples
--------
>>> x = np.arange(6)
>>> x.shape
(6,)
>>> v = sliding_window_view(x, 3)
>>> v.shape
(4, 3)
>>> v
array([[0, 1, 2],
[1, 2, 3],
[2, 3, 4],
[3, 4, 5]])
This also works in more dimensions, e.g.
>>> i, j = np.ogrid[:3, :4]
>>> x = 10*i + j
>>> x.shape
(3, 4)
>>> x
array([[ 0, 1, 2, 3],
[10, 11, 12, 13],
[20, 21, 22, 23]])
>>> shape = (2,2)
>>> v = sliding_window_view(x, shape)
>>> v.shape
(2, 3, 2, 2)
>>> v
array([[[[ 0, 1],
[10, 11]],
[[ 1, 2],
[11, 12]],
[[ 2, 3],
[12, 13]]],
[[[10, 11],
[20, 21]],
[[11, 12],
[21, 22]],
[[12, 13],
[22, 23]]]])
The axis can be specified explicitly:
>>> v = sliding_window_view(x, 3, 0)
>>> v.shape
(1, 4, 3)
>>> v
array([[[ 0, 10, 20],
[ 1, 11, 21],
[ 2, 12, 22],
[ 3, 13, 23]]])
The same axis can be used several times. In that case, every use reduces
the corresponding original dimension:
>>> v = sliding_window_view(x, (2, 3), (1, 1))
>>> v.shape
(3, 1, 2, 3)
>>> v
array([[[[ 0, 1, 2],
[ 1, 2, 3]]],
[[[10, 11, 12],
[11, 12, 13]]],
[[[20, 21, 22],
[21, 22, 23]]]])
Combining with stepped slicing (`::step`), this can be used to take sliding
views which skip elements:
>>> x = np.arange(7)
>>> sliding_window_view(x, 5)[:, ::2]
array([[0, 2, 4],
[1, 3, 5],
[2, 4, 6]])
or views which move by multiple elements
>>> x = np.arange(7)
>>> sliding_window_view(x, 3)[::2, :]
array([[0, 1, 2],
[2, 3, 4],
[4, 5, 6]])
A common application of `sliding_window_view` is the calculation of running
statistics. The simplest example is the
`moving average <https://en.wikipedia.org/wiki/Moving_average>`_:
>>> x = np.arange(6)
>>> x.shape
(6,)
>>> v = sliding_window_view(x, 3)
>>> v.shape
(4, 3)
>>> v
array([[0, 1, 2],
[1, 2, 3],
[2, 3, 4],
[3, 4, 5]])
>>> moving_average = v.mean(axis=-1)
>>> moving_average
array([1., 2., 3., 4.])
Note that a sliding window approach is often **not** optimal (see Notes).
"""
window_shape = (tuple(window_shape)
if np.iterable(window_shape)
else (window_shape,))
# first convert input to array, possibly keeping subclass
x = np.array(x, copy=False, subok=subok)
window_shape_array = np.array(window_shape)
if np.any(window_shape_array < 0):
raise ValueError('`window_shape` cannot contain negative values')
if axis is None:
axis = tuple(range(x.ndim))
if len(window_shape) != len(axis):
raise ValueError(f'Since axis is `None`, must provide '
f'window_shape for all dimensions of `x`; '
f'got {len(window_shape)} window_shape elements '
f'and `x.ndim` is {x.ndim}.')
else:
axis = normalize_axis_tuple(axis, x.ndim, allow_duplicate=True)
if len(window_shape) != len(axis):
raise ValueError(f'Must provide matching length window_shape and '
f'axis; got {len(window_shape)} window_shape '
f'elements and {len(axis)} axes elements.')
out_strides = x.strides + tuple(x.strides[ax] for ax in axis)
# note: same axis can be windowed repeatedly
x_shape_trimmed = list(x.shape)
for ax, dim in zip(axis, window_shape):
if x_shape_trimmed[ax] < dim:
raise ValueError(
'window shape cannot be larger than input array shape')
x_shape_trimmed[ax] -= dim - 1
out_shape = tuple(x_shape_trimmed) + window_shape
return as_strided(x, strides=out_strides, shape=out_shape,
subok=subok, writeable=writeable)
+7
View File
@@ -0,0 +1,7 @@
from typing import Protocol
class SKLearnModel(Protocol):
def fit(self, X, y, sample_weight=None): ...
def predict(self, X): ...
def score(self, X, y, sample_weight=None): ...
def set_params(self, **params): ...
+44
View File
@@ -0,0 +1,44 @@
import pandas as pd
from sklearn.base import clone
from utils.typing import SKLearnModel
import numpy as np
def walk_forward_train_test(
model_name: str,
model: SKLearnModel,
X: pd.DataFrame,
y: pd.Series,
window_size: int,
retrain_every: int
) -> tuple[list[SKLearnModel], list[float]]:
print("Training: ", model_name)
predictions = pd.Series(index=y.index)
models = pd.Series(index=y.index)
train_from = window_size
train_till = y.index[-1]
iterations_since_retrain = 0
for i in range(train_from, train_till):
iterations_since_retrain += 1
window_start = i - window_size
window_end = i
X_slice = X[window_start:window_end]
y_slice = y[window_start:window_end]
if iterations_since_retrain >= retrain_every or pd.isna(models[i-1]):
current_model = clone(model)
current_model.fit(X_slice.to_numpy(), y_slice)
iterations_since_retrain = 0
else:
current_model = models[i-1]
models[window_end] = current_model
next_timestep = X.iloc[window_end+1].to_numpy().reshape(1, -1)
predictions[window_end+1] = current_model.predict(next_timestep).item()
return models, predictions