fix(WalkForward): major bug where we passed in "window of windows of data" is resolved, refactored walk_forward_train_test() and load_data()

.vscode/launch.json

@@ -0,0 +1,14 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Run pipeline",
+            "type": "python",
+            "request": "launch",
+            "module": "model_walk_forward"
+        }
+    ]
+}

load_data.py

@@ -4,33 +4,34 @@ import os
 import numpy as np
 from pandas.core.frame import DataFrame
 from utils.technical_indicators import ROC, RSI, STOK, STOD
-from typing import List, Literal
+from typing import Literal
 
 #%%
 
-def load_files(path: str,
+def load_data(path: str,
-            own_asset: str,
+            target_asset: str,
-            own_asset_lags: List[int],
+            target_asset_lags: list[int],
             load_other_assets: bool,
-            other_asset_lags: List[int],
+            other_asset_lags: list[int],
             log_returns: bool,
             add_date_features: bool,
             own_technical_features: Literal['none', 'level1', 'level2'],
             other_technical_features: Literal['none', 'level1', 'level2'],
             exogenous_features: Literal['none', 'level1'],
             index_column: Literal['date', 'int'],
+            method: Literal['regression', 'classification'],
             narrow_format: bool = False,
-            ) -> pd.DataFrame:
+        ) -> tuple[pd.DataFrame, pd.Series]:
     
     files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
-    files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(own_asset))]
+    files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(target_asset))]
-    def is_own_asset(own_asset: str, file: str): return file.split('.')[0].startswith(own_asset)
+    def is_target_asset(target_asset: str, file: str): return file.split('.')[0].startswith(target_asset)
     dfs = [__load_df(
         path=os.path.join(path,f),
         prefix=f.split('.')[0],
         log_returns=log_returns,
-        technical_features=own_technical_features if is_own_asset(own_asset, f) else other_technical_features,
+        technical_features=own_technical_features if is_target_asset(target_asset, f) else other_technical_features,
-        lags= own_asset_lags if is_own_asset(own_asset, f) else other_asset_lags,
+        lags= target_asset_lags if is_target_asset(target_asset, f) else other_asset_lags,
         narrow_format=narrow_format,
     ) for f in files]
     if narrow_format:
@@ -49,11 +50,22 @@ def load_files(path: str,
         dfs.reset_index(drop=True, inplace=True)
 
     if narrow_format:
-        return dfs
+        dfs = dfs.drop(index=dfs.index[0], axis=0)
-    else:
-        return dfs.drop(index=dfs.index[0], axis=0)
 
-def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Literal['none', 'level1', 'level2'], lags: List[int], narrow_format: bool = False) -> pd.DataFrame:
+    ## Create target 
+    target_col = 'target'
+    returns_col = target_asset + '_returns'
+    if method == 'regression':
+        dfs = create_target_cum_forward_returns(dfs, returns_col, 1)
+    elif method == 'classification':
+        dfs = create_target_classes(dfs, returns_col, 1, 'two')
+        
+    X = dfs.drop(columns=[target_col])
+    y = dfs[target_col]
+
+    return X, y
+
+def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Literal['none', 'level1', 'level2'], lags: list[int], narrow_format: bool = False) -> pd.DataFrame:
     df = pd.read_csv(path, header=0, index_col=0).fillna(0)
 
     if log_returns:
@@ -84,7 +96,7 @@ def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_fe
         df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5)
         df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5)
         df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5)
-        df['vol_60'] = df['returns'].rolling(30).std()*(252**0.5)
+        df['vol_60'] = df['returns'].rolling(60).std()*(252**0.5)
 
         # momentum (10, 20, 30, 60, 90 days)
         if log_returns:

model_walk_forward.py

@@ -3,7 +3,7 @@ from typing import Literal
 from sklearnex import patch_sklearn
 patch_sklearn()
 
-from load_data import create_target_cum_forward_returns, load_files, create_target_classes
+from load_data import create_target_cum_forward_returns, load_data, create_target_classes
 from sktime.forecasting.model_selection import temporal_train_test_split
 from utils.evaluate import evaluate_predictions_regression, evaluate_predictions_classification
 
@@ -21,79 +21,42 @@ from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTree
 from sklearn.metrics import r2_score, mean_absolute_error, confusion_matrix, classification_report, accuracy_score
 from sklearn.preprocessing import MinMaxScaler
 
-from utils.sliding_window import sliding_window_and_flatten
+from utils.walk_forward import walk_forward_train_test
-
-def walk_forward_train_test(
-        model_name: str,
-        create_model,
-        X: pd.DataFrame,
-        y: pd.Series,
-        window_size: int,
-        retrain_every: int
-    ):
-    print("Training: ", model_name)
-    predictions = [None] * (len(y)-1)
-    models = [None] * len(predictions)
-
-    train_from = window_size+1
-    train_till = len(y)-2
-    
-    iterations_since_retrain = 0
-
-    for i in range(train_from, train_till):
-        # if i % 20 == 0: print('Fold: ', i)
-        iterations_since_retrain += 1
-        window_start = i - window_size
-        window_end = i
-        X_train_slice = X[window_start:window_end]
-        y_train_slice = y[window_start:window_end]
-
-        if iterations_since_retrain >= retrain_every or models[i-1] is None:
-            model = create_model()
-            model.fit(X_train_slice, y_train_slice)
-            iterations_since_retrain = 0
-        else:
-            model = models[i-1]
-        models[window_end] = model
-
-        predictions[window_end+1] = model.predict(X[window_end+1].reshape(1, -1)).item()
-    return models, predictions
-
-
 
 regression_models = [
-    ('LR', lambda: LinearRegression(n_jobs=-1)),
+    ('LR', LinearRegression(n_jobs=-1)),
-    ('BayesianRidge', lambda: BayesianRidge()),
+    ('BayesianRidge', BayesianRidge()),
-    ('KNN', lambda: KNeighborsRegressor(n_neighbors=15)),
+    ('KNN', KNeighborsRegressor(n_neighbors=15)),
-    ('MLP', lambda: MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
+    # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
-    ('AB', lambda: AdaBoostRegressor()),
+    ('AB', AdaBoostRegressor()),
     # ('RF', lambda: RandomForestRegressor(n_jobs=-1)),
-    ('SVR', lambda: SVR(kernel='rbf', C=1e3, gamma=0.1))
+    ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1))
 ]
 classification_models = [
-    ('LR', lambda: LogisticRegression(n_jobs=-1)),
+    ('LR', LogisticRegression(n_jobs=-1)),
-    ('LDA', lambda: LinearDiscriminantAnalysis()),
+    ('LDA', LinearDiscriminantAnalysis()),
-    ('KNN', lambda: KNeighborsClassifier()),
+    ('KNN', KNeighborsClassifier()),
-    ('CART', lambda: DecisionTreeClassifier()),
+    ('CART', DecisionTreeClassifier()),
-    ('NB', lambda: GaussianNB()),
+    ('NB', GaussianNB()),
-    ('AB', lambda: AdaBoostClassifier()),
+    ('AB', AdaBoostClassifier()),
-    ('RF', lambda: RandomForestClassifier(n_jobs=-1))
+    ('RF', RandomForestClassifier(n_jobs=-1))
 ]
 
 
 
 def run_whole_pipeline(
-    ticker_to_predict: str,
+                    ticker_to_predict: str,
-    models,
+                    models,
-    method: Literal['regression', 'classification'],
+                    method: Literal['regression', 'classification'],
-    sliding_window_size: int,
+                    sliding_window_size: int,
-    retrain_every: int,
+                    retrain_every: int,
-    ):
+                    scaling: bool,
+                    ):
     print('Predicting: ', ticker_to_predict)
 
-    data = load_files(path='data/',
+    X, y = load_data(path='data/',
-        own_asset=ticker_to_predict,
+        target_asset=ticker_to_predict,
-        own_asset_lags=[1,2,3,4,5,6,8,10,15],
+        target_asset_lags=[1,2,3,4,5,6,8,10,15],
         load_other_assets=False,
         other_asset_lags=[],
         log_returns=True,
@@ -101,34 +64,21 @@ def run_whole_pipeline(
         own_technical_features='level2',
         other_technical_features='none',
         exogenous_features='none',
-        index_column='int'
+        index_column='int',
+        method=method,
     )
 
-    target_col = 'target'
+    if scaling:
-    returns_col = ticker_to_predict + '_returns'
+        # TODO: should move scaling to an expanding window compomenent
-    if method == 'regression':
+        feature_scaler = MinMaxScaler(feature_range= (-1, 1))
-        data = create_target_cum_forward_returns(data, returns_col, 1)
+        X = pd.DataFrame(feature_scaler.fit_transform(X), columns=X.columns, index=X.index)
-    elif method == 'classification':
+        # TODO: should scale y as well probably
-        data = create_target_classes(data, returns_col, 1, 'two')
 
-    X = data.drop(columns=[target_col])
+    for model_name, model in models:
-    y = data[target_col]
-
-    # TODO: should move scaling to an expanding window compomenent
-    feature_scaler = MinMaxScaler(feature_range= (-1, 1))
-    X = feature_scaler.fit_transform(X)
-    # TODO: should scale y as well probably
-
-    X = sliding_window_and_flatten(X, sliding_window_size)
-    y = y[sliding_window_size-1:]
-
-
-
-    for model_name, create_model in models:
 
         model_over_time, preds = walk_forward_train_test(
             model_name=model_name,
-            create_model = create_model,
+            model = model,
             X = X,
             y = y,
             window_size = sliding_window_size,
@@ -146,12 +96,14 @@ run_whole_pipeline(
     models = regression_models,
     method = 'regression',
     sliding_window_size = 120,
-    retrain_every = 50
+    retrain_every = 50,
+    scaling=False
 )
 run_whole_pipeline(
     ticker_to_predict = ticker_to_predict,
     models = classification_models,
     method = 'classification',
     sliding_window_size = 120,
-    retrain_every = 50
+    retrain_every = 50,
+    scaling=False
 )

utils/evaluate.py

@@ -47,7 +47,7 @@ def evaluate_predictions_regression(model_name: str, y_true, y_pred, sliding_win
 def evaluate_predictions_classification(model_name: str, y, preds, sliding_window_size: int):
     print("Model: ", model_name)
     evaluate_from = sliding_window_size+1
-    y = pd.Series(y[evaluate_from:-1])
+    y = pd.Series(y[evaluate_from:])
     preds = pd.Series(preds[evaluate_from:])
     print(accuracy_score(y, preds))
     print(confusion_matrix(y, preds))

utils/sliding_window.py

@@ -1,200 +0,0 @@
-import numpy as np
-from numpy.lib.stride_tricks import as_strided
-from numpy.core.numeric import normalize_axis_tuple
-
-def sliding_window_and_flatten(x: np.array, window_size: int) -> np.array:
-    n_features = x.shape[1]
-    x = np.squeeze(__numpy_sliding_window_view(x, (window_size, n_features)), axis = 1)
-    return x.reshape(x.shape[0], -1)
-
-
-def __numpy_sliding_window_view(x, window_shape, axis=None, *,
-                        subok=False, writeable=False):
-    """
-    Create a sliding window view into the array with the given window shape.
-    Also known as rolling or moving window, the window slides across all
-    dimensions of the array and extracts subsets of the array at all window
-    positions.
-    
-    .. versionadded:: 1.20.0
-    Parameters
-    ----------
-    x : array_like
-        Array to create the sliding window view from.
-    window_shape : int or tuple of int
-        Size of window over each axis that takes part in the sliding window.
-        If `axis` is not present, must have same length as the number of input
-        array dimensions. Single integers `i` are treated as if they were the
-        tuple `(i,)`.
-    axis : int or tuple of int, optional
-        Axis or axes along which the sliding window is applied.
-        By default, the sliding window is applied to all axes and
-        `window_shape[i]` will refer to axis `i` of `x`.
-        If `axis` is given as a `tuple of int`, `window_shape[i]` will refer to
-        the axis `axis[i]` of `x`.
-        Single integers `i` are treated as if they were the tuple `(i,)`.
-    subok : bool, optional
-        If True, sub-classes will be passed-through, otherwise the returned
-        array will be forced to be a base-class array (default).
-    writeable : bool, optional
-        When true, allow writing to the returned view. The default is false,
-        as this should be used with caution: the returned view contains the
-        same memory location multiple times, so writing to one location will
-        cause others to change.
-    Returns
-    -------
-    view : ndarray
-        Sliding window view of the array. The sliding window dimensions are
-        inserted at the end, and the original dimensions are trimmed as
-        required by the size of the sliding window.
-        That is, ``view.shape = x_shape_trimmed + window_shape``, where
-        ``x_shape_trimmed`` is ``x.shape`` with every entry reduced by one less
-        than the corresponding window size.
-    See Also
-    --------
-    lib.stride_tricks.as_strided: A lower-level and less safe routine for
-        creating arbitrary views from custom shape and strides.
-    broadcast_to: broadcast an array to a given shape.
-    Notes
-    -----
-    For many applications using a sliding window view can be convenient, but
-    potentially very slow. Often specialized solutions exist, for example:
-    - `scipy.signal.fftconvolve`
-    - filtering functions in `scipy.ndimage`
-    - moving window functions provided by
-      `bottleneck <https://github.com/pydata/bottleneck>`_.
-    As a rough estimate, a sliding window approach with an input size of `N`
-    and a window size of `W` will scale as `O(N*W)` where frequently a special
-    algorithm can achieve `O(N)`. That means that the sliding window variant
-    for a window size of 100 can be a 100 times slower than a more specialized
-    version.
-    Nevertheless, for small window sizes, when no custom algorithm exists, or
-    as a prototyping and developing tool, this function can be a good solution.
-    Examples
-    --------
-    >>> x = np.arange(6)
-    >>> x.shape
-    (6,)
-    >>> v = sliding_window_view(x, 3)
-    >>> v.shape
-    (4, 3)
-    >>> v
-    array([[0, 1, 2],
-           [1, 2, 3],
-           [2, 3, 4],
-           [3, 4, 5]])
-    This also works in more dimensions, e.g.
-    >>> i, j = np.ogrid[:3, :4]
-    >>> x = 10*i + j
-    >>> x.shape
-    (3, 4)
-    >>> x
-    array([[ 0,  1,  2,  3],
-           [10, 11, 12, 13],
-           [20, 21, 22, 23]])
-    >>> shape = (2,2)
-    >>> v = sliding_window_view(x, shape)
-    >>> v.shape
-    (2, 3, 2, 2)
-    >>> v
-    array([[[[ 0,  1],
-             [10, 11]],
-            [[ 1,  2],
-             [11, 12]],
-            [[ 2,  3],
-             [12, 13]]],
-           [[[10, 11],
-             [20, 21]],
-            [[11, 12],
-             [21, 22]],
-            [[12, 13],
-             [22, 23]]]])
-    The axis can be specified explicitly:
-    >>> v = sliding_window_view(x, 3, 0)
-    >>> v.shape
-    (1, 4, 3)
-    >>> v
-    array([[[ 0, 10, 20],
-            [ 1, 11, 21],
-            [ 2, 12, 22],
-            [ 3, 13, 23]]])
-    The same axis can be used several times. In that case, every use reduces
-    the corresponding original dimension:
-    >>> v = sliding_window_view(x, (2, 3), (1, 1))
-    >>> v.shape
-    (3, 1, 2, 3)
-    >>> v
-    array([[[[ 0,  1,  2],
-             [ 1,  2,  3]]],
-           [[[10, 11, 12],
-             [11, 12, 13]]],
-           [[[20, 21, 22],
-             [21, 22, 23]]]])
-    Combining with stepped slicing (`::step`), this can be used to take sliding
-    views which skip elements:
-    >>> x = np.arange(7)
-    >>> sliding_window_view(x, 5)[:, ::2]
-    array([[0, 2, 4],
-           [1, 3, 5],
-           [2, 4, 6]])
-    or views which move by multiple elements
-    >>> x = np.arange(7)
-    >>> sliding_window_view(x, 3)[::2, :]
-    array([[0, 1, 2],
-           [2, 3, 4],
-           [4, 5, 6]])
-    A common application of `sliding_window_view` is the calculation of running
-    statistics. The simplest example is the
-    `moving average <https://en.wikipedia.org/wiki/Moving_average>`_:
-    >>> x = np.arange(6)
-    >>> x.shape
-    (6,)
-    >>> v = sliding_window_view(x, 3)
-    >>> v.shape
-    (4, 3)
-    >>> v
-    array([[0, 1, 2],
-           [1, 2, 3],
-           [2, 3, 4],
-           [3, 4, 5]])
-    >>> moving_average = v.mean(axis=-1)
-    >>> moving_average
-    array([1., 2., 3., 4.])
-    Note that a sliding window approach is often **not** optimal (see Notes).
-    """
-    window_shape = (tuple(window_shape)
-                    if np.iterable(window_shape)
-                    else (window_shape,))
-    # first convert input to array, possibly keeping subclass
-    x = np.array(x, copy=False, subok=subok)
-
-    window_shape_array = np.array(window_shape)
-    if np.any(window_shape_array < 0):
-        raise ValueError('`window_shape` cannot contain negative values')
-
-    if axis is None:
-        axis = tuple(range(x.ndim))
-        if len(window_shape) != len(axis):
-            raise ValueError(f'Since axis is `None`, must provide '
-                             f'window_shape for all dimensions of `x`; '
-                             f'got {len(window_shape)} window_shape elements '
-                             f'and `x.ndim` is {x.ndim}.')
-    else:
-        axis = normalize_axis_tuple(axis, x.ndim, allow_duplicate=True)
-        if len(window_shape) != len(axis):
-            raise ValueError(f'Must provide matching length window_shape and '
-                             f'axis; got {len(window_shape)} window_shape '
-                             f'elements and {len(axis)} axes elements.')
-
-    out_strides = x.strides + tuple(x.strides[ax] for ax in axis)
-
-    # note: same axis can be windowed repeatedly
-    x_shape_trimmed = list(x.shape)
-    for ax, dim in zip(axis, window_shape):
-        if x_shape_trimmed[ax] < dim:
-            raise ValueError(
-                'window shape cannot be larger than input array shape')
-        x_shape_trimmed[ax] -= dim - 1
-    out_shape = tuple(x_shape_trimmed) + window_shape
-    return as_strided(x, strides=out_strides, shape=out_shape,
-                      subok=subok, writeable=writeable)

utils/typing.py

@@ -0,0 +1,7 @@
+from typing import Protocol
+
+class SKLearnModel(Protocol):
+    def fit(self, X, y, sample_weight=None): ...
+    def predict(self, X): ...
+    def score(self, X, y, sample_weight=None): ...
+    def set_params(self, **params): ...

utils/walk_forward.py

@@ -0,0 +1,44 @@
+import pandas as pd
+from sklearn.base import clone
+from utils.typing import SKLearnModel
+import numpy as np
+
+def walk_forward_train_test(
+                            model_name: str,
+                            model: SKLearnModel,
+                            X: pd.DataFrame,
+                            y: pd.Series,
+                            window_size: int,
+                            retrain_every: int
+                        ) -> tuple[list[SKLearnModel], list[float]]:
+                        
+    print("Training: ", model_name)
+    predictions = pd.Series(index=y.index)
+    models = pd.Series(index=y.index)
+
+    train_from = window_size
+    train_till = y.index[-1]
+    
+    iterations_since_retrain = 0
+
+    for i in range(train_from, train_till):
+
+        iterations_since_retrain += 1
+        window_start = i - window_size
+        window_end = i
+        X_slice = X[window_start:window_end]
+        y_slice = y[window_start:window_end]
+
+        if iterations_since_retrain >= retrain_every or pd.isna(models[i-1]):
+            current_model = clone(model)
+            current_model.fit(X_slice.to_numpy(), y_slice)
+            iterations_since_retrain = 0
+        else:
+            current_model = models[i-1]
+
+        models[window_end] = current_model
+
+        next_timestep = X.iloc[window_end+1].to_numpy().reshape(1, -1)
+        predictions[window_end+1] = current_model.predict(next_timestep).item()
+
+    return models, predictions