feat(Core): ensemble models, correct forward returns calculation, scaling, only train from when asset returns are available, major bug fixed in walk_forward_train_test (#35)

* fix(Core): correct forward returns calculation, classifiers are now working again, only train from when asset returns are available

* feat(Utils): added get_first_valid_return_index()

* feat(Ensemble): return models from `run_whole_pipeline`

* feat(Ensemble): added ensemble step, fixed walk_forward_train_test predictions index confusion,

* chore(Pipeline): remove unnecessary extra ensemble results dataframe

* refactor(Core): removed unnecessary ensemble_train_predict, moved run_single_asset_trainig_pipeline to a separate file

* feat(Training): added scaling on expanding window (the past) to walk_forward_train_test(), now printing out mean sharpe ratio

* feat(CI): added environment.yml file

* chore(Environment): update env.yml

* feat(CI): added testing workflow

* fix(CI): renamed enviroment.yml

* fix(Tests): added missing new parameter to walk_forward_train_test()

.github/workflows/test.yml

@@ -0,0 +1,42 @@
+name: run-test
+on: [push]
+jobs:
+  run:
+    runs-on: [ubuntu-latest]
+    # optionally use a convenient Ubuntu LTS + CUDA + DVC + CML image
+    # container: docker://dvcorg/cml:0-dvc2-base1-gpu
+    # container: docker://ghcr.io/iterative/cml:0-dvc2-base1-gpu
+    steps:
+      - uses: actions/checkout@v2
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: false
+          activate-environment: quant
+          environment-file: environment.yml
+          python-version: 3.9
+      - uses: iterative/setup-cml@v1
+      - name: Run pytest
+        shell: bash -l {0}
+#         env: 
+#           WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+        run: |
+          pytest --junit-xml pytest.xml
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Unit Test Results (Python ${{ matrix.python-version }})
+          path: pytest.xml
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        if: always()
+        with:
+          files: pytest.xml
+
+#       - name: Write CML report
+#         env:
+#           REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#         run: |
+#           # Post reports as comments in GitHub PRs
+#           # cat results.txt >> report.md
+#           # cml-send-comment report.md

README.md

@@ -1,5 +1,5 @@
-# Price prediction models
+# Financial time series prediction models
 
 ## Installation
 
-Make sure you use keras `2.6.0`, otherwise you won't be able to train models.
+Use the conda environment file attached!:)

environment.yml

@@ -0,0 +1,20 @@
+name: quant
+channels:
+  - johnsnowlabs
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.9
+  - seaborn
+  - scikit-learn-intelex
+  - ipython
+  - scipy
+  - scikit-learn
+  - numba
+  - pytorch
+  - matplotlib
+  - numpy
+  - quantstats
+  - pytorch-lightning
+  - pytest
+prefix: /usr/local/anaconda3/envs/quant

load_data.py

@@ -10,7 +10,7 @@ from sklearn.preprocessing import OneHotEncoder
 #%%
 
 def get_crypto_assets(path: str) -> list[str]:
-    return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' in f and not f.startswith('.')])
+    return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')])
 
 def get_etf_assets(path: str) -> list[str]:
     return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' not in f and not f.startswith('.')])
@@ -29,7 +29,14 @@ def load_data(path: str,
             index_column: Literal['date', 'int'],
             method: Literal['regression', 'classification'],
             narrow_format: bool = False,
-        ) -> tuple[pd.DataFrame, pd.Series]:
+        ) -> tuple[pd.DataFrame, pd.Series, pd.Series]:
+    """
+    Loads asset data from the specified path.
+    Returns:
+        - DataFrame `X` with all the training data
+        - Series `y` with the target asset returns shifted by 1 day OR if it's a classification problem, the target class)
+        - Series `forward_returns` with the target asset returns shifted by 1 day
+    """
     
     files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
     files = [f for f in files if load_other_assets == True or (load_other_assets == False and f.startswith(target_asset))]
@@ -37,7 +44,7 @@ def load_data(path: str,
     dfs = [__load_df(
         path=os.path.join(path,f),
         prefix=f.split('.')[0],
-        log_returns=log_returns,
+        returns='log_returns' if log_returns else 'returns',
         technical_features=own_technical_features if is_target_asset(target_asset, f) else other_technical_features,
         lags= target_asset_lags if is_target_asset(target_asset, f) else other_asset_lags,
         narrow_format=narrow_format,
@@ -63,28 +70,60 @@ def load_data(path: str,
     ## Create target 
     target_col = 'target'
     returns_col = target_asset + '_returns'
+    prediction_horizon = 1
+    forward_returns = __create_target_cum_forward_returns(dfs, returns_col, prediction_horizon)
     if method == 'regression':
-        dfs = __create_target_cum_forward_returns(dfs, returns_col, 1)
+        dfs[target_col] = forward_returns
     elif method == 'classification':
-        dfs = __create_target_classes(dfs, returns_col, 1, 'two')
+        dfs[target_col] = __create_target_classes(dfs, returns_col, prediction_horizon, 'two')
+    # we need to drop the last row, because we forward-shift the target (see what happens if you call .shift[-1] on a pd.Series) 
+    dfs = dfs.iloc[:-prediction_horizon]
+    forward_returns = forward_returns.iloc[:-prediction_horizon]
     
     X = dfs.drop(columns=[target_col])
     y = dfs[target_col]
 
-    return X, y
+    return X, y, forward_returns
 
-def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Literal['none', 'level1', 'level2'], lags: list[int], narrow_format: bool = False) -> pd.DataFrame:
+# %%
+
+def load_crypto_only_returns(path: str, index_column: Literal['date', 'int'], returns: Literal['price', 'returns']) -> pd.DataFrame:
+    files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')]
+    dfs = [__load_df(
+        path=os.path.join(path,f),
+        prefix=f.split('.')[0],
+        returns=returns,
+        technical_features='none',
+        lags=[],
+        narrow_format=False,
+    ) for f in files]
+    dfs = pd.concat(dfs, axis=1)
+    dfs = dfs.applymap(lambda x: np.nan if x == 0 else x)
+    dfs.index = pd.DatetimeIndex(dfs.index)
+    dfs.columns = [column.split('_')[0] for column in dfs.columns]
+    if index_column == 'int':
+        dfs.reset_index(drop=True, inplace=True)
+
+    return dfs
+
+def load_crypto_assets_availability(path: str, index_column: Literal['date', 'int']) -> pd.DataFrame:
+    return load_crypto_only_returns(path, index_column, 'returns').applymap(lambda x: 0 if x == 0.0 or x == 0 or np.isnan(x) else 1)
+
+
+def __load_df(path: str, prefix: str, returns: Literal['price', 'returns', 'log_returns'], technical_features: Literal['none', 'level1', 'level2'], lags: list[int], narrow_format: bool = False) -> pd.DataFrame:
     df = pd.read_csv(path, header=0, index_col=0).fillna(0)
 
-    if log_returns:
+    if returns == 'log_returns':
         df['returns'] = np.log(df['close']).diff(1)
+    elif returns == 'price':
+        df['returns'] = df['close']
     else:
         df['returns'] = df['close'].pct_change()
     
     for lag in lags:
         df[f'lag_{lag}'] = df['returns'].shift(lag)
 
-    df = __augment_derived_features(df, log_returns=log_returns, technical_features=technical_features)
+    df = __augment_derived_features(df, log_returns=True if returns == 'log_returns' else False, technical_features=technical_features)
 
     df = df.replace([np.inf, -np.inf], 0.)
     df = df.drop(columns=['open', 'high', 'low', 'close'])
@@ -100,7 +139,7 @@ def __load_df(path: str, prefix: str, log_returns: bool, technical_features: Lit
 
 def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_features: Literal['none', 'level1', 'level2']) -> pd.DataFrame:
     if technical_features == 'level1' or technical_features == 'level2':
-        # volatility (10, 20, 30 days)
+        # volatility (10, 20, 30, 60 days)
         df['vol_10'] = df['returns'].rolling(10).std()*(252**0.5)
         df['vol_20'] = df['returns'].rolling(20).std()*(252**0.5)
         df['vol_30'] = df['returns'].rolling(30).std()*(252**0.5)
@@ -137,16 +176,15 @@ def __augment_derived_features(df: pd.DataFrame, log_returns: bool, technical_fe
     return df
 
 
-# %%
 
 # %%
-def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.DataFrame:
+def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.Series:
-    df['target'] = df[source_column].diff(period).shift(-period)
+    assert period > 0
-    df = df.iloc[:-period]
+    return df[source_column].shift(-period)
-    return df
 
 
-def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.DataFrame:
+def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, no_of_classes: Literal["two", "three"]) -> pd.Series:
+    assert period > 0
 
     def get_class_binary(x):
         return 0 if x <= 0.0 else 1
@@ -162,16 +200,10 @@ def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, n
         else:
             return 1
 
-    if period > 0:
+    target_column = df[source_column].shift(-period)
-        df['target'] = df[source_column].shift(-period)
-    else:
-        df['target'] = df[source_column]
     
     get_class_function = get_class_binary
     if no_of_classes == "three":
         get_class_function = get_class_threeway
 
-    df['target'] = df['target'].map(get_class_function)
+    return target_column.map(get_class_function)
-    if period > 0:
-        df = df.iloc[:-period]
-    return df

run_pipeline.py

@@ -1,12 +1,10 @@
-from logging import log
-from typing import Literal
 from sklearnex import patch_sklearn
 patch_sklearn()
 
 from load_data import get_crypto_assets, get_etf_assets, load_data
-from utils.evaluate import evaluate_predictions
 
 import pandas as pd
+import numpy as np
 from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression, Ridge
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier
@@ -15,106 +13,110 @@ from sklearn.svm import SVR
 from sklearn.naive_bayes import GaussianNB
 from sklearn.neural_network import MLPRegressor, MLPClassifier
 from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
-from sklearn.preprocessing import MinMaxScaler
 
-from utils.walk_forward import walk_forward_train_test
+from training.pipeline import run_single_asset_trainig_pipeline
 
+
+# Parameters
 regression_models = [
-    # ('LR', LinearRegression(n_jobs=-1)),
+    ('Lasso', Lasso(alpha=1.0, max_iter=10000)),
-    ('Lasso', Lasso(alpha=0.1, max_iter=10000)),
     ('Ridge', Ridge(alpha=1.0)),
     ('BayesianRidge', BayesianRidge()),
     ('KNN', KNeighborsRegressor(n_neighbors=15)),
+    # ('AB', AdaBoostRegressor()),
+    # ('LR', LinearRegression(n_jobs=-1)),
     # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
-    ('AB', AdaBoostRegressor()),
     # ('RF', RandomForestRegressor(n_jobs=-1)),
     # ('SVR', SVR(kernel='rbf', C=1e3, gamma=0.1))
 ]
+ensemble_model = [('Ensemble - Lasso', Lasso(alpha=1.0, max_iter=10000, positive=True))]
+
 classification_models = [
     ('LR', LogisticRegression(n_jobs=-1)),
-    ('LDA', LinearDiscriminantAnalysis()),
+    # ('LDA', LinearDiscriminantAnalysis()),
-    ('KNN', KNeighborsClassifier()),
+    # ('KNN', KNeighborsClassifier()),
-    ('CART', DecisionTreeClassifier()),
+    # ('CART', DecisionTreeClassifier()),
-    ('NB', GaussianNB()),
+    # ('NB', GaussianNB()),
-    ('AB', AdaBoostClassifier()),
+    # ('AB', AdaBoostClassifier()),
-    ('RF', RandomForestClassifier(n_jobs=-1))
+    # ('RF', RandomForestClassifier(n_jobs=-1))
 ]
 
 
+path = 'data/'
+all_assets = get_crypto_assets(path)
 
-def run_whole_pipeline(
+sliding_window_size = 200
-                    ticker_to_predict: str,
+retrain_every = 100
-                    load_data_args: dict,
+scaler = 'none' # 'normalize' 'minmax' 'standardize' 'none'
-                    models,
+include_original_data_in_ensemble = True
-                    method: Literal['regression', 'classification'],
+method = 'regression'
-                    sliding_window_size: int,
+data_parameters = dict(path=path,
-                    retrain_every: int,
+    target_asset_lags= [1,2,3,4,5,6,8,10,15],
-                    scaling: bool,
+    load_other_assets= True,
-                    ):
+    other_asset_lags= [],
-    print('--------\nPredicting: ', ticker_to_predict)
+    log_returns= True,
+    add_date_features= True,
+    own_technical_features= 'level2',
+    other_technical_features= 'none',
+    exogenous_features= 'none',
+    index_column= 'int',
+    method= method,
+)
 
-    X, y = load_data(**load_data_args)
 
-    if scaling:
+# Run pipeline
-        # TODO: should move scaling to an expanding window compomenent, probably worth not turning it on for now
-        feature_scaler = MinMaxScaler(feature_range= (-1, 1))
-        X = pd.DataFrame(feature_scaler.fit_transform(X), columns=X.columns, index=X.index)
-        # TODO: should scale y as well probably
-
-    results = pd.DataFrame()
-
-    for model_name, model in models:
-
-        model_over_time, preds = walk_forward_train_test(
-            model_name=model_name,
-            model = model,
-            X = X,
-            y = y,
-            window_size = sliding_window_size,
-            retrain_every = retrain_every
-        )
-        result = evaluate_predictions(
-            model_name = model_name,
-            y_true = y,
-            y_pred = preds,
-            sliding_window_size = sliding_window_size,
-            method = method,
-        )
-        column_name = ticker_to_predict + "_" + model_name
-        results[column_name] = result
-
-    return results
 
 results = pd.DataFrame()
-all_assets = get_crypto_assets('data/')
-
-
 
 for asset in all_assets:
-    for method in ['regression']:
+    print('--------\nPredicting: ', asset)
-        load_data_args = dict(path='data/',
+    all_predictions = pd.DataFrame()
-            target_asset= asset,
-            target_asset_lags= [1,2,3,4,5,6,8,10,15],
-            load_other_assets= False,
-            other_asset_lags= [],
-            log_returns= True,
-            add_date_features= True,
-            own_technical_features= 'level2',
-            other_technical_features= 'none',
-            exogenous_features= 'none',
-            index_column= 'int',
-            method= method,
-        )
 
-        current_result = run_whole_pipeline(
+    # 1. Load data
-            ticker_to_predict = asset,
+    data_params = data_parameters.copy()
-            load_data_args = load_data_args,
+    data_params['target_asset'] = asset
-            models = regression_models if method == 'regression' else classification_models,
+
-            method = method,
+    X, y, target_returns = load_data(**data_params)
-            sliding_window_size = 120,
+
-            retrain_every = 50,
+    # 2. Train Level-1 models
-            scaling = False
+    current_result, current_predictions = run_single_asset_trainig_pipeline(
-        )
+        ticker_to_predict = asset,
-        results = pd.concat([results, current_result], axis=1)
+        X = X,
+        y = y,
+        target_returns = target_returns,
+        models = regression_models if method == 'regression' else classification_models,
+        method = method,
+        sliding_window_size = sliding_window_size,
+        retrain_every = retrain_every,
+        scaler = scaler
+    )
+    results = pd.concat([results, current_result], axis=1)
+    all_predictions = pd.concat([all_predictions, current_predictions], axis=1)
+
+    # 3. Train Level-2 (Ensemble) model
+    ensemble_X = all_predictions
+    if include_original_data_in_ensemble:
+        ensemble_X = pd.concat([ensemble_X, X], axis=1)
+
+    ensemble_result, ensemble_preds = run_single_asset_trainig_pipeline(
+        ticker_to_predict = asset,
+        X = ensemble_X,
+        y = target_returns,
+        target_returns = target_returns,
+        models = ensemble_model,
+        method = 'regression',
+        sliding_window_size = sliding_window_size,
+        retrain_every = retrain_every,
+        scaler = scaler
+    )
+
+    results = pd.concat([results, ensemble_result], axis=1)
+    all_predictions = pd.concat([all_predictions, ensemble_preds], axis=1)
 
 results.to_csv('results.csv')
+
+level1_columns = results[[column for column in results.columns if 'Ensemble' not in column]]
+ensemble_columns = results[[column for column in results.columns if 'Ensemble' in column]]
+
+print("Mean Sharpe ratio for Level-1 models: ", level1_columns.loc['sharpe'].mean())
+print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", ensemble_columns.loc['sharpe'].mean())

tests/test_walk_forward.py

@@ -1,20 +1,21 @@
 import pytest
 import numpy as np
 import pandas as pd
-from utils.walk_forward import walk_forward_train_test
+from training.walk_forward import walk_forward_train_test
 from sklearn.base import BaseEstimator
 
-def __generate_test_data():
+no_of_rows = 100
+
+def __generate_test_data(no_of_rows):
     no_columns = 6
-    no_rows = 100
+    X = [[row] * no_columns for row in range(no_of_rows)]
-    X = [[row] * no_columns for row in range(no_rows)]
     assert X[0][0] == 0
     assert X[1][0] == 1
     assert X[2][0] == 2
     assert X[3][0] == 3
     X = pd.DataFrame(X)
 
-    y = [row+1 for row in range(no_rows)]
+    y = [row+1 for row in range(no_of_rows)]
     assert y[0] == 1
     assert y[1] == 2
     assert y[2] == 3
@@ -24,7 +25,7 @@ def __generate_test_data():
 
 
 def test_walk_forward_train_test():
-    X, y = __generate_test_data()
+    X, y = __generate_test_data(no_of_rows)
 
     window_length = 10
     class StubModel(BaseEstimator):
@@ -38,4 +39,16 @@ def test_walk_forward_train_test():
             return np.array([X[0][0] + 1])
 
     model = StubModel()
-    walk_forward_train_test('test', model, X, y, window_length, 10)
+    scaler = None
+    models, predictions = walk_forward_train_test(
+        model_name='test',
+        model=model,
+        X=X,
+        y=y,
+        target_returns=y,
+        window_size=window_length,
+        retrain_every=10,
+        scaler=scaler)
+    for i in range(window_length, no_of_rows):
+        predictions[i] == y[i]
+    

training/pipeline.py

@@ -0,0 +1,60 @@
+import pandas as pd
+from typing import Literal
+from training.walk_forward import walk_forward_train_test
+from sklearn.preprocessing import MinMaxScaler, Normalizer, StandardScaler
+from utils.evaluate import evaluate_predictions
+from utils.typing import SKLearnModel
+
+def __get_scaler(type: Literal['normalize', 'minmax', 'standardize', 'none']):
+    if type == 'normalize':
+        return Normalizer()
+    elif type == 'minmax':
+        return MinMaxScaler(feature_range= (-1, 1))
+    elif type == 'standardize':
+        return StandardScaler()
+    else:
+        return None
+
+def run_single_asset_trainig_pipeline(
+                    ticker_to_predict: str,
+                    X: pd.DataFrame,
+                    y: pd.Series,
+                    target_returns: pd.Series,
+                    models: list[tuple[str, SKLearnModel]],
+                    method: Literal['regression', 'classification'],
+                    sliding_window_size: int,
+                    retrain_every: int,
+                    scaler: Literal['normalize', 'minmax', 'standardize', 'none'],
+                    ) -> tuple[pd.DataFrame, pd.DataFrame]:
+
+
+    scaler = __get_scaler(scaler)
+
+    results = pd.DataFrame()
+    predictions = pd.DataFrame()
+
+    for model_name, model in models:
+
+        model_over_time, preds = walk_forward_train_test(
+            model_name=model_name,
+            model = model,
+            X = X,
+            y = y,
+            target_returns = target_returns,
+            window_size = sliding_window_size,
+            retrain_every = retrain_every,
+            scaler = scaler
+        )
+        assert len(preds) == len(y)
+        result = evaluate_predictions(
+            model_name = model_name,
+            target_returns = target_returns,
+            y_pred = preds,
+            sliding_window_size = sliding_window_size,
+            method = method,
+        )
+        column_name = ticker_to_predict + "_" + model_name
+        results[column_name] = result
+        predictions[column_name] = preds
+
+    return results, predictions

training/walk_forward.py

@@ -0,0 +1,63 @@
+import pandas as pd
+from sklearn.base import clone
+from utils.typing import SKLearnModel
+import numpy as np
+from utils.helpers import get_first_valid_return_index
+
+def walk_forward_train_test(
+                            model_name: str,
+                            model: SKLearnModel,
+                            X: pd.DataFrame,
+                            y: pd.Series,
+                            target_returns: pd.Series,
+                            window_size: int,
+                            retrain_every: int,
+                            scaler,
+                        ) -> tuple[pd.Series, pd.Series]:
+    assert len(X) == len(y)
+    predictions = pd.Series(index=y.index).rename(model_name)
+    models = pd.Series(index=y.index).rename(model_name)
+
+    first_nonzero_return = max(get_first_valid_return_index(target_returns), get_first_valid_return_index(X.iloc[:,0]))
+    train_from = first_nonzero_return + window_size + 1
+    train_till = len(y)
+    
+    iterations_since_retrain = 0
+    if scaler is not None:
+        scaler = clone(scaler)
+
+    for index in range(train_from, train_till):
+
+        iterations_since_retrain += 1
+
+        if iterations_since_retrain >= retrain_every or pd.isna(models[index-1]):
+            train_window_start = index - window_size - 1
+            train_window_end = index - 1
+            
+            if scaler is not None:
+                # First we need to fit on the expanding window data slice
+                # This is our only way to avoid lookahead bia
+                X_expanding_window = X[first_nonzero_return:train_window_end]
+                scaler.fit(X_expanding_window)
+
+            X_slice = X[train_window_start:train_window_end]
+            y_slice = y[train_window_start:train_window_end]
+
+            if scaler is not None:
+                X_slice = scaler.transform(X_slice)
+            else:
+                X_slice = X_slice.to_numpy()
+
+            current_model = clone(model)
+            current_model.fit(X_slice, y_slice.to_numpy())
+            iterations_since_retrain = 0
+        else:
+            current_model = models[index-1]
+
+        models[index] = current_model
+
+        next_timestep = X.iloc[index].to_numpy().reshape(1, -1)
+        prediction = current_model.predict(next_timestep).item()
+        predictions[index] = prediction
+
+    return models, predictions

utils/evaluate.py

@@ -1,53 +1,63 @@
 from typing import Literal
 from sklearn.metrics import mean_absolute_error, accuracy_score, r2_score, f1_score, precision_score, recall_score
 from quantstats.stats import expected_return, sharpe, skew, sortino
+from utils.helpers import get_first_valid_return_index
 import pandas as pd
 import numpy as np
 
-def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.02) -> pd.Series:
+def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.01) -> pd.Series:
     delta_pos = signal.diff(1).abs().fillna(0.)
     costs = transaction_cost * delta_pos
     return (signal * returns) - costs
 
 
-def __preprocess(y_true: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']):
+def __preprocess(target_returns: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']) -> pd.DataFrame:
     y_pred.name = 'y_pred'
-    y_true.name = 'y_true'
+    target_returns.name = 'target_returns'
-    df = pd.concat([y_pred, y_true],axis=1).dropna()
+    df = pd.concat([y_pred, target_returns],axis=1).dropna()
 
-    if method == 'regression':
+    df['sign_pred'] = df.y_pred.apply(lambda x: 1 if x>0 else -1)
-        df['sign_pred'] = df.y_pred.apply(np.sign)
+    def sign_true(x):
-    else:
+        if x > 0:
-        df['sign_pred'] = df.y_pred.apply(lambda x: 1 if x>0 else -1)
+            return 1
-    df['sign_true'] = df.y_true.apply(np.sign)
+        else:
+            return -1
+    df['sign_true'] = df.target_returns.apply(sign_true)
     df['is_correct'] = 0
     df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct
     df['is_incorrect'] = 0
     df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong
     df['is_predicted'] = df.is_correct + df.is_incorrect
-    df['result'] = backtest(df.y_true, df.sign_pred)
+    df['result'] = backtest(df.target_returns, df.sign_pred)
 
     return df
 
-def evaluate_predictions(model_name: str, y_true: pd.Series, y_pred: pd.Series, sliding_window_size: int, method: Literal['classification', 'regression']):
+def evaluate_predictions(
+                        model_name: str,
+                        target_returns: pd.Series,
+                        y_pred: pd.Series,
+                        sliding_window_size: int,
+                        method: Literal['classification', 'regression']
+                        ) -> pd.Series:
     # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
-    first_nonzero_return = np.where(y_true != 0)[0][0]
+    first_nonzero_return = get_first_valid_return_index(target_returns)
     evaluate_from = first_nonzero_return + sliding_window_size + 1
     
-    y_true = pd.Series(y_true[evaluate_from:])
+    target_returns = pd.Series(target_returns[evaluate_from:])
-    # if there are lots of zeros in the ground truth returns, probably something is wrong, but we can tolerate a couple of days of missing data.
+    if method == 'regression':
-    is_zero = y_true[y_true == 0]
+        # if there are lots of zeros in the ground truth returns, probably something is wrong, but we can tolerate a couple of days of missing data.
-    assert len(is_zero) < 5
+        is_zero = target_returns[target_returns == 0]
-    # we can't deal with 0 returns, so we'll just remap the few examples to 1
+        assert len(is_zero) < 15
-    y_true = y_true.apply(lambda x: 1 if x == 0 else x)
+    # we can't deal with 0 returns, so we'll just remap the few examples to 0.0001 
+    target_returns = target_returns.apply(lambda x: 0.0001 if x == 0 else x)
     y_pred = pd.Series(y_pred[evaluate_from:])
 
-    df = __preprocess(y_true, y_pred, method)
+    df = __preprocess(target_returns, y_pred, method)
     
     scorecard = pd.Series()
     if method == 'regression':
-        scorecard.loc['RSQ'] = r2_score(df.y_true, df.y_pred)
+        scorecard.loc['RSQ'] = r2_score(df.target_returns, df.y_pred)
-        scorecard.loc['MAE'] = mean_absolute_error(df.y_true, df.y_pred)
+        scorecard.loc['MAE'] = mean_absolute_error(df.target_returns, df.y_pred)
     elif method == 'classification':
         scorecard.loc['RSQ'] = 0.
         scorecard.loc['MAE Matrix'] = 0.

utils/get_prices.py

@@ -49,13 +49,3 @@ def get_stock_price_av(symbol: str, start_date: str = None) -> pd.DataFrame:
     df = df.rename_axis('time')
     return df
 
-
-
-# %%
-# btc = get_crypto_price_av(symbol = 'BTC', exchange = 'USD', start_date = '2018-01-01')
-# btc
-
-# %%
-
-# spy = get_stock_price_av(symbol = 'SPY', start_date = '2018-01-01')
-# spy

utils/helpers.py

@@ -0,0 +1,6 @@
+import pandas as pd
+import numpy as np
+
+def get_first_valid_return_index(series: pd.Series) -> int:
+    first_nonzero_return = np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series))))[0][0]
+    return first_nonzero_return

utils/walk_forward.py

@@ -1,48 +0,0 @@
-import pandas as pd
-from sklearn.base import clone
-from utils.typing import SKLearnModel
-import numpy as np
-
-def walk_forward_train_test(
-                            model_name: str,
-                            model: SKLearnModel,
-                            X: pd.DataFrame,
-                            y: pd.Series,
-                            window_size: int,
-                            retrain_every: int
-                        ) -> tuple[pd.Series, pd.Series]:
-                        
-    predictions = pd.Series(index=y.index).rename(model_name)
-    models = pd.Series(index=y.index).rename(model_name)
-
-    train_from = window_size
-    train_till = y.index[-1]
-    
-    iterations_since_retrain = 0
-
-    for i in range(train_from, train_till):
-
-        iterations_since_retrain += 1
-        window_start = i - window_size
-        window_end = i
-        X_slice = X[window_start:window_end]
-        y_slice = y[window_start:window_end]
-
-        if iterations_since_retrain >= retrain_every or pd.isna(models[i-1]):
-            current_model = clone(model)
-            current_model.fit(X_slice.to_numpy(), y_slice.to_numpy())
-            iterations_since_retrain = 0
-        else:
-            current_model = models[i-1]
-
-        models[window_end] = current_model
-
-        next_timestep = X.iloc[window_end+1].to_numpy().reshape(1, -1)
-        prediction = current_model.predict(next_timestep).item()
-        if prediction == 0.:
-            # TODO: we shouldn't feed in zeros to the model, and skip training / predicting when everything is 0
-            # print("Warning: model predicted 0., overriding it with 0.0001")
-            prediction = 0.0001
-        predictions[window_end+1] = prediction
-
-    return models, predictions