fix(Evaluate): ignore empty data at evaluation time, add backtesting metrics (sharpe, etc), fixed crash when predicting 0.0 (#23)

* fix(Evaluate): ignore empty data at evaluation time, so we don't inflate the model's performance

* refactor(Pipeline): pass in data_loader arguments to the pipeline

* feat(Evaluation): added sharpe, sortino, etc

* fix: Took out the method to fill NaN numbers with 0s. This way in evaluation we can ignore NaN values.

* fix: Fix of the fix added fillna back. Either we root out NaN lines in the very beginning or we stick with the method you created.

Co-authored-by: Daniel Szemerey <szemy2@gmail.com>

load_data.py

@@ -9,8 +9,11 @@ from sklearn.preprocessing import OneHotEncoder
 
 #%%
 
-def get_all_assets(path: str) -> list[str]:
+def get_crypto_assets(path: str) -> list[str]:
-    return [f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
+    return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' in f and not f.startswith('.')])
+
+def get_etf_assets(path: str) -> list[str]:
+    return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' not in f and not f.startswith('.')])
 
 
 def load_data(path: str,

run_pipeline.py

@@ -1,12 +1,13 @@
+from logging import log
 from typing import Literal
 from sklearnex import patch_sklearn
 patch_sklearn()
 
-from load_data import get_all_assets, load_data
+from load_data import get_crypto_assets, get_etf_assets, load_data
 from utils.evaluate import evaluate_predictions
 
 import pandas as pd
-from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression
+from sklearn.linear_model import LinearRegression, Lasso, BayesianRidge, LogisticRegression, Ridge
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier
 from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
@@ -19,7 +20,9 @@ from sklearn.preprocessing import MinMaxScaler
 from utils.walk_forward import walk_forward_train_test
 
 regression_models = [
-    ('LR', LinearRegression(n_jobs=-1)),
+    # ('LR', LinearRegression(n_jobs=-1)),
+    ('Lasso', Lasso(alpha=0.1, max_iter=10000)),
+    ('Ridge', Ridge(alpha=1.0)),
     ('BayesianRidge', BayesianRidge()),
     ('KNN', KNeighborsRegressor(n_neighbors=15)),
     # ('MLP', MLPRegressor(hidden_layer_sizes=(100,20), max_iter=1000)),
@@ -41,6 +44,7 @@ classification_models = [
 
 def run_whole_pipeline(
                     ticker_to_predict: str,
+                    load_data_args: dict,
                     models,
                     method: Literal['regression', 'classification'],
                     sliding_window_size: int,
@@ -49,20 +53,7 @@ def run_whole_pipeline(
                     ):
     print('--------\nPredicting: ', ticker_to_predict)
 
-    X, y = load_data(path='data/',
+    X, y = load_data(**load_data_args)
-        target_asset=ticker_to_predict,
-        target_asset_lags=[1,2,3,4,5,6,8,10,15],
-        load_other_assets=False,
-        other_asset_lags=[],
-        log_returns=True,
-        add_date_features=True,
-        own_technical_features='level2',
-        other_technical_features='none',
-        exogenous_features='none',
-        index_column='int',
-        method=method,
-    )
-
 
     if scaling:
         # TODO: should move scaling to an expanding window compomenent, probably worth not turning it on for now
@@ -82,19 +73,42 @@ def run_whole_pipeline(
             window_size = sliding_window_size,
             retrain_every = retrain_every
         )
-        result = evaluate_predictions(model_name, y, preds, sliding_window_size, method)
+        result = evaluate_predictions(
+            model_name = model_name,
+            y_true = y,
+            y_pred = preds,
+            sliding_window_size = sliding_window_size,
+            method = method,
+        )
         column_name = ticker_to_predict + "_" + model_name
         results[column_name] = result
 
     return results
 
 results = pd.DataFrame()
-all_assets = get_all_assets('data/')
+all_assets = get_crypto_assets('data/')
+
+
 
 for asset in all_assets:
     for method in ['regression']:
+        load_data_args = dict(path='data/',
+            target_asset= asset,
+            target_asset_lags= [1,2,3,4,5,6,8,10,15],
+            load_other_assets= False,
+            other_asset_lags= [],
+            log_returns= True,
+            add_date_features= True,
+            own_technical_features= 'level2',
+            other_technical_features= 'none',
+            exogenous_features= 'none',
+            index_column= 'int',
+            method= method,
+        )
+
         current_result = run_whole_pipeline(
             ticker_to_predict = asset,
+            load_data_args = load_data_args,
             models = regression_models if method == 'regression' else classification_models,
             method = method,
             sliding_window_size = 120,

utils/evaluate.py

@@ -1,14 +1,13 @@
 from typing import Literal
-from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, r2_score, classification_report
+from sklearn.metrics import mean_absolute_error, accuracy_score, r2_score, f1_score, precision_score, recall_score
-from sklearn.metrics import confusion_matrix
+from quantstats.stats import expected_return, sharpe, skew, sortino
 import pandas as pd
 import numpy as np
 
-def format_data_for_backtest(aggregated_data: pd.DataFrame, returns_col: str, only_test_data: pd.DataFrame, preds) -> pd.DataFrame:
+def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.02) -> pd.Series:
-    backtest_data = aggregated_data.iloc[-only_test_data.shape[0]:].copy()[returns_col]
+    delta_pos = signal.diff(1).abs().fillna(0.)
-    assert backtest_data.shape[0] == only_test_data.shape[0]
+    costs = transaction_cost * delta_pos
-    backtest_data = backtest_data.reset_index(drop=True)    
+    return (signal * returns) - costs
-    return pd.concat([backtest_data, pd.Series(preds)], axis='columns')
 
 
 def __preprocess(y_true: pd.Series, y_pred: pd.Series, method: Literal['classification', 'regression']):
@@ -26,33 +25,54 @@ def __preprocess(y_true: pd.Series, y_pred: pd.Series, method: Literal['classifi
     df['is_incorrect'] = 0
     df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong
     df['is_predicted'] = df.is_correct + df.is_incorrect
-    df['result'] = df.sign_pred * df.y_true 
+    df['result'] = backtest(df.y_true, df.sign_pred)
+
     return df
 
 def evaluate_predictions(model_name: str, y_true: pd.Series, y_pred: pd.Series, sliding_window_size: int, method: Literal['classification', 'regression']):
-    evaluate_from = sliding_window_size+1
+    # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
+    first_nonzero_return = np.where(y_true != 0)[0][0]
+    evaluate_from = first_nonzero_return + sliding_window_size + 1
+    
     y_true = pd.Series(y_true[evaluate_from:])
+    # if there are lots of zeros in the ground truth returns, probably something is wrong, but we can tolerate a couple of days of missing data.
+    is_zero = y_true[y_true == 0]
+    assert len(is_zero) < 5
+    # we can't deal with 0 returns, so we'll just remap the few examples to 1
+    y_true = y_true.apply(lambda x: 1 if x == 0 else x)
     y_pred = pd.Series(y_pred[evaluate_from:])
 
     df = __preprocess(y_true, y_pred, method)
     
     scorecard = pd.Series()
     if method == 'regression':
-        scorecard.loc['RSQ'] = r2_score(df.y_true,df.y_pred)
+        scorecard.loc['RSQ'] = r2_score(df.y_true, df.y_pred)
-        scorecard.loc['MAE'] = mean_absolute_error(df.y_true,df.y_pred)
+        scorecard.loc['MAE'] = mean_absolute_error(df.y_true, df.y_pred)
     elif method == 'classification':
         scorecard.loc['RSQ'] = 0.
         scorecard.loc['MAE Matrix'] = 0.
-    scorecard.loc['directional_accuracy'] = df.is_correct.sum()*1. / (df.is_predicted.sum()*1.)*100
+    sign_true = df.sign_true.astype(int)
+    sign_pred = df.sign_pred.astype(int)
+    
+    scorecard.loc['sharpe'] = sharpe(df.result)
+    scorecard.loc['sortino'] = sortino(df.result)
+    scorecard.loc['skew'] = skew(df.result)
+
+    scorecard.loc['accuracy'] = accuracy_score(sign_true, sign_pred) * 100
+    scorecard.loc['recall'] = recall_score(sign_true, sign_pred, labels = [1, -1])
+    scorecard.loc['precision'] = precision_score(sign_true, sign_pred, labels = [1, -1])
+    scorecard.loc['f1_score'] = f1_score(sign_true, sign_pred, labels = [1, -1])
     scorecard.loc['edge'] = df.result.mean()
     scorecard.loc['noise'] = df.y_pred.diff().abs().mean()
     scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise']
+
+
     if method == 'regression':
         scorecard.loc['edge_to_mae'] = scorecard.loc['edge'] / scorecard.loc['MAE']
     elif method == 'classification':
         scorecard.loc['edge_to_mae'] = 0.
 
-    # TODO: add confusion matrix, f1 score, precision, recall
+    scorecard = scorecard.round(3)
     print("Model name: ", model_name)
     print(scorecard)
     return scorecard  

utils/walk_forward.py

@@ -38,6 +38,11 @@ def walk_forward_train_test(
         models[window_end] = current_model
 
         next_timestep = X.iloc[window_end+1].to_numpy().reshape(1, -1)
-        predictions[window_end+1] = current_model.predict(next_timestep).item()
+        prediction = current_model.predict(next_timestep).item()
+        if prediction == 0.:
+            # TODO: we shouldn't feed in zeros to the model, and skip training / predicting when everything is 0
+            # print("Warning: model predicted 0., overriding it with 0.0001")
+            prediction = 0.0001
+        predictions[window_end+1] = prediction
 
     return models, predictions