From b8b7375c303efd64ccb4f271a9e1c5a06e23836c Mon Sep 17 00:00:00 2001
From: Mark Aron Szulyovszky <mark.szulyovszky@gmail.com>
Date: Sat, 18 Dec 2021 22:23:07 +0100
Subject: [PATCH] feat(Tests): added test for evaluate_predictions() (#41)

---
 tests/test_evaluation.py   | 87 ++++++++++++++++++++++++++++++++++++++
 tests/test_walk_forward.py | 45 ++++++++++++++------
 2 files changed, 118 insertions(+), 14 deletions(-)
 create mode 100644 tests/test_evaluation.py

diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py
new file mode 100644
index 0000000..58f80e4
--- /dev/null
+++ b/tests/test_evaluation.py
@@ -0,0 +1,87 @@
+
+import numpy as np
+import pandas as pd
+from training.walk_forward import walk_forward_train_test
+from sklearn.base import BaseEstimator
+from utils.evaluate import evaluate_predictions
+
+no_of_rows = 100
+
+def __generate_even_odd_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]:
+    ''' Test data, where X[n][any_column] == 1 if n is even, else 0
+    '''
+    
+    no_columns = 6
+    X = [[-1 if row % 2 == 0 else 1] * no_columns for row in range(no_of_rows)]
+    assert X[0][0] == -1
+    assert X[1][0] == 1
+    assert X[2][0] == -1
+    assert X[3][0] == 1
+    X = pd.DataFrame(X)
+
+    y = [-1 if (row+1) % 2 == 0 else 1 for row in range(no_of_rows)]
+    assert y[0] == 1
+    assert y[1] == -1
+    assert y[2] == 1
+    assert y[3] == -1
+
+    y = pd.Series(y)
+
+    return X, y
+
+class EvenOddStubModel(BaseEstimator):
+    '''
+    A deteministic model that can predict the future with 100% accuracy
+    It verifies that the X[n][any_column] == 1 if n is even,
+    '''
+
+    def __init__(self, window_length) -> None:
+        super().__init__()
+        self.window_length = window_length
+
+    def fit(self, X, y):
+        assert len(X) == self.window_length
+        for i in range(len(X)):
+            assert y[i] == -1 if X[i][0] == 1 else 1
+
+    def predict(self, X):
+        return np.array([-1 if X[0][0] == 1 else 1])
+
+
+def test_evaluation():
+    X, y = __generate_even_odd_test_data(no_of_rows)
+    
+    window_length = 10
+
+    model = EvenOddStubModel(window_length = window_length)
+    scaler = None
+    
+    models, predictions = walk_forward_train_test(
+        model_name='test',
+        model=model,
+        X=X,
+        y=y,
+        target_returns=y,
+        window_size=window_length,
+        retrain_every=10,
+        scaler=scaler
+    )
+    
+    # verify if predictions are the same as y
+    for i in range(window_length+2, no_of_rows):
+        assert predictions[i] == y[i]
+
+    fake_target_returns = y * 0.1
+    processed_predictions_to_match_returns = predictions * 0.1
+
+    result = evaluate_predictions(
+        model_name='test',
+        target_returns=fake_target_returns,
+        y_pred=processed_predictions_to_match_returns,
+        method='classification'
+    )
+
+    assert result['accuracy'] == 100.0
+    
+
+    
diff --git a/tests/test_walk_forward.py b/tests/test_walk_forward.py
index eee5791..f8d601d 100644
--- a/tests/test_walk_forward.py
+++ b/tests/test_walk_forward.py
@@ -5,7 +5,10 @@ from sklearn.base import BaseEstimator
 
 no_of_rows = 100
 
-def __generate_test_data(no_of_rows):
+def __generate_incremental_test_data(no_of_rows) -> tuple[pd.DataFrame, pd.Series]:
+    ''' Test data, where X[n][any_column] == y[n]+1
+    '''
+    
     no_columns = 6
     X = [[row] * no_columns for row in range(no_of_rows)]
     assert X[0][0] == 0
@@ -23,22 +26,34 @@ def __generate_test_data(no_of_rows):
     return X, y
 
 
+
+class IncrementingStubModel(BaseEstimator):
+    '''
+    A deteministic model that can predict the future with 100% accuracy
+    It verifies that the X[n][any_column]+1 == y[n]
+    '''
+
+    def __init__(self, window_length) -> None:
+        super().__init__()
+        self.window_length = window_length
+
+    def fit(self, X, y):
+        assert len(X) == self.window_length
+        for i in range(len(X)):
+            assert X[i][0] + 1 == y[i]
+
+    def predict(self, X):
+        return np.array([X[0][0] + 1])
+
+
 def test_walk_forward_train_test():
-    X, y = __generate_test_data(no_of_rows)
+    X, y = __generate_incremental_test_data(no_of_rows)
 
     window_length = 10
-    class StubModel(BaseEstimator):
 
-        def fit(self, X, y):
-            assert len(X) == window_length
-            for i in range(len(X)):
-                assert X[i][0] + 1 == y[i]
-
-        def predict(self, X):
-            return np.array([X[0][0] + 1])
-
-    model = StubModel()
+    model = IncrementingStubModel(window_length = window_length)
     scaler = None
+
     models, predictions = walk_forward_train_test(
         model_name='test',
         model=model,
@@ -48,6 +63,8 @@ def test_walk_forward_train_test():
         window_size=window_length,
         retrain_every=10,
         scaler=scaler)
-    for i in range(window_length, no_of_rows):
-        predictions[i] == y[i]
     
+    # verify if predictions are the same as y
+    for i in range(window_length+2, no_of_rows):
+        assert predictions[i] == y[i]
+