Refactor(Training): new outcome types, representative pipeline steps, bet-sizing (#187)

* refactor(Training): added InferenceResult & TrainedModel types

* refactor(Pipeline): introduced TrainingOutcome, BetSizingWithMetaOutcome, etc.

* fix(Pipeline): getting it to compile

* refactor(WalkForward): separate preprocessing step

* feat(Pipeline): separate out transformations processing step

* refactor(Pipeline): use the Directional model terminology, put bet_sizing into pipeline instead of hiding it in a step

* refactor(WalkForward): moved functions to separate folder

* fix(WalkForward): use sparse array to store models, process transformations in parallel (lot faster)

* fix(Tests): and evaluation

* fix(Tests): for realz

* fix(Inference): preloading everything now, renamed primary models to directional models

* fix(BetSizing): was running transformations on the wrong data, oops

* fix(BetSizing): concatenated on the wrong axis accidentally

* fix(Reporting): able to use the new Stats type

* fix(BetSizing): renamed int column names

* fix(Portfolio): name the column properly

* fix(Reporting): rename the correct Series, lol

* fix(Inference): walk_forwad_inference() can deal with models not being aligned with the starting index

* fix(WalkForward): accidentally using the wrong index

* fix(WalkForward): use the correct indicies to fetch last model/transformations

* fix(CI): changed the name of the results

utils/evaluate.py

@@ -5,6 +5,8 @@ from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio
 from utils.helpers import get_first_valid_return_index
 import pandas as pd
 import numpy as np
+from data_loader.types import ForwardReturnSeries, ySeries
+from training.types import Stats, WeightsSeries
 
 def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.002) -> pd.Series:
     delta_pos = signal.diff(1).abs().fillna(0.)
@@ -12,7 +14,7 @@ def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.002) ->
     return (signal * returns) - costs
 
 
-def __preprocess(forward_returns: pd.Series, y_pred: pd.Series, y_true: pd.Series, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame:
+def __preprocess(forward_returns: ForwardReturnSeries, y_pred: pd.Series, y_true: pd.Series, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame:
     y_pred.name = 'y_pred'
     forward_returns.name = 'forward_returns'
     df = pd.concat([y_pred, forward_returns],axis=1).dropna()
@@ -27,14 +29,13 @@ def __preprocess(forward_returns: pd.Series, y_pred: pd.Series, y_true: pd.Serie
     return df
 
 def evaluate_predictions(
-                        model_name: str,
-                        forward_returns: pd.Series,
-                        y_pred: pd.Series,
-                        y_true: pd.Series,
+                        forward_returns: ForwardReturnSeries,
+                        y_pred: WeightsSeries,
+                        y_true: ySeries,
                         no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
                         print_results: bool,
                         discretize: bool = False,
-                        ) -> pd.Series:
+                        ) -> Stats:
     # ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
     evaluate_from = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(y_pred))
     
@@ -43,42 +44,41 @@ def evaluate_predictions(
 
     df = __preprocess(forward_returns, y_pred, y_true, no_of_classes, discretize)
     
-    scorecard = pd.Series()
+    scorecard = dict()
     
     def count_non_zero(series: pd.Series) -> int:
         return len(series[series != 0])
     no_of_samples = count_non_zero(df.y_pred)
-    scorecard.loc['no_of_samples'] = no_of_samples
+    scorecard['no_of_samples'] = no_of_samples
     sharpe = sharpe_ratio(df.result)
-    scorecard.loc['sharpe'] = sharpe
+    scorecard['sharpe'] = sharpe
     benchmark_sharpe = sharpe_ratio(df.forward_returns)
-    scorecard.loc['benchmark_sharpe'] = benchmark_sharpe
-    scorecard.loc['prob_sharpe'] = probabilistic_sharpe_ratio(sharpe, benchmark_sharpe, no_of_samples)
-    scorecard.loc['sortino'] = sortino(df.result)
-    scorecard.loc['skew'] = skew(df.result)
+    scorecard['benchmark_sharpe'] = benchmark_sharpe
+    scorecard['prob_sharpe'] = probabilistic_sharpe_ratio(sharpe, benchmark_sharpe, no_of_samples)
+    scorecard['sortino'] = sortino(df.result)
+    scorecard['skew'] = skew(df.result)
 
     labels = [1, -1] if no_of_classes == 'two' else [1, -1, 0]
     avg_type = 'weighted' if no_of_classes == 'two' else 'macro'
 
     if discretize == True:
-        scorecard.loc['accuracy'] = accuracy_score(df.sign_true, df.sign_pred) * 100
-        scorecard.loc['recall'] = recall_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
-        scorecard.loc['precision'] = precision_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
-        scorecard.loc['f1_score'] = f1_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
-    scorecard.loc['edge'] = df.result.mean()
-    scorecard.loc['noise'] = df.y_pred.diff().abs().mean()
-    scorecard.loc['edge_to_noise'] = scorecard.loc['edge'] / scorecard.loc['noise']
+        scorecard['accuracy'] = accuracy_score(df.sign_true, df.sign_pred) * 100
+        scorecard['recall'] = recall_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
+        scorecard['precision'] = precision_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
+        scorecard['f1_score'] = f1_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
+    scorecard['edge'] = df.result.mean()
+    scorecard['noise'] = df.y_pred.diff().abs().mean()
+    scorecard['edge_to_noise'] = scorecard['edge'] / (scorecard['noise'] + 0.00001)
     
     if discretize == True:
         for index, row in df.sign_true.value_counts().iteritems():
-            scorecard.loc['sign_true_ratio_' + str(index)] = row / len(df.sign_true)
+            scorecard['sign_true_ratio_' + str(index)] = row / len(df.sign_true)
         
         for index, row in df.sign_pred.value_counts().iteritems():
-            scorecard.loc['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred)
+            scorecard['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred)
 
-    scorecard = scorecard.round(3)
+    scorecard = {k: round(float(v), 3) for k, v in scorecard.items()}
     if print_results:        
-        print("Model name: ", model_name)
         print(scorecard)
     return scorecard  
 

utils/helpers.py

@@ -3,6 +3,7 @@ import numpy as np
 import os
 import string
 import random
+from itertools import dropwhile
 
 def get_files_from_dir(path: str) -> list[str]:
     return [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
@@ -16,6 +17,8 @@ def get_first_valid_return_index(series: pd.Series) -> int:
         return 0
     return nested_result[0]
 
+def get_last_non_na_index(series: pd.Series, index: int) -> int:
+    return next(dropwhile(lambda x: pd.isna(x[1]), enumerate(reversed(series[:index+1]))))[0]
 
 
 def flatten(list_of_lists: list) -> list: