mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-28 11:17:47 +00:00
Mark Aron Szulyovszky
133 lines
4.5 KiB
Python
133 lines
4.5 KiB
Python
from typing import Literal, Callable
|
|
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
|
|
from quantstats.stats import skew, sortino
|
|
from utils.metrics import probabilistic_sharpe_ratio, sharpe_ratio
|
|
from utils.helpers import get_first_valid_return_index
|
|
import pandas as pd
|
|
import numpy as np
|
|
from data_loader.types import ForwardReturnSeries, ySeries
|
|
from training.types import Stats, WeightsSeries
|
|
|
|
|
|
def backtest(
|
|
returns: pd.Series, signal: pd.Series, transaction_cost=0.002
|
|
) -> pd.Series:
|
|
delta_pos = signal.diff(1).abs().fillna(0.0)
|
|
costs = transaction_cost * delta_pos
|
|
return (signal * returns) - costs
|
|
|
|
|
|
def __preprocess(
|
|
forward_returns: ForwardReturnSeries,
|
|
y_pred: pd.Series,
|
|
y_true: pd.Series,
|
|
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
|
|
discretize: bool,
|
|
) -> pd.DataFrame:
|
|
y_pred.name = "y_pred"
|
|
forward_returns.name = "forward_returns"
|
|
df = pd.concat([y_pred, forward_returns], axis=1).dropna()
|
|
|
|
discretize_func = get_discretize_function(no_of_classes)
|
|
# make sure that we evaluate binary/three-way predictions even if the model is a regression
|
|
df["sign_pred"] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
|
|
df["sign_true"] = y_true
|
|
|
|
df["result"] = backtest(df.forward_returns, df.sign_pred)
|
|
|
|
return df
|
|
|
|
|
|
def evaluate_predictions(
|
|
forward_returns: ForwardReturnSeries,
|
|
y_pred: WeightsSeries,
|
|
y_true: ySeries,
|
|
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
|
|
discretize: bool = False,
|
|
) -> Stats:
|
|
# ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
|
|
evaluate_from = max(
|
|
get_first_valid_return_index(forward_returns),
|
|
get_first_valid_return_index(y_pred),
|
|
)
|
|
|
|
forward_returns = pd.Series(forward_returns[evaluate_from:])
|
|
y_pred = pd.Series(y_pred[evaluate_from:])
|
|
|
|
df = __preprocess(forward_returns, y_pred, y_true, no_of_classes, discretize)
|
|
|
|
scorecard = dict()
|
|
|
|
def count_non_zero(series: pd.Series) -> int:
|
|
return len(series[series != 0])
|
|
|
|
no_of_samples = count_non_zero(df.y_pred)
|
|
scorecard["no_of_samples"] = no_of_samples
|
|
sharpe = sharpe_ratio(df.result + 1e-20)
|
|
scorecard["sharpe"] = sharpe
|
|
benchmark_sharpe = sharpe_ratio(df.forward_returns)
|
|
scorecard["benchmark_sharpe"] = benchmark_sharpe
|
|
scorecard["prob_sharpe"] = probabilistic_sharpe_ratio(
|
|
sharpe, benchmark_sharpe, no_of_samples
|
|
)
|
|
scorecard["sortino"] = sortino(df.result)
|
|
scorecard["skew"] = skew(df.result)
|
|
|
|
labels = [1, -1] if no_of_classes == "two" else [1, -1, 0]
|
|
avg_type = "weighted" if no_of_classes == "two" else "macro"
|
|
|
|
if discretize == True:
|
|
scorecard["accuracy"] = accuracy_score(df.sign_true, df.sign_pred) * 100
|
|
scorecard["recall"] = recall_score(
|
|
df.sign_true, df.sign_pred, labels=labels, average=avg_type
|
|
)
|
|
scorecard["precision"] = precision_score(
|
|
df.sign_true, df.sign_pred, labels=labels, average=avg_type
|
|
)
|
|
scorecard["f1_score"] = f1_score(
|
|
df.sign_true, df.sign_pred, labels=labels, average=avg_type
|
|
)
|
|
scorecard["edge"] = df.result.mean()
|
|
scorecard["noise"] = df.y_pred.diff().abs().mean()
|
|
scorecard["edge_to_noise"] = scorecard["edge"] / (scorecard["noise"] + 0.00001)
|
|
|
|
if discretize == True:
|
|
for index, row in df.sign_true.value_counts().iteritems():
|
|
scorecard["sign_true_ratio_" + str(index)] = row / len(df.sign_true)
|
|
|
|
for index, row in df.sign_pred.value_counts().iteritems():
|
|
scorecard["sign_pred_ratio_" + str(index)] = row / len(df.sign_pred)
|
|
|
|
scorecard = {k: round(float(v), 3) for k, v in scorecard.items()}
|
|
return scorecard
|
|
|
|
|
|
def __discretize_binary(x):
|
|
return 1 if x > 0 else -1
|
|
|
|
|
|
def __discretize_threeway(x):
|
|
return 0 if x == 0 else 1 if x > 0 else -1
|
|
|
|
|
|
def discretize_threeway_threshold(threshold: float) -> Callable:
|
|
def discretize(current_value):
|
|
lower_threshold = -threshold
|
|
upper_threshold = threshold
|
|
if np.isnan(current_value):
|
|
return np.nan
|
|
elif current_value <= lower_threshold:
|
|
return -1
|
|
elif current_value > lower_threshold and current_value < upper_threshold:
|
|
return 0
|
|
else:
|
|
return 1
|
|
|
|
return discretize
|
|
|
|
|
|
def get_discretize_function(
|
|
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"]
|
|
) -> Callable:
|
|
return __discretize_binary if no_of_classes == "two" else __discretize_threeway
|