gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-09 09:07:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(Linter): reformatted code with black (#211)

Browse Source 
* chore(Linter): reformatted code with black

* Create black.yaml
This commit is contained in:
Mark Aron Szulyovszky
2022-02-17 19:22:17 +01:00
committed by GitHub
parent f3fee4a4e1
commit 8dd2d88740
101 changed files with 2596 additions and 2320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/evaluate.py
+78 -48
View File
@@ -8,95 +8,125 @@ import numpy as np
from data_loader.types import ForwardReturnSeries, ySeries
from training.types import Stats, WeightsSeries
def backtest(returns: pd.Series, signal: pd.Series, transaction_cost = 0.002) -> pd.Series:
delta_pos = signal.diff(1).abs().fillna(0.)
def backtest(
returns: pd.Series, signal: pd.Series, transaction_cost=0.002
) -> pd.Series:
delta_pos = signal.diff(1).abs().fillna(0.0)
costs = transaction_cost * delta_pos
return (signal * returns) - costs
def __preprocess(forward_returns: ForwardReturnSeries, y_pred: pd.Series, y_true: pd.Series, no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'], discretize: bool) -> pd.DataFrame:
y_pred.name = 'y_pred'
forward_returns.name = 'forward_returns'
df = pd.concat([y_pred, forward_returns],axis=1).dropna()
def __preprocess(
forward_returns: ForwardReturnSeries,
y_pred: pd.Series,
y_true: pd.Series,
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
discretize: bool,
) -> pd.DataFrame:
y_pred.name = "y_pred"
forward_returns.name = "forward_returns"
df = pd.concat([y_pred, forward_returns], axis=1).dropna()
discretize_func = get_discretize_function(no_of_classes)
# make sure that we evaluate binary/three-way predictions even if the model is a regression
df['sign_pred'] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
df['sign_true'] = y_true
df["sign_pred"] = df.y_pred.apply(discretize_func) if discretize else df.y_pred
df["sign_true"] = y_true
df['result'] = backtest(df.forward_returns, df.sign_pred)
df["result"] = backtest(df.forward_returns, df.sign_pred)
return df
def evaluate_predictions(
forward_returns: ForwardReturnSeries,
y_pred: WeightsSeries,
y_true: ySeries,
no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
discretize: bool = False,
) -> Stats:
forward_returns: ForwardReturnSeries,
y_pred: WeightsSeries,
y_true: ySeries,
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"],
discretize: bool = False,
) -> Stats:
# ignore the predictions until we see a non-zero returns (and definitely skip the first sliding_window_size)
evaluate_from = max(get_first_valid_return_index(forward_returns), get_first_valid_return_index(y_pred))
evaluate_from = max(
get_first_valid_return_index(forward_returns),
get_first_valid_return_index(y_pred),
)
forward_returns = pd.Series(forward_returns[evaluate_from:])
y_pred = pd.Series(y_pred[evaluate_from:])
df = __preprocess(forward_returns, y_pred, y_true, no_of_classes, discretize)
scorecard = dict()
def count_non_zero(series: pd.Series) -> int:
return len(series[series != 0])
no_of_samples = count_non_zero(df.y_pred)
scorecard['no_of_samples'] = no_of_samples
sharpe = sharpe_ratio(df.result + 1e-20)
scorecard['sharpe'] = sharpe
benchmark_sharpe = sharpe_ratio(df.forward_returns)
scorecard['benchmark_sharpe'] = benchmark_sharpe
scorecard['prob_sharpe'] = probabilistic_sharpe_ratio(sharpe, benchmark_sharpe, no_of_samples)
scorecard['sortino'] = sortino(df.result)
scorecard['skew'] = skew(df.result)
labels = [1, -1] if no_of_classes == 'two' else [1, -1, 0]
avg_type = 'weighted' if no_of_classes == 'two' else 'macro'
no_of_samples = count_non_zero(df.y_pred)
scorecard["no_of_samples"] = no_of_samples
sharpe = sharpe_ratio(df.result + 1e-20)
scorecard["sharpe"] = sharpe
benchmark_sharpe = sharpe_ratio(df.forward_returns)
scorecard["benchmark_sharpe"] = benchmark_sharpe
scorecard["prob_sharpe"] = probabilistic_sharpe_ratio(
sharpe, benchmark_sharpe, no_of_samples
)
scorecard["sortino"] = sortino(df.result)
scorecard["skew"] = skew(df.result)
labels = [1, -1] if no_of_classes == "two" else [1, -1, 0]
avg_type = "weighted" if no_of_classes == "two" else "macro"
if discretize == True:
scorecard['accuracy'] = accuracy_score(df.sign_true, df.sign_pred) * 100
scorecard['recall'] = recall_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
scorecard['precision'] = precision_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
scorecard['f1_score'] = f1_score(df.sign_true, df.sign_pred, labels = labels, average=avg_type)
scorecard['edge'] = df.result.mean()
scorecard['noise'] = df.y_pred.diff().abs().mean()
scorecard['edge_to_noise'] = scorecard['edge'] / (scorecard['noise'] + 0.00001)
scorecard["accuracy"] = accuracy_score(df.sign_true, df.sign_pred) * 100
scorecard["recall"] = recall_score(
df.sign_true, df.sign_pred, labels=labels, average=avg_type
)
scorecard["precision"] = precision_score(
df.sign_true, df.sign_pred, labels=labels, average=avg_type
)
scorecard["f1_score"] = f1_score(
df.sign_true, df.sign_pred, labels=labels, average=avg_type
)
scorecard["edge"] = df.result.mean()
scorecard["noise"] = df.y_pred.diff().abs().mean()
scorecard["edge_to_noise"] = scorecard["edge"] / (scorecard["noise"] + 0.00001)
if discretize == True:
for index, row in df.sign_true.value_counts().iteritems():
scorecard['sign_true_ratio_' + str(index)] = row / len(df.sign_true)
scorecard["sign_true_ratio_" + str(index)] = row / len(df.sign_true)
for index, row in df.sign_pred.value_counts().iteritems():
scorecard['sign_pred_ratio_' + str(index)] = row / len(df.sign_pred)
scorecard["sign_pred_ratio_" + str(index)] = row / len(df.sign_pred)
scorecard = {k: round(float(v), 3) for k, v in scorecard.items()}
return scorecard
return scorecard
def __discretize_binary(x):
return 1 if x > 0 else -1
def __discretize_threeway(x):
return 0 if x == 0 else 1 if x > 0 else -1
def __discretize_binary(x): return 1 if x > 0 else -1
def __discretize_threeway(x): return 0 if x == 0 else 1 if x > 0 else -1
def discretize_threeway_threshold(threshold: float) -> Callable:
def discretize(current_value):
lower_threshold = -threshold
upper_threshold = threshold
if np.isnan(current_value):
return np.nan
return np.nan
elif current_value <= lower_threshold:
return -1
elif current_value > lower_threshold and current_value < upper_threshold:
return 0
else:
return 1
return discretize
def get_discretize_function(no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced']) -> Callable:
return __discretize_binary if no_of_classes == 'two' else __discretize_threeway
def get_discretize_function(
no_of_classes: Literal["two", "three-balanced", "three-imbalanced"]
) -> Callable:
return __discretize_binary if no_of_classes == "two" else __discretize_threeway