mirror of
https://github.com/webclinic017/drift.git
synced 2026-08-08 00:27:50 +00:00
Refactor(Training): new outcome types, representative pipeline steps, bet-sizing (#187)
* refactor(Training): added InferenceResult & TrainedModel types * refactor(Pipeline): introduced TrainingOutcome, BetSizingWithMetaOutcome, etc. * fix(Pipeline): getting it to compile * refactor(WalkForward): separate preprocessing step * feat(Pipeline): separate out transformations processing step * refactor(Pipeline): use the Directional model terminology, put bet_sizing into pipeline instead of hiding it in a step * refactor(WalkForward): moved functions to separate folder * fix(WalkForward): use sparse array to store models, process transformations in parallel (lot faster) * fix(Tests): and evaluation * fix(Tests): for realz * fix(Inference): preloading everything now, renamed primary models to directional models * fix(BetSizing): was running transformations on the wrong data, oops * fix(BetSizing): concatenated on the wrong axis accidentally * fix(Reporting): able to use the new Stats type * fix(BetSizing): renamed int column names * fix(Portfolio): name the column properly * fix(Reporting): rename the correct Series, lol * fix(Inference): walk_forwad_inference() can deal with models not being aligned with the starting index * fix(WalkForward): accidentally using the wrong index * fix(WalkForward): use the correct indicies to fetch last model/transformations * fix(CI): changed the name of the results
This commit is contained in:
Mark Aron Szulyovszky
committed by
GitHub
GitHub
parent
42a1bc59cb
commit
3eb3ea94e3
+14
-24
@@ -2,39 +2,29 @@ from reporting.wandb import send_report_to_wandb
|
||||
import pandas as pd
|
||||
from utils.helpers import weighted_average
|
||||
from config.types import Config
|
||||
from training.types import WeightsSeries, Stats
|
||||
|
||||
def report_results(results:pd.DataFrame, all_predictions:pd.DataFrame, config: Config, wandb, sweep: bool, project_name:str):
|
||||
|
||||
primary_results = results[[column for column in results.columns if 'ensemble' not in column]]
|
||||
ensemble_results = results[[column for column in results.columns if 'ensemble' in column]]
|
||||
def report_results(directional_stats: list[Stats], output_stats: Stats, output_weights: WeightsSeries, config: Config, wandb, sweep: bool):
|
||||
|
||||
# Only send the results of the final model to wandb
|
||||
results_to_send = ensemble_results if ensemble_results.shape[1] > 0 else primary_results
|
||||
send_report_to_wandb(results_to_send, wandb)
|
||||
results.to_csv('output/results.csv')
|
||||
send_report_to_wandb(output_stats, wandb)
|
||||
pd.Series(output_stats).to_csv('output/results.csv')
|
||||
|
||||
primary_weights = all_predictions[[column for column in all_predictions.columns if 'ensemble' not in column]]
|
||||
ensemble_weights = all_predictions[[column for column in all_predictions.columns if 'ensemble' in column]]
|
||||
predictions_to_save = ensemble_weights if ensemble_weights.shape[1] > 0 else primary_weights
|
||||
predictions_to_save.to_csv('output/predictions.csv')
|
||||
output_weights.rename(config.target_asset[1]).to_csv('output/predictions.csv')
|
||||
|
||||
print("\n--------\n")
|
||||
all_avg_results = weighted_average(results, 'no_of_samples')
|
||||
primary_avg_results = weighted_average(primary_results, 'no_of_samples')
|
||||
ensemble_avg_results = weighted_average(ensemble_results, 'no_of_samples')
|
||||
directional_avg_stats = weighted_average(pd.concat([pd.Series(stat) for stat in directional_stats], axis = 1), 'no_of_samples')
|
||||
|
||||
print("Benchmark buy-and-hold sharpe: ", round(all_avg_results.loc['benchmark_sharpe'], 3))
|
||||
print("Benchmark buy-and-hold sharpe: ", output_stats['benchmark_sharpe'])
|
||||
|
||||
print("Level-1: Number of samples evaluated: ", primary_results.loc['no_of_samples'].sum())
|
||||
print("Mean Sharpe ratio for Level-1 models: ", round(primary_avg_results.loc['sharpe'], 3))
|
||||
print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(primary_avg_results.loc['prob_sharpe'].mean(), 3))
|
||||
print("Level-1: Number of samples evaluated: ", directional_avg_stats.loc['no_of_samples'].sum())
|
||||
print("Mean Sharpe ratio for Level-1 models: ", round(directional_avg_stats.loc['sharpe'], 3))
|
||||
print("Mean Probabilistic Sharpe ratio for Level-1 models: ", round(directional_avg_stats.loc['prob_sharpe'].mean(), 3))
|
||||
|
||||
if len(config.meta_labeling_models) > 0:
|
||||
print("Level-2 (Ensemble): Number of samples evaluated: ", ensemble_results.loc['no_of_samples'].sum())
|
||||
print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_avg_results.loc['sharpe'].mean(), 3))
|
||||
print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", round(ensemble_avg_results.loc['prob_sharpe'].mean(), 3))
|
||||
|
||||
ensemble_avg_results.to_csv('output/results_level2.csv')
|
||||
if len(config.meta_models) > 0:
|
||||
print("Level-2 (Ensemble): Number of samples evaluated: ", output_stats['no_of_samples'])
|
||||
print("Mean Sharpe ratio for Level-2 (Ensemble) models: ", output_stats['sharpe'])
|
||||
print("Mean Probabilistic Sharpe ratio for Level-2 (Ensemble) models: ", output_stats['prob_sharpe'])
|
||||
|
||||
if sweep:
|
||||
if wandb.run is not None:
|
||||
|
||||
+7
-9
@@ -1,17 +1,15 @@
|
||||
import pickle
|
||||
import datetime
|
||||
from config.types import Config
|
||||
from typing import Optional, Union
|
||||
from typing import Optional
|
||||
import os
|
||||
import warnings
|
||||
from reporting.types import Reporting
|
||||
from training.types import PipelineOutcome
|
||||
|
||||
|
||||
|
||||
def save_models(all_models: Reporting.Asset, config: Config) -> None:
|
||||
def save_models(pipeline_outcome: PipelineOutcome, config: Config) -> None:
|
||||
dict_for_pickle = dict()
|
||||
dict_for_pickle['config'] = config
|
||||
dict_for_pickle['all_models'] = all_models
|
||||
dict_for_pickle['pipeline_outcome'] = pipeline_outcome
|
||||
|
||||
date_string = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")
|
||||
|
||||
@@ -22,7 +20,7 @@ def save_models(all_models: Reporting.Asset, config: Config) -> None:
|
||||
pickle.dump( dict_for_pickle, open( "output/models/{}.p".format(date_string), "wb" ) )
|
||||
|
||||
|
||||
def load_models(file_name:Union[str, None]) -> tuple[Reporting.Asset, Config]:
|
||||
def load_models(file_name: Optional[str]) -> tuple[PipelineOutcome, Config]:
|
||||
|
||||
if file_name is None:
|
||||
warnings.warn("No file name provided, will load latest models and configurations.")
|
||||
@@ -34,7 +32,7 @@ def load_models(file_name:Union[str, None]) -> tuple[Reporting.Asset, Config]:
|
||||
packacked_dict = pickle.load( open( "output/models/{}".format(file_name), "rb" ) )
|
||||
|
||||
config = packacked_dict.pop("config", None)
|
||||
all_models = packacked_dict.pop("all_models", None)
|
||||
pipeline_outcome = packacked_dict.pop("pipeline_outcome", None)
|
||||
|
||||
return all_models, config
|
||||
return pipeline_outcome, config
|
||||
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class Reporting:
|
||||
def __init__(self):
|
||||
self.results: pd.DataFrame = pd.DataFrame()
|
||||
self.all_predictions: pd.DataFrame = pd.DataFrame()
|
||||
self.all_probabilities: pd.DataFrame = pd.DataFrame()
|
||||
self.asset: Reporting.Asset
|
||||
|
||||
def get_results(self) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, Reporting.Asset]:
|
||||
return self.results, self.all_predictions, self.all_probabilities, self.asset
|
||||
|
||||
@dataclass
|
||||
class Single_Model:
|
||||
model_name: str
|
||||
model_over_time: pd.Series
|
||||
transformations_over_time: list[pd.Series]
|
||||
|
||||
|
||||
class Training_Step:
|
||||
def __init__(self, level: str):
|
||||
self.level: str = level
|
||||
self.base: list[Reporting.Single_Model] = []
|
||||
self.metalabeling: list[list[Reporting.Single_Model]] = []
|
||||
|
||||
def get_base(self) -> list[tuple[str, pd.Series, list[pd.Series]]]:
|
||||
return [(x.model_name, x.model_over_time, x.transformations_over_time) for x in self.base ]
|
||||
|
||||
def get_metalabeling(self) -> dict:
|
||||
structured_dict = dict()
|
||||
for i, model in enumerate(self.base):
|
||||
structured_dict[model.model_name] = [(x.model_name, x.model_over_time, x.transformations_over_time) for x in self.metalabeling[i]]
|
||||
return structured_dict
|
||||
|
||||
@dataclass
|
||||
class Asset():
|
||||
name: str
|
||||
primary: Reporting.Training_Step
|
||||
secondary: Reporting.Training_Step
|
||||
|
||||
+3
-3
@@ -2,6 +2,7 @@ import pandas as pd
|
||||
from config.types import RawConfig
|
||||
from typing import Optional
|
||||
from utils.helpers import weighted_average
|
||||
from training.types import Stats
|
||||
|
||||
def launch_wandb(project_name:str, default_config: RawConfig, sweep:bool=False) -> Optional[object]:
|
||||
from wandb_setup import get_wandb
|
||||
@@ -28,14 +29,13 @@ def override_config_with_wandb_values(wandb: Optional[object], raw_config: RawCo
|
||||
|
||||
return RawConfig(**config_dict)
|
||||
|
||||
def send_report_to_wandb(results: pd.DataFrame, wandb:Optional[object]):
|
||||
def send_report_to_wandb(stats: Stats, wandb:Optional[object]):
|
||||
if wandb is None: return
|
||||
|
||||
run = wandb.run
|
||||
run.save()
|
||||
|
||||
mean_results = weighted_average(results, 'no_of_samples')
|
||||
for key, value in mean_results.iteritems():
|
||||
for key, value in stats.items():
|
||||
run.log({ key: value })
|
||||
|
||||
run.finish()
|
||||
|
||||
Reference in New Issue
Block a user