gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-09 00:57:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Project): use SKLearn models directly, removed custom ensembling, use 5 minute data, batch inference, numba cusum filter (#192)

Browse Source 
* feat(Project): use 5 minute data, running training in parallel, sped up cusum filter by 10x with numba

* fix(WalkForward): inference mini-batch parallelization

* fix(WalkForward): don't use the parallel version of any of the functions

* feat(CI): download the data required

* fix(Project): 5min_crypto folder added

* fix(Evaluate): make sure we have numerical stability in returns

* feat(Models): use SKLearn models directly to enable composability

* feat(Inference): batched inference now working, added forecasting_horizon

* fix(Inference): works again

* fix(Inference)

* chore(Models): remove unused Ensemble model

* fix(Labeller): don't just forward shift returns, also take the sum of the data happened until then

* Update test.yml
This commit is contained in:
Mark Aron Szulyovszky
2022-02-17 16:36:35 +01:00
committed by GitHub
parent 5c94af8b01
commit 9d47ee942d
52 changed files with 470 additions and 628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
labeling/event_filters/cusum.py
+26 -17
View File
@@ -1,8 +1,11 @@
from numpy import float32
from ..types import EventFilter
from data_loader.types import ReturnSeries
import pandas as pd
from numba import njit
from numba.typed import List
class CUSUMVolatilityEventFilter(EventFilter):
@@ -11,7 +14,7 @@ class CUSUMVolatilityEventFilter(EventFilter):
def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex:
rolling_vol = returns.rolling(self.vol_period).std() * 0.15
rolling_vol = returns.rolling(self.vol_period).std().mean()
filtered_indices = []
pos_threshold = 0
@@ -39,22 +42,28 @@ class CUSUMFixedEventFilter(EventFilter):
self.threshold = threshold
def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex:
filtered_indices = []
pos_threshold = 0
neg_threshold = 0
diff = returns.diff()
for index in diff.index[1:]:
pos_threshold, neg_threshold = (
max(0, pos_threshold + diff.loc[index]),
min(0, neg_threshold + diff.loc[index]),
)
diffed_returns = returns.diff()
int_indicies = _process(List(diffed_returns.to_list()), abs(returns.mean()) * self.threshold)
if neg_threshold < -self.threshold:
neg_threshold = 0
filtered_indices.append(index)
return pd.DatetimeIndex([returns.index[i] for i in int_indicies])
elif pos_threshold > self.threshold:
pos_threshold = 0
filtered_indices.append(index)
@njit
def _process(diffed_returns: List, threshold: float32) -> List:
pos_threshold: float32 = 0.0 # type: ignore
neg_threshold: float32 = 0.0 # type: ignore
filtered_indicies = List()
for index in range(1, len(diffed_returns[1:])):
pos_threshold, neg_threshold = ( # type: ignore
max(0, pos_threshold + diffed_returns[index]), # type: ignore
min(0, neg_threshold + diffed_returns[index]), # type: ignore
)
return pd.DatetimeIndex(filtered_indices)
if neg_threshold < -threshold:
neg_threshold = 0.0 # type: ignore
filtered_indicies.append(index)
elif pos_threshold > threshold:
pos_threshold = 0.0 # type: ignore
filtered_indicies.append(index)
return filtered_indicies
labeling/eventfilters_map.py
+1 -1
View File
@@ -4,5 +4,5 @@ from .event_filters.cusum import CUSUMVolatilityEventFilter, CUSUMFixedEventFilt
eventfilters_map = dict(
none = NoEventFilter(),
cusum_vol = CUSUMVolatilityEventFilter(vol_period = 20),
cusum_fixed = CUSUMFixedEventFilter(threshold = 0.05)
cusum_fixed = CUSUMFixedEventFilter(threshold = 500)
)
labeling/labellers/fixed_time_three_class_balanced.py
+9 -5
View File
@@ -1,16 +1,20 @@
from data_loader.types import ForwardReturnSeries
from data_loader.types import ReturnSeries, ForwardReturnSeries
from ..types import EventLabeller, EventsDataFrame
import pandas as pd
from .utils import create_forward_returns
class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
time_horizon: int
def __init__(self, time_horizon: int = 1):
def __init__(self, time_horizon: int):
self.time_horizon = time_horizon
def label_events(self, event_start_times: pd.DatetimeIndex, forward_returns: ForwardReturnSeries) -> EventsDataFrame:
def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
event_start_times[event_start_times < cutoff_point]
event_candidates = forward_returns[event_start_times]
def get_bins_threeway(x):
@@ -35,10 +39,10 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
return 1
labels = event_candidates.map(map_class_threeway)
return pd.DataFrame({
return (pd.DataFrame({
'start': event_start_times,
'end': event_start_times + pd.Timedelta(days=self.time_horizon),
'label': labels,
'returns': forward_returns[event_start_times]
})
}), forward_returns[event_start_times])
labeling/labellers/fixed_time_three_class_imbalanced.py
+9 -5
View File
@@ -1,15 +1,19 @@
from ..types import EventLabeller, EventsDataFrame, ForwardReturnSeries
from ..types import EventLabeller, EventsDataFrame, ReturnSeries, ForwardReturnSeries
import pandas as pd
from .utils import create_forward_returns
class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
time_horizon: int
def __init__(self, time_horizon: int = 1):
def __init__(self, time_horizon: int):
self.time_horizon = time_horizon
def label_events(self, event_start_times: pd.DatetimeIndex, forward_returns: ForwardReturnSeries) -> EventsDataFrame:
def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
event_start_times[event_start_times < cutoff_point]
event_candidates = forward_returns[event_start_times]
def get_bins_threeway(x):
@@ -34,11 +38,11 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
return 1
labels = event_candidates.map(map_class_threeway)
return pd.DataFrame({
return (pd.DataFrame({
'start': event_start_times,
'end': event_start_times + pd.Timedelta(days=self.time_horizon),
'label': labels,
'returns': forward_returns[event_start_times]
})
}), forward_returns[event_start_times])
labeling/labellers/fixed_time_two_class.py
+9 -7
View File
@@ -1,29 +1,31 @@
from ..types import EventLabeller, EventsDataFrame, ForwardReturnSeries
from ..types import EventLabeller, EventsDataFrame, ReturnSeries, ForwardReturnSeries
import pandas as pd
from .utils import create_forward_returns
class FixedTimeHorionTwoClassEventLabeller(EventLabeller):
time_horizon: int
def __init__(self, time_horizon: int = 1):
def __init__(self, time_horizon: int):
self.time_horizon = time_horizon
def label_events(self, event_start_times: pd.DatetimeIndex, forward_returns: ForwardReturnSeries) -> EventsDataFrame:
def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
event_start_times[event_start_times < cutoff_point]
event_candidates = forward_returns[event_start_times]
def get_class_binary(x: float) -> int:
return -1 if x <= 0.0 else 1
labels = event_candidates.map(get_class_binary)
return pd.DataFrame({
return (pd.DataFrame({
'start': event_start_times,
'end': event_start_times + pd.Timedelta(days=self.time_horizon),
'label': labels,
'returns': forward_returns[event_start_times]
})
}), forward_returns[event_start_times])
labeling/labellers/utils.py
+9
View File
@@ -0,0 +1,9 @@
import pandas as pd
from data_loader.types import ForwardReturnSeries
def create_forward_returns(series: pd.Series, period: int) -> ForwardReturnSeries:
assert period > 0
indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=period)
forward_returns = series.rolling(window=indexer).sum()
return forward_returns
labeling/labellers_map.py
+3 -3
View File
@@ -3,7 +3,7 @@ from .labellers.fixed_time_three_class_imbalanced import FixedTimeHorionThreeCla
from .labellers.fixed_time_two_class import FixedTimeHorionTwoClassEventLabeller
labellers_map = dict(
two_class = FixedTimeHorionTwoClassEventLabeller(),
three_class_balanced = FixedTimeHorionThreeClassBalancedEventLabeller(),
three_class_imbalanced = FixedTimeHorionThreeClassImbalancedEventLabeller()
two_class = FixedTimeHorionTwoClassEventLabeller,
three_class_balanced = FixedTimeHorionThreeClassBalancedEventLabeller,
three_class_imbalanced = FixedTimeHorionThreeClassImbalancedEventLabeller
)
labeling/process.py
+14 -12
View File
@@ -2,16 +2,18 @@ from .types import EventFilter, EventLabeller, EventsDataFrame
from data_loader.types import ForwardReturnSeries, XDataFrame, ReturnSeries, ySeries
def label_data(
event_filter: EventFilter,
event_labeller: EventLabeller,
X: XDataFrame,
returns: ReturnSeries,
forward_returns: ForwardReturnSeries) -> tuple[EventsDataFrame, XDataFrame, ySeries, ForwardReturnSeries]:
event_start_times = event_filter.get_event_start_times(returns)
events = event_labeller.label_events(event_start_times, forward_returns)
X = X.filter(items = events.index, axis = 0)
y = events['label']
forward_returns = events['returns']
event_filter: EventFilter,
event_labeller: EventLabeller,
X: XDataFrame,
returns: ReturnSeries) -> tuple[EventsDataFrame, XDataFrame, ySeries, ForwardReturnSeries]:
return events, X, y, forward_returns
event_start_times = event_filter.get_event_start_times(returns)
print("| Filtered out ", (1 - (len(event_start_times) / len(returns))) * 100, "% of timestamps" )
events, forward_returns = event_labeller.label_events(event_start_times, returns)
X = X.filter(items = events.index, axis = 0)
y = events['label']
forward_returns = events['returns']
return events, X, y, forward_returns
labeling/types.py
+1 -1
View File
@@ -23,6 +23,6 @@ EventsDataFrame = DataFrame[EventSchema]
class EventLabeller(ABC):
@abstractmethod
def label_events(self, event_start_times: pd.DatetimeIndex, forward_returns: ForwardReturnSeries) -> EventsDataFrame:
def label_events(self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries) -> tuple[EventsDataFrame, ForwardReturnSeries]:
raise NotImplementedError