gavindiaz/drift
1
0
Fork 0
mirror of https://github.com/webclinic017/drift.git synced 2026-08-22 23:38:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Labeling): purge overlapping events, sort dataframe at loading time (#226)

Browse Source 
* feat(Labeling): purge overlapping events, sort dataframe at loading time

* fix(Linter): ran

* refactor(Labeling): moved purge_overlapping_events one abstraction level higher

* fix(Data): renamed class

* fix(Data): corrected parameter name

* fix(Config): parameters

* fix(Data): fixed path

* fix(Data): uncommented required code

* feat(EventFilters): use vol based CUSUM

* fix(Config): only retrain every 2000 samples

* fix(Config): filter out even more events

* fix(Inference): added remove_overlapping_events

* refactor(Types): simplified type hierarchy
This commit is contained in:
Mark Aron Szulyovszky
2022-03-02 00:26:33 +01:00
committed by GitHub
parent 10a0803c91
commit 75157c6285
17 changed files with 120 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
labeling/event_filters/cusum.py
+3 -2
View File
@@ -7,12 +7,13 @@ from numba.typed import List
class CUSUMVolatilityEventFilter(EventFilter):
def __init__(self, vol_period: int):
def __init__(self, vol_period: int, multiplier: float):
self.vol_period = vol_period
self.multiplier = multiplier
def get_event_start_times(self, returns: ReturnSeries) -> pd.DatetimeIndex:
rolling_vol = returns.rolling(self.vol_period).std().mean()
rolling_vol = returns.rolling(self.vol_period).std() * self.multiplier
filtered_indices = []
pos_threshold = 0
labeling/eventfilters_map.py
+2 -2
View File
@@ -3,6 +3,6 @@ from .event_filters.cusum import CUSUMVolatilityEventFilter, CUSUMFixedEventFilt
eventfilters_map = dict(
none=NoEventFilter(),
cusum_vol=CUSUMVolatilityEventFilter(vol_period=20),
cusum_fixed=CUSUMFixedEventFilter(threshold=70),
cusum_vol=CUSUMVolatilityEventFilter(vol_period=100, multiplier=3.5),
cusum_fixed=CUSUMFixedEventFilter(threshold=20),
)
labeling/labellers/fixed_time_three_class_balanced.py
+10 -13
View File
@@ -1,4 +1,4 @@
from data_loader.types import ReturnSeries, ForwardReturnSeries
from data_loader.types import ReturnSeries
from ..types import EventLabeller, EventsDataFrame
import pandas as pd
from .utils import create_forward_returns
@@ -13,7 +13,7 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
def label_events(
self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
) -> tuple[EventsDataFrame, ForwardReturnSeries]:
) -> EventsDataFrame:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
@@ -43,15 +43,12 @@ class FixedTimeHorionThreeClassBalancedEventLabeller(EventLabeller):
return 1
labels = event_candidates.map(map_class_threeway)
return (
pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(days=self.time_horizon),
"label": labels,
"returns": forward_returns[event_start_times],
}
),
forward_returns[event_start_times],
events = pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(minutes=self.time_horizon * 5),
"label": labels,
"returns": forward_returns[event_start_times],
}
)
return events
labeling/labellers/fixed_time_three_class_imbalanced.py
+10 -13
View File
@@ -1,4 +1,4 @@
from ..types import EventLabeller, EventsDataFrame, ReturnSeries, ForwardReturnSeries
from ..types import EventLabeller, EventsDataFrame, ReturnSeries
import pandas as pd
from .utils import create_forward_returns
@@ -12,7 +12,7 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
def label_events(
self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
) -> tuple[EventsDataFrame, ForwardReturnSeries]:
) -> EventsDataFrame:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
@@ -42,15 +42,12 @@ class FixedTimeHorionThreeClassImbalancedEventLabeller(EventLabeller):
return 1
labels = event_candidates.map(map_class_threeway)
return (
pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(days=self.time_horizon),
"label": labels,
"returns": forward_returns[event_start_times],
}
),
forward_returns[event_start_times],
events = pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(minutes=self.time_horizon * 5),
"label": labels,
"returns": forward_returns[event_start_times],
}
)
return events
labeling/labellers/fixed_time_two_class.py
+10 -13
View File
@@ -1,4 +1,4 @@
from ..types import EventLabeller, EventsDataFrame, ReturnSeries, ForwardReturnSeries
from ..types import EventLabeller, EventsDataFrame, ReturnSeries
import pandas as pd
from .utils import create_forward_returns
@@ -12,7 +12,7 @@ class FixedTimeHorionTwoClassEventLabeller(EventLabeller):
def label_events(
self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
) -> tuple[EventsDataFrame, ForwardReturnSeries]:
) -> EventsDataFrame:
forward_returns = create_forward_returns(returns, self.time_horizon)
cutoff_point = returns.index[-self.time_horizon]
@@ -23,15 +23,12 @@ class FixedTimeHorionTwoClassEventLabeller(EventLabeller):
return -1 if x <= 0.0 else 1
labels = event_candidates.map(get_class_binary)
return (
pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(days=self.time_horizon),
"label": labels,
"returns": forward_returns[event_start_times],
}
),
forward_returns[event_start_times],
events = pd.DataFrame(
{
"start": event_start_times,
"end": event_start_times + pd.Timedelta(minutes=self.time_horizon * 5),
"label": labels,
"returns": forward_returns[event_start_times],
}
)
return events
labeling/labellers/utils.py
+14
View File
@@ -1,5 +1,6 @@
import pandas as pd
from data_loader.types import ForwardReturnSeries
from labeling.types import EventsDataFrame
def create_forward_returns(series: pd.Series, period: int) -> ForwardReturnSeries:
@@ -8,3 +9,16 @@ def create_forward_returns(series: pd.Series, period: int) -> ForwardReturnSerie
forward_returns = series.rolling(window=indexer).sum()
return forward_returns
def purge_overlapping_events(events: EventsDataFrame) -> EventsDataFrame:
events = events.copy()
indicies_to_remove = []
last_event_end = events.iloc[0]["start"]
for index, row in events.iterrows():
if row["start"] < last_event_end:
indicies_to_remove.append(index)
else:
last_event_end = row["end"]
events.drop(indicies_to_remove, inplace=True)
return events
labeling/process.py
+10 -1
View File
@@ -1,5 +1,6 @@
from .types import EventFilter, EventLabeller, EventsDataFrame
from data_loader.types import ForwardReturnSeries, XDataFrame, ReturnSeries, ySeries
from .labellers.utils import purge_overlapping_events
def label_data(
@@ -7,6 +8,7 @@ def label_data(
event_labeller: EventLabeller,
X: XDataFrame,
returns: ReturnSeries,
remove_overlapping_events: bool,
) -> tuple[EventsDataFrame, XDataFrame, ySeries, ForwardReturnSeries]:
event_start_times = event_filter.get_event_start_times(returns)
@@ -16,7 +18,14 @@ def label_data(
"% of timestamps",
)
events, forward_returns = event_labeller.label_events(event_start_times, returns)
events = event_labeller.label_events(event_start_times, returns)
if remove_overlapping_events:
events = purge_overlapping_events(events)
print(
"| Purged ",
(1 - (len(events) / len(event_start_times))) * 100,
"% of overlapping events",
)
X = X.filter(items=events.index, axis=0)
y = events["label"]
labeling/types.py
+1 -1
View File
@@ -25,5 +25,5 @@ class EventLabeller(ABC):
@abstractmethod
def label_events(
self, event_start_times: pd.DatetimeIndex, returns: ReturnSeries
) -> tuple[EventsDataFrame, ForwardReturnSeries]:
) -> EventsDataFrame:
raise NotImplementedError