fix(DataLoader): sort the data before merging (this corrupted the pipeline completely)

data_loader/load.py

@@ -6,7 +6,7 @@ from feature_extractors.types import FeatureExtractor
 from utils.helpers import drop_columns_if_exist
 from data_loader.collections import DataCollection
 from typing import Literal, Optional
-from utils.resample import resample_ohlc
+from utils.resample import upsample
 from config.hashing import hash_data_config
 from .types import XDataFrame, ReturnSeries
 from diskcache import Cache
@@ -98,7 +98,6 @@ def __load_data(
     X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.0)
 
     X.index = pd.DatetimeIndex(X.index)
-    X.sort_index(inplace=True)
 
     ## Create target
     returns = df_target_asset_only_returns[target_asset.file_name + "_returns"]
@@ -134,8 +133,10 @@ def __load_df(
 
     df = df.replace([np.inf, -np.inf], 0.0)
     df.index = pd.DatetimeIndex(df.index)
+    df.sort_index(inplace=True)
+
     if resample_to_freq is not None and resample_to_freq != data_source.freq:
-        df = resample_ohlc(df, resample_to_freq)
+        df = upsample(df, resample_to_freq)
 
     df = drop_columns_if_exist(df, ["open", "high", "low", "close", "volume"])
 

feature_extractors/feature_extractor_presets.py

@@ -18,7 +18,9 @@ from .fractional_differentiation import (
 )
 
 __presets = dict(
-    debug_future_lookahead=[("debug_future", feature_debug_future_lookahead, [1])],
+    debug_future_lookahead=[
+        ("debug_future", feature_debug_future_lookahead, [1, 5, 10])
+    ],
     single_mom=[("mom", feature_mom, [30])],
     single_vol=[("vol", feature_vol, [30])],
     mom=[("mom", feature_mom, [100, 300, 600, 900, 1800])],

feature_extractors/feature_extractors.py

@@ -4,7 +4,7 @@ from feature_extractors.utils import get_close_low_high
 
 
 def feature_debug_future_lookahead(df: pd.DataFrame, period: int) -> pd.Series:
-    return df["returns"].shift(-period)
+    return df["returns"].rolling(window=pd.api.indexers.FixedForwardWindowIndexer(window_size=period)).sum()
 
 
 def feature_lag(df: pd.DataFrame, period: int) -> pd.Series:

utils/resample.py

@@ -3,14 +3,16 @@ import pandas as pd
 
 def resample_ohlc(df, period):
     output = pd.DataFrame()
-    period = period.replace("m", "T")
+    output["open"] = df.open.resample(period).first()
+    output["high"] = df.high.resample(period).max()
+    output["low"] = df.low.resample(period).min()
+    output["close"] = df.close.resample(period).last()
+    return output
 
-    if "open" in df.columns:
+
-        output["open"] = df.open.resample(period).first()
+def upsample(df, period):
-        output["high"] = df.high.resample(period).max()
+    output = pd.DataFrame()
-        output["low"] = df.low.resample(period).min()
+    period = period.replace("m", "T")
-        output["close"] = df.close.resample(period).last()
+    output = df.resample(period).ffill()
-    else:
-        output = df.resample(period).ffill()
 
     return output