382 lines
14 KiB
Python
382 lines
14 KiB
Python
|
# feature_engineering.py
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import pandas as pd
|
||
|
|
import math
|
||
|
|
import ta
|
||
|
|
from statsmodels.tsa.stattools import adfuller
|
||
|
|
from scipy.fftpack import fft
|
||
|
|
from sklearn.preprocessing import StandardScaler
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 1) TA-LIB FEATURES (add_all_ta_features)
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def add_all_ta_features(df: pd.DataFrame) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Adds a wide range of technical analysis indicators to the DataFrame
|
||
|
|
using the 'ta' library. Modifies the DataFrame in place.
|
||
|
|
"""
|
||
|
|
df = ta.add_all_ta_features(
|
||
|
|
df, open="open", high="high", low="low", close="close", volume="tick_volume", fillna=True
|
||
|
|
)
|
||
|
|
return df
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def create_custom_feature(df: pd.DataFrame) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Example custom feature. For instance, a rolling mean of the close price.
|
||
|
|
"""
|
||
|
|
df["rolling_mean_10"] = df["close"].rolling(window=10).mean()
|
||
|
|
return df
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 2) MISCELLANEOUS FEATURES
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def spread(df: pd.DataFrame) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Calculates the spread between 'high' and 'low' columns.
|
||
|
|
"""
|
||
|
|
df_copy = df.copy()
|
||
|
|
df_copy["spread"] = df_copy["high"] - df_copy["low"]
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
def auto_corr_multi(df: pd.DataFrame, col: str, n: int = 50, lags: list = [1, 3, 5, 10]) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Computes rolling autocorrelation for multiple lags.
|
||
|
|
"""
|
||
|
|
df_copy = df.copy()
|
||
|
|
for lag in lags:
|
||
|
|
df_copy[f"autocorr_{lag}"] = (
|
||
|
|
df_copy[col]
|
||
|
|
.rolling(window=n, min_periods=n)
|
||
|
|
.apply(lambda x: x.autocorr(lag=lag), raw=False)
|
||
|
|
)
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
|
||
|
|
def candle_information(df: pd.DataFrame) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Adds candle-specific features:
|
||
|
|
- candle_way
|
||
|
|
- fill
|
||
|
|
- amplitude
|
||
|
|
"""
|
||
|
|
df_copy = df.copy()
|
||
|
|
df_copy["candle_way"] = 0
|
||
|
|
df_copy.loc[df_copy["close"] > df_copy["open"], "candle_way"] = 1
|
||
|
|
|
||
|
|
df_copy["fill"] = (
|
||
|
|
np.abs(df_copy["close"] - df_copy["open"])
|
||
|
|
/ (df_copy["high"] - df_copy["low"] + 1e-5)
|
||
|
|
)
|
||
|
|
df_copy["amplitude"] = (
|
||
|
|
np.abs(df_copy["close"] - df_copy["open"])
|
||
|
|
/ (df_copy["open"] + 1e-5)
|
||
|
|
)
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
def log_transform(df: pd.DataFrame, col: str, n: int) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Log-transform a column + compute % change over 'n' bars.
|
||
|
|
"""
|
||
|
|
df_copy = df.copy()
|
||
|
|
df_copy[f"log_{col}"] = np.log(df_copy[col])
|
||
|
|
df_copy[f"ret_log_{n}"] = df_copy[f"log_{col}"].pct_change(periods=n)
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
def mathematical_derivatives(df: pd.DataFrame, col: str) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Adds 'velocity' and 'acceleration' for a given column.
|
||
|
|
"""
|
||
|
|
df_copy = df.copy()
|
||
|
|
df_copy["velocity"] = df_copy[col].diff()
|
||
|
|
df_copy["acceleration"] = df_copy["velocity"].diff()
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 3) VOLATILITY ESTIMATORS
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def parkinson_estimator(window: pd.DataFrame) -> float:
|
||
|
|
n = len(window)
|
||
|
|
if n < 1:
|
||
|
|
return np.nan
|
||
|
|
sum_sq = np.sum(np.log(window['high'] / window['low']) ** 2)
|
||
|
|
return math.sqrt(sum_sq / (4 * math.log(2) * n))
|
||
|
|
|
||
|
|
def moving_parkinson_estimator(df: pd.DataFrame, window_size: int = 30) -> pd.DataFrame:
|
||
|
|
df_copy = df.copy()
|
||
|
|
rolling_vol = pd.Series(dtype="float64", index=df_copy.index)
|
||
|
|
for i in range(window_size, len(df_copy)):
|
||
|
|
w = df_copy.iloc[i - window_size : i]
|
||
|
|
rolling_vol.iloc[i] = parkinson_estimator(w)
|
||
|
|
df_copy["rolling_volatility_parkinson"] = rolling_vol
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
def yang_zhang_estimator(window: pd.DataFrame) -> float:
|
||
|
|
n = len(window)
|
||
|
|
if n < 1:
|
||
|
|
return np.nan
|
||
|
|
term1 = np.log(window['high'] / window['low']) ** 2
|
||
|
|
term2 = np.log(window['close'] / window['open']) ** 2
|
||
|
|
return math.sqrt(np.mean(term1 + term2))
|
||
|
|
|
||
|
|
def moving_yang_zhang_estimator(df: pd.DataFrame, window_size: int = 30) -> pd.DataFrame:
|
||
|
|
df_copy = df.copy()
|
||
|
|
rolling_vol = pd.Series(dtype="float64", index=df_copy.index)
|
||
|
|
for i in range(window_size, len(df_copy)):
|
||
|
|
w = df_copy.iloc[i - window_size : i]
|
||
|
|
rolling_vol.iloc[i] = yang_zhang_estimator(w)
|
||
|
|
df_copy["rolling_volatility_yang_zhang"] = rolling_vol
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 4) MARKET REGIME / DC EVENTS
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def dc_event(P: float, Pext: float, threshold: float) -> int:
|
||
|
|
dc = 0
|
||
|
|
var = (P - Pext) / Pext
|
||
|
|
if var >= threshold:
|
||
|
|
dc = 1
|
||
|
|
elif var <= -threshold:
|
||
|
|
dc = -1
|
||
|
|
return dc
|
||
|
|
|
||
|
|
def calculate_dc(df: pd.DataFrame, threshold: float = 0.01) -> tuple:
|
||
|
|
df_copy = df.copy()
|
||
|
|
prices = df_copy['close'].values
|
||
|
|
dc_events_up, dc_events_down = [], []
|
||
|
|
Pext = prices[0]
|
||
|
|
direction = 0
|
||
|
|
for i in range(1, len(prices)):
|
||
|
|
P = prices[i]
|
||
|
|
dc_flag = dc_event(P, Pext, threshold)
|
||
|
|
if dc_flag == 1:
|
||
|
|
dc_events_up.append(i)
|
||
|
|
direction = 1
|
||
|
|
Pext = P
|
||
|
|
elif dc_flag == -1:
|
||
|
|
dc_events_down.append(i)
|
||
|
|
direction = -1
|
||
|
|
Pext = P
|
||
|
|
else:
|
||
|
|
if direction == 1 and P > Pext:
|
||
|
|
Pext = P
|
||
|
|
elif direction == -1 and P < Pext:
|
||
|
|
Pext = P
|
||
|
|
return dc_events_up, dc_events_down
|
||
|
|
|
||
|
|
def calculate_trend(dc_events_up: list, dc_events_down: list, df: pd.DataFrame):
|
||
|
|
trend_events_down = []
|
||
|
|
trend_events_up = []
|
||
|
|
trend_events_down.extend(sorted(dc_events_down))
|
||
|
|
trend_events_up.extend(sorted(dc_events_up))
|
||
|
|
return trend_events_down, trend_events_up
|
||
|
|
|
||
|
|
def market_regime_dc(df: pd.DataFrame, threshold: float = 0.01) -> pd.DataFrame:
|
||
|
|
df_copy = df.copy()
|
||
|
|
dc_up, dc_down = calculate_dc(df_copy, threshold=threshold)
|
||
|
|
t_down, t_up = calculate_trend(dc_up, dc_down, df_copy)
|
||
|
|
df_copy['market_regime'] = np.nan
|
||
|
|
df_copy.loc[t_up, 'market_regime'] = 1
|
||
|
|
df_copy.loc[t_down, 'market_regime'] = 0
|
||
|
|
df_copy['market_regime'] = df_copy['market_regime'].ffill().bfill()
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
def kama_market_regime(df: pd.DataFrame, col: str = 'close', n1: int = 10, n2: int = 30) -> pd.DataFrame:
|
||
|
|
df_copy = df.copy()
|
||
|
|
short_kama = df_copy[col].ewm(span=n1, adjust=False).mean()
|
||
|
|
long_kama = df_copy[col].ewm(span=n2, adjust=False).mean()
|
||
|
|
df_copy['kama_diff'] = short_kama - long_kama
|
||
|
|
df_copy['kama_trend'] = (df_copy['kama_diff'] >= 0).astype(int)
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 5) GAP & DISPLACEMENT
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def gap_detection(df: pd.DataFrame, lookback: int = 1) -> pd.DataFrame:
|
||
|
|
df_copy = df.copy()
|
||
|
|
df_copy['Bullish_gap_inf'] = np.nan
|
||
|
|
df_copy['Bullish_gap_sup'] = np.nan
|
||
|
|
df_copy['Bullish_gap_size'] = np.nan
|
||
|
|
df_copy['Bearish_gap_inf'] = np.nan
|
||
|
|
df_copy['Bearish_gap_sup'] = np.nan
|
||
|
|
df_copy['Bearish_gap_size'] = np.nan
|
||
|
|
for i in range(lookback, len(df_copy)):
|
||
|
|
prev_high = df_copy['high'].iloc[i - lookback]
|
||
|
|
prev_low = df_copy['low'].iloc[i - lookback]
|
||
|
|
curr_high = df_copy['high'].iloc[i]
|
||
|
|
curr_low = df_copy['low'].iloc[i]
|
||
|
|
if curr_low > prev_high:
|
||
|
|
df_copy.at[df_copy.index[i], 'Bullish_gap_inf'] = prev_high
|
||
|
|
df_copy.at[df_copy.index[i], 'Bullish_gap_sup'] = curr_low
|
||
|
|
df_copy.at[df_copy.index[i], 'Bullish_gap_size'] = curr_low - prev_high
|
||
|
|
if curr_high < prev_low:
|
||
|
|
df_copy.at[df_copy.index[i], 'Bearish_gap_inf'] = curr_high
|
||
|
|
df_copy.at[df_copy.index[i], 'Bearish_gap_sup'] = prev_low
|
||
|
|
df_copy.at[df_copy.index[i], 'Bearish_gap_size'] = prev_low - curr_high
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
def displacement_detection(
|
||
|
|
df: pd.DataFrame,
|
||
|
|
type_range: str = 'standard',
|
||
|
|
strenght: float = 3.0,
|
||
|
|
period: int = 20
|
||
|
|
) -> pd.DataFrame:
|
||
|
|
df_copy = df.copy()
|
||
|
|
if type_range == 'standard':
|
||
|
|
df_copy['candle_range'] = np.abs(df_copy['close'] - df_copy['open'])
|
||
|
|
elif type_range == 'extrem':
|
||
|
|
df_copy['candle_range'] = np.abs(df_copy['high'] - df_copy['low'])
|
||
|
|
else:
|
||
|
|
raise ValueError("Invalid 'type_range'. Use 'standard' or 'extrem'.")
|
||
|
|
|
||
|
|
df_copy['Variation'] = np.abs(df_copy['close'] / df_copy['open'] - 1)
|
||
|
|
df_copy['STD'] = df_copy['candle_range'].rolling(period).std()
|
||
|
|
df_copy['displacement'] = 0
|
||
|
|
mask = df_copy['candle_range'] > strenght * df_copy['STD']
|
||
|
|
df_copy.loc[mask, 'displacement'] = 1
|
||
|
|
df_copy['red_displacement'] = (
|
||
|
|
df_copy['displacement'] & df_copy['displacement'].shift(1).fillna(0)
|
||
|
|
).astype(int)
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 6) ROLLING ADF (Stationarity)
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def rolling_adf_with_flag(df: pd.DataFrame, col: str = 'close', window_size: int = 50, p_value_threshold=0.05) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Computes rolling ADF test and adds a stationarity flag (1=stationary, 0=non-stationary).
|
||
|
|
"""
|
||
|
|
df_copy = df.copy()
|
||
|
|
adf_stat = pd.Series(dtype="float64", index=df_copy.index)
|
||
|
|
adf_pval = pd.Series(dtype="float64", index=df_copy.index)
|
||
|
|
stationarity_flag = pd.Series(dtype="int", index=df_copy.index)
|
||
|
|
|
||
|
|
for i in range(window_size, len(df_copy)):
|
||
|
|
slice_data = df_copy[col].iloc[i - window_size : i].values
|
||
|
|
try:
|
||
|
|
result = adfuller(slice_data, autolag='AIC')
|
||
|
|
adf_stat.iloc[i] = result[0]
|
||
|
|
adf_pval.iloc[i] = result[1]
|
||
|
|
stationarity_flag.iloc[i] = 1 if result[1] < p_value_threshold else 0
|
||
|
|
except:
|
||
|
|
adf_stat.iloc[i] = np.nan
|
||
|
|
adf_pval.iloc[i] = np.nan
|
||
|
|
stationarity_flag.iloc[i] = np.nan
|
||
|
|
|
||
|
|
df_copy['rolling_adf_stat'] = adf_stat
|
||
|
|
df_copy['rolling_adf_pval'] = adf_pval
|
||
|
|
df_copy['stationary_flag'] = stationarity_flag # 1 = stationary, 0 = non-stationary
|
||
|
|
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 7) DOUBLE-BARRIER LABEL
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def set_double_barrier_label(
|
||
|
|
df: pd.DataFrame,
|
||
|
|
up: float = 0.005,
|
||
|
|
down: float = 0.005,
|
||
|
|
horizon: int = 50
|
||
|
|
) -> pd.DataFrame:
|
||
|
|
df_copy = df.copy()
|
||
|
|
closes = df_copy["close"].values
|
||
|
|
labels = np.full(len(closes), np.nan)
|
||
|
|
|
||
|
|
for i in range(len(closes)):
|
||
|
|
current_price = closes[i]
|
||
|
|
upper_barrier = current_price * (1 + up)
|
||
|
|
lower_barrier = current_price * (1 - down)
|
||
|
|
end = min(i + horizon, len(closes))
|
||
|
|
for forward_i in range(i + 1, end):
|
||
|
|
if closes[forward_i] >= upper_barrier:
|
||
|
|
labels[i] = 1
|
||
|
|
break
|
||
|
|
elif closes[forward_i] <= lower_barrier:
|
||
|
|
labels[i] = 0
|
||
|
|
break
|
||
|
|
df_copy["barrier_label"] = labels
|
||
|
|
df_copy.dropna(subset=["barrier_label"], inplace=True)
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 8) FUTURE MARKET REGIME (Directional-Change Example)
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def future_DC_market_regime(df: pd.DataFrame, threshold: float = 0.03, horizon: int = 10) -> pd.DataFrame:
|
||
|
|
df_copy = df.copy()
|
||
|
|
df_copy['future_return'] = df_copy['close'].shift(-horizon) / df_copy['close'] - 1.0
|
||
|
|
df_copy['future_market_regime'] = np.nan
|
||
|
|
df_copy.loc[df_copy['future_return'] >= threshold, 'future_market_regime'] = 1
|
||
|
|
df_copy.loc[df_copy['future_return'] <= -threshold, 'future_market_regime'] = 0
|
||
|
|
df_copy.dropna(subset=['future_market_regime'], inplace=True)
|
||
|
|
return df_copy
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 9) Introduce Fourier & Wavelet Features for Cyclical Pattern Recognition
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
|
||
|
|
def add_fourier_features(df: pd.DataFrame, col: str = "close", n_components: int = 5) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Extracts the top 'n_components' Fourier coefficients from price data.
|
||
|
|
"""
|
||
|
|
fft_vals = np.abs(fft(df[col].values))
|
||
|
|
for i in range(1, n_components + 1):
|
||
|
|
df[f'fft_comp_{i}'] = fft_vals[i]
|
||
|
|
return df
|
||
|
|
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 10) Optimize ADF Test for Model Selection
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
|
||
|
|
def apply_differencing_if_needed(df: pd.DataFrame, col: str = "close", threshold: float = 0.05) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
If ADF p-value > threshold (non-stationary), apply first differencing.
|
||
|
|
"""
|
||
|
|
if df['rolling_adf_pval'].iloc[-1] > threshold: # Check last rolling p-value
|
||
|
|
df[f"{col}_diff"] = df[col] - df[col].shift(1) # First differencing
|
||
|
|
return df.dropna()
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
# 11) Normalize Feature Distributions (Scaling)
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def scale_features(df: pd.DataFrame, cols_to_scale: list) -> pd.DataFrame:
|
||
|
|
scaler = StandardScaler()
|
||
|
|
df[cols_to_scale] = scaler.fit_transform(df[cols_to_scale])
|
||
|
|
return df
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# 12) SINGLE PIPELINE EXAMPLE
|
||
|
|
# --------------------------------------------------------------------
|
||
|
|
def create_features(df: pd.DataFrame, col: str = "close", window_size: int = 30) -> pd.DataFrame:
|
||
|
|
"""
|
||
|
|
Optimized pipeline integrating TA, autocorrelation, stationarity, Fourier transform, and normalization.
|
||
|
|
"""
|
||
|
|
df = add_all_ta_features(df) # Adds TA indicators
|
||
|
|
df = spread(df) # Adds 'spread'
|
||
|
|
df = auto_corr_multi(df, col='close') # Multi-lag autocorrelation
|
||
|
|
df = rolling_adf_with_flag(df) # ADF with stationarity flag
|
||
|
|
|
||
|
|
df = log_transform(df, col, 5) # Log transform
|
||
|
|
df = moving_yang_zhang_estimator(df, window_size)
|
||
|
|
df = moving_parkinson_estimator(df, window_size)
|
||
|
|
|
||
|
|
df = add_fourier_features(df, col="close") # Fourier Transform for cyclic detection
|
||
|
|
df = apply_differencing_if_needed(df, col="close") # Ensure stationarity
|
||
|
|
|
||
|
|
# Normalize all numeric features
|
||
|
|
df = scale_features(df, df.select_dtypes(include=[np.number]).columns.tolist())
|
||
|
|
|
||
|
|
return df
|