mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-27 18:57:55 +00:00
feat(FeatureExtraction): added fractionally differentiated returns to remove lagged returns (#95)
* feat(FeatureExtraction): added fractionally differentiated returns to remove lagged returns * fix(Sweep): config * fix(Sweep): name * fix(Sweep): grid * feat(Config): separated sliding_window_size_level1 & sliding_window_size_level2 * feat(Dependencies): added ray, now using it to parallel process feature extraction * fix(Dependencies): added pip explicitly * fix(Dependencies): removed ray from root * fix(Models): average model was probably not taking the right timestamp to average * feat(Config): separated expanding_window_level1 & expanding_window_level2 * fix(Config): set n_features_to_select to the optimal 30
This commit is contained in:
Mark Aron Szulyovszky
committed by
GitHub
GitHub
parent
cc70d3f907
commit
f762ceed2a
@@ -7,10 +7,13 @@ from models.model_map import model_names_classification, model_names_regression
|
||||
def get_default_level_1_config() -> tuple[dict, dict, dict]:
|
||||
|
||||
training_config = dict(
|
||||
dimensionality_reduction = False,
|
||||
dimensionality_reduction = True,
|
||||
feature_selection = True,
|
||||
expanding_window = False,
|
||||
sliding_window_size = 380,
|
||||
n_features_to_select = 30,
|
||||
expanding_window_level1 = False,
|
||||
expanding_window_level2 = False,
|
||||
sliding_window_size_level1 = 380,
|
||||
sliding_window_size_level2 = 1,
|
||||
retrain_every = 20,
|
||||
scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
|
||||
include_original_data_in_ensemble = False,
|
||||
@@ -46,8 +49,11 @@ def get_default_level_2_config() -> tuple[dict, dict, dict]:
|
||||
training_config = dict(
|
||||
dimensionality_reduction = True,
|
||||
feature_selection = True,
|
||||
expanding_window = True,
|
||||
sliding_window_size = 380,
|
||||
n_features_to_select = 30,
|
||||
expanding_window_level1 = True,
|
||||
expanding_window_level2 = False,
|
||||
sliding_window_size_level1 = 380,
|
||||
sliding_window_size_level2 = 1,
|
||||
retrain_every = 20,
|
||||
scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
|
||||
include_original_data_in_ensemble = False,
|
||||
@@ -59,8 +65,8 @@ def get_default_level_2_config() -> tuple[dict, dict, dict]:
|
||||
load_other_assets= True,
|
||||
log_returns= True,
|
||||
forecasting_horizon = 1,
|
||||
own_features = ['level_2', 'date_days', 'lags_up_to_5'],
|
||||
other_features = ['level_2', 'lags_up_to_5'],
|
||||
own_features = ['level_2', 'date_days', 'fracdiff'],
|
||||
other_features = ['level_2', 'fracdiff'],
|
||||
index_column= 'int',
|
||||
method= 'classification',
|
||||
no_of_classes= 'three-balanced'
|
||||
|
||||
@@ -21,4 +21,8 @@ dependencies:
|
||||
- wandb
|
||||
- python-dotenv
|
||||
- tscv
|
||||
- pip
|
||||
- pip:
|
||||
- fracdiff
|
||||
- ray
|
||||
prefix: /usr/local/anaconda3/envs/quant
|
||||
|
||||
+537
-140
File diff suppressed because one or more lines are too long
@@ -1,6 +1,7 @@
|
||||
from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
|
||||
from utils.typing import FeatureExtractorConfig
|
||||
from utils.helpers import flatten
|
||||
from feature_extractors.fractional_differentiation import feature_fractional_differentiation
|
||||
|
||||
__presets = dict(
|
||||
debug_future_lookahead = [('debug_future', feature_debug_future_lookahead, [1])],
|
||||
@@ -22,6 +23,7 @@ __presets = dict(
|
||||
rsi = [('rsi', feature_ROC, [10, 30, 100])],
|
||||
stod = [('stod', feature_STOD, [10, 30, 200])],
|
||||
stok = [('stok', feature_STOK, [10, 30, 200])],
|
||||
fracdiff = [('fracdiff', feature_fractional_differentiation, [10, 30])],
|
||||
)
|
||||
|
||||
presets = __presets | dict(
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
from fracdiff.sklearn import FracdiffStat
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
def feature_fractional_differentiation(df: pd.DataFrame, period: int, is_log_return: bool) -> pd.Series:
|
||||
feature_selector = FracdiffStat(window = period)
|
||||
input_series = df["close"].to_numpy().reshape(-1, 1)
|
||||
feature_selector.fit(input_series)
|
||||
result = feature_selector.transform(input_series)
|
||||
return pd.Series(np.log(result.squeeze()), index = df.index)
|
||||
@@ -1,240 +0,0 @@
|
||||
"""
|
||||
Fractional differentiation is a technique to make a time series stationary but also
|
||||
retain as much memory as possible. This is done by differencing by a positive real
|
||||
number. Fractionally differenced series can be used as a feature in machine learning
|
||||
process.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class FractionalDifferentiation:
|
||||
"""
|
||||
FractionalDifferentiation class encapsulates the functions that can
|
||||
be used to compute fractionally differentiated series.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_weights(diff_amt, size):
|
||||
"""
|
||||
Advances in Financial Machine Learning, Chapter 5, section 5.4.2, page 79.
|
||||
|
||||
The helper function generates weights that are used to compute fractionally
|
||||
differentiated series. It computes the weights that get used in the computation
|
||||
of fractionally differentiated series. This generates a non-terminating series
|
||||
that approaches zero asymptotically. The side effect of this function is that
|
||||
it leads to negative drift "caused by an expanding window's added weights"
|
||||
(see page 83 AFML)
|
||||
|
||||
When diff_amt is real (non-integer) positive number then it preserves memory.
|
||||
|
||||
The book does not discuss what should be expected if d is a negative real
|
||||
number. Conceptually (from set theory) negative d leads to set of negative
|
||||
number of elements. And that translates into a set whose elements can be
|
||||
selected more than once or as many times as one chooses (multisets with
|
||||
unbounded multiplicity) - see http://faculty.uml.edu/jpropp/msri-up12.pdf.
|
||||
|
||||
:param diff_amt: (float) Differencing amount
|
||||
:param size: (int) Length of the series
|
||||
:return: (np.ndarray) Weight vector
|
||||
"""
|
||||
|
||||
# The algorithm below executes the iterative estimation (section 5.4.2, page 78)
|
||||
weights = [1.] # create an empty list and initialize the first element with 1.
|
||||
for k in range(1, size):
|
||||
weights_ = -weights[-1] * (diff_amt - k + 1) / k # compute the next weight
|
||||
weights.append(weights_)
|
||||
|
||||
# Now, reverse the list, convert into a numpy column vector
|
||||
weights = np.array(weights[::-1]).reshape(-1, 1)
|
||||
return weights
|
||||
|
||||
@staticmethod
|
||||
def frac_diff(series, diff_amt, thresh=0.01):
|
||||
"""
|
||||
Advances in Financial Machine Learning, Chapter 5, section 5.5, page 82.
|
||||
|
||||
References:
|
||||
https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086
|
||||
https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf
|
||||
https://en.wikipedia.org/wiki/Fractional_calculus
|
||||
|
||||
The steps are as follows:
|
||||
- Compute weights (this is a one-time exercise)
|
||||
- Iteratively apply the weights to the price series and generate output points
|
||||
|
||||
This is the expanding window variant of the fracDiff algorithm
|
||||
Note 1: For thresh-1, nothing is skipped
|
||||
Note 2: diff_amt can be any positive fractional, not necessarility bounded [0, 1]
|
||||
|
||||
:param series: (pd.Series) A time series that needs to be differenced
|
||||
:param diff_amt: (float) Differencing amount
|
||||
:param thresh: (float) Threshold or epsilon
|
||||
:return: (pd.DataFrame) Differenced series
|
||||
"""
|
||||
|
||||
# 1. Compute weights for the longest series
|
||||
weights = get_weights(diff_amt, series.shape[0])
|
||||
|
||||
# 2. Determine initial calculations to be skipped based on weight-loss threshold
|
||||
weights_ = np.cumsum(abs(weights))
|
||||
weights_ /= weights_[-1]
|
||||
skip = weights_[weights_ > thresh].shape[0]
|
||||
|
||||
# 3. Apply weights to values
|
||||
output_df = {}
|
||||
for name in series.columns:
|
||||
series_f = series[[name]].fillna(method='ffill').dropna()
|
||||
output_df_ = pd.Series(index=series.index, dtype='float64')
|
||||
|
||||
for iloc in range(skip, series_f.shape[0]):
|
||||
loc = series_f.index[iloc]
|
||||
|
||||
# At this point all entries are non-NAs so no need for the following check
|
||||
# if np.isfinite(series.loc[loc, name]):
|
||||
output_df_[loc] = np.dot(weights[-(iloc + 1):, :].T, series_f.loc[:loc])[0, 0]
|
||||
|
||||
output_df[name] = output_df_.copy(deep=True)
|
||||
output_df = pd.concat(output_df, axis=1)
|
||||
return output_df
|
||||
|
||||
@staticmethod
|
||||
def get_weights_ffd(diff_amt, thresh, lim):
|
||||
"""
|
||||
Advances in Financial Machine Learning, Chapter 5, section 5.4.2, page 83.
|
||||
|
||||
The helper function generates weights that are used to compute fractionally
|
||||
differentiate dseries. It computes the weights that get used in the computation
|
||||
of fractionally differentiated series. The series is of fixed width and same
|
||||
weights (generated by this function) can be used when creating fractional
|
||||
differentiated series.
|
||||
This makes the process more efficient. But the side-effect is that the
|
||||
fractionally differentiated series is skewed and has excess kurtosis. In
|
||||
other words, it is not Gaussian any more.
|
||||
|
||||
The discussion of positive and negative d is similar to that in get_weights
|
||||
(see the function get_weights)
|
||||
|
||||
:param diff_amt: (float) Differencing amount
|
||||
:param thresh: (float) Threshold for minimum weight
|
||||
:param lim: (int) Maximum length of the weight vector
|
||||
:return: (np.ndarray) Weight vector
|
||||
"""
|
||||
|
||||
weights = [1.]
|
||||
k = 1
|
||||
|
||||
# The algorithm below executes the iterativetive estimation (section 5.4.2, page 78)
|
||||
# The output weights array is of the indicated length (specified by lim)
|
||||
ctr = 0
|
||||
while True:
|
||||
# compute the next weight
|
||||
weights_ = -weights[-1] * (diff_amt - k + 1) / k
|
||||
|
||||
if abs(weights_) < thresh:
|
||||
break
|
||||
|
||||
weights.append(weights_)
|
||||
k += 1
|
||||
ctr += 1
|
||||
if ctr == lim - 1: # if we have reached the size limit, exit the loop
|
||||
break
|
||||
|
||||
# Now, reverse the list, convert into a numpy column vector
|
||||
weights = np.array(weights[::-1]).reshape(-1, 1)
|
||||
return weights
|
||||
|
||||
@staticmethod
|
||||
def frac_diff_ffd(series, diff_amt, thresh=1e-5):
|
||||
"""
|
||||
Advances in Financial Machine Learning, Chapter 5, section 5.5, page 83.
|
||||
|
||||
References:
|
||||
|
||||
* https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086
|
||||
* https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf
|
||||
* https://en.wikipedia.org/wiki/Fractional_calculus
|
||||
|
||||
The steps are as follows:
|
||||
|
||||
- Compute weights (this is a one-time exercise)
|
||||
- Iteratively apply the weights to the price series and generate output points
|
||||
|
||||
Constant width window (new solution)
|
||||
Note 1: thresh determines the cut-off weight for the window
|
||||
Note 2: diff_amt can be any positive fractional, not necessarity bounded [0, 1].
|
||||
|
||||
:param series: (pd.Series) A time series that needs to be differenced
|
||||
:param diff_amt: (float) Differencing amount
|
||||
:param thresh: (float) Threshold for minimum weight
|
||||
:return: (pd.DataFrame) A data frame of differenced series
|
||||
"""
|
||||
|
||||
# 1) Compute weights for the longest series
|
||||
weights = get_weights_ffd(diff_amt, thresh, series.shape[0])
|
||||
width = len(weights) - 1
|
||||
|
||||
# 2) Apply weights to values
|
||||
# 2.1) Start by creating a dictionary to hold all the fractionally differenced series
|
||||
output_df = {}
|
||||
|
||||
# 2.2) compute fractionally differenced series for each stock
|
||||
for name in series.columns:
|
||||
series_f = series[[name]].fillna(method='ffill').dropna()
|
||||
temp_df_ = pd.Series(index=series.index, dtype='float64')
|
||||
for iloc1 in range(width, series_f.shape[0]):
|
||||
loc0 = series_f.index[iloc1 - width]
|
||||
loc1 = series.index[iloc1]
|
||||
|
||||
# At this point all entries are non-NAs, hence no need for the following check
|
||||
# if np.isfinite(series.loc[loc1, name]):
|
||||
temp_df_[loc1] = np.dot(weights.T, series_f.loc[loc0:loc1])[0, 0]
|
||||
|
||||
output_df[name] = temp_df_.copy(deep=True)
|
||||
|
||||
# transform the dictionary into a data frame
|
||||
output_df = pd.concat(output_df, axis=1)
|
||||
return output_df
|
||||
|
||||
|
||||
def get_weights(diff_amt, size):
|
||||
""" This is a pass-through function """
|
||||
return FractionalDifferentiation.get_weights(diff_amt, size)
|
||||
|
||||
|
||||
def frac_diff(series, diff_amt, thresh=0.01):
|
||||
""" This is a pass-through function """
|
||||
return FractionalDifferentiation.frac_diff(series, diff_amt, thresh)
|
||||
|
||||
|
||||
def get_weights_ffd(diff_amt, thresh, lim):
|
||||
""" This is a pass-through function """
|
||||
return FractionalDifferentiation.get_weights_ffd(diff_amt, thresh, lim)
|
||||
|
||||
|
||||
def frac_diff_ffd(series, diff_amt, thresh=1e-5):
|
||||
"""
|
||||
Advances in Financial Machine Learning, Chapter 5, section 5.5, page 83.
|
||||
|
||||
References:
|
||||
|
||||
* https://www.wiley.com/en-us/Advances+in+Financial+Machine+Learning-p-9781119482086
|
||||
* https://wwwf.imperial.ac.uk/~ejm/M3S8/Problems/hosking81.pdf
|
||||
* https://en.wikipedia.org/wiki/Fractional_calculus
|
||||
|
||||
The steps are as follows:
|
||||
|
||||
- Compute weights (this is a one-time exercise)
|
||||
- Iteratively apply the weights to the price series and generate output points
|
||||
|
||||
Constant width window (new solution)
|
||||
Note 1: thresh determines the cut-off weight for the window
|
||||
Note 2: diff_amt can be any positive fractional, not necessarity bounded [0, 1].
|
||||
|
||||
:param series: (pd.Series) A time series that needs to be differenced
|
||||
:param diff_amt: (float) Differencing amount
|
||||
:param thresh: (float) Threshold for minimum weight
|
||||
:return: (pd.DataFrame) A data frame of differenced series
|
||||
"""
|
||||
return FractionalDifferentiation.frac_diff_ffd(series, diff_amt, thresh)
|
||||
@@ -4,7 +4,7 @@ import pandas as pd
|
||||
from models.base import Model, SKLearnModel
|
||||
from sklearn.decomposition import PCA
|
||||
|
||||
def select_features(X: pd.DataFrame, y: pd.Series, model: Model, min_features_to_select: int, backup_model: SKLearnModel) -> pd.DataFrame:
|
||||
def select_features(X: pd.DataFrame, y: pd.Series, model: Model, n_features_to_select: int, backup_model: SKLearnModel) -> pd.DataFrame:
|
||||
''' Select features using RFECV, returns a pd.DataFrame (X) with only the selected features.'''
|
||||
if model.model_type != 'ml': return X
|
||||
|
||||
@@ -16,7 +16,7 @@ def select_features(X: pd.DataFrame, y: pd.Series, model: Model, min_features_to
|
||||
feat_selector_model = backup_model.model
|
||||
|
||||
# selector = RFECV(feat_selector_model, cv = cv, step=5, min_features_to_select=min_features_to_select)
|
||||
selector = RFE(feat_selector_model, n_features_to_select=10)
|
||||
selector = RFE(feat_selector_model, n_features_to_select= n_features_to_select)
|
||||
selector = selector.fit(X, y)
|
||||
print("Kept %d features out of %d" % (selector.n_features_, X.shape[1]))
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ class StaticAverageModel(Model):
|
||||
def predict(self, X):
|
||||
# Make sure there's data to average
|
||||
assert X.shape[1] > 0
|
||||
prediction = np.average(X[0])
|
||||
prediction = np.average(X[-1])
|
||||
return np.array([prediction])
|
||||
|
||||
def clone(self):
|
||||
|
||||
+8
-7
@@ -8,6 +8,8 @@ from utils.helpers import get_first_valid_return_index, weighted_average
|
||||
from config import get_default_level_1_config, get_default_level_2_config, validate_config, get_model_name
|
||||
from feature_selection.feature_selection import select_features
|
||||
from feature_selection.dim_reduction import reduce_dimensionality
|
||||
import ray
|
||||
ray.init()
|
||||
|
||||
def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
|
||||
model_config, training_config, data_config = get_default_level_2_config()
|
||||
@@ -39,7 +41,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
|
||||
original_X = X.copy()
|
||||
first_valid_index = get_first_valid_return_index(X.iloc[:,0])
|
||||
samples_to_train = len(y) - first_valid_index
|
||||
if samples_to_train < training_config['sliding_window_size'] * 3:
|
||||
if samples_to_train < training_config['sliding_window_size_level1'] * 3:
|
||||
print("Not enough samples to train")
|
||||
continue
|
||||
|
||||
@@ -52,10 +54,9 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
|
||||
print("Feature Selection started")
|
||||
# TODO: this needs to be done per model!
|
||||
backup_model = default_feature_selector_regression if data_config['method'] == 'regression' else default_feature_selector_classification
|
||||
X = select_features(X, y, model_config['level_1_models'][0][1], min_features_to_select = 10, backup_model = backup_model)
|
||||
X = select_features(X, y, model_config['level_1_models'][0][1], n_features_to_select = training_config['n_features_to_select'], backup_model = backup_model)
|
||||
print("Feature Selection ended")
|
||||
|
||||
|
||||
# 3. Train Level-1 models
|
||||
current_result, current_predictions = run_single_asset_trainig(
|
||||
ticker_to_predict = asset,
|
||||
@@ -65,8 +66,8 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
|
||||
target_returns = target_returns,
|
||||
models = model_config['level_1_models'],
|
||||
method = data_config['method'],
|
||||
expanding_window = training_config['expanding_window'],
|
||||
sliding_window_size = training_config['sliding_window_size'],
|
||||
expanding_window = training_config['expanding_window_level1'],
|
||||
sliding_window_size = training_config['sliding_window_size_level1'],
|
||||
retrain_every = training_config['retrain_every'],
|
||||
scaler = training_config['scaler'],
|
||||
no_of_classes = data_config['no_of_classes'],
|
||||
@@ -90,8 +91,8 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
|
||||
target_returns = target_returns,
|
||||
models = [model_config['level_2_model']],
|
||||
method = data_config['method'],
|
||||
expanding_window = training_config['expanding_window'],
|
||||
sliding_window_size = training_config['sliding_window_size'],
|
||||
expanding_window = training_config['expanding_window_level2'],
|
||||
sliding_window_size = training_config['sliding_window_size_level2'],
|
||||
retrain_every = training_config['retrain_every'],
|
||||
scaler = training_config['scaler'],
|
||||
no_of_classes = data_config['no_of_classes'],
|
||||
|
||||
@@ -1,23 +1,28 @@
|
||||
program: run_sweep.py
|
||||
method: bayes
|
||||
method: grid
|
||||
project: price-forecasting
|
||||
name: Feature selection
|
||||
name: Fractional differentiation / number of features
|
||||
metric:
|
||||
goal: maximize
|
||||
name: sharpe
|
||||
parameters:
|
||||
path :
|
||||
value: 'data/'
|
||||
expanding_window:
|
||||
expanding_window_level1:
|
||||
value: True
|
||||
sliding_window_size:
|
||||
expanding_window_level2:
|
||||
value: False
|
||||
sliding_window_size_level1:
|
||||
value: 380
|
||||
sliding_window_size_level2:
|
||||
value: 1
|
||||
feature_selection:
|
||||
values: [True, False]
|
||||
value: True
|
||||
n_features_to_select:
|
||||
values: [10, 20, 30]
|
||||
distribution: categorical
|
||||
dimensionality_reduction:
|
||||
values: [True, False]
|
||||
distribution: categorical
|
||||
value: True
|
||||
retrain_every:
|
||||
value: 20
|
||||
scaler:
|
||||
@@ -27,8 +32,7 @@ parameters:
|
||||
method:
|
||||
value: 'classification'
|
||||
no_of_classes:
|
||||
values: ['two', 'three-balanced', 'three-imbalanced']
|
||||
distribution: categorical
|
||||
value: 'three-balanced'
|
||||
forecasting_horizon:
|
||||
value: 1
|
||||
load_other_assets:
|
||||
@@ -42,6 +46,8 @@ parameters:
|
||||
level_2_model:
|
||||
value: "Ensemble_Average"
|
||||
own_features:
|
||||
value: ['date_days', 'level_2', 'lags_up_to_5']
|
||||
values: [['date_days', 'level_2', 'lags_up_to_5'], ['date_days', 'level_2', 'fracdiff']]
|
||||
distribution: categorical
|
||||
other_features:
|
||||
value: ['level_2', 'lags_up_to_5']
|
||||
values: [['level_2', 'lags_up_to_5'], ['level_2', 'fracdiff']]
|
||||
distribution: categorical
|
||||
+13
-2
@@ -8,12 +8,23 @@ metric:
|
||||
parameters:
|
||||
path :
|
||||
value: 'data/'
|
||||
expanding_window:
|
||||
expanding_window_level1:
|
||||
values: [True, False]
|
||||
distribution: categorical
|
||||
sliding_window_size:
|
||||
expanding_window_level2:
|
||||
value: False
|
||||
feature_selection:
|
||||
value: True
|
||||
n_features_to_select:
|
||||
values: [10, 20, 30]
|
||||
distribution: categorical
|
||||
dimensionality_reduction:
|
||||
value: True
|
||||
sliding_window_size_level1:
|
||||
values: [180, 280, 380, 480, 580]
|
||||
distribution: categorical
|
||||
sliding_window_size_level2:
|
||||
value: 1
|
||||
retrain_every:
|
||||
values: [10, 20, 30]
|
||||
distribution: categorical
|
||||
|
||||
+15
-2
@@ -8,10 +8,23 @@ metric:
|
||||
parameters:
|
||||
path :
|
||||
value: 'data/'
|
||||
expanding_window:
|
||||
expanding_window_level1:
|
||||
values: [True, False]
|
||||
distribution: categorical
|
||||
sliding_window_size:
|
||||
expanding_window_level2:
|
||||
values: [True, False]
|
||||
distribution: categorical
|
||||
feature_selection:
|
||||
value: True
|
||||
n_features_to_select:
|
||||
values: [10, 20, 30]
|
||||
distribution: categorical
|
||||
dimensionality_reduction:
|
||||
value: True
|
||||
sliding_window_size_level1:
|
||||
values: [180, 280, 380]
|
||||
distribution: categorical
|
||||
sliding_window_size_level2:
|
||||
values: [180, 280, 380]
|
||||
distribution: categorical
|
||||
retrain_every:
|
||||
|
||||
+4
-5
@@ -1,11 +1,9 @@
|
||||
#%%
|
||||
import pandas as pd
|
||||
import os
|
||||
import numpy as np
|
||||
from utils.typing import FeatureExtractor
|
||||
from typing import Literal
|
||||
|
||||
#%%
|
||||
import ray
|
||||
|
||||
def get_crypto_assets(path: str) -> list[str]:
|
||||
return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')])
|
||||
@@ -40,13 +38,14 @@ def load_data(path: str,
|
||||
other_files = [f for f in files if load_other_assets == True and f.startswith(target_asset) == False]
|
||||
files = target_file + other_files
|
||||
def is_target_asset(target_asset: str, file: str): return file.split('.')[0].startswith(target_asset)
|
||||
dfs = [__load_df(
|
||||
futures = [__load_df.remote(
|
||||
path=os.path.join(path,f),
|
||||
prefix=f.split('.')[0],
|
||||
returns='log_returns' if log_returns else 'returns',
|
||||
feature_extractors=own_features if is_target_asset(target_asset, f) else other_features,
|
||||
narrow_format=narrow_format,
|
||||
) for f in files]
|
||||
dfs = ray.get(futures)
|
||||
if narrow_format:
|
||||
dfs = pd.concat(dfs, axis=0).fillna(0.)
|
||||
else:
|
||||
@@ -78,6 +77,7 @@ def load_data(path: str,
|
||||
return X, y, forward_returns
|
||||
|
||||
|
||||
@ray.remote
|
||||
def __load_df(path: str,
|
||||
prefix: str,
|
||||
returns: Literal['price', 'returns', 'log_returns'],
|
||||
@@ -123,7 +123,6 @@ def __apply_feature_extractors(df: pd.DataFrame,
|
||||
return df
|
||||
|
||||
|
||||
# %%
|
||||
def __create_target_cum_forward_returns(df: pd.DataFrame, source_column: str, period: int) -> pd.Series:
|
||||
assert period > 0
|
||||
return df[source_column].shift(-period)
|
||||
|
||||
Reference in New Issue
Block a user