feat(Data): create DataSource, DataCollection, added hourly crypto data (#96)

* feat(Data): create DataSource, DataCollection, added hourly crypto data

* fix(Data): hourly data format, loading & config

archive/forecasting.py

@@ -9,7 +9,7 @@ from pytorch_forecasting.metrics import QuantileLoss
 import sys
 sys.path.insert(0, '..')
 
-from utils.load_data import load_files
+from data_loader.load_data import load_files
 
 print("success")
 

archive/housing_preprocess.py

@@ -1,18 +0,0 @@
-#%%
-import pandas as pd
-
-oecd_housing = pd.read_csv('data/oecd_housing_prices.csv')
-# we're only interested in "real" prices
-oecd_housing = oecd_housing[oecd_housing['SUBJECT'] == 'REAL']
-# we're only interested annual data
-oecd_housing = oecd_housing[oecd_housing['FREQUENCY'] == 'A']
-oecd_housing
-# %%
-countries = oecd_housing['LOCATION'].unique()
-countries
-
-# %%
-oecd_housing[oecd_housing['LOCATION'] == 'HUN'].plot(x = "TIME", y= "Value")
-# %%
-
-# %%

archive/lightning/create_dataset.py

@@ -8,7 +8,7 @@ from pytorch_forecasting.metrics import QuantileLoss
 import sys
 sys.path.insert(0, '..')
 
-from utils.load_data import load_files
+from data_loader.load_data import load_files
 
 print("success")
 

archive/lightning/train_predict.py

@@ -10,7 +10,7 @@ warnings.filterwarnings("ignore")
 import sys
 sys.path.insert(0, '..')
 
-from utils.load_data import load_files
+from data_loader.load_data import load_files
 
 
 

archive/model_classification_keras.py

@@ -1,6 +1,6 @@
 #%% Import all the stuff, load data, define constants
 from sklearn.utils import shuffle
-from utils.load_data import load_files, create_target_classes
+from data_loader.load_data import load_files, create_target_classes
 import pandas as pd
 from tensorflow import keras
 from utils.normalize import normalize
@@ -16,7 +16,7 @@ from keras_models.classification_transformer import create_basic_transformer_mod
 data = load_files(path='data/',
     own_asset='BTC_ETH',
     own_asset_lags=[1,2,3,4,5,6,8,10,15],
-    load_other_assets=True,
+    load_non_target_asset=True,
     other_asset_lags=[1,2,3,4],
     log_returns=False,
     add_date_features=True,

archive/model_classification_sklearn.py

@@ -1,158 +0,0 @@
-#%% Import all the stuff, load data, define constants
-from sklearnex import patch_sklearn
-patch_sklearn()
-
-from utils.load_data import create_target_cum_forward_returns, create_target_classes, load_files
-from sktime.forecasting.model_selection import temporal_train_test_split
-from sklearn.metrics import accuracy_score
-from sklearn.metrics import confusion_matrix
-# from utils.evaluate import print_classification_metrics, format_data_for_backtest
-
-import numpy as np
-import pandas as pd
-from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
-from sklearn.naive_bayes import GaussianNB
-from sklearn.svm import SVC
-from sklearn.neural_network import MLPClassifier
-from sklearn.pipeline import Pipeline
-from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
-from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
-from sklearn.preprocessing import MinMaxScaler
-
-from utils.sliding_window import sliding_window_and_flatten
-
-ticket_to_predict = 'BTC_ETH'
-print('Predicting: ', ticket_to_predict)
-
-data = load_files(path='data/',
-    own_asset=ticket_to_predict,
-    own_asset_lags=[1,2,3,4,5,6,8,10,15],
-    load_other_assets=False,
-    other_asset_lags=[1,2,3,4],
-    log_returns=True,
-    add_date_features=True,
-    own_technical_features='level2',
-    other_technical_features='none',
-    exogenous_features='none',
-    index_column='int'
-)
-
-target_col = 'target'
-returns_col = ticket_to_predict + '_returns'
-data = create_target_classes(data, returns_col, 1, 'three')
-
-X = data.drop(columns=['target'])
-y = data[target_col]
-
-X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.2)
-feature_scaler = MinMaxScaler(feature_range= (-1, 1))
-X_test_orig = X_test.copy()
-X_train = feature_scaler.fit_transform(X_train)
-X_test = feature_scaler.transform(X_test)
-
-
-#%%
-
-sliding_window_size = 10
-X_train = sliding_window_and_flatten(X_train, sliding_window_size)
-X_test = sliding_window_and_flatten(X_test, sliding_window_size)
-X_test_orig = X_test_orig.iloc[sliding_window_size-1:]
-y_train = y_train[sliding_window_size-1:]
-y_test = y_test[sliding_window_size-1:]
-
-assert X_train.shape[0] == y_train.shape[0]
-assert X_test.shape[0] == y_test.shape[0]
-scoring = 'accuracy'
-
-# %%
-num_folds = 10
-
-models = []
-models.append(('LR', LogisticRegression(n_jobs=-1)))
-models.append(('LDA', LinearDiscriminantAnalysis()))
-models.append(('KNN', KNeighborsClassifier()))
-models.append(('CART', DecisionTreeClassifier()))
-models.append(('NB', GaussianNB()))
-# models.append(('NN', MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False)))
-models.append(('AB', AdaBoostClassifier()))
-# models.append(('GBM', GradientBoostingClassifier()))
-models.append(('RF', RandomForestClassifier(n_jobs=-1)))
-
-results = []
-names = []
-for name, model in models:
-    kfold = KFold(n_splits=num_folds, shuffle=False)
-    cv_results = cross_val_score(model, X_train, y_train, cv=kfold, scoring=scoring)
-    results.append(cv_results)
-    names.append(name)
-    msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
-    print(msg)
-
-# # compare algorithms
-# fig = plt.figure()
-# fig.suptitle('Algorithm Comparison')
-# ax = fig.add_subplot(111)
-# plt.boxplot(results)
-# ax.set_xticklabels(names)
-# fig.set_size_inches(15,8)
-# plt.show()
-
-#%%
-# n_estimators = [20,80]
-# max_depth= [5,10, 15]
-# criterion = ["gini","entropy"]
-# param_grid = dict(n_estimators=n_estimators, max_depth=max_depth, criterion = criterion )
-# model = RandomForestClassifier(n_jobs=-1)
-# kfold = KFold(n_splits=10, shuffle=False)
-# grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring=scoring, cv=kfold)
-# grid_result = grid.fit(X_train, y_train)
-
-# #Print Results
-# print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
-# means = grid_result.cv_results_['mean_test_score']
-# stds = grid_result.cv_results_['std_test_score']
-# params = grid_result.cv_results_['params']
-# ranks = grid_result.cv_results_['rank_test_score']
-# for mean, stdev, param, rank in zip(means, stds, params, ranks):
-#     print("#%d %f (%f) with: %r" % (rank, mean, stdev, param))
-
-
-#%% prepare model
-model = RandomForestClassifier(criterion='entropy', n_estimators=80, max_depth=5, n_jobs=-1) 
-# model = LogisticRegression() 
-# model = MLPClassifier(hidden_layer_sizes=[200, 100, 50], shuffle=False, max_iter=1000)
-model = GaussianNB()
-model.fit(X_train, y_train)
-
-
-#%%
-# estimate accuracy on validation set
-predictions = model.predict(X_test)
-print(accuracy_score(y_test, predictions))
-print(confusion_matrix(y_test, predictions))
-print(classification_report(y_test, predictions))
-
-
-#%%
-# feat_importance = pd.DataFrame({'Importance':model.feature_importances_*100}, index=X.columns)
-# feat_importance.sort_values('Importance', axis=0, ascending=True)
-# feat_importance.plot(kind='barh', color='r' )
-# plt.xlabel('Variable Importance')
-# print(feat_importance)
-
-#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day
-backtestdata = pd.DataFrame(index= X_test_orig.index)
-backtestdata['signal_pred'] = predictions
-backtestdata['signal_actual'] = y_test
-backtestdata['returns'] = X_test_orig[returns_col]
-backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1)
-backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1)
-
-# %%
-print(backtestdata.cumsum().apply(np.exp).tail(1))
-
-# %%

archive/model_classification_sklearn_walk_forward.py

@@ -1,142 +0,0 @@
-#%% Import all the stuff, load data, define constants
-from sklearnex import patch_sklearn
-patch_sklearn()
-
-from utils.load_data import create_target_classes, load_files
-from sktime.forecasting.model_selection import temporal_train_test_split
-from sklearn.metrics import accuracy_score
-from sklearn.metrics import confusion_matrix
-# from utils.evaluate import print_classification_metrics, format_data_for_backtest
-
-import numpy as np
-import pandas as pd
-from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
-from sklearn.naive_bayes import GaussianNB
-from sklearn.svm import SVC
-from sklearn.neural_network import MLPClassifier
-from sklearn.pipeline import Pipeline
-from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
-from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
-from sklearn.preprocessing import MinMaxScaler
-
-from utils.sliding_window import sliding_window_and_flatten
-
-ticket_to_predict = 'BTC_ETH'
-print('Predicting: ', ticket_to_predict)
-
-data = load_files(path='data/',
-    own_asset=ticket_to_predict,
-    own_asset_lags=[1,2,3,4,5,6,8,10,15],
-    load_other_assets=False,
-    other_asset_lags=[1,2,3,4],
-    log_returns=True,
-    add_date_features=True,
-    own_technical_features='level1',
-    other_technical_features='none',
-    exogenous_features='none',
-    index_column='int'
-)
-
-target_col = 'target'
-returns_col = ticket_to_predict + '_returns'
-data = create_target_classes(data, returns_col, 1, 'two')
-    
-X = data.drop(columns=[target_col])
-y = data[target_col]
-
-X_train, X_test, y_train, y_test = temporal_train_test_split(X, y, test_size=0.1)
-feature_scaler = MinMaxScaler(feature_range= (-1, 1))
-X_test_orig = X_test.copy()
-X_train = feature_scaler.fit_transform(X_train)
-X_test = feature_scaler.transform(X_test)
-
-
-#%%
-
-sliding_window_size = 120
-retrain_every = 60
-X_train = sliding_window_and_flatten(X_train, sliding_window_size)
-# X_test = sliding_window_and_flatten(X_test, sliding_window_size)
-X_test_orig = X_test_orig.iloc[sliding_window_size-1:]
-y_train = y_train[sliding_window_size-1:]
-# y_test = y_test[sliding_window_size-1:]
-
-
-
-
-def evaluate_predictions(model_name: str, y: pd.Series, preds: pd.Series, sliding_window_size: int):
-    print("Model: ", model_name)
-    evaluate_from = sliding_window_size*2
-    print(accuracy_score(y[evaluate_from:-1], preds[evaluate_from:]))
-    print(confusion_matrix(y[evaluate_from:-1], preds[evaluate_from:]))
-    print(classification_report(y[evaluate_from:-1], preds[evaluate_from:]))
-
-
-def walk_forward_train_test(
-        create_model,
-        X_train: pd.DataFrame,
-        y_train: pd.Series,
-        window_size: int,
-        retrain_every: int
-    ):
-    predictions = [None] * (len(y_train)-1)
-    models = [None] * len(predictions)
-
-    train_from = sliding_window_size+1
-    train_till = len(y_train)-2
-    
-    iterations_since_retrain = 0
-
-    for i in range(train_from, train_till):
-        # if i % 20 == 0: print('Fold: ', i)
-        iterations_since_retrain += 1
-        window_start = i - window_size
-        window_end = i
-        X_train_slice = X_train[window_start:window_end]
-        y_train_slice = y_train[window_start:window_end]
-
-        if iterations_since_retrain >= retrain_every or models[i-1] is None:
-            model = create_model()
-            model.fit(X_train_slice, y_train_slice)
-            iterations_since_retrain = 0
-        else:
-            model = models[i-1]
-        models[window_end] = model
-
-        predictions[window_end+1] = model.predict(X_train[window_end+1].reshape(1, -1)).item()
-    return models, predictions
-
-
-#%%
-for model_name, create_model in models_to_try:
-
-    model_over_time, preds = walk_forward_train_test(
-        create_model = create_model,
-        X_train = X_train,
-        y_train = y_train,
-        window_size = sliding_window_size,
-        retrain_every = retrain_every
-    )
-
-    evaluate_predictions(model_name, y_train, preds)
-
-
-#%%
-
-
-#%% Create column for Strategy Returns by multiplying the daily returns by the position that was held at close of business the previous day
-# backtestdata = pd.DataFrame(index= X_test_orig.index)
-# backtestdata['signal_pred'] = predictions
-# backtestdata['signal_actual'] = y_test
-# backtestdata['returns'] = X_test_orig[returns_col]
-# backtestdata['only_positive_returns'] = backtestdata['returns'] * backtestdata['signal_actual'].shift(1)
-# backtestdata['strategy_returns'] = backtestdata['returns'] * backtestdata['signal_pred'].shift(1)
-
-# %%
-# print(backtestdata.cumsum().apply(np.exp).tail(1))
-
-# %%

archive/model_classification_sktime.py

@@ -1,5 +1,5 @@
 #%% Import all the stuff, load data, define constants
-from utils.load_data import load_files, create_target_classes
+from data_loader.load_data import load_files, create_target_classes
 import pandas as pd
 import numpy as np
 from utils.sktime import from_df_to_sktime_data
@@ -36,7 +36,7 @@ print('Predicting: ', ticket_to_predict)
 data = load_files(path='data/',
     own_asset=ticket_to_predict,
     own_asset_lags=[1,2,3,4,5,6,8,10,15],
-    load_other_assets=False,
+    load_non_target_asset=False,
     other_asset_lags=[1,2,3,4],
     log_returns=True,
     add_date_features=True,

archive/model_regression_keras.py

@@ -1,116 +0,0 @@
-#%% Import all the stuff, load data, define constants
-from sklearn.utils import shuffle
-from utils.load_data import load_files, create_target_cum_forward_returns
-import pandas as pd
-from tensorflow import keras
-from utils.normalize import normalize
-import tensorflow as tf
-from utils.visualize import visualize_loss
-from sklearn.preprocessing import MinMaxScaler
-from utils.evaluate import print_regression_metrics
-import numpy as np
-from utils.rolling import rolling_window
-
-
-data = load_files('data/', add_features=True, log_returns=False)
-data.reset_index(drop=True, inplace=True)
-data = data[[column for column in data.columns if not column.endswith('volume')]]
-data = data[["BTC_returns", "BTC_mom_10", "BTC_mom_20", "BTC_mom_30", "BTC_mom_60", "BTC_vol_10", "BTC_vol_20", "BTC_vol_60", "day_month", "day_week", "month"]]
-
-target_col = 'target'
-data = create_target_cum_forward_returns(data, 'BTC_returns', 10)
-
-learning_rate = 0.002
-batch_size = 64
-epochs = 100
-
-split_fraction = 0.715
-train_split = int(split_fraction * int(data.shape[0]))
-
-past = 10
-future = 1
-
-start = past + future
-end = start + train_split
-
-#%% split data into training - validation sets
-train_data = data.loc[0 : train_split - 1]
-val_data = data.loc[train_split:]
-
-#%% create features and target for training set & keras dataset
-feature_scaler = MinMaxScaler(feature_range= (-1, 1))
-target_scaler = MinMaxScaler(feature_range= (-1, 1))
-
-x_train = feature_scaler.fit_transform(train_data.drop(target_col, axis=1).values) # you get the mean and std
-y_train = target_scaler.fit_transform(data.iloc[start:end][target_col].values.reshape(-1, 1))
-
-dataset_train = keras.preprocessing.timeseries_dataset_from_array(
-    x_train,
-    y_train,
-    sequence_length=past,
-    batch_size=batch_size,
-)
-
-
-#%% create features and target for validation set & keras dataset
-x_end = len(val_data) - past - future
-label_start = train_split + past + future
-
-x_val = feature_scaler.transform(val_data.drop(target_col, axis=1).iloc[:x_end].values) # you use the training data's mean and std
-y_val = target_scaler.transform(data.iloc[label_start:][target_col].values.reshape(-1, 1))
-
-dataset_val = keras.utils.timeseries_dataset_from_array(
-    x_val,
-    y_val,
-    sequence_length=past,
-    batch_size=batch_size,
-    shuffle=False,
-)
-
-
-#%%
-
-for batch in dataset_train.take(10):
-    batch_inputs, batch_targets = batch
-
-print("Input shape:", batch_inputs.shape)
-print("Target shape:", batch_targets.shape)
-
-# print(batch_inputs)
-# print(batch_targets)
-
-# %%
-model = keras.Sequential()
-model.add(keras.layers.LSTM(units = 10, return_sequences = True, activation = 'sigmoid', input_shape=(batch_inputs.shape[1], batch_inputs.shape[2])))
-model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 64, activation = 'sigmoid'))
-model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 32, activation = 'sigmoid'))
-model.add(keras.layers.Dropout(0.4))
-model.add(keras.layers.Dense(units = 1))
-
-optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
-model.compile(optimizer=optimizer, loss="mean_squared_error")
-model.summary()
-
-# %%
-
-path_checkpoint = "model_checkpoint.h5"
-
-history = model.fit(
-    dataset_train,
-    epochs=epochs,
-    validation_data=dataset_val,
-)
-
-#%%
-
-pred = model.predict(rolling_window(x_val, 11))
-pred = pred.reshape(pred.shape[0], 1)
-pred = target_scaler.inverse_transform(pred)
-
-print_regression_metrics(y_val, pred)
-
-#%%
-
-visualize_loss(history, "Training and Validation Loss")

archive/run_lightning.py

@@ -1,4 +1,4 @@
-from utils.load_data import load_files
+from data_loader.load_data import load_files
 import pandas as pd
 # from tensorflow import keras
 from utils.normalize import normalize

config.py

@@ -1,10 +1,7 @@
-from collections import defaultdict
-from utils.load_data import get_crypto_assets
-from feature_extractors.feature_extractor_presets import presets
 from models.model_map import model_names_classification, model_names_regression
 
 
-def get_default_level_1_config() -> tuple[dict, dict, dict]:
+def get_default_level_1_daily_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
         dimensionality_reduction = True,
@@ -20,9 +17,10 @@ def get_default_level_1_config() -> tuple[dict, dict, dict]:
     )
 
     data_config = dict(
-        path='data/',
-        all_assets = get_crypto_assets('data/'),
-        load_other_assets= True,
+        assets = ['hourly_crypto'],
+        other_assets = [],
+        # exogenous_data = [],
+        load_non_target_asset= True,
         log_returns= True,
         forecasting_horizon = 1,
         own_features = ['level_2', 'date_days'],
@@ -43,8 +41,49 @@ def get_default_level_1_config() -> tuple[dict, dict, dict]:
     return model_config, training_config, data_config
 
 
+def get_default_level_2_hourly_config() -> tuple[dict, dict, dict]:
+  
+    training_config = dict(
+        dimensionality_reduction = True,
+        feature_selection = True,
+        n_features_to_select = 30,
+        expanding_window_level1 = True,
+        expanding_window_level2 = False,
+        sliding_window_size_level1 = 2480,
+        sliding_window_size_level2 = 1,
+        retrain_every = 100,
+        scaler = 'minmax', # 'normalize' 'minmax' 'standardize' 'none'
+        include_original_data_in_ensemble = False,
+    )
 
-def get_default_level_2_config() -> tuple[dict, dict, dict]:
+    data_config = dict(
+        assets = ['hourly_crypto'],
+        other_assets = [],
+        # exogenous_data = [],
+        load_non_target_asset= True,
+        log_returns= True,
+        forecasting_horizon = 1,
+        own_features = ['level_2', 'date_days', 'lags_up_to_5'],
+        other_features = ['level_2'],
+        index_column= 'int',
+        method= 'classification',
+        no_of_classes= 'three-balanced'
+    )
+
+    regression_models = ["Lasso", "KNN", "RF"]
+    regression_ensemble_model = 'KNN'
+    classification_models = ["LDA", "KNN", "CART", "RF", "StaticMom"]
+    classification_ensemble_model = 'Ensemble_Average'
+
+    model_config = dict(
+        level_1_models = regression_models if data_config['method'] == 'regression' else classification_models,
+        level_2_model = regression_ensemble_model if data_config['method'] == 'regression' else classification_ensemble_model
+    )
+    
+    return model_config, training_config, data_config
+
+
+def get_default_level_2_daily_config() -> tuple[dict, dict, dict]:
   
     training_config = dict(
         dimensionality_reduction = True,
@@ -60,9 +99,10 @@ def get_default_level_2_config() -> tuple[dict, dict, dict]:
     )
 
     data_config = dict(
-        path='data/',
-        all_assets = get_crypto_assets('data/'),
-        load_other_assets= True,
+        assets = ['daily_crypto'],
+        other_assets = ['daily_etf'],
+        # exogenous_data = [],
+        load_non_target_asset= True,
         log_returns= True,
         forecasting_horizon = 1,
         own_features = ['level_2', 'date_days', 'fracdiff'],
@@ -74,7 +114,7 @@ def get_default_level_2_config() -> tuple[dict, dict, dict]:
 
     regression_models = ["Lasso", "KNN", "RF"]
     regression_ensemble_model = 'KNN'
-    classification_models = ["LR", "LDA", "KNN", "CART", "RF", "StaticMom"]
+    classification_models = ["LDA", "KNN", "CART", "RF", "StaticMom"]
     classification_ensemble_model = 'Ensemble_Average'
 
     model_config = dict(

data_loader/collections.py

@@ -0,0 +1,43 @@
+from utils.types import Path, FileName, DataSource, DataCollection
+from utils.helpers import flatten
+
+daily_etf = ["GLD", "IEF", "QQQ", "SPY", "TLT"]
+daily_etf = list(zip(["data/daily_etf"] * len(daily_etf), daily_etf))
+
+daily_crypto = ["ADA_USD",
+    "BCH_USD",
+    "BNB_USD",
+    "BTC_USD",
+    "DOT_USD",
+    "ETC_USD",
+    "ETH_USD",
+    "FIL_USD",
+    "LTC_USD",
+    "SOL_USD",
+    "THETA_USD",
+    "TRX_USD",
+    "UNI_USD",
+    "XLM_USD",
+    "XRP_USD",
+    "XTZ_USD"]
+daily_crypto = list(zip(["data/daily_crypto"] * len(daily_crypto), daily_crypto))
+
+
+hourly_crypto = ["BTC_USD", "DASH_USD", "ETC_USD", "ETH_USD", "LTC_USD", "TRX_USD", "XLM_USD", "XMR_USD", "XRP_USD"]
+hourly_crypto = list(zip(["data/hourly_crypto"] * len(hourly_crypto), hourly_crypto))
+
+
+data_collections = dict(
+    daily_crypto = daily_crypto,
+    daily_etf = daily_etf,
+    hourly_crypto = hourly_crypto
+)
+
+def preprocess_data_collections_config(data_dict: dict) -> dict:
+    data_dict = data_dict.copy()
+    keys = ['assets', 'other_assets']
+    # keys = ['assets', 'other_assets', 'exogenous_data']
+    for key in keys:
+        preset_names = data_dict[key]
+        data_dict[key] = flatten([data_collections[preset_name] for preset_name in preset_names])
+    return data_dict

data_loader/get_prices.py

@@ -15,7 +15,6 @@ ada = get_crypto_price_crypto_compare('ADA', 'USD', 1500)
 ada
 
 
-# %%
 
 def get_crypto_price_av(symbol: str, exchange: str, start_date = None) -> pd.DataFrame:
     api_url = f'https://www.alphavantage.co/query?function=DIGITAL_CURRENCY_DAILY&symbol={symbol}&market={exchange}&apikey={AV_API_KEY}'

data_loader/load_data.py

@@ -1,20 +1,17 @@
 import pandas as pd
-import os
 import numpy as np
-from utils.typing import FeatureExtractor
+from utils.types import DataSource, FeatureExtractor
+from utils.helpers import deduplicate_indexes
+from data_loader.collections import DataCollection
 from typing import Literal
 import ray
+import os
 
-def get_crypto_assets(path: str) -> list[str]:
-    return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and 'USD' in f and not f.startswith('.')])
-
-def get_etf_assets(path: str) -> list[str]:
-    return sorted([f.split('.')[0] for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and '_' not in f and not f.startswith('.')])
-
-
-def load_data(path: str,
+def load_data(assets: DataCollection,
+            other_assets: DataCollection,
+            # exogenous_data: DataCollection,
             target_asset: str,
-            load_other_assets: bool,
+            load_non_target_asset: bool,
             log_returns: bool,
             forecasting_horizon: int,
             own_features: list[tuple[str, FeatureExtractor, list[int]]],
@@ -22,8 +19,7 @@ def load_data(path: str,
             index_column: Literal['date', 'int'],
             method: Literal['regression', 'classification'],
             no_of_classes: Literal['two', 'three-balanced', 'three-imbalanced'],
-            narrow_format: bool = False,
-            all_assets:list=[]
+            narrow_format: bool = False
         ) -> tuple[pd.DataFrame, pd.Series, pd.Series]:
     """
     Loads asset data from the specified path.
@@ -33,23 +29,25 @@ def load_data(path: str,
         - Series `forward_returns` with the target asset returns shifted by 1 day
     """
 
-    files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
-    target_file = [f for f in files if f.startswith(target_asset)]
-    other_files = [f for f in files if load_other_assets == True and f.startswith(target_asset) == False]
-    files = target_file + other_files
+    target_file = [f for f in assets if f[1].startswith(target_asset)]
+    other_files = [f for f in assets if load_non_target_asset == True and f[1].startswith(target_asset) == False]
+    files = target_file + other_files + other_assets
     def is_target_asset(target_asset: str, file: str): return file.split('.')[0].startswith(target_asset)
     futures = [__load_df.remote(
-        path=os.path.join(path,f),
-        prefix=f.split('.')[0],
+        data_source=data_source,
+        prefix=data_source[1],
         returns='log_returns' if log_returns else 'returns',
-        feature_extractors=own_features if is_target_asset(target_asset, f) else other_features,
+        feature_extractors=own_features if is_target_asset(target_asset[1], data_source[1]) else other_features,
         narrow_format=narrow_format,
-    ) for f in files]
+    ) for data_source in files]
     dfs = ray.get(futures)
+
+    dfs = [deduplicate_indexes(df) for df in dfs]
+    longest_df = max(dfs, key=lambda df: df.shape[0])
     if narrow_format:
         dfs = pd.concat(dfs, axis=0).fillna(0.)
     else:
-        dfs = pd.concat(dfs, axis=1).fillna(0.)
+        dfs = pd.concat([df.reindex(longest_df.index) for df in dfs], axis=1).fillna(0.)
 
     dfs.index = pd.DatetimeIndex(dfs.index)
 
@@ -61,7 +59,7 @@ def load_data(path: str,
 
     ## Create target 
     target_col = 'target'
-    returns_col = target_asset + '_returns'
+    returns_col = target_asset[1] + '_returns'
     forward_returns = __create_target_cum_forward_returns(dfs, returns_col, forecasting_horizon)
     if method == 'regression':
         dfs[target_col] = forward_returns
@@ -78,12 +76,12 @@ def load_data(path: str,
 
 
 @ray.remote
-def __load_df(path: str,
+def __load_df(data_source: DataSource,
             prefix: str,
             returns: Literal['price', 'returns', 'log_returns'],
             feature_extractors: list[tuple[str, FeatureExtractor, list[int]]],
             narrow_format: bool = False) -> pd.DataFrame:
-    df = pd.read_csv(path, header=0, index_col=0).fillna(0)
+    df = pd.read_csv(os.path.join(data_source[0], data_source[1] + '.csv'), header=0, index_col=0).fillna(0)
 
     if returns == 'log_returns':
         df['returns'] = np.log(df['close']).diff(1)
@@ -192,6 +190,8 @@ def __create_target_classes(df: pd.DataFrame, source_column: str, period: int, n
         return target_column.map(get_class_binary)
 
 
+def datasource_to_file(data_source: DataSource) -> str:
+    return data_source[0] + '/' + data_source[1] + '.csv'
 
 
 # These are needed for the portfolio feature, maybe we can do this in a more elegant way

environment.yml

@@ -21,6 +21,7 @@ dependencies:
   - wandb
   - python-dotenv
   - tscv
+  - tqdm
   - pip
   - pip:
     - fracdiff

exploration.ipynb

@@ -14,7 +14,7 @@
     "import numpy as np\n",
     "from feature_extractors.feature_extractor_presets import presets\n",
     "\n",
-    "X, y, target_returns = load_data(path = './data', target_asset = 'BTC_USD', load_other_assets = True, log_returns=False, forecasting_horizon= 1, own_features= presets[\"level_2\"],\n",
+    "X, y, target_returns = load_data(path = './data', target_asset = 'BTC_USD', load_non_target_asset = True, log_returns=False, forecasting_horizon= 1, own_features= presets[\"level_2\"],\n",
     "            other_features= presets[\"level_2\"],\n",
     "            index_column= 'int',\n",
     "            method= 'classification',\n",

feature_extractors/feature_extractor_presets.py

@@ -1,5 +1,5 @@
 from feature_extractors.feature_extractors import feature_lag, feature_mom, feature_ROC, feature_RSI, feature_STOD, feature_STOK, feature_vol, feature_day_of_month, feature_day_of_week, feature_month, feature_debug_future_lookahead
-from utils.typing import FeatureExtractorConfig
+from utils.types import FeatureExtractorConfig
 from utils.helpers import flatten
 from feature_extractors.fractional_differentiation import feature_fractional_differentiation
 
@@ -38,15 +38,3 @@ def preprocess_feature_extractors_config(data_dict: dict) -> dict:
         preset_names = data_dict[key]
         data_dict[key] = flatten([presets[preset_name] for preset_name in preset_names])
     return data_dict
-
-
-# Use this if ever we want to create an independent boolean for each featureextractor
-# def preprocess_feature_extractors_config(data_dict: dict) -> dict:
-#     prefixes = ['own_features', 'other_features']
-#     features_dict = dict()
-#     for prefix in prefixes:
-#         features_to_include = [key.replace(prefix + "_", "") for key, value in data_dict.items() if key.startswith(prefix) and value == True]
-#         features_dict[prefix] = flatten([presets[feature_name] for feature_name in features_to_include])
-
-#     data_dict = {k: v for k, v in data_dict.items() if not (k.startswith(prefixes[0]) or k.startswith(prefixes[1]))}
-#     return (data_dict | features_dict)

models/model_map.py

@@ -45,7 +45,7 @@ model_names_regression = list(model_map["regression_models"].keys())
 default_feature_selector_regression = model_map['regression_models']['RF']
 default_feature_selector_classification = model_map['classification_models']['RF']
 
-def map_model_name_to_function(model_config:dict, method:str) -> dict:
+def preprocess_model_config(model_config:dict, method:str) -> dict:
     model_config['level_1_models'] = [(model_name, model_map[method + '_models'][model_name]) for model_name in  model_config['level_1_models']]
     if model_config['level_2_model'] is not None:
         model_config['level_2_model'] = (model_config['level_2_model'], model_map[method + '_models'][model_config['level_2_model']])

run_fetch_data_daily.py

@@ -1,22 +1,24 @@
 #%%
 import pandas as pd
-from utils.get_prices import get_crypto_price_crypto_compare, get_stock_price_av
+from data_loader.get_prices import get_crypto_price_crypto_compare, get_stock_price_av
 
 #%%
 crypto_tickers = ["BTC", "ETH", "BNB", "ADA", "SOL", "XRP", "DOT", "LTC", "UNI", "TRX", "XLM", "BCH", "FIL", "ETC", "THETA", "XTZ"]
 etf_tickers = ["GLD", "IEF", "TLT", "SPY", "QQQ"]
+crypto_path = "data/daily_crypto"
+etf_path = "data/daily_etf"
 
 #%%
 for ticker in etf_tickers:
     print("Fetching ", ticker)
     df = get_stock_price_av(ticker, "2017-11-10")
 
-    df.to_csv(f"data/{ticker}.csv", index=True)
+    df.to_csv(f"{etf_path}/{ticker}.csv", index=True)
 
 for src_ticker in crypto_tickers:
     print("Fetching ", src_ticker, "USD")
     df = get_crypto_price_crypto_compare(src_ticker, "USD", 1500)
 
-    df.to_csv(f"data/{src_ticker}_USD.csv", index=True)
+    df.to_csv(f"{crypto_path}/{src_ticker}_USD.csv", index=True)
 
 

run_fetch_data_hourly.py

@@ -0,0 +1,34 @@
+import pandas as pd
+import ssl
+from tqdm import tqdm
+
+base_url = "https://www.cryptodatadownload.com/cdd/"
+
+exchange_name = "Bitfinex"
+files_to_download = [
+ exchange_name + '_TRXUSD_1h.csv',
+ exchange_name + '_ETHUSD_1h.csv',
+ exchange_name + '_XLMUSD_1h.csv',
+ exchange_name + '_XMRUSD_1h.csv',
+ exchange_name + '_LTCUSD_1h.csv',
+#  exchange_name + '_FILUSD_1h.csv',
+ exchange_name + '_DASHUSD_1h.csv',
+#  exchange_name + '_LINKUSD_1h.csv',
+#  exchange_name + '_SOLUSD_1h.csv',
+ exchange_name + '_BTCUSD_1h.csv',
+ exchange_name + '_ETCUSD_1h.csv',
+#  exchange_name + '_VETUSD_1h.csv',
+#  exchange_name + '_DOTUSD_1h.csv'
+ ]
+
+
+for file in tqdm(files_to_download):
+    data_location = base_url + file
+    ssl._create_default_https_context = ssl._create_unverified_context
+    data = pd.read_csv(data_location, skiprows=1, index_col=1, parse_dates=True).drop(columns=['unix'])
+    volume_column_to_delete = [c for c in data.columns if c.startswith('Volume') and 'USD' not in c]
+    data.drop(volume_column_to_delete + ['symbol'], axis=1, inplace=True)
+    data.rename({'Volume USD': 'volume'}, axis=1, inplace=True)
+    data.index.rename('time', inplace=True)
+    target_file = file.split('_')[1].replace('USD', '') + '_USD'
+    data.to_csv(f"data/hourly_crypto/{target_file}.csv")

run_pipeline.py

@@ -1,26 +1,28 @@
-from utils.load_data import load_data
+from data_loader.collections import preprocess_data_collections_config
+from data_loader.load_data import load_data
 import pandas as pd
 from training.training import run_single_asset_trainig
 from reporting.wandb import launch_wandb, send_report_to_wandb, register_config_with_wandb
-from models.model_map import map_model_name_to_function, default_feature_selector_regression, default_feature_selector_classification
+from models.model_map import preprocess_model_config, default_feature_selector_regression, default_feature_selector_classification
 from feature_extractors.feature_extractor_presets import preprocess_feature_extractors_config
 from utils.helpers import get_first_valid_return_index, weighted_average
-from config import get_default_level_1_config, get_default_level_2_config, validate_config, get_model_name
+from config import get_default_level_1_daily_config, get_default_level_2_daily_config, get_default_level_2_hourly_config, validate_config, get_model_name
 from feature_selection.feature_selection import select_features
 from feature_selection.dim_reduction import reduce_dimensionality
 import ray
 ray.init()
 
 def setup_pipeline(project_name:str, with_wandb: bool, sweep: bool):
-    model_config, training_config, data_config = get_default_level_2_config()
+    model_config, training_config, data_config = get_default_level_2_daily_config()
     
     wandb = None
     if with_wandb: 
         wandb = launch_wandb(project_name=project_name, default_config=dict(**model_config, **training_config, **data_config), sweep=sweep)
         register_config_with_wandb(wandb, model_config, training_config, data_config)
 
-    model_config = map_model_name_to_function(model_config, data_config['method'])
+    model_config = preprocess_model_config(model_config, data_config['method'])
     data_config = preprocess_feature_extractors_config(data_config)
+    data_config = preprocess_data_collections_config(data_config)
     pipeline(project_name, wandb, sweep, model_config, training_config, data_config)  
     
 
@@ -29,8 +31,8 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
     results = pd.DataFrame()
     validate_config(model_config, training_config, data_config)
 
-    for asset in data_config['all_assets']:
-        print('--------\nPredicting: ', asset)
+    for asset in data_config['assets']:
+        print('--------\nPredicting: ', asset[1])
         all_predictions = pd.DataFrame()
 
         # 1. Load data
@@ -59,7 +61,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
 
         # 3. Train Level-1 models
         current_result, current_predictions = run_single_asset_trainig(
-            ticker_to_predict = asset,
+            ticker_to_predict = asset[1],
             original_X = original_X,
             X = X,
             y = y,
@@ -84,7 +86,7 @@ def pipeline(project_name:str, wandb, sweep:bool, model_config:dict, training_co
                 ensemble_X = pd.concat([ensemble_X, X], axis=1)
 
             ensemble_result, ensemble_preds = run_single_asset_trainig(
-                ticker_to_predict = asset,
+                ticker_to_predict = asset[1],
                 original_X = ensemble_X,
                 X = ensemble_X,
                 y = y,

sweep_fracdiff.yaml

@@ -35,7 +35,7 @@ parameters:
       value: 'three-balanced'
     forecasting_horizon: 
       value: 1
-    load_other_assets: 
+    load_non_target_asset: 
       value: True
     log_returns: 
       value: True

sweep_level_1.yaml

@@ -39,7 +39,7 @@ parameters:
       distribution: categorical
     forecasting_horizon: 
       value: 1
-    load_other_assets: 
+    load_non_target_asset: 
       values: [True, False]
       distribution: categorical
     log_returns: 

sweep_level_2.yaml

@@ -42,7 +42,7 @@ parameters:
       distribution: categorical
     forecasting_horizon: 
       value: 1
-    load_other_assets: 
+    load_non_target_asset: 
       values: [True, False]
       distribution: categorical
     log_returns: 

training/walk_forward.py

@@ -2,6 +2,7 @@ import pandas as pd
 from models.base import Model
 import numpy as np
 from utils.helpers import get_first_valid_return_index
+from tqdm import tqdm
 
 from sklearn.base import clone
 
@@ -33,7 +34,7 @@ def walk_forward_train_test(
     if is_scaling_on:
         scaler = clone(scaler)
 
-    for index in range(train_from, train_till):
+    for index in tqdm(range(train_from, train_till)):
 
         if iterations_before_retrain <= 0 or pd.isna(models[index-1]):
             if expanding_window:

utils/helpers.py

@@ -1,6 +1,9 @@
-from typing import Literal
 import pandas as pd
 import numpy as np
+import os
+
+def get_files_from_dir(path: str) -> list[str]:
+    return [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f)) and not f.startswith('.')]
 
 def get_first_valid_return_index(series: pd.Series) -> int:
     double_nested_results = np.where(np.logical_and(series != 0, np.logical_not(np.isnan(series))))
@@ -26,3 +29,4 @@ def weighted_average(df: pd.DataFrame, weights_source: str) -> pd.DataFrame:
 
     return mean_df
 
+def deduplicate_indexes(df: pd.DataFrame) -> pd.DataFrame: return df[~df.index.duplicated(keep='last')]

utils/types.py

@@ -1,9 +1,12 @@
 from typing import Callable, Union
 import pandas as pd
 
-
 Period = int
 IsLogReturn = bool
 FeatureExtractor = Callable[[pd.DataFrame, Period, IsLogReturn], Union[pd.DataFrame, pd.Series]]
 Name = str
-FeatureExtractorConfig = tuple[Name, FeatureExtractor, list[Period]]
+FeatureExtractorConfig = tuple[Name, FeatureExtractor, list[Period]]
+Path = str
+FileName = str
+DataSource = list[tuple[Path, FileName]]
+DataCollection = list[DataSource]