mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-28 03:08:01 +00:00
Mark Aron Szulyovszky
9d47ee942d
* feat(Project): use 5 minute data, running training in parallel, sped up cusum filter by 10x with numba * fix(WalkForward): inference mini-batch parallelization * fix(WalkForward): don't use the parallel version of any of the functions * feat(CI): download the data required * fix(Project): 5min_crypto folder added * fix(Evaluate): make sure we have numerical stability in returns * feat(Models): use SKLearn models directly to enable composability * feat(Inference): batched inference now working, added forecasting_horizon * fix(Inference): works again * fix(Inference) * chore(Models): remove unused Ensemble model * fix(Labeller): don't just forward shift returns, also take the sum of the data happened until then * Update test.yml
38 lines
1.5 KiB
Python
38 lines
1.5 KiB
Python
import pandas as pd
|
|
import ssl
|
|
from tqdm import tqdm
|
|
from data_loader.utils import deduplicate_indexes
|
|
from utils.resample import resample_ohlc
|
|
|
|
base_url = "https://www.cryptodatadownload.com/cdd/"
|
|
|
|
exchange_name = "Bitfinex"
|
|
period = "minute" # 1h
|
|
files_to_download = [
|
|
exchange_name + '_TRXUSD_' + period + '.csv',
|
|
exchange_name + '_ETHUSD_' + period + '.csv',
|
|
exchange_name + '_XLMUSD_' + period + '.csv',
|
|
exchange_name + '_XMRUSD_' + period + '.csv',
|
|
exchange_name + '_LTCUSD_' + period + '.csv',
|
|
exchange_name + '_DASHUSD_' + period + '.csv',
|
|
exchange_name + '_BTCUSD_' + period + '.csv',
|
|
exchange_name + '_ETCUSD_' + period + '.csv',
|
|
]
|
|
|
|
|
|
for file in tqdm(files_to_download):
|
|
data_location = base_url + file
|
|
ssl._create_default_https_context = ssl._create_unverified_context
|
|
data = pd.read_csv(data_location, skiprows=1, index_col=1, parse_dates=True).drop(columns=['unix'])
|
|
volume_column_to_delete = [c for c in data.columns if c.startswith('Volume') and 'USD' not in c]
|
|
data.drop(volume_column_to_delete + ['symbol'], axis=1, inplace=True)
|
|
data.rename({'Volume USD': 'volume'}, axis=1, inplace=True)
|
|
data.index.rename('time', inplace=True)
|
|
data.sort_index(inplace=True)
|
|
data = deduplicate_indexes(data)
|
|
data.index = pd.to_datetime(data.index)
|
|
data = data.resample('1Min').ffill()
|
|
data = resample_ohlc(data, '5Min')
|
|
target_file = file.split('_')[1].replace('USD', '') + '_USD'
|
|
data.to_csv(f"data/5min_crypto/{target_file}.csv")
|