mirror of
https://github.com/webclinic017/drift.git
synced 2026-07-27 18:57:55 +00:00
Mark Aron Szulyovszky
75157c6285
* feat(Labeling): purge overlapping events, sort dataframe at loading time * fix(Linter): ran * refactor(Labeling): moved purge_overlapping_events one abstraction level higher * fix(Data): renamed class * fix(Data): corrected parameter name * fix(Config): parameters * fix(Data): fixed path * fix(Data): uncommented required code * feat(EventFilters): use vol based CUSUM * fix(Config): only retrain every 2000 samples * fix(Config): filter out even more events * fix(Inference): added remove_overlapping_events * refactor(Types): simplified type hierarchy
64 lines
1.5 KiB
Python
64 lines
1.5 KiB
Python
from binance_historical_data import BinanceDataDumper
|
|
import datetime
|
|
|
|
data_dumper = BinanceDataDumper(
|
|
path_dir_where_to_dump="./data/5min_crypto/",
|
|
data_frequency="5m",
|
|
)
|
|
|
|
assets = [
|
|
"TRXUSDT",
|
|
"XRPUSDT",
|
|
"ADAUSDT",
|
|
"SOLUSDT",
|
|
"AVAXUSDT",
|
|
"DOTUSDT",
|
|
"ETHUSDT",
|
|
"LTCUSDT",
|
|
"BNBUSDT",
|
|
"BTCUSDT",
|
|
"ETCUSDT",
|
|
]
|
|
|
|
data_dumper.dump_data(
|
|
list_tickers=assets,
|
|
date_start=datetime.date(2018, 1, 1),
|
|
date_end=datetime.date(2022, 1, 1),
|
|
is_to_update_existing=False,
|
|
)
|
|
|
|
import os
|
|
from tqdm import tqdm
|
|
import pandas as pd
|
|
|
|
for asset in tqdm(assets):
|
|
path = f"./data/5min_crypto/spot/monthly/klines/{asset}/5m/"
|
|
files = os.listdir(path)
|
|
|
|
def load_df(path):
|
|
df = pd.read_csv(
|
|
path,
|
|
names=[
|
|
"timestamp",
|
|
"open",
|
|
"low",
|
|
"high",
|
|
"close",
|
|
"volume",
|
|
"Closetime",
|
|
"Quote asset volume",
|
|
"Number of trades",
|
|
"Taker buy base asset volume",
|
|
"Taker buy quote asset volume",
|
|
"Ignore",
|
|
],
|
|
)
|
|
df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms")
|
|
df = df[["timestamp", "open", "high", "low", "close", "volume"]]
|
|
df.set_index("timestamp", inplace=True)
|
|
df.sort_index(inplace=True)
|
|
return df
|
|
|
|
dfs = pd.concat([load_df(path + file) for file in files], axis=0)
|
|
dfs.to_parquet(f"./data/5min_crypto/{asset}.parquet")
|