mirror of
https://github.com/manifoldbt/manifoldbt.git
synced 2026-11-02 07:56:11 +00:00
96 lines
3.1 KiB
Python
96 lines
3.1 KiB
Python
"""Populate the shared data store the examples read. Not an example itself.
|
|
|
|
Twenty of the files here are marked `Data: shared store`: they read real
|
|
market data from `data/` + `metadata/`, which are not in the repository
|
|
because market data does not belong in a git repository. This script
|
|
downloads what those files name, once.
|
|
|
|
python examples/setup_data.py # everything, ~10 symbols
|
|
python examples/setup_data.py BTC # just the symbols an ID matches
|
|
|
|
Everything is ingested at 1h. The engine aggregates upwards, so one 1h store
|
|
serves every example here, including the ones running on 12h or daily bars.
|
|
|
|
The connectors used (binance, dydx) are free on all tiers and need no API key.
|
|
Re-running is safe: an interval already present is not downloaded twice.
|
|
|
|
Usage:
|
|
python examples/setup_data.py
|
|
"""
|
|
import sys
|
|
|
|
import manifoldbt as mbt
|
|
|
|
from _bootstrap import DATA_ROOT, METADATA_DB
|
|
|
|
# Every symbol the `Data: shared store` examples name, with the id it gets in
|
|
# the store. The ids only have to be unique -- the examples address symbols by
|
|
# name (`"BTC-USDT:perp"`), not by id.
|
|
#
|
|
# `asset_class="crypto_perp"` is the part that is easy to miss: the examples
|
|
# ask for `BTC-USDT:perp`, and an ingest left on the default `crypto_spot`
|
|
# stores a symbol they will not find.
|
|
BINANCE_PERPS = [
|
|
("BTCUSDT", 1),
|
|
("ETHUSDT", 2),
|
|
("LTCUSDT", 3),
|
|
("BNBUSDT", 4),
|
|
("DOTUSDT", 5),
|
|
("XRPUSDT", 6),
|
|
("ADAUSDT", 7),
|
|
("LINKUSDT", 8),
|
|
("DOGEUSDT", 9),
|
|
("AVAXUSDT", 10),
|
|
]
|
|
|
|
# 15_cross_exchange executes on dydx and takes its signal from binance. dydx
|
|
# v4 history starts in 2024, which is why this one has its own start date.
|
|
DYDX_PERPS = [("BTC-USD", 11)]
|
|
|
|
START = "2021-01-01T00:00:00Z"
|
|
DYDX_START = "2024-02-01T00:00:00Z"
|
|
END = "2026-03-01T00:00:00Z"
|
|
INTERVAL = "1h"
|
|
|
|
|
|
def _ingest(provider, symbol, symbol_id, start):
|
|
try:
|
|
mbt.ingest(
|
|
provider=provider,
|
|
symbol=symbol,
|
|
symbol_id=symbol_id,
|
|
start=start,
|
|
end=END,
|
|
interval=INTERVAL,
|
|
asset_class="crypto_perp",
|
|
data_root=DATA_ROOT,
|
|
metadata_db=METADATA_DB,
|
|
)
|
|
return True
|
|
except Exception as exc: # noqa: BLE001 - reported, not raised
|
|
print(f" {provider}:{symbol} FAILED -- {exc}")
|
|
return False
|
|
|
|
|
|
def main(argv):
|
|
wanted = [a.upper() for a in argv]
|
|
jobs = [("binance", s, i, START) for s, i in BINANCE_PERPS]
|
|
jobs += [("dydx", s, i, DYDX_START) for s, i in DYDX_PERPS]
|
|
if wanted:
|
|
jobs = [j for j in jobs if any(w in j[1].upper() for w in wanted)]
|
|
if not jobs:
|
|
raise SystemExit(f"No symbol matches {', '.join(wanted)}")
|
|
|
|
print(f"Ingesting {len(jobs)} symbols at {INTERVAL} into {DATA_ROOT}\n")
|
|
ok = sum(_ingest(*job) for job in jobs)
|
|
|
|
print(f"\n{ok}/{len(jobs)} symbols in the store.")
|
|
if ok:
|
|
print("The `Data: shared store` examples can run now, e.g.")
|
|
print(" python examples/01_trend_following.py")
|
|
return 0 if ok == len(jobs) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|