manifoldbt/python/tests/test_ingest_trades.py

"""`bt.ingest_trades` reading Bybit's derivatives archive into a stored tape.

No network: with ``cache_dir`` an archive already in the cache under its venue
name is read exactly as a downloaded one would be, so a hand-written one stands
in for a real day. What is checked is the conversion, not the numbers of any
real day: the aggressor's side, the stamp to the digit, the venue's order
(newest-first files included), the kind of contract, and one market and one id
per symbol, refused before the network.
"""
import gzip

import pytest

pa = pytest.importorskip("pyarrow")

import manifoldbt as bt  # noqa: E402

DAY = "2023-03-01"
# 2023-03-01T00:00:00Z
T0 = 1677628800
SYMBOL = "TESTUSDT"
HEADER = ("timestamp,symbol,side,size,price,tickDirection,trdMatchID,"
          "grossValue,homeNotional,foreignNotional")


def _tape_unlocked():
    """Probe the gate, not the banner: a Pro licence does not open this."""
    try:
        bt.ticks.tape_info(__file__)
    except PermissionError:
        return False
    except Exception:
        return True
    return True


pytestmark = pytest.mark.skipif(
    not _tape_unlocked(),
    reason="tick layer locked (runs in the engine's own debug builds)",
)


def _line(ts, side, size, price, symbol=SYMBOL, inverse=False):
    """One trade as Bybit writes it: the coin value of an inverse contract is
    size / price, the quote value of a linear one size * price."""
    foreign = size / price if inverse else size * price
    return (f"{ts},{symbol},{side},{size},{price},PlusTick,"
            f"00000000-0000-0000-0000-000000000000,1,{size},{foreign!r}")


def _archive(cache, lines, symbol=SYMBOL, day=DAY):
    cache.mkdir(parents=True, exist_ok=True)
    path = cache / f"{symbol}{day}.csv.gz"
    path.write_bytes(gzip.compress(("\n".join([HEADER, *lines]) + "\n").encode()))
    return path


def _ingest(tmp_path, category="linear", symbol=SYMBOL, symbol_id=7, day=DAY, **kw):
    return bt.ingest_trades(
        "bybit", symbol, symbol_id=symbol_id, start=day, end=day,
        data_root=str(tmp_path / "data"), metadata_db=str(tmp_path / "meta.sqlite"),
        category=category, cache_dir=str(tmp_path / "cache"), progress=False, **kw,
    )


def _stored(tmp_path, symbol=SYMBOL, day=DAY):
    path = tmp_path / "data" / "mega" / "bybit" / "ticks" / symbol / f"{day}.arrow"
    with pa.ipc.open_file(str(path)) as f:
        return f.read_all(), {k.decode(): v.decode() for k, v in (f.schema.metadata or {}).items()}


def test_a_linear_day_is_stored_print_by_print_with_its_market(tmp_path):
    _archive(tmp_path / "cache", [
        _line(f"{T0 + 3}.1129", "Sell", 111, 0.3761),
        _line(f"{T0 + 3}.1129", "Buy", 105, 0.3762),
        _line(f"{T0 + 3}.96", "Buy", 360, 0.3762),
    ])
    _ingest(tmp_path)
    table, meta = _stored(tmp_path)
    assert (meta["category"], meta["source"]) == ("linear", "bybit/linear")
    ns = [t.value for t in table.column("timestamp")]
    # The stamp to the digit: `.96` is `.9600`, and no float on the way.
    assert ns == [(T0 + 3) * 10**9 + 112_900_000] * 2 + [(T0 + 3) * 10**9 + 960_000_000]
    assert table.column("qty").to_pylist() == [111.0, 105.0, 360.0]
    # The side is the aggressor's: a sell aggressor means the buyer made.
    assert table.column("is_buyer_maker").to_pylist() == [True, False, False]
    # A UUID has no place in the integer id.
    assert table.column("trade_id").null_count == 3


def test_a_day_listed_newest_first_comes_back_in_the_venue_order(tmp_path):
    """Up to 2021-12-06 Bybit lists a day newest first: reversed, the prints
    sharing a stamp keep the order they were matched in."""
    _archive(tmp_path / "cache", [
        _line(f"{T0 + 9}.5", "Buy", 30, 1.5),
        _line(f"{T0 + 5}.25", "Sell", 21, 1.4),
        _line(f"{T0 + 5}.25", "Sell", 20, 1.4),
        _line(f"{T0 + 1}", "Sell", 18, 1.5),
    ])
    _ingest(tmp_path)
    table, _ = _stored(tmp_path)
    assert table.column("qty").to_pylist() == [18.0, 20.0, 21.0, 30.0]


def test_prints_stamped_outside_the_day_stay_out_of_its_file(tmp_path):
    _archive(tmp_path / "cache", [
        _line(f"{T0 - 1}.5", "Buy", 1, 1.5),
        _line(f"{T0 + 1}", "Buy", 2, 1.5),
        _line(f"{T0 + 86_400}", "Buy", 3, 1.5),
    ])
    _ingest(tmp_path)
    table, _ = _stored(tmp_path)
    assert table.column("qty").to_pylist() == [2.0]


def test_an_inverse_archive_asked_as_linear_is_refused_by_name(tmp_path):
    _archive(tmp_path / "cache", [_line(f"{T0 + 1}", "Buy", 517, 23134.5, inverse=True)],
             symbol="TESTUSD")
    with pytest.raises((ValueError, RuntimeError), match="inverse contract.*category='inverse'"):
        _ingest(tmp_path, symbol="TESTUSD")
    assert not (tmp_path / "data" / "mega" / "bybit" / "ticks" / "TESTUSD").exists()
    # Under its own name it is stored, sizes in contracts.
    _ingest(tmp_path, category="inverse", symbol="TESTUSD")
    table, meta = _stored(tmp_path, symbol="TESTUSD")
    assert meta["category"] == "inverse"
    assert table.column("qty").to_pylist() == [517.0]


def test_one_symbol_is_one_market_refused_before_the_network(tmp_path):
    """The day asked for is in no cache: were the market checked after the
    download, the error would be the network's."""
    _archive(tmp_path / "cache", [_line(f"{T0 + 1}", "Buy", 1, 1.5)])
    _ingest(tmp_path)
    with pytest.raises((ValueError, RuntimeError), match="linear tape of TESTUSDT.*one market"):
        _ingest(tmp_path, category="spot", day="2023-03-02")
    with pytest.raises((ValueError, RuntimeError), match="linear tape of TESTUSDT.*one market"):
        bt.ingest_book(
            "bybit", SYMBOL, symbol_id=7, start="2023-03-02", end="2023-03-02",
            data_root=str(tmp_path / "data"), metadata_db=str(tmp_path / "meta.sqlite"),
            category="spot", cache_dir=str(tmp_path / "cache"), progress=False,
        )


def test_the_tape_and_the_book_of_a_symbol_share_one_id(tmp_path):
    _archive(tmp_path / "cache", [_line(f"{T0 + 1}", "Buy", 1, 1.5)])
    _ingest(tmp_path, symbol_id=7)
    with pytest.raises((ValueError, RuntimeError), match="symbol_id 7.*symbol_id=7"):
        _ingest(tmp_path, symbol_id=8)


@pytest.mark.parametrize("provider, kwargs, match", [
    ("bybit", {"category": "option"}, "no option trade archive"),
    ("binance", {"category": "linear"}, "provider must be 'bybit'"),
    ("binance", {"cache_dir": "x"}, "cache_dir keeps Bybit"),
    ("bybit", {"category": "linear", "exchange": "BINANCE"}, "filed under BYBIT"),
    ("bybit", {"category": "linear", "asset_class": "crypto_spot"}, "perpetual"),
])
def test_what_cannot_be_read_is_refused_before_the_network(tmp_path, provider, kwargs, match):
    with pytest.raises(ValueError, match=match):
        bt.ingest_trades(
            provider, SYMBOL, symbol_id=7, start=DAY, end=DAY,
            data_root=str(tmp_path / "data"), metadata_db=str(tmp_path / "meta.sqlite"),
            progress=False, **kwargs,
        )
    assert not (tmp_path / "data").exists()