"""`bt.reconcile`: a broker's fills against the backtest of the same days. Three halves, the split `test_fill_marks.py` already makes. The half that runs everywhere needs no tape: how a journal is read (the column spellings, the sides, the sort), the refusals a wrong journal hits first, and the chart, which reads nothing but the dict it is handed. The end-to-end half needs a store holding a tape, because a mark is a price the market printed and a tape enters a store only through `bt.ingest_trades`. It skips by name unless `MANIFOLDBT_TEST_TAPE_STORE` points at one, as `|||` -- the same variable `test_fill_marks.py` reads. The measurement itself is pinned by the engine's own tests, unit and end to end. """ import os import pytest pd = pytest.importorskip("pandas") np = pytest.importorskip("numpy") import manifoldbt as bt # noqa: E402 from manifoldbt.expr import col, lit # noqa: E402 from manifoldbt.helpers import Interval, Slippage # noqa: E402 from manifoldbt.reconcile import Reconciliation, fills_to_columns # noqa: E402 SYMBOL_ID = 1 HORIZONS = ("100ms", "1s", "10s") # --------------------------------------------------------------------------- # Reading a journal # --------------------------------------------------------------------------- def _journal(**over): base = { "timestamp": pd.to_datetime( ["2021-03-01T00:00:02Z", "2021-03-01T00:00:01Z"], utc=True ), "side": ["buy", "sell"], "price": [100.0, 101.0], "qty": [1.0, 2.0], } base.update(over) return pd.DataFrame(base) def test_a_journal_is_read_in_time_order_whatever_order_it_was_written_in(): cols = fills_to_columns(_journal(), SYMBOL_ID) assert cols["ts"] == sorted(cols["ts"]) # The row that was second is now first, with its own price and side. assert cols["price"] == [101.0, 100.0] assert cols["side"] == [2, 1] assert cols["qty"] == [2.0, 1.0] assert cols["symbol"] == [SYMBOL_ID, SYMBOL_ID] def test_the_common_broker_spellings_are_read_without_a_rename(): frame = pd.DataFrame( { "time": pd.to_datetime(["2021-03-01T00:00:01Z"], utc=True), "direction": ["B"], "avg_price": [100.0], "quantity": [3.0], } ) cols = fills_to_columns(frame, SYMBOL_ID) assert cols["side"] == [1] assert cols["price"] == [100.0] assert cols["qty"] == [3.0] def test_a_side_that_reads_as_neither_is_refused_rather_than_taken_for_a_sell(): with pytest.raises(ValueError, match="neither a buy nor a sell"): fills_to_columns(_journal(side=["buy", "hedge"]), SYMBOL_ID) def test_the_engines_own_side_codes_round_trip(): cols = fills_to_columns(_journal(side=[1, 2]), SYMBOL_ID) assert cols["side"] == [2, 1] # sorted by time: the second row comes first def test_a_missing_column_names_what_it_wanted_and_what_it_saw(): frame = _journal().drop(columns=["price"]) with pytest.raises(ValueError, match="no 'price' column"): fills_to_columns(frame, SYMBOL_ID) def test_a_journal_with_no_symbol_asks_for_one(): with pytest.raises(ValueError, match="symbol_id"): fills_to_columns(_journal(), None) def test_a_journal_carrying_its_own_symbol_column_keeps_it(): cols = fills_to_columns(_journal(symbol_id=[7, 9]), None) assert cols["symbol"] == [9, 7] def test_a_dict_of_columns_is_a_journal_too(): cols = fills_to_columns( { "timestamp": np.array([2_000_000_000, 1_000_000_000], dtype="int64"), "side": np.array([1, 2], dtype="int64"), "price": np.array([100.0, 101.0]), "qty": np.array([1.0, 2.0]), }, SYMBOL_ID, ) assert cols["ts"] == [1_000_000_000, 2_000_000_000] def test_something_that_is_not_a_table_is_refused_by_name(): with pytest.raises(TypeError, match="DataFrame"): fills_to_columns([1, 2, 3], SYMBOL_ID) def test_a_tolerance_that_is_not_a_duration_is_refused_by_name(): with pytest.raises(TypeError, match="duration"): bt.reconcile(None, _journal(), store=None, symbol_id=1, tolerance="soon") # --------------------------------------------------------------------------- # The accessors and the chart # --------------------------------------------------------------------------- def _stub(): """The engine's answer, hand-made: the wrapper reads it and nothing else.""" horizons = [ { "horizon": h, "horizon_ns": ns, "live_mean_bps": -1.2, "live_stderr_bps": 0.05, "live_marked_fills": 400, "sim_mean_bps": -0.7, "sim_stderr_bps": 0.04, "sim_marked_fills": 660, "diff_mean_bps": -0.5, "paired_mean_bps": -0.5, "paired_stderr_bps": 0.01, "paired_fills": 400, } for h, ns in zip(HORIZONS, (100_000_000, 1_000_000_000, 10_000_000_000)) ] return { "tolerance_ns": 1_000_000_000, "price_tolerance": None, "live_fills": 400, "sim_fills": 660, "matched": 400, "live_only": 0, "sim_only": 260, "matched_share_live": 1.0, "matched_share_sim": 400 / 660, "service": {"orders_posted": 5000, "live_fills": 400, "sim_fills": 660, "live_rate": 0.08, "sim_rate": 0.132, "served_ratio": 400 / 660}, "by_day": {"key": [0], "live_fills": [400], "sim_fills": [660], "matched": [400], "live_only": [0], "sim_only": [260], "served_ratio": [400 / 660]}, "by_hour": {"key": [0, 1], "live_fills": [200, 200], "sim_fills": [330, 330], "matched": [200, 200], "live_only": [0, 0], "sim_only": [130, 130], "served_ratio": [0.606, 0.606]}, "horizons": horizons, "half_spread": {"live_captured_bps": 0.001, "live_fills": 400, "sim_captured_bps": 0.006, "sim_fills": 660, "diff_bps": -0.005, "book_half_spread_bps": 0.0064}, "price_gap_bps": {"count": 400, "mean": -0.5, "min": -0.5, "p05": -0.5, "p25": -0.5, "median": -0.5, "p75": -0.5, "p95": -0.5, "max": -0.5}, "time_gap_ns": {"count": 400, "mean": 0.0, "min": 0.0, "p05": 0.0, "p25": 0.0, "median": 0.0, "p75": 0.0, "p95": 0.0, "max": 0.0}, "verdict": ["service: the backtest serves 13.2% ... where the journal serves 8.0%"], "pairs": {"live_index": [0], "sim_index": [0], "symbol_id": [1], "side": [1], "live_timestamp": [0], "sim_timestamp": [0], "time_gap_ns": [0], "live_price": [100.005], "sim_price": [100.0], "price_edge_bps": [-0.5], "qty_gap": [0.0], "live_mark_100ms": [99.9], "sim_mark_100ms": [99.9], "live_mark_1s": [99.9], "sim_mark_1s": [99.9], "live_mark_10s": [99.9], "sim_mark_10s": [99.9]}, } def test_the_tables_come_back_shaped_and_named(): rec = Reconciliation(_stub()) assert rec.live_fills == 400 and rec.sim_fills == 660 and rec.matched == 400 day = rec.by_day_df() assert list(day.columns)[0] == "day" assert str(day["day"].dtype).startswith("datetime64[ns") hour = rec.by_hour_df() assert list(hour.columns)[0] == "hour" assert list(hour["hour"]) == [0, 1] horizons = rec.horizons_df() assert list(horizons["horizon"]) == list(HORIZONS) assert "paired_stderr_bps" in horizons.columns pairs = rec.pairs_df() assert str(pairs["live_timestamp"].dtype).startswith("datetime64[ns") for h in HORIZONS: assert f"live_mark_{h}" in pairs.columns assert f"sim_mark_{h}" in pairs.columns def test_the_summary_is_the_verdict_one_line_at_a_time(): rec = Reconciliation(_stub()) assert rec.summary() == "\n".join(rec.verdict) assert "Reconciliation(" in repr(rec) def test_the_chart_is_exported_and_refuses_what_is_not_a_reconciliation(): plot = pytest.importorskip("manifoldbt.plot") assert "reconcile" in plot.__all__ with pytest.raises(ValueError, match="bt.reconcile"): plot.reconcile(object()) def test_the_chart_draws_both_sides_on_both_panels(): pytest.importorskip("plotly") plot = pytest.importorskip("manifoldbt.plot") fig = plot.reconcile(Reconciliation(_stub()), show=False) # Two bars a panel, two panels: service by hour and markout by horizon. assert len(fig.data) == 4 assert {t.name for t in fig.data} == {"backtest", "journal"} def test_the_chart_refuses_a_reconciliation_of_two_empty_sides(): plot = pytest.importorskip("manifoldbt.plot") empty = _stub() empty["live_fills"] = 0 empty["sim_fills"] = 0 with pytest.raises(ValueError, match="nothing"): plot.reconcile(Reconciliation(empty)) def test_the_name_is_exported(): assert "reconcile" in bt.__all__ assert "Reconciliation" in bt.__all__ assert callable(bt.reconcile) # --------------------------------------------------------------------------- # End to end, on a store that holds a tape # --------------------------------------------------------------------------- _TAPE = os.environ.get("MANIFOLDBT_TEST_TAPE_STORE", "") _needs_tape = pytest.mark.skipif( not _TAPE, reason="set MANIFOLDBT_TEST_TAPE_STORE=||| " "to reconcile real fills (a tape enters a store only through bt.ingest_trades, " "which needs the network)", ) def _tape_store(): root, meta, sid, day = _TAPE.split("|") store = bt.DataStore( data_root=root, metadata_db=meta, arrow_dir=os.path.join(root, "mega") ) start = int(bt.date_to_ns(day)) return store, int(sid), start, start + 86_400_000_000_000 def _tape_run(store, sid, start, end): strategy = ( bt.Strategy.create("maker") .signal("level", col("close") * lit(1.0 - 1e-4)) .size(lit(0.01)) .limit_entry(signal="level", time_in_force={"GTB": 1}) .take_profit(pct=0.02, side="long") .stop_loss(pct=0.06, side="long") ) cfg = bt.BacktestConfig( universe=[sid], time_range_start=start, time_range_end=end, bar_interval=Interval.seconds(1), initial_capital=100_000.0, # Daily output, the way a run without a licence sees it: nothing here # asserts on an intraday row. output_resolution=Interval.days(1), warmup_bars=60, execution=bt.ExecutionConfig( signal_delay=1, allow_short=False, position_sizing_mode="Units", fill_resolution="ticks", fill_marks=True, ), fees=bt.FeeConfig.zero(), slippage=Slippage.none(), ) return bt.run(strategy, cfg, store) def _as_journal(trades, *, drop=0.0, worse_bps=0.0, seed=20260908): kept = trades if drop: rng = np.random.default_rng(seed) kept = trades.loc[rng.random(len(trades)) >= drop] side = kept["side"].to_numpy() price = kept["fill_price"].to_numpy() if worse_bps: price = price * (1.0 + np.where(side == 1, 1.0, -1.0) * worse_bps * 1e-4) return pd.DataFrame({ "timestamp": kept["execution_timestamp"].to_numpy(), "side": np.where(side == 1, "buy", "sell"), "price": price, "qty": kept["quantity"].to_numpy(), }) @_needs_tape def test_a_run_reconciled_with_its_own_fill_log_is_zero_everywhere(): store, sid, start, end = _tape_store() result = _tape_run(store, sid, start, end) trades = result.trades_df() assert len(trades) > 0, "the day has to trade for this to say anything" rec = bt.reconcile(result, _as_journal(trades), store=store, symbol_id=sid, tolerance=Interval.seconds(1)) assert rec.matched == rec.live_fills == rec.sim_fills == len(trades) assert rec.live_only == 0 and rec.sim_only == 0 for h in rec.horizons: assert h["paired_mean_bps"] == 0.0 assert h["diff_mean_bps"] == 0.0 assert h["live_mean_bps"] == h["sim_mean_bps"] assert rec.price_gap_bps["mean"] == 0.0 assert rec.time_gap_ns["max"] == 0.0 # The marks it took for the backtest side ARE the run's own marks. for own, both in zip(result.fill_marks["horizons"], rec.horizons): assert own["mean_bps"] == both["sim_mean_bps"] assert own["stderr_bps"] == both["sim_stderr_bps"] assert own["marked_fills"] == both["sim_marked_fills"] @_needs_tape def test_a_degraded_journal_reads_back_the_degradation_that_was_injected(): store, sid, start, end = _tape_store() result = _tape_run(store, sid, start, end) trades = result.trades_df() journal = _as_journal(trades, drop=0.40, worse_bps=0.5) rec = bt.reconcile(result, journal, store=store, symbol_id=sid, tolerance=Interval.seconds(1)) assert rec.matched == rec.live_fills, "a kept fill lost its twin" assert rec.sim_only == rec.sim_fills - rec.live_fills assert abs(rec.live_fills / rec.sim_fills - 0.60) < 0.05 assert abs(rec.price_gap_bps["mean"] + 0.5) < 1e-2 for h in rec.horizons: if h["paired_fills"] < 20: continue assert abs(h["paired_mean_bps"] + 0.5) < 1e-2, h["horizon"] # The tables come back with rows in them, and the day is the run's day. assert len(rec.by_day_df()) >= 1 assert len(rec.by_hour_df()) >= 1 assert len(rec.pairs_df()) == rec.matched assert any(line.startswith("service:") for line in rec.verdict) @_needs_tape def test_the_service_rates_share_the_backtests_own_posting_count(): store, sid, start, end = _tape_store() result = _tape_run(store, sid, start, end) posted = result.order_activity["orders_posted"] rec = bt.reconcile(result, _as_journal(result.trades_df(), drop=0.40), store=store, symbol_id=sid) assert rec.service["orders_posted"] == posted assert rec.service["sim_rate"] == pytest.approx(rec.sim_fills / posted) assert rec.service["live_rate"] == pytest.approx(rec.live_fills / posted)