manifoldbt/python/tests/test_import_dataframe.py

"""Tests for bt.import_dataframe — in-memory DataFrame → Arrow IPC store.

The contract under test: import_dataframe is the in-memory twin of
import_csv. Same data through either path must produce an identical store
(same backtest results), and the normalisation layer must give clear errors
for bad inputs instead of a Rust panic.
"""
import os

import pytest

import manifoldbt as bt

pd = pytest.importorskip("pandas")

N_BARS = 120
START_MS = 1_577_836_800_000  # 2020-01-01T00:00:00Z


def _bars_df(n=N_BARS, tz="UTC"):
    """Synthetic 1m bars as a pandas DataFrame."""
    ts = pd.date_range("2020-01-01", periods=n, freq="1min", tz=tz)
    close = [100.0 + i * 0.5 for i in range(n)]
    return pd.DataFrame(
        {
            "timestamp": ts,
            "open": close,
            "high": [c + 1.0 for c in close],
            "low": [c - 1.0 for c in close],
            "close": close,
            "volume": [10.0] * n,
        }
    )


def _store_paths(tmp_path, name):
    root = tmp_path / name
    return str(root / "data"), str(root / "metadata.sqlite")


def _import_df(df, tmp_path, name="df", **kw):
    data_root, metadata_db = _store_paths(tmp_path, name)
    os.makedirs(os.path.dirname(metadata_db), exist_ok=True)
    return bt.import_dataframe(
        df, symbol="BTCUSDT", symbol_id=1,
        data_root=data_root, metadata_db=metadata_db, **kw
    )


def _run_buy_and_hold(store):
    strategy = bt.Strategy(
        name="bh",
        signals={"signal": bt.lit(1.0)},
        position_sizing=bt.col("signal"),
    )
    config = bt.BacktestConfig(
        universe=[1],
        time_range_start=0,
        time_range_end=START_MS * 1_000_000 + N_BARS * 60_000_000_000,
        bar_interval={"Minutes": 1},
        initial_capital=1000.0,
        currency="USD",
        execution=bt.ExecutionConfig(
            signal_delay=1,
            execution_price="AtClose",
            max_position_pct=1.0,
            allow_short=False,
            allow_fractional=True,
            skip_gap_bars=False,
            position_sizing_mode="Units",
        ),
        fees=bt.FeeConfig(),
        slippage={"FixedBps": {"bps": 0.0}},
        rng_seed=7,
    )
    return bt.run(strategy, config, store)


def test_import_dataframe_roundtrip(tmp_path):
    """DataFrame → store → run produces a usable backtest."""
    store = _import_df(_bars_df(), tmp_path)
    assert store.resolve_symbol("BTCUSDT") == 1

    result = _run_buy_and_hold(store)
    equity = result.equity_curve.to_pylist()
    assert len(equity) > 0
    # Price rises monotonically → buy & hold ends above initial capital.
    assert equity[-1] > 1000.0


def test_import_dataframe_matches_import_csv(tmp_path):
    """Same bars through import_csv and import_dataframe → identical results."""
    df = _bars_df()

    # CSV path (standard format: epoch-ms timestamp).
    #
    # Built from START_MS rather than derived from the datetime column:
    # `.astype("int64")` returns the underlying integer in the COLUMN's
    # resolution, which pandas picks for itself. Locally that was ns (so
    # //1e6 gave ms), on CI it was us (so //1e6 gave seconds) and the import
    # rejected the row. The bars are 1 minute apart by construction here, so
    # spelling the epoch out keeps the CSV identical on every pandas.
    csv_df = df.copy()
    csv_df["timestamp"] = [START_MS + i * 60_000 for i in range(len(csv_df))]
    csv_path = tmp_path / "bars.csv"
    csv_df.to_csv(csv_path, index=False)
    csv_root, csv_meta = _store_paths(tmp_path, "csv")
    os.makedirs(os.path.dirname(csv_meta), exist_ok=True)
    store_csv = bt.import_csv(
        str(csv_path), symbol="BTCUSDT", symbol_id=1,
        data_root=csv_root, metadata_db=csv_meta,
    )

    store_df = _import_df(df, tmp_path)

    res_csv = _run_buy_and_hold(store_csv)
    res_df = _run_buy_and_hold(store_df)
    assert res_df.equity_curve.to_pylist() == res_csv.equity_curve.to_pylist()
    assert res_df.metrics == res_csv.metrics


def test_import_dataframe_naive_timestamps_assumed_utc(tmp_path):
    """tz-naive datetimes are accepted and treated as UTC."""
    naive = _bars_df(tz=None)
    aware = _bars_df(tz="UTC")
    store_naive = _import_df(naive, tmp_path, name="naive")
    store_aware = _import_df(aware, tmp_path, name="aware")
    assert _run_buy_and_hold(store_naive).equity_curve.to_pylist() == \
        _run_buy_and_hold(store_aware).equity_curve.to_pylist()


def test_import_dataframe_datetime_index_promoted(tmp_path):
    """A pandas DatetimeIndex is used as the timestamp column."""
    df = _bars_df().set_index("timestamp")
    assert "timestamp" not in df.columns
    store = _import_df(df, tmp_path)
    assert store.resolve_symbol("BTCUSDT") == 1


def test_import_dataframe_polars(tmp_path):
    """Polars DataFrames go through the zero-copy to_arrow path."""
    pl = pytest.importorskip("polars")
    df = pl.from_pandas(_bars_df())
    store = _import_df(df, tmp_path)
    assert store.resolve_symbol("BTCUSDT") == 1


def test_import_dataframe_missing_column_raises(tmp_path):
    df = _bars_df().drop(columns=["volume"])
    with pytest.raises(bt.DataError, match="volume"):
        _import_df(df, tmp_path)


def test_import_dataframe_integer_timestamp_raises(tmp_path):
    """Epoch integers are ambiguous (ms? ns?) — require datetimes."""
    df = _bars_df()
    df["timestamp"] = df["timestamp"].astype("int64")
    with pytest.raises(bt.DataError, match="datetime"):
        _import_df(df, tmp_path)


def test_import_dataframe_empty_raises(tmp_path):
    with pytest.raises(bt.DataError, match="no data rows"):
        _import_df(_bars_df(0), tmp_path)


def test_import_dataframe_daily_interval_runs(tmp_path):
    """Daily bars import AND backtest.

    Regression: the resolution table listed only 1m/1h, so a daily store
    resolved to the (empty) 1m directory and the run died with "empty bar
    dataset for symbol". A ``1d`` entry in the table lets the daily provider
    layout be found. 1m/1h were unaffected, which is exactly why this slipped.
    """
    n = 30
    ts = pd.date_range("2021-01-01", periods=n, freq="1D", tz="UTC")
    close = [100.0 + i for i in range(n)]  # strictly rising → buy & hold profits
    df = pd.DataFrame(
        {
            "timestamp": ts,
            "open": close,
            "high": [c + 1.0 for c in close],
            "low": [c - 1.0 for c in close],
            "close": close,
            "volume": [10.0] * n,
        }
    )
    store = _import_df(df, tmp_path, name="daily", interval="1d")
    assert store.resolve_symbol("BTCUSDT") == 1

    strategy = bt.Strategy(
        name="bh",
        signals={"signal": bt.lit(1.0)},
        position_sizing=bt.col("signal"),
    )
    config = bt.BacktestConfig(
        universe=[1],
        time_range_start=0,
        time_range_end=int(ts[-1].value) + 5 * 86_400_000_000_000,
        bar_interval={"Days": 1},
        initial_capital=1000.0,
        execution=bt.ExecutionConfig(
            signal_delay=1, execution_price="AtClose",
            position_sizing_mode="Units",
        ),
        fees=bt.FeeConfig(),
        slippage={"FixedBps": {"bps": 0.0}},
    )
    result = bt.run(strategy, config, store)
    equity = result.equity_curve.to_pylist()
    assert len(equity) > 0
    assert equity[-1] > 1000.0