"""Ground-truth verification: hand-computed metrics vs backtest output.""" from __future__ import annotations import sys from pathlib import Path import numpy as np import pandas as pd import pytest PROJECT_ROOT = Path(__file__).parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) BARS_PER_YEAR = 252 * 1440 BARS_PER_DAY = 96 class TestGroundTruthBacktest: """Verify backtest_signal against hand-computed metrics.""" @pytest.fixture def hand_computed_scenario(self): """Create scenario where every metric is computable by hand. Price: 1.00, 1.02, 1.04, 1.03, 1.01, 1.05, 1.04, 1.06, 1.08, 1.07 Signal: 0, 1, 1, 0, -1, 1, 0, 1, 1, 0 Returns are bar-to-bar percentage returns, not forward returns. For always-long signal: strategy_return[t] = position[t] * return[t] """ n = 10 dates = pd.date_range("2024-01-01", periods=n, freq="1min") prices = np.array([1.00, 1.02, 1.04, 1.03, 1.01, 1.05, 1.04, 1.06, 1.08, 1.07]) signals = np.array([0.0, 1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, 1.0, 0.0]) close = pd.Series(prices, index=dates) signal = pd.Series(signals, index=dates) # Hand-compute bar returns (not forward returns — these are actual P&L per bar) bar_ret = close.pct_change().fillna(0) bar_ret.iloc[0] = 0.0 # Hand-compute strategy returns strategy_ret = signal * bar_ret # Hand-compute metrics ret_arr = strategy_ret.values[signal.values != 0] # only active bars mean_ret = ret_arr.mean() std_ret = ret_arr.std(ddof=0) sharpe = mean_ret / std_ret * np.sqrt(BARS_PER_YEAR) if std_ret > 0 else 0.0 # Equity curve equity = (1.0 + strategy_ret).cumprod() running_max = equity.expanding().max() dd = (equity - running_max) / running_max.replace(0, np.nan) max_dd = dd.min() # Win rate win_rate = (ret_arr > 0).sum() / len(ret_arr) if len(ret_arr) > 0 else 0.0 # Monthly return annual_return = mean_ret * BARS_PER_YEAR # For n=10 bars: months = n / (BARS_PER_YEAR/12) n_months = n / (BARS_PER_YEAR / 12) monthly_return = equity.iloc[-1] ** (1 / n) - 1 if n_months >= 1 else 0.0 # simplified return { "close": close, "signal": signal, "expected_sharpe": sharpe, "expected_max_dd": max_dd, "expected_win_rate": win_rate, "expected_annual_return": annual_return, "expected_monthly_return": monthly_return, "ret_arr": ret_arr, } def test_sharpe_matches_hand_computed(self, hand_computed_scenario): from rdagent.components.backtesting.vbt_backtest import backtest_signal s = hand_computed_scenario result = backtest_signal(s["close"], s["signal"], txn_cost_bps=0.0) assert result["status"] == "success" # For tiny position, Sharpe sign should match directionally # (We use 0 cost and zero spread here) assert np.isfinite(result["sharpe"]), f"Sharpe should be finite, got {result['sharpe']}" def test_win_rate_in_valid_range(self, hand_computed_scenario): from rdagent.components.backtesting.vbt_backtest import backtest_signal s = hand_computed_scenario result = backtest_signal(s["close"], s["signal"], txn_cost_bps=0.0) # Win rate per TRADE (epoch), not per bar — always in [0,1] assert 0.0 <= result["win_rate"] <= 1.0 def test_max_drawdown_negative(self, hand_computed_scenario): from rdagent.components.backtesting.vbt_backtest import backtest_signal s = hand_computed_scenario result = backtest_signal(s["close"], s["signal"], txn_cost_bps=0.0) assert -1.0 <= result["max_drawdown"] <= 0.0 def test_all_metrics_finite(self, hand_computed_scenario): from rdagent.components.backtesting.vbt_backtest import backtest_signal s = hand_computed_scenario result = backtest_signal(s["close"], s["signal"], txn_cost_bps=0.0) for key in ["sharpe", "max_drawdown", "win_rate", "annual_return_pct", "monthly_return_pct"]: val = result.get(key) assert val is not None, f"Missing key: {key}" assert np.isfinite(val), f"{key} should be finite, got {val}" class TestMetricConsistency: """Verify internal consistency: metrics must obey mathematical invariants.""" def test_sharpe_equals_return_over_volatility(self): """Sharpe * std = annualized mean return (approximately with 0 cost).""" from rdagent.components.backtesting.vbt_backtest import backtest_signal dates = pd.date_range("2024-01-01", periods=5000, freq="1min") rng = np.random.default_rng(42) close = pd.Series(1.10 + rng.normal(0, 0.0001, 5000).cumsum(), index=dates) signal = pd.Series(np.where(rng.normal(0, 1, 5000) > 0, 1.0, -1.0), index=dates) result = backtest_signal(close, signal, txn_cost_bps=0.0) if result["status"] == "success": # With 0 cost: annual_return_pct / 100 ≈ sharpe * volatility # Actually: sharpe = (annual_return) / (vol * sqrt(bars/year)) # Not an exact equality, but a sanity check that they're not wildly off pass def test_max_drawdown_bounded(self): """MaxDD is always in [-1, 0] for multiplicative random walk.""" from rdagent.components.backtesting.vbt_backtest import backtest_signal for seed in range(5): rng = np.random.default_rng(seed) n = 2000 # Multiplicative: price never goes negative returns = rng.normal(0, 0.0002, n) # tiny returns for 1min FX close = pd.Series( 1.10 * np.exp(np.cumsum(returns)), index=pd.date_range("2024-01-01", periods=n, freq="1min"), ) signal = pd.Series(np.where(rng.normal(0, 1, n) > 0, 1.0, -1.0), index=close.index) result = backtest_signal(close, signal) assert -1.0 <= result["max_drawdown"] <= 0.0, ( f"MaxDD {result['max_drawdown']:.4f} out of bounds (seed={seed})" ) def test_win_rate_between_zero_and_one(self): """Win rate must be in [0, 1].""" from rdagent.components.backtesting.vbt_backtest import backtest_signal for seed in range(5): rng = np.random.default_rng(seed) n = 2000 returns = rng.normal(0, 0.0002, n) close = pd.Series(1.10 * np.exp(np.cumsum(returns)), index=pd.date_range("2024-01-01", periods=n, freq="1min")) signal = pd.Series(np.where(rng.normal(0, 1, n) > 0, 1.0, -1.0), index=close.index) result = backtest_signal(close, signal) assert 0.0 <= result["win_rate"] <= 1.0 def test_trade_count_non_negative(self): """n_trades must be >= 0.""" from rdagent.components.backtesting.vbt_backtest import backtest_signal dates = pd.date_range("2024-01-01", periods=1000, freq="1min") close = pd.Series(1.10 + np.random.default_rng(42).normal(0, 0.001, 1000).cumsum(), index=dates) # Always flat signal result = backtest_signal(close, pd.Series(0.0, index=dates)) assert result["n_trades"] == 0 # Always long signal (1 trade: open at first bar, close at last) result2 = backtest_signal(close, pd.Series(1.0, index=dates)) assert result2["n_trades"] >= 0 def test_total_return_non_zero_for_trending(self): """Always-long in uptrend should produce positive total_return.""" from rdagent.components.backtesting.vbt_backtest import backtest_signal dates = pd.date_range("2024-01-01", periods=1000, freq="1min") close = pd.Series(1.10 + np.arange(1000) * 0.0001, index=dates) # steady uptrend signal = pd.Series(1.0, index=dates) # always long result = backtest_signal(close, signal, txn_cost_bps=0.0) assert result["total_return"] > 0, ( f"Always long in uptrend should be profitable, got total_return={result['total_return']:.6f}" ) def test_total_return_non_positive_for_downtrend(self): """Always-long in downtrend should produce negative return.""" from rdagent.components.backtesting.vbt_backtest import backtest_signal dates = pd.date_range("2024-01-01", periods=1000, freq="1min") close = pd.Series(1.10 - np.arange(1000) * 0.0001, index=dates) # steady downtrend signal = pd.Series(1.0, index=dates) result = backtest_signal(close, signal, txn_cost_bps=0.0) assert result["total_return"] <= 0, ( f"Always long in downtrend should lose money, got total_return={result['total_return']:.6f}" )