NexQuant/test/qlib/test_verify_runtime_deep.py

"""Deep tests for verify_runtime — property-based, fuzzing, edge cases.

Extends test_verify_runtime.py with property-based tests using hypothesis
and exhaustive combinatorial checking of all 10 invariants.
"""

from __future__ import annotations

import sys
from pathlib import Path

import numpy as np
import pytest

PROJECT_ROOT = Path(__file__).parent.parent.parent
sys.path.insert(0, str(PROJECT_ROOT))

from hypothesis import strategies as st
from hypothesis import given, settings
from hypothesis.extra.numpy import arrays

from rdagent.components.backtesting.verify import verify_and_log, verify_backtest_result


GOOD = {
    "sharpe": 1.5, "max_drawdown": -0.15, "win_rate": 0.55,
    "total_return": 0.25, "annual_return_pct": 15.0, "monthly_return_pct": 1.2,
    "n_trades": 50, "status": "success",
}


class TestVerifyPropertyBased:
    @given(
        sharpe=st.floats(allow_nan=False, allow_infinity=False),
        dd=st.floats(allow_nan=False, allow_infinity=False),
        wr=st.floats(allow_nan=False, allow_infinity=False),
        trades=st.integers(),
    )
    @settings(max_examples=500, deadline=5000)
    def test_edge_detection_invariant(self, sharpe, dd, wr, trades):
        """Every combination of edge values must produce warnings or pass cleanly."""
        result = {**GOOD, "sharpe": sharpe, "max_drawdown": dd, "win_rate": wr, "n_trades": trades}
        warnings = verify_backtest_result(result)
        assert isinstance(warnings, list)

    @given(st.lists(st.text(min_size=1, max_size=20), min_size=0, max_size=10))
    @settings(max_examples=100, deadline=5000)
    def test_arbitrary_keys_no_crash(self, keys):
        """Arbitrary dict keys must not crash the verifier."""
        d = {}
        for i, k in enumerate(keys):
            d[k] = 1.0
        res = verify_backtest_result(d)
        assert isinstance(res, list)


class TestVerifyFuzzing:
    @pytest.mark.parametrize("field,vals", [
        ("sharpe", [float("inf"), float("-inf"), float("nan"), 1e308, -1e308, 0.0, -0.0, 1e-16, 1e16]),
        ("max_drawdown", [-10, -2, -1.01, -1.0, -0.5, 0.0, 0.5, 1.0, float("nan")]),
        ("win_rate", [-1, -0.01, 0.0, 1.0, 1.01, 2.0, 0.3333333, float("nan")]),
        ("total_return", [-100, -1, 0, 1, 100, float("nan"), float("inf")]),
        ("n_trades", [-100, -1, 0, 1, 1000000, 2**63 - 1]),
        ("monthly_return_pct", [-10000, -100, 0, 100, 10000, float("nan")]),
        ("annual_return_pct", [-10000, -100, 0, 100, 10000, float("nan")]),
    ])
    def test_fuzz_individual_field(self, field, vals):
        """Each field individually fuzzed — verifier must not crash."""
        for v in vals:
            r = {**GOOD, field: v}
            warnings = verify_backtest_result(r)
            assert isinstance(warnings, list)

    def test_random_results_no_crash(self):
        """1000 random result dicts — verifier must handle all."""
        rng = np.random.default_rng(777)
        for _ in range(1000):
            d = {
                "sharpe": float(rng.choice([rng.normal(1, 5), rng.exponential(2), float("nan"), float("inf")])),
                "max_drawdown": float(rng.uniform(-5, 1)),
                "win_rate": float(rng.beta(5, 5)),
                "total_return": float(rng.normal(0, 10)),
                "annual_return_pct": float(rng.normal(0, 50)),
                "monthly_return_pct": float(rng.normal(0, 5)),
                "n_trades": int(rng.integers(-10, 10000)),
                "status": rng.choice(["success", "error", "timeout", "unknown"]),
            }
            res = verify_backtest_result(d)
            assert isinstance(res, list)


class TestVerifyInvariantIndependence:
    def test_all_10_invariants_trigger_independently(self):
        """Each of the 10 invariants should be independently triggerable."""
        bad_cases = [
            ({}, "Missing"),
            ({**GOOD, "sharpe": float("inf")}, "infinite"),
            ({**GOOD, "max_drawdown": -1.5}, "range"),
            ({**GOOD, "max_drawdown": 0.5}, "range"),
            ({**GOOD, "win_rate": -0.1}, "range"),
            ({**GOOD, "win_rate": 1.5}, "range"),
            ({**GOOD, "total_return": float("nan")}, "NaN"),
            ({**GOOD, "n_trades": -1}, "negative"),
            ({**GOOD, "sharpe": 5.0, "annual_return_pct": -50.0}, "opposite"),
            ({**GOOD, "monthly_return_pct": float("nan")}, "NaN"),
            ({**GOOD, "monthly_return_pct": float("inf")}, "infinite"),
            ({**GOOD, "annual_return_pct": float("inf")}, "infinite"),
            ({**GOOD, "status": "crashed"}, "status"),
        ]
        for bad, _expected_word in bad_cases:
            warnings = verify_backtest_result(bad)
            assert len(warnings) > 0, f"Expected warning for: {bad}"

    def test_verify_and_log_never_raises(self):
        """verify_and_log must never raise, even on pathological inputs."""
        for malicious in [
            {},
            {"sharpe": "not_a_number"},
            {"sharpe": None},
            {1: 2},
        ]:
            try:
                verify_and_log(malicious)
            except Exception as e:
                pytest.fail(f"verify_and_log raised on {malicious!r}: {e}")


class TestVerifyDeep:
    def test_sharpe_annual_return_sign_invariant(self):
        """If annual_return_pct > 0, sharpe should not be negative (statistically unlikely edge)."""
        # This is a soft check — the verifier should catch clear contradictions
        r = {**GOOD, "sharpe": -2.0, "annual_return_pct": 20.0}
        w = verify_backtest_result(r)
        assert len(w) > 0

    def test_drawdown_bounded_by_total_return(self):
        """max_drawdown should not imply losing more than -100% (impossible)."""
        # DD can be -2.0 meaning -200% of equity — mathematically possible with leverage
        r = {**GOOD, "max_drawdown": -2.5}
        w = verify_backtest_result(r)
        assert len(w) > 0

    def test_monthly_total_return_consistency(self):
        """Massive monthly return should be flagged but not crash."""
        r = {**GOOD, "monthly_return_pct": 50.0, "total_return": 0.01}
        w = verify_backtest_result(r)
        assert isinstance(w, list)

    @given(
        dd=st.floats(min_value=-0.99, max_value=-0.0001),
        sharpe=st.floats(min_value=-100, max_value=100, allow_nan=False, allow_infinity=False),
    )
    @settings(max_examples=200, deadline=5000)
    def test_property_clean_inputs_pass(self, dd, sharpe):
        """Numerically clean inputs should pass verification."""
        assume(not np.isnan(dd) and not np.isinf(dd))
        assume(not np.isnan(sharpe) and not np.isinf(sharpe))
        r = {
            "sharpe": sharpe, "max_drawdown": dd, "win_rate": 0.5,
            "total_return": 0.1, "annual_return_pct": 10.0,
            "monthly_return_pct": 0.8, "n_trades": 100, "status": "success",
        }
        w = verify_backtest_result(r)
        # Might get 0 warnings if all clean, or 1 (opposite signs)
        assert isinstance(w, list)