"""Deep tests for verify_runtime — property-based, fuzzing, edge cases. Extends test_verify_runtime.py with property-based tests using hypothesis and exhaustive combinatorial checking of all 10 invariants. """ from __future__ import annotations import sys from pathlib import Path import numpy as np import pytest PROJECT_ROOT = Path(__file__).parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) from hypothesis import strategies as st from hypothesis import given, settings from hypothesis.extra.numpy import arrays from rdagent.components.backtesting.verify import verify_and_log, verify_backtest_result GOOD = { "sharpe": 1.5, "max_drawdown": -0.15, "win_rate": 0.55, "total_return": 0.25, "annual_return_pct": 15.0, "monthly_return_pct": 1.2, "n_trades": 50, "status": "success", } class TestVerifyPropertyBased: @given( sharpe=st.floats(allow_nan=False, allow_infinity=False), dd=st.floats(allow_nan=False, allow_infinity=False), wr=st.floats(allow_nan=False, allow_infinity=False), trades=st.integers(), ) @settings(max_examples=500, deadline=5000) def test_edge_detection_invariant(self, sharpe, dd, wr, trades): """Every combination of edge values must produce warnings or pass cleanly.""" result = {**GOOD, "sharpe": sharpe, "max_drawdown": dd, "win_rate": wr, "n_trades": trades} warnings = verify_backtest_result(result) assert isinstance(warnings, list) @given(st.lists(st.text(min_size=1, max_size=20), min_size=0, max_size=10)) @settings(max_examples=100, deadline=5000) def test_arbitrary_keys_no_crash(self, keys): """Arbitrary dict keys must not crash the verifier.""" d = {} for i, k in enumerate(keys): d[k] = 1.0 res = verify_backtest_result(d) assert isinstance(res, list) class TestVerifyFuzzing: @pytest.mark.parametrize("field,vals", [ ("sharpe", [float("inf"), float("-inf"), float("nan"), 1e308, -1e308, 0.0, -0.0, 1e-16, 1e16]), ("max_drawdown", [-10, -2, -1.01, -1.0, -0.5, 0.0, 0.5, 1.0, float("nan")]), ("win_rate", [-1, -0.01, 0.0, 1.0, 1.01, 2.0, 0.3333333, float("nan")]), ("total_return", [-100, -1, 0, 1, 100, float("nan"), float("inf")]), ("n_trades", [-100, -1, 0, 1, 1000000, 2**63 - 1]), ("monthly_return_pct", [-10000, -100, 0, 100, 10000, float("nan")]), ("annual_return_pct", [-10000, -100, 0, 100, 10000, float("nan")]), ]) def test_fuzz_individual_field(self, field, vals): """Each field individually fuzzed — verifier must not crash.""" for v in vals: r = {**GOOD, field: v} warnings = verify_backtest_result(r) assert isinstance(warnings, list) def test_random_results_no_crash(self): """1000 random result dicts — verifier must handle all.""" rng = np.random.default_rng(777) for _ in range(1000): d = { "sharpe": float(rng.choice([rng.normal(1, 5), rng.exponential(2), float("nan"), float("inf")])), "max_drawdown": float(rng.uniform(-5, 1)), "win_rate": float(rng.beta(5, 5)), "total_return": float(rng.normal(0, 10)), "annual_return_pct": float(rng.normal(0, 50)), "monthly_return_pct": float(rng.normal(0, 5)), "n_trades": int(rng.integers(-10, 10000)), "status": rng.choice(["success", "error", "timeout", "unknown"]), } res = verify_backtest_result(d) assert isinstance(res, list) class TestVerifyInvariantIndependence: def test_all_10_invariants_trigger_independently(self): """Each of the 10 invariants should be independently triggerable.""" bad_cases = [ ({}, "Missing"), ({**GOOD, "sharpe": float("inf")}, "infinite"), ({**GOOD, "max_drawdown": -1.5}, "range"), ({**GOOD, "max_drawdown": 0.5}, "range"), ({**GOOD, "win_rate": -0.1}, "range"), ({**GOOD, "win_rate": 1.5}, "range"), ({**GOOD, "total_return": float("nan")}, "NaN"), ({**GOOD, "n_trades": -1}, "negative"), ({**GOOD, "sharpe": 5.0, "annual_return_pct": -50.0}, "opposite"), ({**GOOD, "monthly_return_pct": float("nan")}, "NaN"), ({**GOOD, "monthly_return_pct": float("inf")}, "infinite"), ({**GOOD, "annual_return_pct": float("inf")}, "infinite"), ({**GOOD, "status": "crashed"}, "status"), ] for bad, _expected_word in bad_cases: warnings = verify_backtest_result(bad) assert len(warnings) > 0, f"Expected warning for: {bad}" def test_verify_and_log_never_raises(self): """verify_and_log must never raise, even on pathological inputs.""" for malicious in [ {}, {"sharpe": "not_a_number"}, {"sharpe": None}, {1: 2}, ]: try: verify_and_log(malicious) except Exception as e: pytest.fail(f"verify_and_log raised on {malicious!r}: {e}") class TestVerifyDeep: def test_sharpe_annual_return_sign_invariant(self): """If annual_return_pct > 0, sharpe should not be negative (statistically unlikely edge).""" # This is a soft check — the verifier should catch clear contradictions r = {**GOOD, "sharpe": -2.0, "annual_return_pct": 20.0} w = verify_backtest_result(r) assert len(w) > 0 def test_drawdown_bounded_by_total_return(self): """max_drawdown should not imply losing more than -100% (impossible).""" # DD can be -2.0 meaning -200% of equity — mathematically possible with leverage r = {**GOOD, "max_drawdown": -2.5} w = verify_backtest_result(r) assert len(w) > 0 def test_monthly_total_return_consistency(self): """Massive monthly return should be flagged but not crash.""" r = {**GOOD, "monthly_return_pct": 50.0, "total_return": 0.01} w = verify_backtest_result(r) assert isinstance(w, list) @given( dd=st.floats(min_value=-0.99, max_value=-0.0001), sharpe=st.floats(min_value=-100, max_value=100, allow_nan=False, allow_infinity=False), ) @settings(max_examples=200, deadline=5000) def test_property_clean_inputs_pass(self, dd, sharpe): """Numerically clean inputs should pass verification.""" assume(not np.isnan(dd) and not np.isinf(dd)) assume(not np.isnan(sharpe) and not np.isinf(sharpe)) r = { "sharpe": sharpe, "max_drawdown": dd, "win_rate": 0.5, "total_return": 0.1, "annual_return_pct": 10.0, "monthly_return_pct": 0.8, "n_trades": 100, "status": "success", } w = verify_backtest_result(r) # Might get 0 warnings if all clean, or 1 (opposite signs) assert isinstance(w, list)