mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 23:47:46 +00:00
TPTBusiness
160ac96130
- test_verify_runtime_deep: hypothesis property tests, fuzzing 1000 random results, invariant independence checking, edge cases (NaN, inf, negative trades) - test_vbt_backtest_deep: property tests (cost monotonicity, signal inversion, max_dd invariants), edge cases (1 bar, empty, NaN, inf, mismatched lengths) - test_autopilot: mocked orchestrator tests (failure recovery, counting logic, ensemble building, style cycling, hypothesis property tests)
166 lines
6.9 KiB
Python
166 lines
6.9 KiB
Python
"""Deep tests for verify_runtime — property-based, fuzzing, edge cases.
|
|
|
|
Extends test_verify_runtime.py with property-based tests using hypothesis
|
|
and exhaustive combinatorial checking of all 10 invariants.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from hypothesis import assume, given, settings
|
|
from hypothesis import strategies as st
|
|
from hypothesis.extra.numpy import arrays
|
|
|
|
from rdagent.components.backtesting.verify import verify_and_log, verify_backtest_result
|
|
|
|
|
|
GOOD = {
|
|
"sharpe": 1.5, "max_drawdown": -0.15, "win_rate": 0.55,
|
|
"total_return": 0.25, "annual_return_pct": 15.0, "monthly_return_pct": 1.2,
|
|
"n_trades": 50, "status": "success",
|
|
}
|
|
|
|
|
|
class TestVerifyPropertyBased:
|
|
@given(
|
|
sharpe=st.floats(allow_nan=False, allow_infinity=False),
|
|
dd=st.floats(allow_nan=False, allow_infinity=False),
|
|
wr=st.floats(allow_nan=False, allow_infinity=False),
|
|
trades=st.integers(),
|
|
)
|
|
@settings(max_examples=500, deadline=5000)
|
|
def test_edge_detection_invariant(self, sharpe, dd, wr, trades):
|
|
"""Every combination of edge values must produce warnings or pass cleanly."""
|
|
result = {**GOOD, "sharpe": sharpe, "max_drawdown": dd, "win_rate": wr, "n_trades": trades}
|
|
warnings = verify_backtest_result(result)
|
|
assert isinstance(warnings, list)
|
|
|
|
@given(st.lists(st.text(min_size=1, max_size=20), min_size=0, max_size=10))
|
|
@settings(max_examples=100, deadline=5000)
|
|
def test_arbitrary_keys_no_crash(self, keys):
|
|
"""Arbitrary dict keys must not crash the verifier."""
|
|
d = {}
|
|
for i, k in enumerate(keys):
|
|
d[k] = 1.0
|
|
res = verify_backtest_result(d)
|
|
assert isinstance(res, list)
|
|
|
|
|
|
class TestVerifyFuzzing:
|
|
@pytest.mark.parametrize("field,vals", [
|
|
("sharpe", [float("inf"), float("-inf"), float("nan"), 1e308, -1e308, 0.0, -0.0, 1e-16, 1e16]),
|
|
("max_drawdown", [-10, -2, -1.01, -1.0, -0.5, 0.0, 0.5, 1.0, float("nan")]),
|
|
("win_rate", [-1, -0.01, 0.0, 1.0, 1.01, 2.0, 0.3333333, float("nan")]),
|
|
("total_return", [-100, -1, 0, 1, 100, float("nan"), float("inf")]),
|
|
("n_trades", [-100, -1, 0, 1, 1000000, 2**63 - 1]),
|
|
("monthly_return_pct", [-10000, -100, 0, 100, 10000, float("nan")]),
|
|
("annual_return_pct", [-10000, -100, 0, 100, 10000, float("nan")]),
|
|
])
|
|
def test_fuzz_individual_field(self, field, vals):
|
|
"""Each field individually fuzzed — verifier must not crash."""
|
|
for v in vals:
|
|
r = {**GOOD, field: v}
|
|
warnings = verify_backtest_result(r)
|
|
assert isinstance(warnings, list)
|
|
|
|
def test_random_results_no_crash(self):
|
|
"""1000 random result dicts — verifier must handle all."""
|
|
rng = np.random.default_rng(777)
|
|
for _ in range(1000):
|
|
d = {
|
|
"sharpe": float(rng.choice([rng.normal(1, 5), rng.exponential(2), float("nan"), float("inf")])),
|
|
"max_drawdown": float(rng.uniform(-5, 1)),
|
|
"win_rate": float(rng.beta(5, 5)),
|
|
"total_return": float(rng.normal(0, 10)),
|
|
"annual_return_pct": float(rng.normal(0, 50)),
|
|
"monthly_return_pct": float(rng.normal(0, 5)),
|
|
"n_trades": int(rng.integers(-10, 10000)),
|
|
"status": rng.choice(["success", "error", "timeout", "unknown"]),
|
|
}
|
|
res = verify_backtest_result(d)
|
|
assert isinstance(res, list)
|
|
|
|
|
|
class TestVerifyInvariantIndependence:
|
|
def test_all_10_invariants_trigger_independently(self):
|
|
"""Each of the 10 invariants should be independently triggerable."""
|
|
bad_cases = [
|
|
({}, "Missing"),
|
|
({**GOOD, "sharpe": float("inf")}, "infinite"),
|
|
({**GOOD, "max_drawdown": -1.5}, "range"),
|
|
({**GOOD, "max_drawdown": 0.5}, "range"),
|
|
({**GOOD, "win_rate": -0.1}, "range"),
|
|
({**GOOD, "win_rate": 1.5}, "range"),
|
|
({**GOOD, "total_return": float("nan")}, "NaN"),
|
|
({**GOOD, "n_trades": -1}, "negative"),
|
|
({**GOOD, "sharpe": 5.0, "annual_return_pct": -50.0}, "opposite"),
|
|
({**GOOD, "monthly_return_pct": float("nan")}, "NaN"),
|
|
({**GOOD, "monthly_return_pct": float("inf")}, "infinite"),
|
|
({**GOOD, "annual_return_pct": float("inf")}, "infinite"),
|
|
({**GOOD, "status": "crashed"}, "status"),
|
|
]
|
|
for bad, _expected_word in bad_cases:
|
|
warnings = verify_backtest_result(bad)
|
|
assert len(warnings) > 0, f"Expected warning for: {bad}"
|
|
|
|
def test_verify_and_log_never_raises(self):
|
|
"""verify_and_log must never raise, even on pathological inputs."""
|
|
for malicious in [
|
|
{},
|
|
{"sharpe": "not_a_number"},
|
|
{"sharpe": None},
|
|
{1: 2},
|
|
]:
|
|
try:
|
|
verify_and_log(malicious)
|
|
except Exception as e:
|
|
pytest.fail(f"verify_and_log raised on {malicious!r}: {e}")
|
|
|
|
|
|
class TestVerifyDeep:
|
|
def test_sharpe_annual_return_sign_invariant(self):
|
|
"""If annual_return_pct > 0, sharpe should not be negative (statistically unlikely edge)."""
|
|
# This is a soft check — the verifier should catch clear contradictions
|
|
r = {**GOOD, "sharpe": -2.0, "annual_return_pct": 20.0}
|
|
w = verify_backtest_result(r)
|
|
assert len(w) > 0
|
|
|
|
def test_drawdown_bounded_by_total_return(self):
|
|
"""max_drawdown should not imply losing more than -100% (impossible)."""
|
|
# DD can be -2.0 meaning -200% of equity — mathematically possible with leverage
|
|
r = {**GOOD, "max_drawdown": -2.5}
|
|
w = verify_backtest_result(r)
|
|
assert len(w) > 0
|
|
|
|
def test_monthly_total_return_consistency(self):
|
|
"""Massive monthly return should be flagged but not crash."""
|
|
r = {**GOOD, "monthly_return_pct": 50.0, "total_return": 0.01}
|
|
w = verify_backtest_result(r)
|
|
assert isinstance(w, list)
|
|
|
|
@given(
|
|
dd=st.floats(min_value=-0.99, max_value=-0.0001),
|
|
sharpe=st.floats(min_value=-100, max_value=100, allow_nan=False, allow_infinity=False),
|
|
)
|
|
@settings(max_examples=200, deadline=5000)
|
|
def test_property_clean_inputs_pass(self, dd, sharpe):
|
|
"""Numerically clean inputs should pass verification."""
|
|
assume(not np.isnan(dd) and not np.isinf(dd))
|
|
assume(not np.isnan(sharpe) and not np.isinf(sharpe))
|
|
r = {
|
|
"sharpe": sharpe, "max_drawdown": dd, "win_rate": 0.5,
|
|
"total_return": 0.1, "annual_return_pct": 10.0,
|
|
"monthly_return_pct": 0.8, "n_trades": 100, "status": "success",
|
|
}
|
|
w = verify_backtest_result(r)
|
|
# Might get 0 warnings if all clean, or 1 (opposite signs)
|
|
assert isinstance(w, list)
|