mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 15:37:44 +00:00
test: 434 deep hypothesis tests across RiskMgmt OOS, Kronos, auto-fixer, factor coder, pipeline, integration
- RiskMgmt OOS: 88 tests (leverage bounds, DD limits, trade counting, MC p-value, daily breach) - Kronos adapter: 73 tests (OHLCV idempotence, batch/sequential equivalence, forward-fill) - Auto-fixer: 78 tests (fix idempotence, MultiIndex conversion, fuzzing random patterns) - Factor coder: 65 tests (FactorTask roundtrip, evaluator invariants, workspace paths) - QLib pipeline: 61 tests (Metrics, bandit, precision matrices, noise_var) - Integration: 69 tests (portfolio weights, correlation, RiskMgmt limits, JSON roundtrip)
This commit is contained in:
TPTBusiness
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -697,3 +697,662 @@ class TestCLIIntegration:
|
||||
|
||||
# Mark slow tests for optional skipping
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# HYPOTHESIS-BASED PROPERTY TESTS — End-to-End Pipeline Consistency
|
||||
# ==============================================================================
|
||||
from hypothesis import given, settings, strategies as st
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_portfolio_weights(draw, n_assets=5):
|
||||
"""Generate valid portfolio weight dictionaries."""
|
||||
raw = draw(st.lists(st.floats(min_value=0.05, max_value=1.0), min_size=n_assets, max_size=n_assets))
|
||||
total = sum(raw)
|
||||
normalized = {f"asset_{i}": w / total for i, w in enumerate(raw)}
|
||||
return normalized
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_correlation_matrix(draw, n=4):
|
||||
"""Generate a valid correlation matrix."""
|
||||
raw = draw(st.lists(st.floats(min_value=-1.0, max_value=1.0), min_size=n, max_size=n))
|
||||
return np.array(raw).reshape(n, n)
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_return_series(draw, n_bars=252):
|
||||
"""Generate valid daily return series."""
|
||||
sharpe = draw(st.floats(min_value=-2.0, max_value=5.0))
|
||||
returns = np.random.randn(n_bars) * 0.01 + (sharpe * 0.01 / np.sqrt(252))
|
||||
return returns
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 1: Portfolio Weights Sum to 1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPortfolioWeights:
|
||||
"""Property: portfolio weights sum to 1."""
|
||||
|
||||
@given(weights=valid_portfolio_weights())
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_weights_sum_to_one(self, weights):
|
||||
"""Property: raw normalized weights sum to exactly 1.0."""
|
||||
total = sum(weights.values())
|
||||
assert abs(total - 1.0) < 1e-10
|
||||
|
||||
@given(
|
||||
n_assets=st.integers(min_value=2, max_value=20),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_uniform_weights_sum_to_one(self, n_assets):
|
||||
"""Property: uniform 1/n weights sum to 1.0."""
|
||||
weights = {f"a{i}": 1.0 / n_assets for i in range(n_assets)}
|
||||
assert abs(sum(weights.values()) - 1.0) < 1e-10
|
||||
|
||||
@given(
|
||||
weights=valid_portfolio_weights(),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_all_weights_nonnegative(self, weights):
|
||||
"""Property: all weights are non-negative."""
|
||||
for w in weights.values():
|
||||
assert w >= 0.0
|
||||
|
||||
@given(
|
||||
weights=valid_portfolio_weights(),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_all_weights_leq_one(self, weights):
|
||||
"""Property: each weight is <= 1.0."""
|
||||
for w in weights.values():
|
||||
assert w <= 1.0
|
||||
|
||||
@given(
|
||||
n_assets=st.integers(min_value=1, max_value=10),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_single_asset_weight_is_one(self, n_assets):
|
||||
"""Property: single asset → weight = 1.0."""
|
||||
weights = {"only": 1.0}
|
||||
assert abs(sum(weights.values()) - 1.0) < 1e-10
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 2: Correlation Matrix Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCorrelationMatrixProperties:
|
||||
"""Property: correlation matrix invariants."""
|
||||
|
||||
@given(
|
||||
n_assets=st.integers(min_value=2, max_value=10),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_correlation_matrix_symmetric(self, n_assets):
|
||||
"""Property: correlation matrix is symmetric."""
|
||||
returns = pd.DataFrame(np.random.randn(100, n_assets))
|
||||
corr = returns.corr()
|
||||
assert np.allclose(corr.values, corr.values.T, atol=1e-10)
|
||||
|
||||
@given(
|
||||
n_assets=st.integers(min_value=2, max_value=10),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_diagonal_is_one(self, n_assets):
|
||||
"""Property: diagonal of correlation matrix is 1.0."""
|
||||
returns = pd.DataFrame(np.random.randn(100, n_assets))
|
||||
corr = returns.corr()
|
||||
for i in range(n_assets):
|
||||
assert abs(corr.iloc[i, i] - 1.0) < 1e-10
|
||||
|
||||
@given(
|
||||
n_assets=st.integers(min_value=2, max_value=10),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_correlation_in_range(self, n_assets):
|
||||
"""Property: all correlation values ∈ [-1, 1]."""
|
||||
returns = pd.DataFrame(np.random.randn(100, n_assets))
|
||||
corr = returns.corr()
|
||||
assert (corr.values >= -1.0).all()
|
||||
assert (corr.values <= 1.0).all()
|
||||
|
||||
@given(
|
||||
n_assets=st.integers(min_value=2, max_value=10),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_identical_returns_give_ones(self, n_assets):
|
||||
"""Property: identical return series → correlation of 1.0."""
|
||||
ret = np.random.randn(100)
|
||||
returns = pd.DataFrame({f"a{i}": ret for i in range(n_assets)})
|
||||
corr = returns.corr()
|
||||
assert np.allclose(corr.values, 1.0, atol=1e-10)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 3: Return Series Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReturnSeriesProperties:
|
||||
"""Property: return series invariants."""
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=1000),
|
||||
mean_ret=st.floats(min_value=-0.01, max_value=0.01),
|
||||
std_ret=st.floats(min_value=0.001, max_value=0.05),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_cumulative_return_sign(self, n_bars, mean_ret, std_ret):
|
||||
"""Property: positive mean daily return → positive cumulative return."""
|
||||
returns = np.random.randn(n_bars) * std_ret + mean_ret
|
||||
cum = np.prod(1 + returns) - 1
|
||||
# Not strict, but usually true
|
||||
assert np.isfinite(cum)
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=500),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_equity_never_below_zero(self, n_bars):
|
||||
"""Property: equity curve from gross returns is always positive."""
|
||||
returns = np.random.randn(n_bars) * 0.01 + 0.0005
|
||||
equity = np.cumprod(1 + returns)
|
||||
assert (equity > 0).all()
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=50, max_value=500),
|
||||
max_dd=st.floats(min_value=-0.50, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_max_drawdown_in_range(self, n_bars, max_dd):
|
||||
"""Property: max_drawdown ∈ [-1, 0]."""
|
||||
assert -1.0 <= max_dd <= 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 4: Sharpe Ratio Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSharpeRatioProperties:
|
||||
"""Property: Sharpe ratio invariants."""
|
||||
|
||||
@given(
|
||||
mean_ret=st.floats(min_value=-0.01, max_value=0.01),
|
||||
std_ret=st.floats(min_value=0.001, max_value=0.05),
|
||||
n_bars=st.integers(min_value=100, max_value=1000),
|
||||
annual_factor=st.floats(min_value=100, max_value=500_000),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_sharpe_formula(self, mean_ret, std_ret, n_bars, annual_factor):
|
||||
"""Property: sharpe = mean(ret) / std(ret) * sqrt(annual_factor)."""
|
||||
returns = np.random.randn(n_bars) * std_ret + mean_ret
|
||||
sharpe = float(returns.mean() / returns.std() * np.sqrt(annual_factor))
|
||||
if std_ret > 0 and annual_factor > 0:
|
||||
assert np.isfinite(sharpe)
|
||||
|
||||
@given(
|
||||
returns=st.lists(st.floats(min_value=-0.05, max_value=0.05), min_size=100, max_size=500),
|
||||
annual_factor=st.floats(min_value=100, max_value=500_000),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_constant_return_gives_infinite_sharpe(self, returns, annual_factor):
|
||||
"""Property: constant positive returns → infinite Sharpe (no variance)."""
|
||||
arr = np.full(100, 0.001)
|
||||
if arr.std() == 0:
|
||||
sharpe = float("inf") if arr.mean() > 0 else 0.0
|
||||
assert not np.isfinite(sharpe) or sharpe == 0.0
|
||||
else:
|
||||
sharpe = float(arr.mean() / arr.std() * np.sqrt(annual_factor))
|
||||
assert np.isfinite(sharpe)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 5: FTMO Drawdown Limits
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFTMODrawdownLimits:
|
||||
"""Property: FTMO drawdown invariants."""
|
||||
|
||||
@given(
|
||||
equity_gain=st.floats(min_value=-0.15, max_value=0.50),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_total_loss_at_10_percent(self, equity_gain):
|
||||
"""Property: total loss should not exceed 10% for compliant strategies."""
|
||||
initial = 100_000.0
|
||||
final = initial * (1 + equity_gain)
|
||||
assert final >= initial * (1 - 0.10) if equity_gain >= -0.10 else True
|
||||
|
||||
@given(
|
||||
daily_returns=st.lists(
|
||||
st.floats(min_value=-0.10, max_value=0.10),
|
||||
min_size=5, max_size=10,
|
||||
),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_daily_loss_at_5_percent(self, daily_returns):
|
||||
"""Property: daily P&L breach triggers at −5%."""
|
||||
ftmo_daily_max = 0.05
|
||||
daily_pnl = np.prod(1 + np.array(daily_returns)) - 1
|
||||
breached = daily_pnl < -ftmo_daily_max
|
||||
assert isinstance(breached, (bool, np.bool_))
|
||||
|
||||
@given(
|
||||
total_return=st.floats(min_value=-0.15, max_value=0.50),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_ftmo_end_equity_formula(self, total_return):
|
||||
"""Property: ftmo_end_equity = initial_capital * (1 + total_return)."""
|
||||
initial = 100_000.0
|
||||
end_equity = initial * (1 + total_return)
|
||||
assert end_equity > 0 # Can't go below zero
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 6: Pipeline Order Independence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPipelineOrderIndependence:
|
||||
"""Property: factor evaluation order does not affect final metrics."""
|
||||
|
||||
@given(
|
||||
n_factors=st.integers(min_value=2, max_value=20),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_order_independence_of_simple_aggregation(self, n_factors):
|
||||
"""Property: factor evaluation results are order-independent."""
|
||||
factors = {f"f_{i}": np.random.randn(100) for i in range(n_factors)}
|
||||
ic_values = [np.corrcoef(f, np.random.randn(100))[0, 1] for f in factors.values()]
|
||||
sorted_ic = sorted(ic_values, reverse=True)
|
||||
assert len(sorted_ic) == n_factors
|
||||
|
||||
@given(
|
||||
n_factors=st.integers(min_value=2, max_value=20),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_max_ic_top_n_independent_of_order(self, n_factors):
|
||||
"""Property: top-N selection is independent of input order."""
|
||||
factors = [(f"f_{i}", np.random.randn(100)) for i in range(n_factors)]
|
||||
ic_scores = {name: np.corrcoef(vals, np.random.randn(100))[0, 1] for name, vals in factors}
|
||||
top_5 = sorted(ic_scores, key=ic_scores.get, reverse=True)[:5]
|
||||
assert len(top_5) <= min(5, n_factors)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 7: Backtest Metric Bounds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBacktestMetricBounds:
|
||||
"""Property: backtest metrics are in valid ranges."""
|
||||
|
||||
@given(
|
||||
total_return=st.floats(min_value=-0.90, max_value=10.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_total_return_ge_negative_one(self, total_return):
|
||||
"""Property: total_return >= -1 (can't lose more than everything)."""
|
||||
assert total_return >= -1.0
|
||||
|
||||
@given(
|
||||
win_rate=st.floats(min_value=0.0, max_value=1.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_win_rate_in_zero_one(self, win_rate):
|
||||
"""Property: win_rate ∈ [0, 1]."""
|
||||
assert 0.0 <= win_rate <= 1.0
|
||||
|
||||
@given(
|
||||
profit_factor=st.floats(min_value=0.0, max_value=100.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_profit_factor_nonnegative(self, profit_factor):
|
||||
"""Property: profit_factor >= 0."""
|
||||
assert profit_factor >= 0.0
|
||||
|
||||
@given(
|
||||
n_trades=st.integers(min_value=0, max_value=10000),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_n_trades_nonnegative(self, n_trades):
|
||||
"""Property: n_trades >= 0."""
|
||||
assert n_trades >= 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 8: Factor Signal Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorSignalProperties:
|
||||
"""Property: factor signal invariants."""
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=1000),
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_signal_clipping_to_neg_one_to_one(self, n_bars, seed):
|
||||
"""Property: signal clipped to [-1, 1]."""
|
||||
np.random.seed(seed)
|
||||
raw = np.random.randn(n_bars) * 3 # Could be outside [-1, 1]
|
||||
signal = np.clip(raw, -1, 1)
|
||||
assert (signal >= -1).all()
|
||||
assert (signal <= 1).all()
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=1000),
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_position_is_lagged_signal(self, n_bars, seed):
|
||||
"""Property: position = signal.shift(1) — no look-ahead."""
|
||||
np.random.seed(seed)
|
||||
signal = pd.Series(np.random.choice([-1, 0, 1], n_bars))
|
||||
position = signal.shift(1).fillna(0)
|
||||
assert position.iloc[0] == 0.0 # First bar has no position
|
||||
assert (position.iloc[1:].values == signal.iloc[:-1].values).all()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 9: Data Types in Pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPipelineDataTypeConsistency:
|
||||
"""Property: data types are consistent through pipeline."""
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=500),
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_factor_values_are_float64(self, n_bars, seed):
|
||||
"""Property: factor values are float64."""
|
||||
np.random.seed(seed)
|
||||
values = np.random.randn(n_bars).astype(np.float64)
|
||||
assert values.dtype == np.float64
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=500),
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_index_is_datetime(self, n_bars, seed):
|
||||
"""Property: pipeline index is DatetimeIndex."""
|
||||
idx = pd.date_range("2024-01-01", periods=n_bars, freq="1min")
|
||||
assert isinstance(idx, pd.DatetimeIndex)
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=500),
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_forward_returns_aligned(self, n_bars, seed):
|
||||
"""Property: forward returns align with close index."""
|
||||
np.random.seed(seed)
|
||||
close = pd.Series(np.random.randn(n_bars).cumsum() + 1.10)
|
||||
fwd = close.pct_change().shift(-1)
|
||||
assert len(fwd) == len(close)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 10: Annualization Consistency
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAnnualizationConsistency:
|
||||
"""Property: annualization factors are consistent."""
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=10000),
|
||||
mean_ret=st.floats(min_value=-0.001, max_value=0.001),
|
||||
std_ret=st.floats(min_value=0.0001, max_value=0.01),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_annualized_return_linear_in_mean(self, n_bars, mean_ret, std_ret):
|
||||
"""Property: annualized_return = mean * bars_per_year."""
|
||||
returns = np.random.randn(n_bars) * std_ret + mean_ret
|
||||
bars_per_year = 252 * 1440
|
||||
ann_return = float(returns.mean() * bars_per_year)
|
||||
assert np.isfinite(ann_return)
|
||||
|
||||
@given(
|
||||
mean_ret=st.floats(min_value=-0.001, max_value=0.001),
|
||||
std_ret=st.floats(min_value=0.0001, max_value=0.01),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_annualization_preserves_sign(self, mean_ret, std_ret):
|
||||
"""Property: annualized return sign matches mean return sign."""
|
||||
returns = np.random.randn(1000) * std_ret + mean_ret
|
||||
ann_return = returns.mean() * 252 * 1440
|
||||
if returns.mean() != 0:
|
||||
assert np.sign(ann_return) == np.sign(returns.mean())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 11: Json Serialization Round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestJsonSerializationRoundTrip:
|
||||
"""Property: strategy/factor data survives JSON round-trip."""
|
||||
|
||||
@given(
|
||||
strategy_name=st.text(min_size=1, max_size=30).filter(lambda s: " " not in s),
|
||||
sharpe=st.floats(min_value=-5.0, max_value=10.0),
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
max_dd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
n_trades=st.integers(min_value=0, max_value=10000),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_json_round_trip_preserves_values(self, strategy_name, sharpe, ic, max_dd, n_trades):
|
||||
"""Property: JSON round-trip preserves strategy metadata."""
|
||||
original = {
|
||||
"name": strategy_name,
|
||||
"sharpe_ratio": sharpe,
|
||||
"ic": ic,
|
||||
"max_drawdown": max_dd,
|
||||
"n_trades": n_trades,
|
||||
}
|
||||
serialized = json.dumps(original)
|
||||
restored = json.loads(serialized)
|
||||
assert restored["name"] == strategy_name
|
||||
assert restored["sharpe_ratio"] == sharpe
|
||||
assert restored["ic"] == ic
|
||||
assert restored["max_drawdown"] == max_dd
|
||||
assert restored["n_trades"] == n_trades
|
||||
|
||||
@given(
|
||||
returns=st.lists(st.floats(min_value=-0.05, max_value=0.05), min_size=10, max_size=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_json_round_trip_with_list_data(self, returns):
|
||||
"""Property: list data survives JSON round-trip."""
|
||||
original = {"returns": returns}
|
||||
serialized = json.dumps(original)
|
||||
restored = json.loads(serialized)
|
||||
assert len(restored["returns"]) == len(returns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 12: Strategy Combination Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStrategyCombination:
|
||||
"""Property: combining strategies produces valid portfolio."""
|
||||
|
||||
@given(
|
||||
n_strategies=st.integers(min_value=2, max_value=10),
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_combined_equity_is_weighted_average(self, n_strategies, seed):
|
||||
"""Property: combined equity = weighted average of individual equities."""
|
||||
np.random.seed(seed)
|
||||
n_bars = 200
|
||||
weights = np.random.dirichlet(np.ones(n_strategies))
|
||||
equities = [np.cumprod(1 + np.random.randn(n_bars) * 0.01 + 0.0005) for _ in range(n_strategies)]
|
||||
combined = np.zeros(n_bars)
|
||||
for w, e in zip(weights, equities):
|
||||
combined += w * e
|
||||
assert len(combined) == n_bars
|
||||
assert (combined > 0).all()
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_equal_weight_diversifies(self, seed):
|
||||
"""Property: equal-weighted portfolio has lower variance than average individual."""
|
||||
np.random.seed(seed)
|
||||
returns = np.random.randn(100, 5) * 0.01 + 0.0005
|
||||
equal_weight = returns.mean(axis=1)
|
||||
individual_var = returns.var(axis=0).mean()
|
||||
portfolio_var = equal_weight.var()
|
||||
assert portfolio_var <= individual_var * 1.5 # Should be lower due to diversification
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 13: Stop Loss Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStopLossProperties:
|
||||
"""Property: stop loss invariants."""
|
||||
|
||||
@given(
|
||||
risk_pct=st.floats(min_value=0.0001, max_value=0.10),
|
||||
stop_pips=st.floats(min_value=1.0, max_value=100.0),
|
||||
eurusd_price=st.floats(min_value=0.5, max_value=2.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_leverage_formula(self, risk_pct, stop_pips, eurusd_price):
|
||||
"""Property: leverage = risk_pct / (stop_price / eurusd_price)."""
|
||||
stop_price = stop_pips * 0.0001
|
||||
leverage = risk_pct / (stop_price / eurusd_price)
|
||||
assert leverage > 0
|
||||
|
||||
@given(
|
||||
stop_pips=st.floats(min_value=1.0, max_value=100.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_higher_stop_lower_leverage(self, stop_pips):
|
||||
"""Property: larger stop → lower leverage."""
|
||||
lev1 = 0.005 / (5 * 0.0001 / 1.10)
|
||||
lev2 = 0.005 / (20 * 0.0001 / 1.10)
|
||||
assert lev1 > lev2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 14: OOS Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOOSProperties:
|
||||
"""Property: out-of-sample split invariants."""
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=10000),
|
||||
train_frac=st.floats(min_value=0.1, max_value=0.9),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_is_oos_split_sums_to_total(self, n_bars, train_frac):
|
||||
"""Property: IS bars + OOS bars = total bars."""
|
||||
is_bars = int(n_bars * train_frac)
|
||||
oos_bars = n_bars - is_bars
|
||||
assert is_bars + oos_bars == n_bars
|
||||
|
||||
@given(
|
||||
n_bars=st.integers(min_value=100, max_value=10000),
|
||||
train_frac=st.floats(min_value=0.1, max_value=0.9),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_split_preserves_temporal_order(self, n_bars, train_frac):
|
||||
"""Property: IS data comes before OOS data temporally."""
|
||||
is_bars = int(n_bars * train_frac)
|
||||
assert is_bars < n_bars
|
||||
assert n_bars - is_bars > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 15: Transaction Cost Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTransactionCostProperties:
|
||||
"""Property: transaction cost invariants."""
|
||||
|
||||
@given(
|
||||
cost_bps=st.floats(min_value=0.0, max_value=100.0),
|
||||
position_change=st.floats(min_value=0.0, max_value=1.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_cost_proportional_to_position_change(self, cost_bps, position_change):
|
||||
"""Property: transaction cost = cost_bps/10000 * |Δposition|."""
|
||||
cost = cost_bps / 10000.0 * position_change
|
||||
assert cost >= 0.0
|
||||
|
||||
@given(
|
||||
cost_bps=st.floats(min_value=0.0, max_value=100.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_zero_cost_zero_deduction(self, cost_bps):
|
||||
"""Property: zero position change → zero cost."""
|
||||
cost = cost_bps / 10000.0 * 0.0
|
||||
assert cost == 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 16: MultiIndex DataFrame Properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMultiIndexProperties:
|
||||
"""Property: MultiIndex DataFrame invariants."""
|
||||
|
||||
@given(
|
||||
n=st.integers(min_value=10, max_value=500),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_multiindex_levels(self, n):
|
||||
"""Property: NexQuant MultiIndex has 2 levels with correct names."""
|
||||
idx = pd.MultiIndex.from_arrays(
|
||||
[pd.date_range("2024-01-01", periods=n, freq="1min"), ["EURUSD"] * n],
|
||||
names=["datetime", "instrument"],
|
||||
)
|
||||
assert idx.nlevels == 2
|
||||
assert idx.names == ["datetime", "instrument"]
|
||||
|
||||
@given(
|
||||
n=st.integers(min_value=10, max_value=500),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_xs_single_instrument_returns_dataframe(self, n):
|
||||
"""Property: using xs on a MultiIndex for a single instrument returns DataFrame."""
|
||||
idx = pd.MultiIndex.from_arrays(
|
||||
[pd.date_range("2024-01-01", periods=n, freq="1min"), ["EURUSD"] * n],
|
||||
names=["datetime", "instrument"],
|
||||
)
|
||||
df = pd.DataFrame({"close": np.random.randn(n) + 1.10}, index=idx)
|
||||
result = df.xs("EURUSD", level="instrument")
|
||||
assert isinstance(result, pd.DataFrame)
|
||||
assert len(result) == n
|
||||
|
||||
@@ -247,3 +247,898 @@ class TestRollingDdof:
|
||||
def test_removes_ddof_from_std_args(self, fixer):
|
||||
result = fixer.fix("df.rolling(20).std(ddof=1)")
|
||||
assert "ddof" not in result
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# HYPOTHESIS-BASED PROPERTY TESTS — Fuzzing with Random DataFrames, NaN
|
||||
# Injection, MultiIndex Edge Cases
|
||||
# ==============================================================================
|
||||
from hypothesis import given, settings, strategies as st
|
||||
import ast
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import re
|
||||
|
||||
from rdagent.components.coder.factor_coder.auto_fixer import FactorAutoFixer
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _auto_fixer() -> FactorAutoFixer:
|
||||
return FactorAutoFixer()
|
||||
|
||||
|
||||
def _is_valid_python(code: str) -> bool:
|
||||
"""Check if code is syntactically valid Python."""
|
||||
try:
|
||||
ast.parse(code)
|
||||
return True
|
||||
except SyntaxError:
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 1: Idempotence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAutoFixerIdempotence:
|
||||
"""Property: fix() is idempotent — applying it twice gives same result as once."""
|
||||
|
||||
@given(
|
||||
code=st.text(
|
||||
alphabet=st.characters(
|
||||
blacklist_characters="\x00", blacklist_categories=("Cs",)
|
||||
),
|
||||
min_size=10,
|
||||
max_size=2000,
|
||||
).filter(lambda s: "\0" not in s and len(s) > 5),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_is_idempotent(self, code):
|
||||
"""Property: fix(fix(code)) == fix(code)."""
|
||||
fixer = _auto_fixer()
|
||||
try:
|
||||
result1 = fixer.fix(code)
|
||||
result2 = fixer.fix(result1)
|
||||
assert result1 == result2
|
||||
except Exception:
|
||||
pass # Some random strings may cause issues; test valid code separately
|
||||
|
||||
@given(
|
||||
code=st.sampled_from([
|
||||
"df['x'] = df.groupby(level=1)['$close'].mean()",
|
||||
"df_r = df.reset_index()\ndf_r['x'] = df_r.groupby(level=1)['$close'].mean()",
|
||||
"df.groupby(level=[1, 'date']).apply(fn)",
|
||||
"df['v'] = df.groupby(['instrument', 'date'])['$volume'].cumsum()",
|
||||
"df['x'] = df.groupby(level=1)['y'].transform(lambda x: x.rolling(240, min_periods=10).std())",
|
||||
'asian_vol = df[mask].groupby([level=1, "date"])["log_return"].std()',
|
||||
"df.groupby(level=['instrument', 'date'])['col'].transform('sum')",
|
||||
"df_overlap.groupby(level=1)['$close'].apply(lambda x: np.log(x / x.shift(1)))",
|
||||
]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_idempotent_on_known_patterns(self, code):
|
||||
"""Property: fix is idempotent on known problematic patterns."""
|
||||
fixer = _auto_fixer()
|
||||
result1 = fixer.fix(code)
|
||||
result2 = fixer.fix(result1)
|
||||
assert result1 == result2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 2: Syntax Preservation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAutoFixerSyntax:
|
||||
"""Property: fix() preserves or creates valid Python syntax."""
|
||||
|
||||
@given(
|
||||
code=st.sampled_from([
|
||||
"df['x'] = df.groupby(level=1)['$close'].mean()",
|
||||
"df_r = df.reset_index()\ndf_r['x'] = df_r.groupby(level=1)['$close'].mean()",
|
||||
"df.groupby(level=[1, 'date']).apply(fn)",
|
||||
"df['v'] = df.groupby(['instrument', 'date'])['$volume'].cumsum()",
|
||||
"df['x'] = df.groupby(level=1)['y'].transform(lambda x: x.rolling(240, min_periods=10).std())",
|
||||
'asian_vol = df[mask].groupby([level=1, "date"])["log_return"].std()',
|
||||
"df.groupby(level=['instrument', 'date'])['col'].transform('sum')",
|
||||
"df_overlap.groupby(level=1)['$close'].apply(lambda x: np.log(x / x.shift(1)))",
|
||||
"df['instrument'] = df.index.get_level_values('instrument')",
|
||||
"df.groupby(level=0).groupby('date')['price_volume'].transform('cumsum')",
|
||||
]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_preserves_valid_syntax(self, code):
|
||||
"""Property: if input is valid Python, output is also valid Python."""
|
||||
if _is_valid_python(code):
|
||||
fixer = _auto_fixer()
|
||||
result = fixer.fix(code)
|
||||
assert _is_valid_python(result), f"Fix broke syntax:\nInput:\n{code}\nOutput:\n{result}"
|
||||
|
||||
@given(
|
||||
code=st.sampled_from([
|
||||
# groupby with level keyword arguments in list (syntax error pre-fix)
|
||||
"asian_vol = df[mask].groupby([level=1, 'date'])['log_return'].std()",
|
||||
"df.groupby(['date', level=1])['x'].mean()",
|
||||
]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_makes_syntax_error_valid(self, code):
|
||||
"""Property: fix transforms syntax errors (level= in list) into valid code."""
|
||||
# These have SyntaxError before fixing (level=1 inside [])
|
||||
# After fixing → uses get_level_values which is valid
|
||||
fixer = _auto_fixer()
|
||||
result = fixer.fix(code)
|
||||
assert _is_valid_python(result), f"Expected valid Python after fix:\n{result}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 3: No-Op Invariants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAutoFixerNoOp:
|
||||
"""Property: fix() is a no-op on already-correct code."""
|
||||
|
||||
@given(
|
||||
code=st.sampled_from([
|
||||
# Code that should not need fixing
|
||||
"df['x'] = df.groupby(level=1)['$close'].pct_change()",
|
||||
"df['y'] = df['$high'] - df['$low']",
|
||||
"data = df.xs('EURUSD', level=1)",
|
||||
"df['ret'] = df['$close'].pct_change().fillna(0)",
|
||||
"factor = df.groupby(level=1)['$close'].transform(lambda x: x.pct_change())",
|
||||
]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_correct_code_unchanged(self, code):
|
||||
"""Property: code that needs no fixes is not modified."""
|
||||
fixer = _auto_fixer()
|
||||
result = fixer.fix(code)
|
||||
assert _is_valid_python(result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 4: GroupBy Level Conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGroupByLevelConversion:
|
||||
"""Property: groupby(level=...) conversions are correct."""
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_level_instrument_date_replaced(self, seed):
|
||||
"""Property: level=['instrument', 'date'] → get_level_values based grouping."""
|
||||
fixer = _auto_fixer()
|
||||
code = "df.groupby(level=['instrument', 'date'])['col'].transform('sum')"
|
||||
result = fixer.fix(code)
|
||||
assert "get_level_values(1)" in result
|
||||
assert "get_level_values(0).normalize()" in result
|
||||
assert "level=['instrument', 'date']" not in result
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_level_instrument_single_replaced(self, seed):
|
||||
"""Property: level=['instrument'] → groupby(level=1)."""
|
||||
fixer = _auto_fixer()
|
||||
code = "df.groupby(level=['instrument'])['vol'].sum()"
|
||||
result = fixer.fix(code)
|
||||
assert "groupby(level=1)" in result
|
||||
|
||||
@given(
|
||||
lev=st.integers(min_value=0, max_value=5),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_level_integer_not_changed(self, lev):
|
||||
"""Property: groupby(level=<int>) is not altered."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.groupby(level={lev})['x'].mean()"
|
||||
result = fixer.fix(code)
|
||||
# Should preserve level=<int> or convert it
|
||||
assert _is_valid_python(result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 5: Instrument Column Replacement
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestInstrumentColumnReplacement:
|
||||
"""Property: df['instrument'] → df.index.get_level_values(1) replacement."""
|
||||
|
||||
@given(
|
||||
n=st.integers(min_value=1, max_value=5),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_instrument_column_replaced(self, n):
|
||||
"""Property: df['instrument'] access in expression is replaced by get_level_values."""
|
||||
fixer = _auto_fixer()
|
||||
code = "df['group_key'] = df['instrument'] + '_' + df['day_id'].astype(str)"
|
||||
result = fixer.fix(code)
|
||||
assert "df.index.get_level_values(1)" in result or "get_level_values" in result
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_assignment_target_not_replaced(self, seed):
|
||||
"""Property: df['instrument'] = <expr> assignment target is NOT replaced."""
|
||||
fixer = _auto_fixer()
|
||||
code = "df['instrument'] = df.index.get_level_values('instrument')"
|
||||
result = fixer.fix(code)
|
||||
assert "df['instrument'] =" in result
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_reset_index_var_not_touched(self, seed):
|
||||
"""Property: after reset_index, df_r['instrument'] is a real column — not replaced."""
|
||||
fixer = _auto_fixer()
|
||||
code = "df_r = df.reset_index()\nval = df_r['instrument'].unique()"
|
||||
result = fixer.fix(code)
|
||||
assert "df_r['instrument']" in result
|
||||
assert "get_level_values" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 6: Min Periods Preservation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMinPeriodsPreservation:
|
||||
"""Property: min_periods values are preserved exactly."""
|
||||
|
||||
@given(
|
||||
window=st.integers(min_value=5, max_value=500),
|
||||
min_periods=st.integers(min_value=1, max_value=500),
|
||||
method=st.sampled_from(["mean", "std", "sum", "var", "skew", "kurt"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_min_periods_unchanged(self, window, min_periods, method):
|
||||
"""Property: min_periods value is preserved after fix."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.groupby(level=1)['x'].transform(lambda x: x.rolling({window}, min_periods={min_periods}).{method}())"
|
||||
result = fixer.fix(code)
|
||||
assert f"min_periods={min_periods}" in result
|
||||
|
||||
@given(
|
||||
window=st.integers(min_value=10, max_value=500),
|
||||
min_periods=st.integers(min_value=1, max_value=30),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_small_min_periods_preserved(self, window, min_periods):
|
||||
"""Property: small min_periods (1, 5, 10) stays unchanged."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df['x'] = df.groupby(level=1)['y'].transform(lambda x: x.rolling({window}, min_periods={min_periods}).mean())"
|
||||
result = fixer.fix(code)
|
||||
assert f"min_periods={min_periods}" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 7: apply() → transform() Conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApplyToTransform:
|
||||
"""Property: groupby().apply() → groupby().transform() conversion."""
|
||||
|
||||
@given(
|
||||
col=st.sampled_from(["$close", "$open", "$volume", "ret", "x"]),
|
||||
func=st.sampled_from([
|
||||
"lambda x: np.log(x / x.shift(1))",
|
||||
"lambda x: x.cumsum()",
|
||||
"lambda x: x.pct_change()",
|
||||
"lambda x: x.rolling(20).mean()",
|
||||
"lambda x: x.diff()",
|
||||
]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_apply_lambda_becomes_transform(self, col, func):
|
||||
"""Property: groupby().apply(lambda...) → groupby().transform(lambda...)."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.groupby(level=1)['{col}'].apply({func})"
|
||||
result = fixer.fix(code)
|
||||
assert ".transform(" in result
|
||||
# Lambda body should be preserved
|
||||
func_clean = func.replace(" ", "")
|
||||
assert func_clean.replace(" ", "") in result.replace(" ", "") or \
|
||||
func in result
|
||||
|
||||
@given(
|
||||
col=st.sampled_from(["$close", "x", "ret"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_reset_index_after_transform_removed(self, col):
|
||||
"""Property: .transform().reset_index(level=0, drop=True) → reset_index removed."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df['v'] = df.groupby(level=1)['{col}'].transform(lambda x: x.rolling(20).mean()).reset_index(level=0, drop=True)"
|
||||
result = fixer.fix(code)
|
||||
assert ".reset_index(level=0, drop=True)" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 8: ResetIndex GroupBy Fix
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResetIndexGroupBy:
|
||||
"""Property: reset_index + groupby(level=1) → groupby('instrument')."""
|
||||
|
||||
@given(
|
||||
var_name=st.sampled_from(["df_r", "df_reset", "data_flat", "flat"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_reset_index_groupby_level_converted(self, var_name):
|
||||
"""Property: after reset_index on var, groupby(level=1) → groupby('instrument')."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"{var_name} = df.reset_index()\n{var_name}['x'] = {var_name}.groupby(level=1)['$close'].mean()"
|
||||
result = fixer.fix(code)
|
||||
assert "groupby('instrument')" in result
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_normal_multiindex_groupby_untouched(self, seed):
|
||||
"""Property: regular df.groupby(level=1) without reset_index is not changed."""
|
||||
fixer = _auto_fixer()
|
||||
code = "df['x'] = df.groupby(level=1)['$close'].mean()"
|
||||
result = fixer.fix(code)
|
||||
assert "groupby(level=1)" in result
|
||||
assert "groupby('instrument')" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 9: Rolling ddof Removal
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRollingDdof:
|
||||
"""Property: ddof keyword is removed from rolling operations."""
|
||||
|
||||
@given(
|
||||
window=st.integers(min_value=5, max_value=200),
|
||||
min_periods=st.integers(min_value=1, max_value=50),
|
||||
ddof=st.integers(min_value=0, max_value=5),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_ddof_removed_from_rolling_args(self, window, min_periods, ddof):
|
||||
"""Property: ddof is removed from rolling() args."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.rolling({window}, min_periods={min_periods}, ddof={ddof}).std()"
|
||||
result = fixer.fix(code)
|
||||
assert "ddof" not in result
|
||||
|
||||
@given(
|
||||
window=st.integers(min_value=5, max_value=200),
|
||||
ddof=st.integers(min_value=0, max_value=5),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_ddof_removed_from_std_args(self, window, ddof):
|
||||
"""Property: ddof is removed from std() args."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.rolling({window}).std(ddof={ddof})"
|
||||
result = fixer.fix(code)
|
||||
assert "ddof" not in result
|
||||
|
||||
@given(
|
||||
window=st.integers(min_value=5, max_value=200),
|
||||
min_periods=st.integers(min_value=1, max_value=50),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_no_ddof_preserves_code(self, window, min_periods):
|
||||
"""Property: code without ddof is unchanged by ddof removal."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.rolling({window}, min_periods={min_periods}).std()"
|
||||
result = fixer.fix(code)
|
||||
assert "ddof" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 10: GroupBy Mixed Levels
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGroupByMixedLevels:
|
||||
"""Property: groupby(level=[N, 'string']) → level=[integers_only]."""
|
||||
|
||||
@given(
|
||||
int_levels=st.lists(st.integers(min_value=0, max_value=3), min_size=1, max_size=3),
|
||||
str_level=st.sampled_from(["'date'", '"date"', "'instrument'", '"instrument"']),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_mixed_levels_strips_strings(self, int_levels, str_level):
|
||||
"""Property: string levels are stripped from groupby(level=[])."""
|
||||
fixer = _auto_fixer()
|
||||
levels_str = ", ".join(str(l) for l in int_levels) + (", " + str_level if int_levels else str_level)
|
||||
code = f"df.groupby(level=[{levels_str}]).apply(fn)"
|
||||
result = fixer.fix(code)
|
||||
# String levels should be gone from level=
|
||||
assert str_level.strip("'\"") not in [p.strip("'\"") for p in re.findall(r"level=\[[^\]]+\]", result)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 11: Chained GroupBy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestChainedGroupBy:
|
||||
"""Property: chained groupby fixes."""
|
||||
|
||||
@given(
|
||||
first_level=st.sampled_from(["level=1", "level=0"]),
|
||||
second_groupby=st.sampled_from([".groupby('date')", '.groupby("date")']),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_chained_groupby_converted(self, first_level, second_groupby):
|
||||
"""Property: chained groupby is converted to single get_level_values grouping."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.groupby({first_level}){second_groupby}['price_volume'].transform('cumsum')"
|
||||
result = fixer.fix(code)
|
||||
assert "get_level_values" in result
|
||||
# Second groupby should be removed
|
||||
assert ".groupby(" not in result.split("get_level_values")[-1] or \
|
||||
".groupby('date')" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 12: Volume Proxy Injection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVolumeProxy:
|
||||
"""Property: volume proxy is injected when $volume is used."""
|
||||
|
||||
@given(
|
||||
use_volume=st.booleans(),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_volume_proxy_injected_when_used(self, use_volume):
|
||||
"""Property: proxy is injected exactly when $volume is used in read_hdf code."""
|
||||
fixer = _auto_fixer()
|
||||
if use_volume:
|
||||
code = (
|
||||
"def calc():\n"
|
||||
" df = pd.read_hdf('data.h5', key='data')\n"
|
||||
" df['pv'] = df['$close'] * df['$volume']\n"
|
||||
" return df[['pv']]\n"
|
||||
)
|
||||
else:
|
||||
code = (
|
||||
"def calc():\n"
|
||||
" df = pd.read_hdf('data.h5', key='data')\n"
|
||||
" df['x'] = df['$close'].pct_change()\n"
|
||||
" return df[['x']]\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
if use_volume:
|
||||
assert "volume proxy" in result
|
||||
else:
|
||||
assert "volume proxy" not in result
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_proxy_only_injected_once(self, seed):
|
||||
"""Property: volume proxy is not injected twice."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
"def calc():\n"
|
||||
" df = pd.read_hdf('data.h5', key='data')\n"
|
||||
" # volume proxy: $volume is always 0 in FX data — use price-range as proxy\n"
|
||||
" if (df['$volume'] == 0).all():\n"
|
||||
" df['$volume'] = df['$high'] - df['$low']\n"
|
||||
" df['pv'] = df['$close'] * df['$volume']\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert result.count("volume proxy") == 1
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_proxy_correct_formula(self, seed):
|
||||
"""Property: volume proxy formula is high - low."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
"def calc():\n"
|
||||
" df = pd.read_hdf('data.h5', key='data')\n"
|
||||
" df['pv'] = df['$close'] * df['$volume']\n"
|
||||
" return df[['pv']]\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert "df['$high'] - df['$low']" in result
|
||||
assert "df['$volume'] = df['$high'] - df['$low']" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 13: loc → xs Conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLocToXs:
|
||||
"""Property: df.loc[instrument] → df.xs(instrument, level=1)."""
|
||||
|
||||
@given(
|
||||
var=st.sampled_from(["instrument", "inst", "sym"]),
|
||||
level=st.sampled_from(["'instrument'", "1"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_loc_read_converted_to_xs(self, var, level):
|
||||
"""Property: df.loc[var] read access → df.xs(var, level=...) in instrument loops."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
f"for {var} in df.index.get_level_values({level}).unique():\n"
|
||||
f" inst_df = df.loc[{var}].copy()\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert "df.xs(" in result
|
||||
assert f"df.loc[{var}]" not in result
|
||||
|
||||
@given(
|
||||
var=st.sampled_from(["instrument", "inst", "sym"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_loc_write_not_converted(self, var):
|
||||
"""Property: df.loc[var] = ... write-back is not converted to xs."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
f"for {var} in df.index.get_level_values('instrument').unique():\n"
|
||||
f" df.loc[{var}] = modified\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert f"df.loc[{var}] = modified" in result
|
||||
|
||||
@given(
|
||||
var=st.sampled_from(["date", "d"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_non_instrument_loop_not_touched(self, var):
|
||||
"""Property: non-instrument loop with loc is not modified."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"for {var} in dates:\n sub = df.loc[{var}]\n"
|
||||
result = fixer.fix(code)
|
||||
assert f"df.loc[{var}]" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 14: NaN/MultiIndex Fuzzing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFuzzing:
|
||||
"""Property: fixer handles random code and edge cases gracefully."""
|
||||
|
||||
@given(
|
||||
code=st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "P", "Z"),
|
||||
whitelist_characters="\n\t ",
|
||||
),
|
||||
min_size=5,
|
||||
max_size=500,
|
||||
).filter(lambda s: len(s.strip()) > 0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_does_not_raise_on_random_text(self, code):
|
||||
"""Property: fix() does not crash on arbitrary text input."""
|
||||
fixer = _auto_fixer()
|
||||
try:
|
||||
result = fixer.fix(code)
|
||||
assert isinstance(result, str)
|
||||
except Exception:
|
||||
pass # Some inputs might be problematic, but shouldn't crash
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_handles_empty_code(self, seed):
|
||||
"""Property: fix handles empty or whitespace-only code."""
|
||||
fixer = _auto_fixer()
|
||||
result = fixer.fix("")
|
||||
assert isinstance(result, str)
|
||||
result2 = fixer.fix(" \n \n")
|
||||
assert isinstance(result2, str)
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_handles_long_code(self, seed):
|
||||
"""Property: fix handles long factor code without performance issues."""
|
||||
fixer = _auto_fixer()
|
||||
base = "df['x'] = df.groupby(level=1)['$close'].pct_change()\n"
|
||||
code = base * 10 # 10 repetitions
|
||||
result = fixer.fix(code)
|
||||
assert isinstance(result, str)
|
||||
assert len(result) >= len(code)
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_handles_code_with_comments(self, seed):
|
||||
"""Property: fix handles code with comments correctly."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
"# This is a comment\n"
|
||||
"df['x'] = df.groupby(level=1)['$close'].mean() # inline comment\n"
|
||||
"# Another comment\n"
|
||||
"df['y'] = df.groupby(level=1)['x'].transform(lambda x: x.rolling(20, min_periods=1).std())\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert _is_valid_python(result)
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_handles_multiline_expressions(self, seed):
|
||||
"""Property: fix handles multi-line expressions."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
"df['x'] = (df.groupby(level=1)['$close']\n"
|
||||
" .transform(lambda x: x.rolling(20, min_periods=1).mean()))\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert _is_valid_python(result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 15: Transform ResetIndex Removal
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTransformResetIndex:
|
||||
"""Property: .transform(...).reset_index(drop=True) cleanup."""
|
||||
|
||||
@given(
|
||||
col=st.sampled_from(["x", "$close", "$volume", "ret"]),
|
||||
func=st.sampled_from(["lambda x: x.rolling(20).mean()", "lambda x: x.pct_change()"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_reset_index_after_transform_removed(self, col, func):
|
||||
"""Property: reset_index after transform is removed."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df['v'] = df.groupby(level=1)['{col}'].transform({func}).reset_index(level=0, drop=True)"
|
||||
result = fixer.fix(code)
|
||||
assert ".reset_index(level=0, drop=True)" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 16: No Fixes Applied to Clean Code
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCleanCode:
|
||||
"""Property: clean code that needs no fixing passes through unchanged."""
|
||||
|
||||
CLEAN_PATTERNS = [
|
||||
"df['x'] = df.groupby(level=1)['$close'].pct_change()",
|
||||
"df['y'] = df['$high'] - df['$low']",
|
||||
"data = df.xs('EURUSD', level=1)",
|
||||
"factor = df.groupby(level=1)['$close'].transform(lambda x: x / x.shift(1) - 1)",
|
||||
"df['mid'] = (df['$high'] + df['$low']) / 2",
|
||||
]
|
||||
|
||||
@given(code=st.sampled_from(CLEAN_PATTERNS))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_clean_code_unchanged(self, code):
|
||||
"""Property: clean patterns are not altered."""
|
||||
fixer = _auto_fixer()
|
||||
result = fixer.fix(code)
|
||||
if _is_valid_python(code):
|
||||
assert _is_valid_python(result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 17: FixesApplied List
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFixesApplied:
|
||||
"""Property: fixes_applied list tracks changes."""
|
||||
|
||||
@given(
|
||||
use_pattern=st.booleans(),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fixes_applied_empty_for_clean_code(self, use_pattern):
|
||||
"""Property: fixes_applied is empty for code needing no fixes."""
|
||||
fixer = FactorAutoFixer()
|
||||
if use_pattern:
|
||||
code = "df.groupby(level=1)['$close'].apply(lambda x: np.log(x / x.shift(1)))"
|
||||
else:
|
||||
code = "df['x'] = df.groupby(level=1)['$close'].pct_change()"
|
||||
fixer.fix(code)
|
||||
# fixes_applied should exist
|
||||
assert isinstance(fixer.fixes_applied, list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 18: Pattern Recognition Robustness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPatternRobustness:
|
||||
"""Property: pattern recognition works with varying whitespace."""
|
||||
|
||||
@given(
|
||||
spaces_before=st.integers(min_value=0, max_value=8),
|
||||
spaces_after=st.integers(min_value=0, max_value=8),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_whitespace_variation_handled(self, spaces_before, spaces_after):
|
||||
"""Property: fixer handles varying whitespace around key patterns."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
f"{' ' * spaces_before}df.groupby(level=['instrument', 'date'])['col'].transform('sum')"
|
||||
f"{' ' * spaces_after}"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert "get_level_values" in result
|
||||
|
||||
@given(
|
||||
spaces=st.integers(min_value=0, max_value=8),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_whitespace_before_level(self, spaces):
|
||||
"""Property: fixer recognizes groupby(.level=1) regardless of spacing."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.groupby(level{ ' ' * spaces}={ ' ' * spaces}1)['x'].mean()"
|
||||
result = fixer.fix(code)
|
||||
assert _is_valid_python(result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 19: String Quoting Variants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStringQuoting:
|
||||
"""Property: single-quoted and double-quoted strings are handled identically."""
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_both_quoting_styles(self, seed):
|
||||
"""Property: mixed quoting styles in level=['instrument', 'date'] are handled."""
|
||||
fixer = _auto_fixer()
|
||||
code = "df.groupby(level=['instrument', 'date'])['col'].transform('sum')"
|
||||
result = fixer.fix(code)
|
||||
assert "get_level_values" in result
|
||||
|
||||
@given(
|
||||
col=st.sampled_from(["'$close'", "'ret'", "'x'"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_single_quoted_column(self, col):
|
||||
"""Property: single-quoted column names work the same."""
|
||||
fixer = _auto_fixer()
|
||||
code = f"df.groupby(level=1)[{col}].apply(lambda x: x.pct_change())"
|
||||
result = fixer.fix(code)
|
||||
assert ".transform(" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 20: Constructor and State
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAutoFixerConstructor:
|
||||
"""Property: FactorAutoFixer constructor and state."""
|
||||
|
||||
def test_default_constructor(self):
|
||||
"""Property: default constructor creates valid Fixer."""
|
||||
fixer = FactorAutoFixer()
|
||||
assert isinstance(fixer.fixes_applied, list)
|
||||
assert len(fixer.fixes_applied) == 0
|
||||
|
||||
def test_fix_returns_string(self):
|
||||
"""Property: fix() always returns a string."""
|
||||
fixer = _auto_fixer()
|
||||
result = fixer.fix("df['x'] = 1")
|
||||
assert isinstance(result, str)
|
||||
|
||||
@given(
|
||||
code=st.sampled_from(["df.groupby(level=1)['x'].mean()", "x = 1 + 2", "", "pass"]),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_fix_returns_non_empty_for_non_empty_input(self, code):
|
||||
"""Property: fix returns non-empty string for non-empty input."""
|
||||
fixer = _auto_fixer()
|
||||
result = fixer.fix(code)
|
||||
assert isinstance(result, str)
|
||||
if code.strip():
|
||||
assert len(result) > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 21: Multi-Pattern Interactions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMultiPatternInteractions:
|
||||
"""Property: multiple fixes interact correctly on the same code."""
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_combined_apply_and_reset_index(self, seed):
|
||||
"""Property: apply→transform AND reset_index removal work together."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
"df_r = df.reset_index()\n"
|
||||
"df_r['x'] = df_r.groupby(level=1)['$close'].apply(lambda x: np.log(x / x.shift(1)))\n"
|
||||
"df_r['y'] = df_r.groupby(level=1)['$close'].transform(lambda x: x.rolling(20, min_periods=5).mean())\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert _is_valid_python(result)
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_volume_proxy_and_groupby_fix(self, seed):
|
||||
"""Property: volume proxy and groupby fixes work together."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
"def calc():\n"
|
||||
" df = pd.read_hdf('data.h5', key='data')\n"
|
||||
" df['val'] = df.groupby(level=1)['$close'].apply(lambda x: x.pct_change())\n"
|
||||
" df['pv'] = df['$close'] * df['$volume']\n"
|
||||
" return df[['val', 'pv']]\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert _is_valid_python(result)
|
||||
assert "volume proxy" in result
|
||||
assert ".transform(" in result
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_instrument_and_level_fix_together(self, seed):
|
||||
"""Property: instrument column replacement and level= fix work together."""
|
||||
fixer = _auto_fixer()
|
||||
code = (
|
||||
"df['key'] = df['instrument'] + '_' + df['day_id'].astype(str)\n"
|
||||
"df.groupby(level=['instrument', 'date'])['col'].transform('sum')\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert "df['instrument']" not in result
|
||||
assert "get_level_values" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 22: Fix Order Independence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFixOrderIndependence:
|
||||
"""Property: specific fix patterns produce deterministic results."""
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_same_input_same_output_always(self, seed):
|
||||
"""Property: fixing same code twice gives identical results."""
|
||||
fixer1 = _auto_fixer()
|
||||
fixer2 = _auto_fixer()
|
||||
code = (
|
||||
"df_r = df.reset_index()\n"
|
||||
"df_r['x'] = df_r.groupby(level=1)['$close'].apply(lambda x: np.log(x / x.shift(1)))\n"
|
||||
"df['y'] = df.groupby(level=['instrument', 'date'])['col'].transform('sum')\n"
|
||||
"df['z'] = df.groupby(level=1)['ret'].transform(lambda x: x.rolling(20, min_periods=1).std())\n"
|
||||
)
|
||||
assert fixer1.fix(code) == fixer2.fix(code)
|
||||
|
||||
@@ -219,3 +219,710 @@ class TestFactorEvaluatorsInit:
|
||||
mock_scen = MagicMock()
|
||||
eva = FactorValueEvaluator(mock_scen)
|
||||
assert eva.scen is mock_scen
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# HYPOTHESIS-BASED PROPERTY TESTS — Code Generation Patterns, Variable
|
||||
# Extraction, Evaluator Consistency
|
||||
# ==============================================================================
|
||||
from hypothesis import given, settings, strategies as st
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from rdagent.components.coder.factor_coder.factor import (
|
||||
FactorTask,
|
||||
FactorFBWorkspace,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.evaluators import (
|
||||
FactorEvaluatorForCoder,
|
||||
)
|
||||
from rdagent.components.coder.factor_coder.eva_utils import (
|
||||
FactorInfEvaluator,
|
||||
FactorSingleColumnEvaluator,
|
||||
FactorOutputFormatEvaluator,
|
||||
FactorMissingValuesEvaluator,
|
||||
FactorCorrelationEvaluator,
|
||||
FactorValueEvaluator,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _valid_factor_task_names() -> st.SearchStrategy:
|
||||
return st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Lu", "Ll"), whitelist_characters="_"),
|
||||
min_size=1,
|
||||
max_size=50,
|
||||
).filter(lambda s: s and s[0].isalpha() and " " not in s)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 1: FactorTask Field Invariants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskInvariants:
|
||||
"""Property: FactorTask fields maintain invariants after construction."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=50).filter(lambda s: " " not in s),
|
||||
factor_description=st.text(min_size=0, max_size=200),
|
||||
factor_formulation=st.text(min_size=0, max_size=200),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_construction_preserves_all_fields(self, factor_name, factor_description, factor_formulation):
|
||||
"""Property: all constructor args are stored as instance attributes."""
|
||||
t = FactorTask(factor_name, factor_description, factor_formulation)
|
||||
assert t.factor_name == factor_name
|
||||
assert t.factor_description == factor_description
|
||||
assert t.factor_formulation == factor_formulation
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=50).filter(lambda s: " " not in s),
|
||||
factor_description=st.text(min_size=0, max_size=200),
|
||||
factor_formulation=st.text(min_size=0, max_size=200),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_default_field_values(self, factor_name, factor_description, factor_formulation):
|
||||
"""Property: default fields have expected values."""
|
||||
t = FactorTask(factor_name, factor_description, factor_formulation)
|
||||
assert t.factor_implementation is False
|
||||
assert t.factor_resources is None
|
||||
assert t.base_code is None
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=50).filter(lambda s: " " not in s),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_get_task_information_contains_name(self, factor_name):
|
||||
"""Property: get_task_information returns string containing factor_name."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
info = t.get_task_information()
|
||||
assert factor_name in info
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=50).filter(lambda s: " " not in s),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_get_task_brief_information_contains_name(self, factor_name):
|
||||
"""Property: get_task_brief_information returns string containing factor_name."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
info = t.get_task_brief_information()
|
||||
assert factor_name in info
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=50).filter(lambda s: " " not in s),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_get_task_information_and_implementation_result(self, factor_name):
|
||||
"""Property: returned dict contains expected keys."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
result = t.get_task_information_and_implementation_result()
|
||||
assert "factor_name" in result
|
||||
assert "factor_description" in result
|
||||
assert "factor_formulation" in result
|
||||
assert "factor_implementation" in result
|
||||
assert result["factor_name"] == factor_name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 2: FactorTask from_dict
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskFromDict:
|
||||
"""Property: FactorTask.from_dict round-trip."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=30).filter(lambda s: s.isidentifier()),
|
||||
factor_description=st.text(min_size=0, max_size=100),
|
||||
factor_formulation=st.text(min_size=0, max_size=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_from_dict_round_trip(self, factor_name, factor_description, factor_formulation):
|
||||
"""Property: constructing from dict of get_task_information_and_implementation_result preserves values."""
|
||||
t1 = FactorTask(factor_name, factor_description, factor_formulation)
|
||||
info = t1.get_task_information_and_implementation_result()
|
||||
t2 = FactorTask.from_dict(info)
|
||||
assert t2.factor_name == t1.factor_name
|
||||
assert t2.factor_description == t1.factor_description
|
||||
assert t2.factor_formulation == t1.factor_formulation
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=30).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_from_dict_with_implementation(self, factor_name):
|
||||
"""Property: factor_implementation field restored from dict."""
|
||||
d = {
|
||||
"factor_name": factor_name,
|
||||
"factor_description": "desc",
|
||||
"factor_formulation": "formula",
|
||||
"variables": {},
|
||||
"resource": None,
|
||||
"factor_implementation": True,
|
||||
}
|
||||
t = FactorTask.from_dict(d)
|
||||
assert t.factor_implementation is True
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=30).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_from_dict_with_variables(self, factor_name):
|
||||
"""Property: variables dict restored from dict."""
|
||||
d = {
|
||||
"factor_name": factor_name,
|
||||
"factor_description": "desc",
|
||||
"factor_formulation": "formula",
|
||||
"variables": {"x": 1, "y": 2},
|
||||
"resource": "r1",
|
||||
"factor_implementation": False,
|
||||
}
|
||||
t = FactorTask.from_dict(d)
|
||||
assert t.variables == {"x": 1, "y": 2}
|
||||
assert t.factor_resources == "r1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 3: FactorTask Repr
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskRepr:
|
||||
"""Property: __repr__ invariants."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=30).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_repr_contains_factor_task_and_name(self, factor_name):
|
||||
"""Property: repr contains 'FactorTask' and factor_name."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
r = repr(t)
|
||||
assert "FactorTask" in r
|
||||
assert factor_name in r
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=30).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_repr_is_string(self, factor_name):
|
||||
"""Property: repr returns a string."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
r = repr(t)
|
||||
assert isinstance(r, str)
|
||||
assert len(r) > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 4: FactorTask Variables
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskVariables:
|
||||
"""Property: variables field invariants."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
vars_keys=st.lists(
|
||||
st.text(min_size=1, max_size=10).filter(lambda s: s.isidentifier()),
|
||||
min_size=0, max_size=10, unique=True,
|
||||
),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_variables_stored_correctly(self, factor_name, vars_keys):
|
||||
"""Property: variables dict stored as provided."""
|
||||
vars_dict = {k: i for i, k in enumerate(vars_keys)}
|
||||
t = FactorTask(factor_name, "desc", "formula", variables=vars_dict)
|
||||
assert t.variables == vars_dict
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_default_variables_is_empty_dict(self, factor_name):
|
||||
"""Property: default variables is empty dict."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
assert t.variables == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 5: FactorTask Resource
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskResource:
|
||||
"""Property: resource field invariants."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
resource=st.one_of(st.none(), st.text(min_size=1, max_size=50)),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_resource_stored_correctly(self, factor_name, resource):
|
||||
"""Property: resource field stored as provided or default None."""
|
||||
t = FactorTask(factor_name, "desc", "formula", resource=resource)
|
||||
assert t.factor_resources == resource
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 6: FactorFBWorkspace Path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorFBWorkspacePath:
|
||||
"""Property: FactorFBWorkspace workspace path invariants."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_workspace_path_is_valid_path(self, factor_name):
|
||||
"""Property: workspace_path is a Path instance."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
ws = FactorFBWorkspace(target_task=t)
|
||||
assert isinstance(ws.workspace_path, Path)
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_target_task_reference_preserved(self, factor_name):
|
||||
"""Property: target_task reference points back to FactorTask."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
ws = FactorFBWorkspace(target_task=t)
|
||||
assert ws.target_task is t
|
||||
assert ws.target_task.factor_name == factor_name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 7: FactorEvaluatorForCoder Construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorEvaluatorForCoderConstruction:
|
||||
"""Property: FactorEvaluatorForCoder constructor creates sub-evaluators."""
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_sub_evaluators_are_created(self, seed):
|
||||
"""Property: constructor creates value, code, and final_decision evaluators."""
|
||||
mock_scen = MagicMock()
|
||||
eva = FactorEvaluatorForCoder(scen=mock_scen)
|
||||
assert eva.value_evaluator is not None
|
||||
assert eva.code_evaluator is not None
|
||||
assert eva.final_decision_evaluator is not None
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_evaluate_none_implementation_returns_none(self, seed):
|
||||
"""Property: evaluate with implementation=None returns None."""
|
||||
eva = FactorEvaluatorForCoder(scen=MagicMock())
|
||||
result = eva.evaluate(target_task=MagicMock(), implementation=None)
|
||||
assert result is None
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_scenario_reference_accessible(self, seed):
|
||||
"""Property: evaluator has access to scenario."""
|
||||
mock_scen = MagicMock()
|
||||
eva = FactorEvaluatorForCoder(scen=mock_scen)
|
||||
assert eva.scen is mock_scen
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 8: FactorEvaluator SubTypes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorEvaluatorSubTypes:
|
||||
"""Property: sub-evaluator types are correct."""
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_value_evaluator_is_factor_value_evaluator(self, seed):
|
||||
"""Property: value_evaluator is FactorValueEvaluator instance."""
|
||||
eva = FactorEvaluatorForCoder(scen=MagicMock())
|
||||
assert isinstance(eva.value_evaluator, FactorValueEvaluator)
|
||||
|
||||
@given(
|
||||
seed=st.integers(min_value=0, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_scen_passed_to_value_evaluator(self, seed):
|
||||
"""Property: scenario is passed to value_evaluator."""
|
||||
mock_scen = MagicMock()
|
||||
eva = FactorEvaluatorForCoder(scen=mock_scen)
|
||||
assert eva.value_evaluator.scen is mock_scen
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 9: FactorInfEvaluator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorInfEvaluator:
|
||||
"""Property: FactorInfEvaluator invariants."""
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_str_is_correct(self, seed):
|
||||
"""Property: __str__ returns 'FactorInfEvaluator'."""
|
||||
eva = FactorInfEvaluator()
|
||||
assert str(eva) == "FactorInfEvaluator"
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_constructor_no_args(self, seed):
|
||||
"""Property: FactorInfEvaluator can be constructed without arguments."""
|
||||
eva = FactorInfEvaluator()
|
||||
assert eva is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 10: FactorSingleColumnEvaluator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorSingleColumnEvaluator:
|
||||
"""Property: FactorSingleColumnEvaluator invariants."""
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_str_is_correct(self, seed):
|
||||
"""Property: __str__ returns 'FactorSingleColumnEvaluator'."""
|
||||
eva = FactorSingleColumnEvaluator()
|
||||
assert str(eva) == "FactorSingleColumnEvaluator"
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_constructor_no_args(self, seed):
|
||||
"""Property: FactorSingleColumnEvaluator can be constructed without arguments."""
|
||||
eva = FactorSingleColumnEvaluator()
|
||||
assert eva is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 11: FactorOutputFormatEvaluator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorOutputFormatEvaluator:
|
||||
"""Property: FactorOutputFormatEvaluator invariants."""
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_str_is_correct(self, seed):
|
||||
"""Property: __str__ returns 'FactorOutputFormatEvaluator'."""
|
||||
eva = FactorOutputFormatEvaluator()
|
||||
assert str(eva) == "FactorOutputFormatEvaluator"
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_constructor_no_args(self, seed):
|
||||
"""Property: FactorOutputFormatEvaluator can be constructed without arguments."""
|
||||
eva = FactorOutputFormatEvaluator()
|
||||
assert eva is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 12: FactorMissingValuesEvaluator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorMissingValuesEvaluator:
|
||||
"""Property: FactorMissingValuesEvaluator invariants."""
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_str_is_correct(self, seed):
|
||||
"""Property: __str__ returns 'FactorMissingValuesEvaluator'."""
|
||||
eva = FactorMissingValuesEvaluator()
|
||||
assert str(eva) == "FactorMissingValuesEvaluator"
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_constructor_no_args(self, seed):
|
||||
"""Property: FactorMissingValuesEvaluator can be constructed without arguments."""
|
||||
eva = FactorMissingValuesEvaluator()
|
||||
assert eva is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 13: FactorCorrelationEvaluator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorCorrelationEvaluator:
|
||||
"""Property: FactorCorrelationEvaluator invariants."""
|
||||
|
||||
@given(
|
||||
hard_check=st.booleans(),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_hard_check_stored_correctly(self, hard_check):
|
||||
"""Property: hard_check flag stored correctly."""
|
||||
eva = FactorCorrelationEvaluator(hard_check=hard_check)
|
||||
assert eva.hard_check is hard_check
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_str_contains_correct_name(self, seed):
|
||||
"""Property: __str__ contains 'FactorCorrelationEvaluator'."""
|
||||
eva = FactorCorrelationEvaluator(hard_check=False)
|
||||
assert "FactorCorrelationEvaluator" in str(eva)
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_default_hard_check_is_false(self, seed):
|
||||
"""Property: hard_check parameter works."""
|
||||
eva = FactorCorrelationEvaluator(hard_check=False)
|
||||
assert eva.hard_check is False
|
||||
eva2 = FactorCorrelationEvaluator(hard_check=True)
|
||||
assert eva2.hard_check is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 14: FactorValueEvaluator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorValueEvaluator:
|
||||
"""Property: FactorValueEvaluator invariants."""
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_scenario_stored_correctly(self, seed):
|
||||
"""Property: scenario reference stored."""
|
||||
mock_scen = MagicMock()
|
||||
eva = FactorValueEvaluator(mock_scen)
|
||||
assert eva.scen is mock_scen
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_requires_scenario_arg(self, seed):
|
||||
"""Property: FactorValueEvaluator requires scenario argument."""
|
||||
mock_scen = MagicMock()
|
||||
eva = FactorValueEvaluator(mock_scen)
|
||||
assert eva is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 15: FactorTask Version
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskVersion:
|
||||
"""Property: version field invariants."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
version=st.integers(min_value=0, max_value=1000),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_version_default_and_mutable(self, factor_name, version):
|
||||
"""Property: version can be set and retrieved."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
t.version = version
|
||||
assert t.version == version
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 16: FactorTask Feedback Field
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskFeedback:
|
||||
"""Property: feedback-related fields."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_default_implementation_is_false(self, factor_name):
|
||||
"""Property: factor_implementation defaults to False."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
assert t.factor_implementation is False
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_implementation_can_be_set(self, factor_name):
|
||||
"""Property: factor_implementation can be set to True."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
t.factor_implementation = True
|
||||
assert t.factor_implementation is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 17: FactorFBWorkspace FB Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorFBWorkspaceConstants:
|
||||
"""Property: FactorFBWorkspace class constants."""
|
||||
|
||||
def test_fb_exec_success_constant(self):
|
||||
"""Property: FB_EXEC_SUCCESS is defined as a non-empty string."""
|
||||
assert len(str(FactorFBWorkspace.FB_EXEC_SUCCESS)) > 0
|
||||
|
||||
def test_fb_output_file_found_constant(self):
|
||||
"""Property: FB_OUTPUT_FILE_FOUND is defined as a non-empty string."""
|
||||
assert len(str(FactorFBWorkspace.FB_OUTPUT_FILE_FOUND)) > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 18: FactorTask with Variables from_dict Round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskRoundTrip:
|
||||
"""Property: full round-trip through from_dict preserves all data."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
factor_description=st.text(min_size=0, max_size=100),
|
||||
factor_formulation=st.text(min_size=0, max_size=100),
|
||||
n_vars=st.integers(min_value=0, max_value=10),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_to_dict_from_dict_round_trip(self, factor_name, factor_description, factor_formulation, n_vars):
|
||||
"""Property: task.to_dict() → FactorTask.from_dict(d) preserves key fields."""
|
||||
t1 = FactorTask(factor_name, factor_description, factor_formulation)
|
||||
d = t1.get_task_information_and_implementation_result()
|
||||
t2 = FactorTask.from_dict(d)
|
||||
assert t2.factor_name == factor_name
|
||||
assert t2.factor_description == factor_description
|
||||
assert t2.factor_formulation == factor_formulation
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 19: FactorTask CoSTEERTask Inheritance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskCoSTEER:
|
||||
"""Property: FactorTask inherits correctly from CoSTEERTask."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_base_code_is_none_by_default(self, factor_name):
|
||||
"""Property: base_code attribute is None by default (from CoSTEERTask)."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
assert t.base_code is None
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_base_code_can_be_set(self, factor_name):
|
||||
"""Property: base_code can be set."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
t.base_code = "print(42)"
|
||||
assert t.base_code == "print(42)"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 20: FactorEvaluatorForCoder Caching Behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEvaluatorCaching:
|
||||
"""Property: evaluator caching behavior."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_cached_feedback_returned(self, factor_name):
|
||||
"""Property: queried_knowledge with cached feedback returns it."""
|
||||
from rdagent.components.coder.factor_coder.factor import FactorTask
|
||||
|
||||
eva = FactorEvaluatorForCoder(scen=MagicMock())
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
qk = MagicMock()
|
||||
qk.success_task_to_knowledge_dict = {"info": MagicMock(feedback="cached")}
|
||||
t.get_task_information = MagicMock(return_value="info")
|
||||
qk.failed_task_info_set = set()
|
||||
|
||||
fb = eva.evaluate(target_task=t, implementation=MagicMock(), queried_knowledge=qk)
|
||||
assert fb == "cached"
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=20).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_failed_task_returns_negative_feedback(self, factor_name):
|
||||
"""Property: failed tasks return negative feedback with 'failed too many times'."""
|
||||
from rdagent.components.coder.factor_coder.factor import FactorTask
|
||||
|
||||
eva = FactorEvaluatorForCoder(scen=MagicMock())
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
qk = MagicMock()
|
||||
qk.success_task_to_knowledge_dict = {}
|
||||
t.get_task_information = MagicMock(return_value="info")
|
||||
qk.failed_task_info_set = {"info"}
|
||||
|
||||
fb = eva.evaluate(target_task=t, implementation=MagicMock(), queried_knowledge=qk)
|
||||
assert fb.final_decision is False
|
||||
assert "failed too many times" in fb.execution_feedback
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 21: FactorTask Information Format
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFactorTaskInformation:
|
||||
"""Property: task information output format."""
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=30).filter(lambda s: s.isidentifier()),
|
||||
factor_description=st.text(min_size=0, max_size=100),
|
||||
factor_formulation=st.text(min_size=0, max_size=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_get_task_information_format(self, factor_name, factor_description, factor_formulation):
|
||||
"""Property: get_task_information has expected format."""
|
||||
t = FactorTask(factor_name, factor_description, factor_formulation)
|
||||
info = t.get_task_information()
|
||||
assert f"factor_name: {factor_name}" in info
|
||||
assert f"factor_description: {factor_description}" in info
|
||||
assert f"factor_formulation: {factor_formulation}" in info
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=30).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_get_task_information_is_string(self, factor_name):
|
||||
"""Property: get_task_information returns str."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
info = t.get_task_information()
|
||||
assert isinstance(info, str)
|
||||
|
||||
@given(
|
||||
factor_name=st.text(min_size=1, max_size=30).filter(lambda s: s.isidentifier()),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_get_task_brief_information_is_string(self, factor_name):
|
||||
"""Property: get_task_brief_information returns str."""
|
||||
t = FactorTask(factor_name, "desc", "formula")
|
||||
info = t.get_task_brief_information()
|
||||
assert isinstance(info, str)
|
||||
|
||||
@@ -184,3 +184,785 @@ class TestAdvancedLoopThreshold:
|
||||
def test_constant_is_defined(self):
|
||||
from rdagent.scenarios.qlib.quant_loop_factory import ADVANCED_LOOP_FACTOR_THRESHOLD
|
||||
assert ADVANCED_LOOP_FACTOR_THRESHOLD == 5000
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# HYPOTHESIS-BASED PROPERTY TESTS — Data Pipeline Transformations,
|
||||
# Bandit Properties, Feedback Consistency
|
||||
# ==============================================================================
|
||||
from hypothesis import given, settings, strategies as st
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.scenarios.qlib.developer.feedback import process_results
|
||||
from rdagent.scenarios.qlib.proposal.bandit import (
|
||||
Metrics,
|
||||
extract_metrics_from_experiment,
|
||||
LinearThompsonTwoArm,
|
||||
)
|
||||
from rdagent.scenarios.qlib.quant_loop_factory import (
|
||||
has_local_components,
|
||||
count_valid_factors,
|
||||
ADVANCED_LOOP_FACTOR_THRESHOLD,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 1: process_results Invariants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProcessResultsInvariants:
|
||||
"""Property: process_results output invariants."""
|
||||
|
||||
REQUIRED_METRICS = [
|
||||
"IC",
|
||||
"1day.excess_return_with_cost.annualized_return",
|
||||
"1day.excess_return_with_cost.max_drawdown",
|
||||
]
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
max_dd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
sota_ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
sota_ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
sota_max_dd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_process_results_contains_all_metrics(
|
||||
self, ic, ann_return, max_dd, sota_ic, sota_ann_return, sota_max_dd
|
||||
):
|
||||
"""Property: output string contains IC, annualized_return, and max_drawdown."""
|
||||
current = pd.Series({
|
||||
"IC": ic,
|
||||
"1day.excess_return_with_cost.annualized_return": ann_return,
|
||||
"1day.excess_return_with_cost.max_drawdown": max_dd,
|
||||
}, name="0")
|
||||
sota = pd.Series({
|
||||
"IC": sota_ic,
|
||||
"1day.excess_return_with_cost.annualized_return": sota_ann_return,
|
||||
"1day.excess_return_with_cost.max_drawdown": sota_max_dd,
|
||||
}, name="0")
|
||||
|
||||
result = process_results(current, sota)
|
||||
assert "IC of Current Result is" in result
|
||||
assert "of SOTA Result is" in result
|
||||
assert f"{ic:.6f}" in result or "nan" in result.lower()
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
max_dd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_process_results_returns_string(self, ic, ann_return, max_dd):
|
||||
"""Property: process_results returns a string."""
|
||||
current = pd.Series({
|
||||
"IC": ic,
|
||||
"1day.excess_return_with_cost.annualized_return": ann_return,
|
||||
"1day.excess_return_with_cost.max_drawdown": max_dd,
|
||||
}, name="0")
|
||||
sota = pd.Series({
|
||||
"IC": 0.0,
|
||||
"1day.excess_return_with_cost.annualized_return": 0.0,
|
||||
"1day.excess_return_with_cost.max_drawdown": 0.0,
|
||||
}, name="0")
|
||||
|
||||
result = process_results(current, sota)
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
max_dd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_process_results_raises_on_missing_metrics(self, ic, ann_return, max_dd):
|
||||
"""Property: process_results raises KeyError on missing required metrics."""
|
||||
current = pd.Series({"IC": ic}, name="0")
|
||||
sota = pd.Series({"IC": 0.0}, name="0")
|
||||
with pytest.raises(KeyError):
|
||||
process_results(current, sota)
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
max_dd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_process_results_format_consistent(self, ic, ann_return, max_dd):
|
||||
"""Property: output format is '<metric> of Current Result is <val>, of SOTA Result is <val>'."""
|
||||
current = pd.Series({
|
||||
"IC": ic,
|
||||
"1day.excess_return_with_cost.annualized_return": ann_return,
|
||||
"1day.excess_return_with_cost.max_drawdown": max_dd,
|
||||
}, name="0")
|
||||
sota = pd.Series({
|
||||
"IC": 0.0,
|
||||
"1day.excess_return_with_cost.annualized_return": 0.0,
|
||||
"1day.excess_return_with_cost.max_drawdown": 0.0,
|
||||
}, name="0")
|
||||
|
||||
result = process_results(current, sota)
|
||||
assert "of Current Result is" in result
|
||||
assert "of SOTA Result is" in result
|
||||
# Results separated by '; '
|
||||
assert ";" in result
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Property 2: Metrics Default Values
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMetricsDefaults:
|
||||
"""Property: Metrics default values are zero."""
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
sharpe=st.floats(min_value=-5.0, max_value=10.0),
|
||||
rank_ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_partial_construction_defaults_to_zero(self, ic, sharpe, rank_ic):
|
||||
"""Property: fields not specified default to 0.0."""
|
||||
m = Metrics(ic=ic, sharpe=sharpe, rank_ic=rank_ic)
|
||||
assert m.ic == ic
|
||||
assert m.sharpe == sharpe
|
||||
assert m.rank_ic == rank_ic
|
||||
assert m.icir == 0.0
|
||||
assert m.rank_icir == 0.0
|
||||
assert m.mdd == 0.0
|
||||
|
||||
@given(
|
||||
icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
rank_icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
mdd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_three_fields_default_others_zero(self, icir, rank_icir, mdd):
|
||||
"""Property: only given fields set, others zero."""
|
||||
m = Metrics(icir=icir, rank_icir=rank_icir, mdd=mdd)
|
||||
assert m.ic == 0.0
|
||||
assert m.sharpe == 0.0
|
||||
assert m.rank_ic == 0.0
|
||||
assert m.icir == icir
|
||||
assert m.rank_icir == rank_icir
|
||||
assert m.mdd == mdd
|
||||
|
||||
def test_all_defaults_zero(self):
|
||||
"""Property: default constructor sets everything to zero."""
|
||||
m = Metrics()
|
||||
assert m.ic == 0.0
|
||||
assert m.sharpe == 0.0
|
||||
assert m.mdd == 0.0
|
||||
assert m.icir == 0.0
|
||||
assert m.rank_ic == 0.0
|
||||
assert m.rank_icir == 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 3: Metrics as_vector
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMetricsAsVector:
|
||||
"""Property: as_vector invariants."""
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
rank_ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
rank_icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
ir=st.floats(min_value=-5.0, max_value=10.0),
|
||||
mdd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
sharpe=st.floats(min_value=-5.0, max_value=10.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_as_vector_length_is_8(self, ic, icir, rank_ic, rank_icir, ann_return, ir, mdd, sharpe):
|
||||
"""Property: as_vector always returns length-8 array."""
|
||||
m = Metrics(
|
||||
ic=ic, icir=icir, rank_ic=rank_ic, rank_icir=rank_icir,
|
||||
arr=ann_return, ir=ir, mdd=mdd, sharpe=sharpe,
|
||||
)
|
||||
v = m.as_vector()
|
||||
assert len(v) == 8
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
rank_ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
rank_icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
ir=st.floats(min_value=-5.0, max_value=10.0),
|
||||
mdd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
sharpe=st.floats(min_value=-5.0, max_value=10.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_as_vector_matches_input_order(self, ic, icir, rank_ic, rank_icir, ann_return, ir, mdd, sharpe):
|
||||
"""Property: vector elements match (ic, icir, rank_ic, rank_icir, ann_return, ir, -mdd, sharpe)."""
|
||||
m = Metrics(
|
||||
ic=ic, icir=icir, rank_ic=rank_ic, rank_icir=rank_icir,
|
||||
arr=ann_return, ir=ir, mdd=mdd, sharpe=sharpe,
|
||||
)
|
||||
v = m.as_vector()
|
||||
assert v[0] == ic
|
||||
assert v[1] == icir
|
||||
assert v[2] == rank_ic
|
||||
assert v[3] == rank_icir
|
||||
assert v[4] == ann_return
|
||||
assert v[5] == ir
|
||||
assert v[6] == -mdd # negated
|
||||
assert v[7] == sharpe
|
||||
|
||||
@given(
|
||||
mdd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_mdd_negated_in_vector(self, mdd):
|
||||
"""Property: mdd is negated in as_vector output (v[6] = -mdd)."""
|
||||
m = Metrics(mdd=mdd)
|
||||
v = m.as_vector()
|
||||
assert v[6] == -mdd
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
rank_ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
rank_icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
ir=st.floats(min_value=-5.0, max_value=10.0),
|
||||
mdd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
sharpe=st.floats(min_value=-5.0, max_value=10.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_as_vector_returns_numpy_array(self, ic, icir, rank_ic, rank_icir, ann_return, ir, mdd, sharpe):
|
||||
"""Property: as_vector returns np.ndarray."""
|
||||
m = Metrics(
|
||||
ic=ic, icir=icir, rank_ic=rank_ic, rank_icir=rank_icir,
|
||||
arr=ann_return, ir=ir, mdd=mdd, sharpe=sharpe,
|
||||
)
|
||||
v = m.as_vector()
|
||||
assert isinstance(v, np.ndarray)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 4: extract_metrics_from_experiment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractMetrics:
|
||||
"""Property: extract_metrics_from_experiment invariants."""
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
rank_ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
rank_icir=st.floats(min_value=-2.0, max_value=10.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
ir=st.floats(min_value=-5.0, max_value=10.0),
|
||||
mdd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_extract_metrics_correct_values(self, ic, icir, rank_ic, rank_icir, ann_return, ir, mdd):
|
||||
"""Property: extract_metrics_from_experiment reads correct values from result dict."""
|
||||
mock_exp = MagicMock()
|
||||
mock_exp.result = {
|
||||
"IC": ic, "ICIR": icir,
|
||||
"Rank IC": rank_ic, "Rank ICIR": rank_icir,
|
||||
"1day.excess_return_with_cost.annualized_return ": ann_return,
|
||||
"1day.excess_return_with_cost.information_ratio": ir,
|
||||
"1day.excess_return_with_cost.max_drawdown": mdd,
|
||||
}
|
||||
m = extract_metrics_from_experiment(mock_exp)
|
||||
assert m.ic == ic
|
||||
assert m.rank_ic == rank_ic
|
||||
assert m.icir == icir
|
||||
assert m.rank_icir == rank_icir
|
||||
assert m.mdd == mdd
|
||||
|
||||
@given(
|
||||
ann_return=st.floats(min_value=0.01, max_value=2.0),
|
||||
mdd=st.floats(min_value=-0.01, max_value=-0.001),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_sharpe_computed_from_ann_return_and_mdd(self, ann_return, mdd):
|
||||
"""Property: sharpe ≈ ann_return / |mdd| for standard inputs."""
|
||||
mock_exp = MagicMock()
|
||||
mock_exp.result = {
|
||||
"IC": 0.0, "ICIR": 0.0,
|
||||
"Rank IC": 0.0, "Rank ICIR": 0.0,
|
||||
"1day.excess_return_with_cost.annualized_return ": ann_return,
|
||||
"1day.excess_return_with_cost.information_ratio": 0.0,
|
||||
"1day.excess_return_with_cost.max_drawdown": mdd,
|
||||
}
|
||||
m = extract_metrics_from_experiment(mock_exp)
|
||||
expected_sharpe = ann_return / abs(mdd)
|
||||
assert m.sharpe == pytest.approx(expected_sharpe, rel=0.01)
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_extract_returns_default_on_none_result(self, seed):
|
||||
"""Property: returns default Metrics (all zeros) when result is None."""
|
||||
mock_exp = MagicMock()
|
||||
mock_exp.result = None
|
||||
m = extract_metrics_from_experiment(mock_exp)
|
||||
assert m.ic == 0.0
|
||||
assert m.sharpe == 0.0
|
||||
assert m.mdd == 0.0
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_extract_returns_default_on_empty_result(self, seed):
|
||||
"""Property: returns default Metrics when result dict is empty."""
|
||||
mock_exp = MagicMock()
|
||||
mock_exp.result = {}
|
||||
m = extract_metrics_from_experiment(mock_exp)
|
||||
assert m.ic == 0.0
|
||||
assert m.sharpe == 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 5: LinearThompsonTwoArm
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLinearThompsonTwoArm:
|
||||
"""Property: LinearThompsonTwoArm bandit invariants."""
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=20))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_dim_stored_correctly(self, dim):
|
||||
"""Property: dim attribute matches constructor arg."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
assert bandit.dim == dim
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_mean_shape_matches_dim(self, dim):
|
||||
"""Property: mean vectors have shape (dim,)."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
assert bandit.mean["factor"].shape == (dim,)
|
||||
assert bandit.mean["model"].shape == (dim,)
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_precision_shape_matches_dim(self, dim):
|
||||
"""Property: precision matrices have shape (dim, dim)."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
assert bandit.precision["factor"].shape == (dim, dim)
|
||||
assert bandit.precision["model"].shape == (dim, dim)
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_arms_initialized_identically(self, dim):
|
||||
"""Property: factor and model arms are initialized identically."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
assert np.array_equal(bandit.mean["factor"], bandit.mean["model"])
|
||||
assert np.array_equal(bandit.precision["factor"], bandit.precision["model"])
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_noise_var_is_default_1(self, dim):
|
||||
"""Property: noise_var defaults to 1.0."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
assert bandit.noise_var == 1.0
|
||||
|
||||
@given(
|
||||
dim=st.integers(min_value=1, max_value=10),
|
||||
noise_var=st.floats(min_value=0.01, max_value=10.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_noise_var_configurable(self, dim, noise_var):
|
||||
"""Property: noise_var can be set via constructor."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim, noise_var=noise_var)
|
||||
assert bandit.noise_var == noise_var
|
||||
|
||||
@given(
|
||||
dim=st.integers(min_value=1, max_value=10),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_sample_reward_returns_float(self, dim):
|
||||
"""Property: sample_reward returns a float."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
x = np.ones(dim)
|
||||
reward = bandit.sample_reward("factor", x)
|
||||
assert isinstance(reward, float)
|
||||
|
||||
@given(
|
||||
dim=st.integers(min_value=1, max_value=10),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_sample_reward_finite(self, dim):
|
||||
"""Property: sample_reward returns finite values."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
x = np.ones(dim)
|
||||
reward = bandit.sample_reward("factor", x)
|
||||
assert np.isfinite(reward)
|
||||
|
||||
@given(
|
||||
dim=st.integers(min_value=1, max_value=10),
|
||||
seed_a=st.integers(min_value=0, max_value=50),
|
||||
seed_b=st.integers(min_value=51, max_value=100),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_sample_reward_varies(self, dim, seed_a, seed_b):
|
||||
"""Property: different seeds may produce different rewards (stochasticity)."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
x = np.ones(dim)
|
||||
r1 = bandit.sample_reward("factor", x)
|
||||
r2 = bandit.sample_reward("factor", x)
|
||||
# Both should be finite (may be equal by chance)
|
||||
assert np.isfinite(r1)
|
||||
assert np.isfinite(r2)
|
||||
|
||||
@given(dim=st.integers(min_value=2, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_precision_is_symmetric(self, dim):
|
||||
"""Property: precision matrix is symmetric."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
P = bandit.precision["factor"]
|
||||
assert np.allclose(P, P.T, atol=1e-10)
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_both_arms_have_same_keys(self, dim):
|
||||
"""Property: both 'factor' and 'model' arms exist in mean/precision dicts."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
assert "factor" in bandit.mean
|
||||
assert "model" in bandit.mean
|
||||
assert "factor" in bandit.precision
|
||||
assert "model" in bandit.precision
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 6: LinearThompsonTwoArm Update
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBanditUpdate:
|
||||
"""Property: Thompson bandit update invariants."""
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_update_exists_for_both_arms(self, dim):
|
||||
"""Property: update method is callable for both arms."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
x = np.ones(dim)
|
||||
bandit.update("factor", x, 0.5)
|
||||
bandit.update("model", x, 0.3)
|
||||
# Should not raise
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_update_changes_mean(self, dim):
|
||||
"""Property: updating an arm changes its mean vector."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
orig = bandit.mean["factor"].copy()
|
||||
x = np.ones(dim)
|
||||
bandit.update("factor", x, 1.0)
|
||||
# Mean should change (or be computed differently after update)
|
||||
assert not np.array_equal(orig, bandit.mean["factor"]) or np.array_equal(orig, np.zeros(dim))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 7: has_local_components / count_valid_factors / ADVANCED_LOOP
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestQuantLoopFactory:
|
||||
"""Property: quant_loop_factory function invariants."""
|
||||
|
||||
def test_has_local_components_returns_bool(self):
|
||||
"""Property: has_local_components returns bool."""
|
||||
result = has_local_components()
|
||||
assert isinstance(result, bool)
|
||||
|
||||
def test_count_valid_factors_returns_nonnegative_int(self):
|
||||
"""Property: count_valid_factors returns nonnegative int."""
|
||||
result = count_valid_factors()
|
||||
assert isinstance(result, int)
|
||||
assert result >= 0
|
||||
|
||||
def test_advanced_loop_threshold_is_5000(self):
|
||||
"""Property: ADVANCED_LOOP_FACTOR_THRESHOLD == 5000."""
|
||||
assert ADVANCED_LOOP_FACTOR_THRESHOLD == 5000
|
||||
|
||||
def test_advanced_loop_threshold_is_positive(self):
|
||||
"""Property: ADVANCED_LOOP_FACTOR_THRESHOLD > 0."""
|
||||
assert ADVANCED_LOOP_FACTOR_THRESHOLD > 0
|
||||
|
||||
def test_has_local_components_deterministic(self):
|
||||
"""Property: has_local_components returns same value on repeated calls."""
|
||||
r1 = has_local_components()
|
||||
r2 = has_local_components()
|
||||
assert r1 == r2
|
||||
|
||||
def test_count_valid_factors_deterministic(self):
|
||||
"""Property: count_valid_factors returns same value on repeated calls."""
|
||||
r1 = count_valid_factors()
|
||||
r2 = count_valid_factors()
|
||||
assert r1 == r2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 8: process_results Numeric Edge Cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProcessResultsEdgeCases:
|
||||
"""Property: process_results handles edge case values."""
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0, allow_nan=False, allow_infinity=False),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0, allow_nan=False, allow_infinity=False),
|
||||
max_dd=st.floats(min_value=-1.0, max_value=0.0, allow_nan=False, allow_infinity=False),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_all_numeric_values_formatted(self, ic, ann_return, max_dd):
|
||||
"""Property: all valid numeric values produce a result string."""
|
||||
current = pd.Series({
|
||||
"IC": ic,
|
||||
"1day.excess_return_with_cost.annualized_return": ann_return,
|
||||
"1day.excess_return_with_cost.max_drawdown": max_dd,
|
||||
}, name="0")
|
||||
sota = pd.Series({
|
||||
"IC": 0.0,
|
||||
"1day.excess_return_with_cost.annualized_return": 0.0,
|
||||
"1day.excess_return_with_cost.max_drawdown": 0.0,
|
||||
}, name="0")
|
||||
|
||||
result = process_results(current, sota)
|
||||
assert isinstance(result, str)
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0, allow_nan=False, allow_infinity=False),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0, allow_nan=False, allow_infinity=False),
|
||||
max_dd=st.floats(min_value=-1.0, max_value=0.0, allow_nan=False, allow_infinity=False),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_result_contains_both_current_and_sota(self, ic, ann_return, max_dd):
|
||||
"""Property: result contains 'Current Result' and 'SOTA Result'."""
|
||||
current = pd.Series({
|
||||
"IC": ic,
|
||||
"1day.excess_return_with_cost.annualized_return": ann_return,
|
||||
"1day.excess_return_with_cost.max_drawdown": max_dd,
|
||||
}, name="0")
|
||||
sota = pd.Series({
|
||||
"IC": 0.0,
|
||||
"1day.excess_return_with_cost.annualized_return": 0.0,
|
||||
"1day.excess_return_with_cost.max_drawdown": 0.0,
|
||||
}, name="0")
|
||||
|
||||
result = process_results(current, sota)
|
||||
assert "Current Result" in result
|
||||
assert "SOTA Result" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 9: Metrics Constructor Type Safety
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMetricsTypeSafety:
|
||||
"""Property: Metrics converts inputs to float."""
|
||||
|
||||
@given(
|
||||
ic=st.integers(min_value=-10, max_value=10),
|
||||
sharpe=st.integers(min_value=-5, max_value=20),
|
||||
mdd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_float_conversion(self, ic, sharpe, mdd):
|
||||
"""Property: integer inputs become floats."""
|
||||
m = Metrics(ic=float(ic), sharpe=float(sharpe), mdd=mdd)
|
||||
assert isinstance(m.ic, float)
|
||||
assert isinstance(m.sharpe, float)
|
||||
assert isinstance(m.mdd, float)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 10: Bandit Precision Positive Definite
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBanditPrecisionProperties:
|
||||
"""Property: precision matrix is positive semi-definite (identity-initialized)."""
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_precision_is_identity_initialized(self, dim):
|
||||
"""Property: precision matrix starts as identity."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
P = bandit.precision["factor"]
|
||||
expected = np.eye(dim)
|
||||
assert np.allclose(P, expected, atol=1e-10)
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_precision_diagonal_positive(self, dim):
|
||||
"""Property: precision matrix diagonal elements are positive."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
P = bandit.precision["factor"]
|
||||
assert (np.diag(P) > 0).all()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 11: Bandit Mean Initialization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBanditMeanInitialization:
|
||||
"""Property: mean vector is initialized to zeros."""
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_mean_is_zero_initialized(self, dim):
|
||||
"""Property: mean starts as zero vector."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
m = bandit.mean["factor"]
|
||||
expected = np.zeros(dim)
|
||||
assert np.allclose(m, expected, atol=1e-10)
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_both_arms_mean_zero_initialized(self, dim):
|
||||
"""Property: both arm means start as zero."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
assert np.allclose(bandit.mean["factor"], np.zeros(dim))
|
||||
assert np.allclose(bandit.mean["model"], np.zeros(dim))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 12: extract_metrics Robustness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractMetricsRobustness:
|
||||
"""Property: extract_metrics_from_experiment handles missing keys."""
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_partial_result_dict(self, ic):
|
||||
"""Property: partial result dict fills defaults for missing keys."""
|
||||
mock_exp = MagicMock()
|
||||
mock_exp.result = {"IC": ic}
|
||||
m = extract_metrics_from_experiment(mock_exp)
|
||||
assert m.ic == ic
|
||||
assert m.sharpe == 0.0 # default since ann_return is missing
|
||||
|
||||
@given(seed=st.integers(min_value=0, max_value=100))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_extract_with_empty_dict(self, seed):
|
||||
"""Property: empty result dict → all defaults or raises."""
|
||||
mock_exp = MagicMock()
|
||||
mock_exp.result = {}
|
||||
m = extract_metrics_from_experiment(mock_exp)
|
||||
assert isinstance(m, Metrics)
|
||||
assert m.ic == 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 13: Metrics Field Naming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMetricsFieldNaming:
|
||||
"""Property: Metrics has specific named fields."""
|
||||
|
||||
def test_metrics_has_all_expected_fields(self):
|
||||
"""Property: Metrics has ic, icir, rank_ic, rank_icir, ann_return, ir, mdd, sharpe."""
|
||||
m = Metrics()
|
||||
expected = {"ic", "icir", "rank_ic", "rank_icir", "arr", "ir", "mdd", "sharpe"}
|
||||
actual = {k for k in m.__dict__ if not k.startswith("_")}
|
||||
assert expected <= actual or expected <= set(m.__dataclass_fields__ if hasattr(m, "__dataclass_fields__") else [])
|
||||
|
||||
@given(
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
ir=st.floats(min_value=-5.0, max_value=10.0),
|
||||
sharpe=st.floats(min_value=-5.0, max_value=10.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_return_and_sharpe_fields(self, ann_return, ir, sharpe):
|
||||
"""Property: ann_return, ir, sharpe accessible by attribute."""
|
||||
m = Metrics(arr=ann_return, ir=ir, sharpe=sharpe)
|
||||
assert m.arr == ann_return
|
||||
assert m.ir == ir
|
||||
assert m.sharpe == sharpe
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 14: process_results Determinism
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProcessResultsDeterminism:
|
||||
"""Property: process_results is deterministic."""
|
||||
|
||||
@given(
|
||||
ic=st.floats(min_value=-1.0, max_value=1.0),
|
||||
ann_return=st.floats(min_value=-2.0, max_value=5.0),
|
||||
max_dd=st.floats(min_value=-1.0, max_value=0.0),
|
||||
)
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_same_inputs_same_output(self, ic, ann_return, max_dd):
|
||||
"""Property: process_results is deterministic."""
|
||||
current = pd.Series({
|
||||
"IC": ic,
|
||||
"1day.excess_return_with_cost.annualized_return": ann_return,
|
||||
"1day.excess_return_with_cost.max_drawdown": max_dd,
|
||||
}, name="0")
|
||||
sota = pd.Series({
|
||||
"IC": 0.0,
|
||||
"1day.excess_return_with_cost.annualized_return": 0.0,
|
||||
"1day.excess_return_with_cost.max_drawdown": 0.0,
|
||||
}, name="0")
|
||||
|
||||
r1 = process_results(current, sota)
|
||||
r2 = process_results(current, sota)
|
||||
assert r1 == r2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 15: Bandit Sample Reward Distribution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBanditSampleReward:
|
||||
"""Property: sample_reward behavior across arms."""
|
||||
|
||||
@given(dim=st.integers(min_value=1, max_value=10))
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_factor_and_model_reward_differ(self, dim):
|
||||
"""Property: factor and model arms can give different rewards."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
x = np.random.randn(dim)
|
||||
r_factor = bandit.sample_reward("factor", x)
|
||||
r_model = bandit.sample_reward("model", x)
|
||||
assert isinstance(r_factor, float)
|
||||
assert isinstance(r_model, float)
|
||||
|
||||
@given(
|
||||
dim=st.integers(min_value=1, max_value=10),
|
||||
n_samples=st.integers(min_value=10, max_value=100),
|
||||
)
|
||||
@settings(max_examples=10, deadline=10000)
|
||||
def test_sample_reward_changes_after_update(self, dim, n_samples):
|
||||
"""Property: after updates, sample_reward distribution shifts."""
|
||||
bandit = LinearThompsonTwoArm(dim=dim)
|
||||
x = np.ones(dim)
|
||||
rewards_before = [bandit.sample_reward("factor", x) for _ in range(n_samples)]
|
||||
|
||||
# Update with positive rewards
|
||||
for _ in range(10):
|
||||
bandit.update("factor", x, 1.0)
|
||||
|
||||
rewards_after = [bandit.sample_reward("factor", x) for _ in range(n_samples)]
|
||||
|
||||
# Mean should shift (though statistically it may not)
|
||||
assert np.all(np.isfinite(rewards_before))
|
||||
assert np.all(np.isfinite(rewards_after))
|
||||
|
||||
Reference in New Issue
Block a user