Patrick SelamyandClaude Opus 4.5
1f4f1fa557
- Use contextlib.suppress instead of try/except/pass (SIM105) - Prefix unused fixture arguments with underscore (ARG002) - Replace asyncio.TimeoutError with TimeoutError (UP041) - Apply ruff formatting to all files 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
614 lines
20 KiB
Python
614 lines
20 KiB
Python
"""Tests for the connection health monitor."""
|
|
|
|
import asyncio
|
|
import time
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
from aiohttp import web
|
|
|
|
from polymarket_insider_tracker.ingestor.health import (
|
|
DEFAULT_HEALTH_CHECK_INTERVAL,
|
|
DEFAULT_STALE_THRESHOLD_SECONDS,
|
|
HealthMonitor,
|
|
HealthReport,
|
|
HealthStatus,
|
|
StreamHealth,
|
|
StreamStatus,
|
|
)
|
|
|
|
|
|
class TestStreamHealth:
|
|
"""Tests for the StreamHealth dataclass."""
|
|
|
|
def test_stream_health_defaults(self) -> None:
|
|
"""Test default values."""
|
|
health = StreamHealth(name="test-stream")
|
|
|
|
assert health.name == "test-stream"
|
|
assert health.status == StreamStatus.DISCONNECTED
|
|
assert health.last_event_time is None
|
|
assert health.events_received == 0
|
|
assert health.events_per_second == 0.0
|
|
assert health.connected_since is None
|
|
assert health.last_error is None
|
|
|
|
def test_stream_health_custom_values(self) -> None:
|
|
"""Test with custom values."""
|
|
now = time.time()
|
|
health = StreamHealth(
|
|
name="trades",
|
|
status=StreamStatus.ACTIVE,
|
|
last_event_time=now,
|
|
events_received=100,
|
|
events_per_second=5.0,
|
|
connected_since=now - 3600,
|
|
last_error=None,
|
|
)
|
|
|
|
assert health.name == "trades"
|
|
assert health.status == StreamStatus.ACTIVE
|
|
assert health.events_received == 100
|
|
|
|
|
|
class TestHealthReport:
|
|
"""Tests for the HealthReport dataclass."""
|
|
|
|
def test_health_report_defaults(self) -> None:
|
|
"""Test default values."""
|
|
report = HealthReport(status=HealthStatus.HEALTHY)
|
|
|
|
assert report.status == HealthStatus.HEALTHY
|
|
assert report.streams == {}
|
|
assert report.total_events_received == 0
|
|
assert report.total_events_per_second == 0.0
|
|
assert report.uptime_seconds == 0.0
|
|
assert report.timestamp > 0
|
|
|
|
def test_health_report_with_streams(self) -> None:
|
|
"""Test with stream data."""
|
|
stream = StreamHealth(name="trades", events_received=100)
|
|
report = HealthReport(
|
|
status=HealthStatus.DEGRADED,
|
|
streams={"trades": stream},
|
|
total_events_received=100,
|
|
total_events_per_second=5.0,
|
|
uptime_seconds=3600.0,
|
|
)
|
|
|
|
assert report.status == HealthStatus.DEGRADED
|
|
assert "trades" in report.streams
|
|
assert report.total_events_received == 100
|
|
|
|
|
|
class TestHealthMonitor:
|
|
"""Tests for the HealthMonitor class."""
|
|
|
|
def test_init(self) -> None:
|
|
"""Test initialization."""
|
|
monitor = HealthMonitor()
|
|
|
|
assert monitor._stale_threshold == DEFAULT_STALE_THRESHOLD_SECONDS
|
|
assert monitor._health_check_interval == DEFAULT_HEALTH_CHECK_INTERVAL
|
|
assert not monitor.is_running
|
|
|
|
def test_init_custom_config(self) -> None:
|
|
"""Test initialization with custom config."""
|
|
monitor = HealthMonitor(
|
|
stale_threshold_seconds=30,
|
|
health_check_interval=10,
|
|
)
|
|
|
|
assert monitor._stale_threshold == 30
|
|
assert monitor._health_check_interval == 10
|
|
|
|
def test_register_stream(self) -> None:
|
|
"""Test registering a stream."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.register_stream("trades")
|
|
|
|
assert "trades" in monitor._streams
|
|
assert monitor._streams["trades"].name == "trades"
|
|
assert monitor._streams["trades"].status == StreamStatus.DISCONNECTED
|
|
|
|
def test_register_stream_idempotent(self) -> None:
|
|
"""Test that registering the same stream twice is idempotent."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.register_stream("trades")
|
|
monitor.record_event("trades") # Adds an event
|
|
monitor.register_stream("trades") # Should not reset
|
|
|
|
assert monitor._streams["trades"].events_received == 1
|
|
|
|
def test_set_stream_connected(self) -> None:
|
|
"""Test marking a stream as connected."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.set_stream_connected("trades")
|
|
|
|
assert monitor._streams["trades"].status == StreamStatus.ACTIVE
|
|
assert monitor._streams["trades"].connected_since is not None
|
|
assert monitor._streams["trades"].last_error is None
|
|
|
|
def test_set_stream_disconnected(self) -> None:
|
|
"""Test marking a stream as disconnected."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.set_stream_connected("trades")
|
|
monitor.set_stream_disconnected("trades", error="Connection reset")
|
|
|
|
assert monitor._streams["trades"].status == StreamStatus.DISCONNECTED
|
|
assert monitor._streams["trades"].connected_since is None
|
|
assert monitor._streams["trades"].last_error == "Connection reset"
|
|
|
|
def test_record_event(self) -> None:
|
|
"""Test recording an event."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.record_event("trades")
|
|
|
|
stream = monitor._streams["trades"]
|
|
assert stream.events_received == 1
|
|
assert stream.last_event_time is not None
|
|
assert stream.status == StreamStatus.ACTIVE
|
|
|
|
def test_record_event_multiple(self) -> None:
|
|
"""Test recording multiple events."""
|
|
monitor = HealthMonitor()
|
|
|
|
for _ in range(10):
|
|
monitor.record_event("trades")
|
|
|
|
assert monitor._streams["trades"].events_received == 10
|
|
|
|
def test_record_event_with_processing_time(self) -> None:
|
|
"""Test recording event with processing time."""
|
|
monitor = HealthMonitor()
|
|
|
|
# Should not raise
|
|
monitor.record_event("trades", processing_time=0.001)
|
|
|
|
assert monitor._streams["trades"].events_received == 1
|
|
|
|
def test_calculate_throughput_empty(self) -> None:
|
|
"""Test throughput calculation with no events."""
|
|
monitor = HealthMonitor()
|
|
|
|
rate = monitor._calculate_throughput("nonexistent")
|
|
|
|
assert rate == 0.0
|
|
|
|
def test_calculate_throughput(self) -> None:
|
|
"""Test throughput calculation."""
|
|
monitor = HealthMonitor()
|
|
|
|
# Add events
|
|
for _ in range(10):
|
|
monitor.record_event("trades")
|
|
|
|
rate = monitor._calculate_throughput("trades")
|
|
|
|
# Should have ~10 events in the window
|
|
assert rate > 0
|
|
|
|
def test_check_stream_staleness_active(self) -> None:
|
|
"""Test that active stream is not marked stale."""
|
|
monitor = HealthMonitor(stale_threshold_seconds=60)
|
|
|
|
monitor.record_event("trades")
|
|
monitor._check_stream_staleness()
|
|
|
|
assert monitor._streams["trades"].status == StreamStatus.ACTIVE
|
|
|
|
def test_check_stream_staleness_stale(self) -> None:
|
|
"""Test that stream becomes stale after threshold."""
|
|
monitor = HealthMonitor(stale_threshold_seconds=1)
|
|
|
|
monitor.record_event("trades")
|
|
# Simulate time passing
|
|
monitor._streams["trades"].last_event_time = time.time() - 2
|
|
|
|
monitor._check_stream_staleness()
|
|
|
|
assert monitor._streams["trades"].status == StreamStatus.STALE
|
|
|
|
def test_check_stream_staleness_connected_no_events(self) -> None:
|
|
"""Test staleness when connected but no events received."""
|
|
monitor = HealthMonitor(stale_threshold_seconds=1)
|
|
|
|
monitor.set_stream_connected("trades")
|
|
# Simulate time passing since connection
|
|
monitor._streams["trades"].connected_since = time.time() - 2
|
|
|
|
monitor._check_stream_staleness()
|
|
|
|
assert monitor._streams["trades"].status == StreamStatus.STALE
|
|
|
|
def test_determine_overall_status_no_streams(self) -> None:
|
|
"""Test overall status with no streams."""
|
|
monitor = HealthMonitor()
|
|
|
|
status = monitor._determine_overall_status()
|
|
|
|
assert status == HealthStatus.HEALTHY
|
|
|
|
def test_determine_overall_status_all_active(self) -> None:
|
|
"""Test overall status with all active streams."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.record_event("trades")
|
|
monitor.record_event("orderbook")
|
|
|
|
status = monitor._determine_overall_status()
|
|
|
|
assert status == HealthStatus.HEALTHY
|
|
|
|
def test_determine_overall_status_some_stale(self) -> None:
|
|
"""Test overall status with some stale streams."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.record_event("trades")
|
|
monitor.register_stream("orderbook")
|
|
monitor._streams["orderbook"].status = StreamStatus.STALE
|
|
|
|
status = monitor._determine_overall_status()
|
|
|
|
assert status == HealthStatus.DEGRADED
|
|
|
|
def test_determine_overall_status_some_disconnected(self) -> None:
|
|
"""Test overall status with some disconnected streams."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.record_event("trades")
|
|
monitor.set_stream_disconnected("orderbook")
|
|
|
|
status = monitor._determine_overall_status()
|
|
|
|
assert status == HealthStatus.DEGRADED
|
|
|
|
def test_determine_overall_status_all_disconnected(self) -> None:
|
|
"""Test overall status with all disconnected streams."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.set_stream_disconnected("trades")
|
|
monitor.set_stream_disconnected("orderbook")
|
|
|
|
status = monitor._determine_overall_status()
|
|
|
|
assert status == HealthStatus.UNHEALTHY
|
|
|
|
def test_get_health_report(self) -> None:
|
|
"""Test getting a health report."""
|
|
monitor = HealthMonitor()
|
|
monitor._start_time = time.time() - 100
|
|
|
|
monitor.record_event("trades")
|
|
monitor.record_event("trades")
|
|
|
|
report = monitor.get_health_report()
|
|
|
|
assert report.status == HealthStatus.HEALTHY
|
|
assert "trades" in report.streams
|
|
assert report.total_events_received == 2
|
|
assert report.uptime_seconds >= 100
|
|
assert report.timestamp > 0
|
|
|
|
def test_get_health_report_calculates_throughput(self) -> None:
|
|
"""Test that health report calculates throughput."""
|
|
monitor = HealthMonitor()
|
|
|
|
for _ in range(10):
|
|
monitor.record_event("trades")
|
|
|
|
report = monitor.get_health_report()
|
|
|
|
assert report.streams["trades"].events_per_second > 0
|
|
assert report.total_events_per_second > 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_stop(self) -> None:
|
|
"""Test starting and stopping the monitor."""
|
|
monitor = HealthMonitor()
|
|
|
|
await monitor.start()
|
|
assert monitor.is_running
|
|
assert monitor._health_task is not None
|
|
|
|
await monitor.stop()
|
|
assert not monitor.is_running
|
|
assert monitor._health_task is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_idempotent(self) -> None:
|
|
"""Test that starting twice is safe."""
|
|
monitor = HealthMonitor()
|
|
|
|
await monitor.start()
|
|
await monitor.start() # Should not raise
|
|
|
|
assert monitor.is_running
|
|
|
|
await monitor.stop()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stop_when_not_running(self) -> None:
|
|
"""Test that stopping when not running is safe."""
|
|
monitor = HealthMonitor()
|
|
|
|
await monitor.stop() # Should not raise
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_context_manager(self) -> None:
|
|
"""Test async context manager."""
|
|
async with HealthMonitor() as monitor:
|
|
assert monitor.is_running
|
|
|
|
assert not monitor.is_running
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_check_loop_updates_report(self) -> None:
|
|
"""Test that health check loop updates the report."""
|
|
monitor = HealthMonitor(health_check_interval=0.1)
|
|
|
|
await monitor.start()
|
|
monitor.record_event("trades")
|
|
|
|
await asyncio.sleep(0.2)
|
|
|
|
# Health should have been checked
|
|
report = monitor.get_health_report()
|
|
assert report.status == HealthStatus.HEALTHY
|
|
|
|
await monitor.stop()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_change_callback(self) -> None:
|
|
"""Test that health change callback is invoked."""
|
|
callback = AsyncMock()
|
|
monitor = HealthMonitor(
|
|
health_check_interval=0.1,
|
|
on_health_change=callback,
|
|
)
|
|
|
|
await monitor.start()
|
|
monitor.record_event("trades")
|
|
|
|
# Wait for health check
|
|
await asyncio.sleep(0.2)
|
|
|
|
await monitor.stop()
|
|
|
|
# Callback should have been called at least once
|
|
assert callback.called
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_change_callback_error_handling(self) -> None:
|
|
"""Test that callback errors don't crash the loop."""
|
|
callback = AsyncMock(side_effect=ValueError("test error"))
|
|
monitor = HealthMonitor(
|
|
health_check_interval=0.1,
|
|
on_health_change=callback,
|
|
)
|
|
|
|
await monitor.start()
|
|
monitor.record_event("trades")
|
|
|
|
# Should not crash
|
|
await asyncio.sleep(0.2)
|
|
|
|
await monitor.stop()
|
|
|
|
|
|
class TestHealthMonitorHTTPEndpoints:
|
|
"""Tests for HTTP endpoints."""
|
|
|
|
@pytest.fixture
|
|
def monitor(self) -> HealthMonitor:
|
|
"""Create a monitor instance."""
|
|
return HealthMonitor()
|
|
|
|
@pytest.fixture
|
|
def app(self, monitor: HealthMonitor) -> web.Application:
|
|
"""Create the aiohttp application."""
|
|
return monitor._create_app()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_endpoint_healthy(
|
|
self, monitor: HealthMonitor, app: web.Application
|
|
) -> None:
|
|
"""Test /health endpoint when healthy."""
|
|
from aiohttp.test_utils import TestClient, TestServer
|
|
|
|
monitor.record_event("trades")
|
|
|
|
async with TestClient(TestServer(app)) as client:
|
|
resp = await client.get("/health")
|
|
assert resp.status == 200
|
|
|
|
data = await resp.json()
|
|
assert data["status"] == "healthy"
|
|
assert "trades" in data["streams"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_endpoint_unhealthy(
|
|
self, monitor: HealthMonitor, app: web.Application
|
|
) -> None:
|
|
"""Test /health endpoint when unhealthy."""
|
|
from aiohttp.test_utils import TestClient, TestServer
|
|
|
|
monitor.set_stream_disconnected("trades")
|
|
|
|
async with TestClient(TestServer(app)) as client:
|
|
resp = await client.get("/health")
|
|
assert resp.status == 503
|
|
|
|
data = await resp.json()
|
|
assert data["status"] == "unhealthy"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_metrics_endpoint(self, monitor: HealthMonitor, app: web.Application) -> None:
|
|
"""Test /metrics endpoint returns Prometheus format."""
|
|
from aiohttp.test_utils import TestClient, TestServer
|
|
|
|
monitor.record_event("trades")
|
|
|
|
async with TestClient(TestServer(app)) as client:
|
|
resp = await client.get("/metrics")
|
|
assert resp.status == 200
|
|
|
|
content_type = resp.headers.get("Content-Type", "")
|
|
assert "text/plain" in content_type
|
|
|
|
text = await resp.text()
|
|
assert "polymarket_events_total" in text
|
|
assert "polymarket_health_status" in text
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ready_endpoint_ready(self, monitor: HealthMonitor, app: web.Application) -> None:
|
|
"""Test /ready endpoint when ready."""
|
|
from aiohttp.test_utils import TestClient, TestServer
|
|
|
|
monitor.record_event("trades")
|
|
|
|
async with TestClient(TestServer(app)) as client:
|
|
resp = await client.get("/ready")
|
|
assert resp.status == 200
|
|
|
|
data = await resp.json()
|
|
assert data["ready"] is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ready_endpoint_not_ready(
|
|
self, monitor: HealthMonitor, app: web.Application
|
|
) -> None:
|
|
"""Test /ready endpoint when not ready."""
|
|
from aiohttp.test_utils import TestClient, TestServer
|
|
|
|
monitor.set_stream_disconnected("trades")
|
|
|
|
async with TestClient(TestServer(app)) as client:
|
|
resp = await client.get("/ready")
|
|
assert resp.status == 503
|
|
|
|
data = await resp.json()
|
|
assert data["ready"] is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_live_endpoint(self, app: web.Application) -> None:
|
|
"""Test /live endpoint always returns 200."""
|
|
from aiohttp.test_utils import TestClient, TestServer
|
|
|
|
async with TestClient(TestServer(app)) as client:
|
|
resp = await client.get("/live")
|
|
assert resp.status == 200
|
|
|
|
data = await resp.json()
|
|
assert data["live"] is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_stop_http_server(self, monitor: HealthMonitor) -> None:
|
|
"""Test starting and stopping HTTP server."""
|
|
await monitor.start_http_server(port=18080)
|
|
assert monitor._runner is not None
|
|
|
|
await monitor.stop_http_server()
|
|
assert monitor._runner is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_http_server_idempotent(self, monitor: HealthMonitor) -> None:
|
|
"""Test that starting HTTP server twice is safe."""
|
|
await monitor.start_http_server(port=18081)
|
|
await monitor.start_http_server(port=18081) # Should not raise
|
|
|
|
await monitor.stop_http_server()
|
|
|
|
|
|
class TestPrometheusMetrics:
|
|
"""Tests for Prometheus metric updates."""
|
|
|
|
def test_events_total_incremented(self) -> None:
|
|
"""Test that events_total counter is incremented."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.record_event("test-metrics")
|
|
monitor.record_event("test-metrics")
|
|
|
|
# Counter should have been incremented
|
|
# (We can't easily test prometheus metrics directly, but at least verify no errors)
|
|
|
|
def test_stream_status_updated(self) -> None:
|
|
"""Test that stream_status gauge is updated."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.set_stream_connected("test-status")
|
|
# Gauge should be 1.0
|
|
|
|
monitor.set_stream_disconnected("test-status")
|
|
# Gauge should be 0.0
|
|
|
|
def test_health_status_updated(self) -> None:
|
|
"""Test that health_status gauge is updated."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.record_event("test-health")
|
|
report = monitor.get_health_report()
|
|
|
|
assert report.status == HealthStatus.HEALTHY
|
|
|
|
|
|
class TestEdgeCases:
|
|
"""Tests for edge cases and error handling."""
|
|
|
|
def test_throughput_with_old_events(self) -> None:
|
|
"""Test throughput calculation ignores old events."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.record_event("trades")
|
|
# Manually add old event to window
|
|
monitor._event_windows["trades"].append(time.time() - 100)
|
|
|
|
rate = monitor._calculate_throughput("trades")
|
|
|
|
# Old event should be filtered out
|
|
# Rate should only count recent events
|
|
assert rate >= 0
|
|
|
|
def test_multiple_streams_independent(self) -> None:
|
|
"""Test that multiple streams are tracked independently."""
|
|
monitor = HealthMonitor()
|
|
|
|
monitor.record_event("trades")
|
|
monitor.record_event("trades")
|
|
monitor.set_stream_disconnected("orderbook")
|
|
|
|
assert monitor._streams["trades"].events_received == 2
|
|
assert monitor._streams["trades"].status == StreamStatus.ACTIVE
|
|
assert monitor._streams["orderbook"].events_received == 0
|
|
assert monitor._streams["orderbook"].status == StreamStatus.DISCONNECTED
|
|
|
|
def test_report_streams_are_copied(self) -> None:
|
|
"""Test that report streams are a copy."""
|
|
monitor = HealthMonitor()
|
|
monitor.record_event("trades")
|
|
|
|
report = monitor.get_health_report()
|
|
|
|
# Modifying report should not affect monitor
|
|
report.streams["trades"].events_received = 999
|
|
assert monitor._streams["trades"].events_received == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stop_cleans_up_http_server(self) -> None:
|
|
"""Test that stop() also stops HTTP server."""
|
|
monitor = HealthMonitor()
|
|
|
|
await monitor.start()
|
|
await monitor.start_http_server(port=18082)
|
|
|
|
await monitor.stop()
|
|
|
|
assert not monitor.is_running
|
|
assert monitor._runner is None
|