feat: add connection health monitor with metrics (#6)

Implements the HealthMonitor class for tracking connection health: - Tracks connection states per stream (active, stale, disconnected) - Records events and calculates throughput (events/second) - Detects stale streams (no events for configurable threshold) - Exposes Prometheus-compatible metrics endpoint (/metrics) - Provides HTTP health endpoints (/health, /ready, /live) Exports: HealthMonitor, HealthReport, HealthStatus, StreamHealth, StreamStatus Tests: 47 comprehensive unit tests covering all functionality 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-04 15:22:10 -05:00
parent 3bf8e228b8
commit 7d32135bb9
4 changed files with 1143 additions and 0 deletions
@@ -0,0 +1,617 @@
+"""Tests for the connection health monitor."""
+
+import asyncio
+import time
+from unittest.mock import AsyncMock
+
+import pytest
+from aiohttp import web
+
+from polymarket_insider_tracker.ingestor.health import (
+    DEFAULT_HEALTH_CHECK_INTERVAL,
+    DEFAULT_STALE_THRESHOLD_SECONDS,
+    HealthMonitor,
+    HealthReport,
+    HealthStatus,
+    StreamHealth,
+    StreamStatus,
+)
+
+
+class TestStreamHealth:
+    """Tests for the StreamHealth dataclass."""
+
+    def test_stream_health_defaults(self) -> None:
+        """Test default values."""
+        health = StreamHealth(name="test-stream")
+
+        assert health.name == "test-stream"
+        assert health.status == StreamStatus.DISCONNECTED
+        assert health.last_event_time is None
+        assert health.events_received == 0
+        assert health.events_per_second == 0.0
+        assert health.connected_since is None
+        assert health.last_error is None
+
+    def test_stream_health_custom_values(self) -> None:
+        """Test with custom values."""
+        now = time.time()
+        health = StreamHealth(
+            name="trades",
+            status=StreamStatus.ACTIVE,
+            last_event_time=now,
+            events_received=100,
+            events_per_second=5.0,
+            connected_since=now - 3600,
+            last_error=None,
+        )
+
+        assert health.name == "trades"
+        assert health.status == StreamStatus.ACTIVE
+        assert health.events_received == 100
+
+
+class TestHealthReport:
+    """Tests for the HealthReport dataclass."""
+
+    def test_health_report_defaults(self) -> None:
+        """Test default values."""
+        report = HealthReport(status=HealthStatus.HEALTHY)
+
+        assert report.status == HealthStatus.HEALTHY
+        assert report.streams == {}
+        assert report.total_events_received == 0
+        assert report.total_events_per_second == 0.0
+        assert report.uptime_seconds == 0.0
+        assert report.timestamp > 0
+
+    def test_health_report_with_streams(self) -> None:
+        """Test with stream data."""
+        stream = StreamHealth(name="trades", events_received=100)
+        report = HealthReport(
+            status=HealthStatus.DEGRADED,
+            streams={"trades": stream},
+            total_events_received=100,
+            total_events_per_second=5.0,
+            uptime_seconds=3600.0,
+        )
+
+        assert report.status == HealthStatus.DEGRADED
+        assert "trades" in report.streams
+        assert report.total_events_received == 100
+
+
+class TestHealthMonitor:
+    """Tests for the HealthMonitor class."""
+
+    def test_init(self) -> None:
+        """Test initialization."""
+        monitor = HealthMonitor()
+
+        assert monitor._stale_threshold == DEFAULT_STALE_THRESHOLD_SECONDS
+        assert monitor._health_check_interval == DEFAULT_HEALTH_CHECK_INTERVAL
+        assert not monitor.is_running
+
+    def test_init_custom_config(self) -> None:
+        """Test initialization with custom config."""
+        monitor = HealthMonitor(
+            stale_threshold_seconds=30,
+            health_check_interval=10,
+        )
+
+        assert monitor._stale_threshold == 30
+        assert monitor._health_check_interval == 10
+
+    def test_register_stream(self) -> None:
+        """Test registering a stream."""
+        monitor = HealthMonitor()
+
+        monitor.register_stream("trades")
+
+        assert "trades" in monitor._streams
+        assert monitor._streams["trades"].name == "trades"
+        assert monitor._streams["trades"].status == StreamStatus.DISCONNECTED
+
+    def test_register_stream_idempotent(self) -> None:
+        """Test that registering the same stream twice is idempotent."""
+        monitor = HealthMonitor()
+
+        monitor.register_stream("trades")
+        monitor.record_event("trades")  # Adds an event
+        monitor.register_stream("trades")  # Should not reset
+
+        assert monitor._streams["trades"].events_received == 1
+
+    def test_set_stream_connected(self) -> None:
+        """Test marking a stream as connected."""
+        monitor = HealthMonitor()
+
+        monitor.set_stream_connected("trades")
+
+        assert monitor._streams["trades"].status == StreamStatus.ACTIVE
+        assert monitor._streams["trades"].connected_since is not None
+        assert monitor._streams["trades"].last_error is None
+
+    def test_set_stream_disconnected(self) -> None:
+        """Test marking a stream as disconnected."""
+        monitor = HealthMonitor()
+
+        monitor.set_stream_connected("trades")
+        monitor.set_stream_disconnected("trades", error="Connection reset")
+
+        assert monitor._streams["trades"].status == StreamStatus.DISCONNECTED
+        assert monitor._streams["trades"].connected_since is None
+        assert monitor._streams["trades"].last_error == "Connection reset"
+
+    def test_record_event(self) -> None:
+        """Test recording an event."""
+        monitor = HealthMonitor()
+
+        monitor.record_event("trades")
+
+        stream = monitor._streams["trades"]
+        assert stream.events_received == 1
+        assert stream.last_event_time is not None
+        assert stream.status == StreamStatus.ACTIVE
+
+    def test_record_event_multiple(self) -> None:
+        """Test recording multiple events."""
+        monitor = HealthMonitor()
+
+        for _ in range(10):
+            monitor.record_event("trades")
+
+        assert monitor._streams["trades"].events_received == 10
+
+    def test_record_event_with_processing_time(self) -> None:
+        """Test recording event with processing time."""
+        monitor = HealthMonitor()
+
+        # Should not raise
+        monitor.record_event("trades", processing_time=0.001)
+
+        assert monitor._streams["trades"].events_received == 1
+
+    def test_calculate_throughput_empty(self) -> None:
+        """Test throughput calculation with no events."""
+        monitor = HealthMonitor()
+
+        rate = monitor._calculate_throughput("nonexistent")
+
+        assert rate == 0.0
+
+    def test_calculate_throughput(self) -> None:
+        """Test throughput calculation."""
+        monitor = HealthMonitor()
+
+        # Add events
+        for _ in range(10):
+            monitor.record_event("trades")
+
+        rate = monitor._calculate_throughput("trades")
+
+        # Should have ~10 events in the window
+        assert rate > 0
+
+    def test_check_stream_staleness_active(self) -> None:
+        """Test that active stream is not marked stale."""
+        monitor = HealthMonitor(stale_threshold_seconds=60)
+
+        monitor.record_event("trades")
+        monitor._check_stream_staleness()
+
+        assert monitor._streams["trades"].status == StreamStatus.ACTIVE
+
+    def test_check_stream_staleness_stale(self) -> None:
+        """Test that stream becomes stale after threshold."""
+        monitor = HealthMonitor(stale_threshold_seconds=1)
+
+        monitor.record_event("trades")
+        # Simulate time passing
+        monitor._streams["trades"].last_event_time = time.time() - 2
+
+        monitor._check_stream_staleness()
+
+        assert monitor._streams["trades"].status == StreamStatus.STALE
+
+    def test_check_stream_staleness_connected_no_events(self) -> None:
+        """Test staleness when connected but no events received."""
+        monitor = HealthMonitor(stale_threshold_seconds=1)
+
+        monitor.set_stream_connected("trades")
+        # Simulate time passing since connection
+        monitor._streams["trades"].connected_since = time.time() - 2
+
+        monitor._check_stream_staleness()
+
+        assert monitor._streams["trades"].status == StreamStatus.STALE
+
+    def test_determine_overall_status_no_streams(self) -> None:
+        """Test overall status with no streams."""
+        monitor = HealthMonitor()
+
+        status = monitor._determine_overall_status()
+
+        assert status == HealthStatus.HEALTHY
+
+    def test_determine_overall_status_all_active(self) -> None:
+        """Test overall status with all active streams."""
+        monitor = HealthMonitor()
+
+        monitor.record_event("trades")
+        monitor.record_event("orderbook")
+
+        status = monitor._determine_overall_status()
+
+        assert status == HealthStatus.HEALTHY
+
+    def test_determine_overall_status_some_stale(self) -> None:
+        """Test overall status with some stale streams."""
+        monitor = HealthMonitor()
+
+        monitor.record_event("trades")
+        monitor.register_stream("orderbook")
+        monitor._streams["orderbook"].status = StreamStatus.STALE
+
+        status = monitor._determine_overall_status()
+
+        assert status == HealthStatus.DEGRADED
+
+    def test_determine_overall_status_some_disconnected(self) -> None:
+        """Test overall status with some disconnected streams."""
+        monitor = HealthMonitor()
+
+        monitor.record_event("trades")
+        monitor.set_stream_disconnected("orderbook")
+
+        status = monitor._determine_overall_status()
+
+        assert status == HealthStatus.DEGRADED
+
+    def test_determine_overall_status_all_disconnected(self) -> None:
+        """Test overall status with all disconnected streams."""
+        monitor = HealthMonitor()
+
+        monitor.set_stream_disconnected("trades")
+        monitor.set_stream_disconnected("orderbook")
+
+        status = monitor._determine_overall_status()
+
+        assert status == HealthStatus.UNHEALTHY
+
+    def test_get_health_report(self) -> None:
+        """Test getting a health report."""
+        monitor = HealthMonitor()
+        monitor._start_time = time.time() - 100
+
+        monitor.record_event("trades")
+        monitor.record_event("trades")
+
+        report = monitor.get_health_report()
+
+        assert report.status == HealthStatus.HEALTHY
+        assert "trades" in report.streams
+        assert report.total_events_received == 2
+        assert report.uptime_seconds >= 100
+        assert report.timestamp > 0
+
+    def test_get_health_report_calculates_throughput(self) -> None:
+        """Test that health report calculates throughput."""
+        monitor = HealthMonitor()
+
+        for _ in range(10):
+            monitor.record_event("trades")
+
+        report = monitor.get_health_report()
+
+        assert report.streams["trades"].events_per_second > 0
+        assert report.total_events_per_second > 0
+
+    @pytest.mark.asyncio
+    async def test_start_stop(self) -> None:
+        """Test starting and stopping the monitor."""
+        monitor = HealthMonitor()
+
+        await monitor.start()
+        assert monitor.is_running
+        assert monitor._health_task is not None
+
+        await monitor.stop()
+        assert not monitor.is_running
+        assert monitor._health_task is None
+
+    @pytest.mark.asyncio
+    async def test_start_idempotent(self) -> None:
+        """Test that starting twice is safe."""
+        monitor = HealthMonitor()
+
+        await monitor.start()
+        await monitor.start()  # Should not raise
+
+        assert monitor.is_running
+
+        await monitor.stop()
+
+    @pytest.mark.asyncio
+    async def test_stop_when_not_running(self) -> None:
+        """Test that stopping when not running is safe."""
+        monitor = HealthMonitor()
+
+        await monitor.stop()  # Should not raise
+
+    @pytest.mark.asyncio
+    async def test_context_manager(self) -> None:
+        """Test async context manager."""
+        async with HealthMonitor() as monitor:
+            assert monitor.is_running
+
+        assert not monitor.is_running
+
+    @pytest.mark.asyncio
+    async def test_health_check_loop_updates_report(self) -> None:
+        """Test that health check loop updates the report."""
+        monitor = HealthMonitor(health_check_interval=0.1)
+
+        await monitor.start()
+        monitor.record_event("trades")
+
+        await asyncio.sleep(0.2)
+
+        # Health should have been checked
+        report = monitor.get_health_report()
+        assert report.status == HealthStatus.HEALTHY
+
+        await monitor.stop()
+
+    @pytest.mark.asyncio
+    async def test_health_change_callback(self) -> None:
+        """Test that health change callback is invoked."""
+        callback = AsyncMock()
+        monitor = HealthMonitor(
+            health_check_interval=0.1,
+            on_health_change=callback,
+        )
+
+        await monitor.start()
+        monitor.record_event("trades")
+
+        # Wait for health check
+        await asyncio.sleep(0.2)
+
+        await monitor.stop()
+
+        # Callback should have been called at least once
+        assert callback.called
+
+    @pytest.mark.asyncio
+    async def test_health_change_callback_error_handling(self) -> None:
+        """Test that callback errors don't crash the loop."""
+        callback = AsyncMock(side_effect=ValueError("test error"))
+        monitor = HealthMonitor(
+            health_check_interval=0.1,
+            on_health_change=callback,
+        )
+
+        await monitor.start()
+        monitor.record_event("trades")
+
+        # Should not crash
+        await asyncio.sleep(0.2)
+
+        await monitor.stop()
+
+
+class TestHealthMonitorHTTPEndpoints:
+    """Tests for HTTP endpoints."""
+
+    @pytest.fixture
+    def monitor(self) -> HealthMonitor:
+        """Create a monitor instance."""
+        return HealthMonitor()
+
+    @pytest.fixture
+    def app(self, monitor: HealthMonitor) -> web.Application:
+        """Create the aiohttp application."""
+        return monitor._create_app()
+
+    @pytest.mark.asyncio
+    async def test_health_endpoint_healthy(
+        self, monitor: HealthMonitor, app: web.Application
+    ) -> None:
+        """Test /health endpoint when healthy."""
+        from aiohttp.test_utils import TestClient, TestServer
+
+        monitor.record_event("trades")
+
+        async with TestClient(TestServer(app)) as client:
+            resp = await client.get("/health")
+            assert resp.status == 200
+
+            data = await resp.json()
+            assert data["status"] == "healthy"
+            assert "trades" in data["streams"]
+
+    @pytest.mark.asyncio
+    async def test_health_endpoint_unhealthy(
+        self, monitor: HealthMonitor, app: web.Application
+    ) -> None:
+        """Test /health endpoint when unhealthy."""
+        from aiohttp.test_utils import TestClient, TestServer
+
+        monitor.set_stream_disconnected("trades")
+
+        async with TestClient(TestServer(app)) as client:
+            resp = await client.get("/health")
+            assert resp.status == 503
+
+            data = await resp.json()
+            assert data["status"] == "unhealthy"
+
+    @pytest.mark.asyncio
+    async def test_metrics_endpoint(
+        self, monitor: HealthMonitor, app: web.Application
+    ) -> None:
+        """Test /metrics endpoint returns Prometheus format."""
+        from aiohttp.test_utils import TestClient, TestServer
+
+        monitor.record_event("trades")
+
+        async with TestClient(TestServer(app)) as client:
+            resp = await client.get("/metrics")
+            assert resp.status == 200
+
+            content_type = resp.headers.get("Content-Type", "")
+            assert "text/plain" in content_type
+
+            text = await resp.text()
+            assert "polymarket_events_total" in text
+            assert "polymarket_health_status" in text
+
+    @pytest.mark.asyncio
+    async def test_ready_endpoint_ready(
+        self, monitor: HealthMonitor, app: web.Application
+    ) -> None:
+        """Test /ready endpoint when ready."""
+        from aiohttp.test_utils import TestClient, TestServer
+
+        monitor.record_event("trades")
+
+        async with TestClient(TestServer(app)) as client:
+            resp = await client.get("/ready")
+            assert resp.status == 200
+
+            data = await resp.json()
+            assert data["ready"] is True
+
+    @pytest.mark.asyncio
+    async def test_ready_endpoint_not_ready(
+        self, monitor: HealthMonitor, app: web.Application
+    ) -> None:
+        """Test /ready endpoint when not ready."""
+        from aiohttp.test_utils import TestClient, TestServer
+
+        monitor.set_stream_disconnected("trades")
+
+        async with TestClient(TestServer(app)) as client:
+            resp = await client.get("/ready")
+            assert resp.status == 503
+
+            data = await resp.json()
+            assert data["ready"] is False
+
+    @pytest.mark.asyncio
+    async def test_live_endpoint(self, app: web.Application) -> None:
+        """Test /live endpoint always returns 200."""
+        from aiohttp.test_utils import TestClient, TestServer
+
+        async with TestClient(TestServer(app)) as client:
+            resp = await client.get("/live")
+            assert resp.status == 200
+
+            data = await resp.json()
+            assert data["live"] is True
+
+    @pytest.mark.asyncio
+    async def test_start_stop_http_server(self, monitor: HealthMonitor) -> None:
+        """Test starting and stopping HTTP server."""
+        await monitor.start_http_server(port=18080)
+        assert monitor._runner is not None
+
+        await monitor.stop_http_server()
+        assert monitor._runner is None
+
+    @pytest.mark.asyncio
+    async def test_start_http_server_idempotent(self, monitor: HealthMonitor) -> None:
+        """Test that starting HTTP server twice is safe."""
+        await monitor.start_http_server(port=18081)
+        await monitor.start_http_server(port=18081)  # Should not raise
+
+        await monitor.stop_http_server()
+
+
+class TestPrometheusMetrics:
+    """Tests for Prometheus metric updates."""
+
+    def test_events_total_incremented(self) -> None:
+        """Test that events_total counter is incremented."""
+        monitor = HealthMonitor()
+
+        monitor.record_event("test-metrics")
+        monitor.record_event("test-metrics")
+
+        # Counter should have been incremented
+        # (We can't easily test prometheus metrics directly, but at least verify no errors)
+
+    def test_stream_status_updated(self) -> None:
+        """Test that stream_status gauge is updated."""
+        monitor = HealthMonitor()
+
+        monitor.set_stream_connected("test-status")
+        # Gauge should be 1.0
+
+        monitor.set_stream_disconnected("test-status")
+        # Gauge should be 0.0
+
+    def test_health_status_updated(self) -> None:
+        """Test that health_status gauge is updated."""
+        monitor = HealthMonitor()
+
+        monitor.record_event("test-health")
+        report = monitor.get_health_report()
+
+        assert report.status == HealthStatus.HEALTHY
+
+
+class TestEdgeCases:
+    """Tests for edge cases and error handling."""
+
+    def test_throughput_with_old_events(self) -> None:
+        """Test throughput calculation ignores old events."""
+        monitor = HealthMonitor()
+
+        monitor.record_event("trades")
+        # Manually add old event to window
+        monitor._event_windows["trades"].append(time.time() - 100)
+
+        rate = monitor._calculate_throughput("trades")
+
+        # Old event should be filtered out
+        # Rate should only count recent events
+        assert rate >= 0
+
+    def test_multiple_streams_independent(self) -> None:
+        """Test that multiple streams are tracked independently."""
+        monitor = HealthMonitor()
+
+        monitor.record_event("trades")
+        monitor.record_event("trades")
+        monitor.set_stream_disconnected("orderbook")
+
+        assert monitor._streams["trades"].events_received == 2
+        assert monitor._streams["trades"].status == StreamStatus.ACTIVE
+        assert monitor._streams["orderbook"].events_received == 0
+        assert monitor._streams["orderbook"].status == StreamStatus.DISCONNECTED
+
+    def test_report_streams_are_copied(self) -> None:
+        """Test that report streams are a copy."""
+        monitor = HealthMonitor()
+        monitor.record_event("trades")
+
+        report = monitor.get_health_report()
+
+        # Modifying report should not affect monitor
+        report.streams["trades"].events_received = 999
+        assert monitor._streams["trades"].events_received == 1
+
+    @pytest.mark.asyncio
+    async def test_stop_cleans_up_http_server(self) -> None:
+        """Test that stop() also stops HTTP server."""
+        monitor = HealthMonitor()
+
+        await monitor.start()
+        await monitor.start_http_server(port=18082)
+
+        await monitor.stop()
+
+        assert not monitor.is_running
+        assert monitor._runner is None