feat: implement real-time SSE event architecture with Redis stream integration and add associated validation tests

tests/test_country_networks.py

@@ -306,6 +306,58 @@ def test_hko_provider_marks_explicit_official_station_as_anchor():
     assert snapshot["official_nearby"][0]["station_code"] == "LFS"
 
 
+def test_hong_kong_cowin_primary_uses_station_6087_history(monkeypatch):
+    from src.database import runtime_state
+
+    requested = {}
+
+    class FakeOfficialIntradayObservationRepository:
+        def load_points(self, *, source_code, station_code, target_date):
+            requested["source_code"] = source_code
+            requested["station_code"] = station_code
+            requested["target_date"] = target_date
+            if source_code == "cowin_obs" and station_code == "6087":
+                return [
+                    {"time": "10:40", "temp": 31.1},
+                    {"time": "10:41", "temp": 31.3},
+                ]
+            return []
+
+    monkeypatch.setattr(
+        runtime_state,
+        "OfficialIntradayObservationRepository",
+        FakeOfficialIntradayObservationRepository,
+    )
+
+    snapshot = build_country_network_snapshot(
+        "hong kong",
+        {
+            "cowin_current": {
+                "temp": 31.3,
+                "obs_time": "2026-05-27T02:41:00Z",
+                "istNo": "6087",
+                "icao": "COWIN6087",
+                "station_label": "保良局陳守仁小學 1min (CoWIN)",
+            },
+            "settlement_current": {
+                "station_code": "HKO",
+                "station_name": "HK Observatory",
+                "observation_time": "2026-05-27T10:40:00+08:00",
+                "current": {"temp": 31.2},
+            },
+        },
+    )
+
+    assert snapshot["airport_primary_current"]["source_code"] == "cowin_obs"
+    assert snapshot["airport_primary_current"]["station_code"] == "6087"
+    assert "陳守仁" in snapshot["airport_primary_current"]["station_label"]
+    assert requested["station_code"] == "6087"
+    assert snapshot["airport_primary_today_obs"] == [
+        {"time": "10:40", "temp": 31.1},
+        {"time": "10:41", "temp": 31.3},
+    ]
+
+
 def test_moscow_provider_uses_realtime_metar_cluster_not_station_archive_rows():
     raw = {
         "metar": {

tests/test_realtime_event_store_factory.py

@@ -0,0 +1,55 @@
+import pytest
+
+from web.realtime_event_store import RealtimeEventStore
+from web.realtime_event_store_factory import create_realtime_event_store
+from web.redis_realtime_event_store import RedisRealtimeEventStore
+
+
+class FakeRedis:
+    pass
+
+
+def test_event_store_factory_uses_sqlite_by_default(monkeypatch, tmp_path):
+    monkeypatch.delenv("POLYWEATHER_EVENT_STORE", raising=False)
+
+    store = create_realtime_event_store(db_path=str(tmp_path / "polyweather.db"))
+
+    assert isinstance(store, RealtimeEventStore)
+
+
+def test_event_store_factory_uses_redis_when_configured(monkeypatch):
+    monkeypatch.setenv("POLYWEATHER_EVENT_STORE", "redis")
+
+    store = create_realtime_event_store(redis_client=FakeRedis())
+
+    assert isinstance(store, RedisRealtimeEventStore)
+
+
+def test_event_store_factory_falls_back_to_sqlite_when_redis_is_optional(monkeypatch, tmp_path):
+    monkeypatch.setenv("POLYWEATHER_EVENT_STORE", "redis")
+    monkeypatch.setenv("POLYWEATHER_REDIS_REQUIRED", "false")
+
+    def broken_redis_store(**_kwargs):
+        raise RuntimeError("redis down")
+
+    store = create_realtime_event_store(
+        db_path=str(tmp_path / "polyweather.db"),
+        redis_store_builder=broken_redis_store,
+    )
+
+    assert isinstance(store, RealtimeEventStore)
+    assert getattr(store, "degraded_from", None) == "redis"
+
+
+def test_event_store_factory_raises_when_redis_is_required(monkeypatch, tmp_path):
+    monkeypatch.setenv("POLYWEATHER_EVENT_STORE", "redis")
+    monkeypatch.setenv("POLYWEATHER_REDIS_REQUIRED", "true")
+
+    def broken_redis_store(**_kwargs):
+        raise RuntimeError("redis down")
+
+    with pytest.raises(RuntimeError, match="redis down"):
+        create_realtime_event_store(
+            db_path=str(tmp_path / "polyweather.db"),
+            redis_store_builder=broken_redis_store,
+        )

tests/test_redis_realtime_event_store.py

@@ -0,0 +1,125 @@
+from web.redis_realtime_event_store import RedisRealtimeEventStore
+from web.realtime_patch_schema import normalize_observation_patch
+
+
+class FakeRedis:
+    def __init__(self):
+        self.counter = 0
+        self.entries = []
+
+    def eval(self, _script, _numkeys, stream_key, counter_key, *args):
+        (
+            maxlen,
+            event_type,
+            schema_type,
+            schema_version,
+            city,
+            source,
+            obs_time,
+            payload_json,
+            created_at_ms,
+            ts,
+            producer_id,
+        ) = args
+        self.counter += 1
+        stream_id = f"{created_at_ms}-{self.counter}"
+        fields = {
+            "revision": str(self.counter),
+            "type": event_type,
+            "schema_type": schema_type,
+            "schema_version": str(schema_version),
+            "city": city,
+            "source": source,
+            "obs_time": obs_time,
+            "payload_json": payload_json,
+            "created_at_ms": str(created_at_ms),
+            "ts": str(ts),
+            "producer_id": producer_id,
+        }
+        self.entries.append((stream_id, fields))
+        trim_to = int(maxlen)
+        if trim_to > 0 and len(self.entries) > trim_to:
+            self.entries = self.entries[-trim_to:]
+        return [self.counter, stream_id]
+
+    def get(self, key):
+        if key == "counter:city_observation_revision":
+            return str(self.counter)
+        return None
+
+    def xrange(self, stream_key, min="-", max="+", count=None):
+        rows = list(self.entries)
+        if count is not None:
+            rows = rows[: int(count)]
+        return rows
+
+
+def _event(city: str, temp: float, source: str = "cwa"):
+    return normalize_observation_patch(
+        {
+            "city": city,
+            "changes": {
+                "temp": temp,
+                "obs_time": "2026-05-27T10:00:00+08:00",
+                "source": source,
+            },
+        }
+    )
+
+
+def test_redis_event_store_appends_monotonic_revisions_and_replays_by_city():
+    store = RedisRealtimeEventStore(redis_client=FakeRedis(), maxlen=10, producer_id="test")
+
+    taipei = store.append_event(_event("taipei", 34.2))
+    seoul = store.append_event(_event("seoul", 21.5, source="amos"))
+    taipei_next = store.append_event(_event("taipei", 34.4))
+
+    assert taipei["revision"] == 1
+    assert seoul["revision"] == 2
+    assert taipei_next["revision"] == 3
+    assert store.latest_revision() == 3
+
+    replay = store.replay_events(cities={"taipei"}, since_revision=1, limit=10)
+
+    assert [event["revision"] for event in replay] == [3]
+    assert replay[0]["city"] == "taipei"
+    assert replay[0]["payload"]["temp"] == 34.4
+
+
+def test_redis_event_store_preserves_time_contract_on_replay():
+    store = RedisRealtimeEventStore(redis_client=FakeRedis(), maxlen=10, producer_id="test")
+
+    stored = store.append_event(
+        normalize_observation_patch(
+            {
+                "city": "toronto",
+                "changes": {
+                    "temp": 26,
+                    "obs_time": "2026-05-27T23:16:00Z",
+                    "source": "metar",
+                },
+            }
+        )
+    )
+    replayed = store.replay_events(cities={"toronto"}, since_revision=0, limit=10)[0]
+
+    assert stored["observed_at_utc"] == "2026-05-27T23:16:00Z"
+    assert replayed["observed_at_local"] == "2026-05-27T19:16:00-04:00"
+    assert replayed["city_timezone"] == "America/Toronto"
+
+
+def test_redis_event_store_reports_replay_gap_when_limit_is_exceeded():
+    store = RedisRealtimeEventStore(redis_client=FakeRedis(), maxlen=10, producer_id="test")
+
+    for temp in [30.0, 30.5, 31.0]:
+        store.append_event(_event("hong kong", temp, source="hko"))
+
+    replay = store.replay_events(cities={"hong kong"}, since_revision=0, limit=2)
+
+    assert [event["revision"] for event in replay] == [1, 2]
+    assert store.replay_requires_resync(
+        cities={"hong kong"},
+        since_revision=0,
+        replay_count=len(replay),
+        limit=2,
+    )

tests/test_sse_replay.py

@@ -131,3 +131,52 @@ def test_replay_limit_is_bounded():
     assert sse_router._bounded_replay_limit(0) == 1
     assert sse_router._bounded_replay_limit(500) == 500
     assert sse_router._bounded_replay_limit(5000) == 2000
+
+
+def test_ingest_patch_uses_external_fanout_without_direct_broadcast(monkeypatch):
+    class FakeExternalStore:
+        uses_external_live_fanout = True
+
+        def __init__(self):
+            self.started = 0
+
+        def start_live_subscription(self, callback):
+            self.started += 1
+            self.callback = callback
+
+        def append_event(self, event):
+            return {
+                **event,
+                "revision": 12,
+            }
+
+    class FakeManager:
+        def __init__(self):
+            self.broadcasted = []
+
+        def broadcast_event(self, event):
+            self.broadcasted.append(event)
+            return event
+
+    store = FakeExternalStore()
+    manager = FakeManager()
+    monkeypatch.setattr(sse_router, "event_store", store)
+    monkeypatch.setattr(sse_router, "sse_manager", manager)
+    monkeypatch.setattr(sse_router, "_live_subscription_started", False)
+
+    response = TestClient(app).post(
+        "/api/internal/collector-patch",
+        json={
+            "city": "taipei",
+            "changes": {
+                "temp": 34.2,
+                "source": "cwa",
+                "obs_time": "2026-05-27T10:00:00+08:00",
+            },
+        },
+    )
+
+    assert response.status_code == 200
+    assert response.json()["revision"] == 12
+    assert store.started == 1
+    assert manager.broadcasted == []

tests/test_web_observability.py

@@ -36,6 +36,10 @@ def test_system_status_returns_summary_shape():
     assert payload['probability']['engine_mode'] == 'legacy'
     assert 'training_data' in payload
     assert 'station_networks' in payload
+    assert 'realtime' in payload
+    assert payload['realtime']['store'] in {'sqlite', 'redis', 'degraded_sqlite'}
+    assert 'latest_revision' in payload['realtime']
+    assert 'sse_connections' in payload['realtime']
     assert 'truth_records' in payload['training_data']
     assert 'training_features' in payload['training_data']
     assert 'city_coverage' in payload['training_data']