Unify runtime state in SQLite and add rollout observability

tests/test_probability_rollout.py

@@ -0,0 +1,65 @@
+from src.analysis.probability_rollout import judge_probability_rollout
+
+
+def test_judge_probability_rollout_holds_on_shadow_brier_regression():
+    evaluation_report = {
+        "summary": {
+            "sample_count": 105,
+            "delta": {
+                "crps": -0.09,
+                "mae": 0.0,
+                "bucket_hit_rate": 0.0,
+            },
+        }
+    }
+    shadow_report = {
+        "summary": {
+            "samples": 103,
+            "delta_mae": 0.01,
+            "delta_bucket_hit_rate": 0.01,
+            "delta_bucket_brier": 0.29,
+        },
+        "by_city": {
+            "miami": {
+                "samples": 4,
+                "delta_mae": 0.24,
+                "delta_bucket_hit_rate": -0.5,
+                "delta_bucket_brier": 0.47,
+            }
+        },
+    }
+
+    payload = judge_probability_rollout(evaluation_report, shadow_report)
+
+    assert payload["decision"] == "hold"
+    assert payload["ready_for_primary"] is False
+    assert payload["blocking_reasons"]
+    assert payload["worst_shadow_regressions"][0]["city"] == "miami"
+
+
+def test_judge_probability_rollout_promotes_on_clean_metrics():
+    evaluation_report = {
+        "summary": {
+            "sample_count": 120,
+            "delta": {
+                "crps": -0.08,
+                "mae": 0.0,
+                "bucket_hit_rate": 0.02,
+            },
+        }
+    }
+    shadow_report = {
+        "summary": {
+            "samples": 110,
+            "delta_mae": 0.0,
+            "delta_bucket_hit_rate": 0.01,
+            "delta_bucket_brier": 0.01,
+        },
+        "by_city": {},
+    }
+
+    payload = judge_probability_rollout(evaluation_report, shadow_report)
+
+    assert payload["decision"] == "promote"
+    assert payload["ready_for_primary"] is True
+    assert payload["blocking_reasons"] == []

tests/test_probability_snapshot_archive.py

@@ -1,9 +1,15 @@
 import json
 from pathlib import Path
+import pytest
 
 from src.analysis.probability_snapshot_archive import append_probability_snapshot
 
 
+@pytest.fixture(autouse=True)
+def _force_file_mode(monkeypatch):
+    monkeypatch.setenv("POLYWEATHER_STATE_STORAGE_MODE", "file")
+
+
 def test_append_probability_snapshot_writes_jsonl(tmp_path: Path):
     archive_path = tmp_path / "probability_training_snapshots.jsonl"
 

tests/test_runtime_state_storage.py

@@ -0,0 +1,74 @@
+
+import time
+
+from src.database.runtime_state import (
+    DailyRecordRepository,
+    OpenMeteoCacheRepository,
+    ProbabilitySnapshotRepository,
+    RuntimeStateDB,
+    TelegramAlertStateRepository,
+)
+
+
+def test_daily_record_repository_roundtrip(tmp_path, monkeypatch):
+    monkeypatch.setenv('POLYWEATHER_DB_PATH', str(tmp_path / 'polyweather.db'))
+    repo = DailyRecordRepository(RuntimeStateDB(str(tmp_path / 'polyweather.db')))
+    repo.upsert_record('ankara', '2026-03-20', {'actual_high': 15.2, 'deb_prediction': 14.8, 'mu': 15.0})
+    data = repo.load_all()
+    assert data['ankara']['2026-03-20']['actual_high'] == 15.2
+
+
+def test_telegram_alert_state_repository_roundtrip(tmp_path, monkeypatch):
+    monkeypatch.setenv('POLYWEATHER_DB_PATH', str(tmp_path / 'polyweather.db'))
+    repo = TelegramAlertStateRepository(RuntimeStateDB(str(tmp_path / 'polyweather.db')))
+    state = {
+        'last_by_city': {
+            'ankara': {
+                'signature': 'sig-1',
+                'trigger_key': 'mkt:test',
+                'severity': 'medium',
+                'ts': 123,
+                'active': True,
+                'evidence': {'x': 1},
+            }
+        },
+        'by_signature': {'sig-1': 123},
+    }
+    repo.save_state(state)
+    loaded = repo.load_state()
+    assert loaded == state
+
+
+def test_probability_snapshot_repository_recent_rows(tmp_path, monkeypatch):
+    monkeypatch.setenv('POLYWEATHER_DB_PATH', str(tmp_path / 'polyweather.db'))
+    repo = ProbabilitySnapshotRepository(RuntimeStateDB(str(tmp_path / 'polyweather.db')))
+    repo.append_snapshot({
+        'city': 'ankara',
+        'date': '2026-03-20',
+        'timestamp': '2026-03-20T12:00:00Z',
+        'raw_mu': 15.2,
+        'raw_sigma': 1.1,
+        'max_so_far': 14.9,
+        'peak_status': 'before',
+        'probability_mode': 'emos_shadow',
+        'prob_snapshot': [{'v': 15, 'p': 0.6}],
+        'shadow_prob_snapshot': [{'v': 15, 'p': 0.4}],
+    })
+    rows = repo.load_recent_rows('ankara', '2026-03-20', 5)
+    assert len(rows) == 1
+    assert rows[0]['raw_mu'] == 15.2
+
+
+def test_open_meteo_cache_repository_roundtrip(tmp_path, monkeypatch):
+    monkeypatch.setenv('POLYWEATHER_DB_PATH', str(tmp_path / 'polyweather.db'))
+    repo = OpenMeteoCacheRepository(RuntimeStateDB(str(tmp_path / 'polyweather.db')))
+    payload = {
+        'forecast': {'ankara': {'t': time.time(), 'temp': 15}},
+        'ensemble': {'ankara': {'t': time.time(), 'spread': 1.5}},
+        'multi_model': {},
+        'saved_at': 1000,
+    }
+    repo.replace_payload(payload, 86400)
+    loaded = repo.load_payload(86400)
+    assert loaded['forecast']['ankara']['temp'] == 15
+    assert loaded['ensemble']['ankara']['spread'] == 1.5

tests/test_web_observability.py

@@ -0,0 +1,37 @@
+
+from fastapi.testclient import TestClient
+
+from web.app import app
+
+
+client = TestClient(app)
+
+
+def test_healthz_returns_ok_shape():
+    response = client.get('/healthz')
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload['status'] in {'ok', 'degraded'}
+    assert 'db' in payload
+    assert 'state_storage_mode' in payload
+    assert 'cities_count' in payload
+
+
+def test_system_status_returns_summary_shape():
+    response = client.get('/api/system/status')
+    assert response.status_code == 200
+    payload = response.json()
+    assert 'db' in payload
+    assert 'features' in payload
+    assert 'integrations' in payload
+    assert 'cache' in payload
+    assert 'probability' in payload
+    assert 'rollout' in payload['probability']
+    assert payload['probability']['rollout']['decision']['decision'] in {'hold', 'observe', 'promote'}
+    assert 'cities_count' in payload
+
+
+def test_metrics_endpoint_returns_prometheus_payload():
+    response = client.get('/metrics')
+    assert response.status_code == 200
+    assert 'polyweather_http_requests_total' in response.text