Decouple live weather reads from source collection

2026-06-14 17:53:24 +08:00
parent a098a5ab20
commit e111031119
18 changed files with 2248 additions and 117 deletions
@@ -15,6 +15,7 @@ from loguru import logger

 import web.routes as legacy_routes
 from web.analysis_service import _runway_history_temp_for_city
+from web.services.canonical_temperature import build_city_weather_from_canonical
 from web.services.request_timing import ServerTimingRecorder

 _RECENT_DEB_CACHE: Optional[Dict[str, Dict[str, object]]] = None
@@ -129,6 +130,132 @@ async def _get_cached_city_payload(city: str, kind: str) -> Dict[str, Any]:
    return payload if isinstance(payload, dict) else {}


+async def _get_canonical_city_payload(city: str, *, detail_depth: str = "panel") -> Dict[str, Any]:
+    try:
+        row = await run_in_threadpool(legacy_routes._CACHE_DB.get_canonical_temperature, city)
+    except Exception:
+        return {}
+    if not isinstance(row, dict):
+        return {}
+    canonical = row.get("payload") or row
+    if not isinstance(canonical, dict):
+        return {}
+    payload = build_city_weather_from_canonical(city, canonical)
+    if not isinstance(payload, dict) or not payload:
+        return {}
+    city_meta = legacy_routes.CITY_REGISTRY.get(city, {}) or {}
+    city_info = legacy_routes.CITIES.get(city, {}) or {}
+    risk = legacy_routes.CITY_RISK_PROFILES.get(city, {}) or {}
+    payload.update(
+        {
+            "detail_depth": detail_depth,
+            "display_name": str(city_meta.get("display_name") or city_meta.get("name") or city.title()),
+            "lat": city_info.get("lat"),
+            "lon": city_info.get("lon"),
+            "temp_symbol": canonical.get("temp_symbol") or payload.get("temp_symbol") or ("°F" if city_info.get("f") else "°C"),
+            "risk": {
+                "level": risk.get("risk_level", "low"),
+                "emoji": risk.get("risk_emoji", "🟢"),
+                "airport": risk.get("airport_name", ""),
+                "icao": risk.get("icao", ""),
+                "distance_km": risk.get("distance_km", 0),
+                "warning": risk.get("warning", ""),
+            },
+            "probabilities": {"mu": None, "distribution": []},
+        }
+    )
+    return payload
+
+
+def _enqueue_collector_refresh_request(
+    city: str,
+    kind: str,
+    *,
+    reason: str = "canonical_fallback",
+) -> bool:
+    try:
+        enqueue = getattr(legacy_routes._CACHE_DB, "enqueue_observation_refresh_request", None)
+        if not callable(enqueue):
+            return False
+        return bool(
+            enqueue(
+                city=city,
+                kind=kind,
+                priority="high",
+                reason=reason,
+            )
+        )
+    except Exception as exc:
+        logger.debug("collector refresh enqueue failed city={} kind={}: {}", city, kind, exc)
+        return False
+
+
+def _request_city_cache_refresh(
+    city: str,
+    kind: str,
+    refresh_fn: Callable[[str, bool], Dict[str, Any]],
+) -> None:
+    if _enqueue_collector_refresh_request(city, kind):
+        return
+    _start_city_cache_stale_refresh(city, kind, refresh_fn)
+
+
+def _request_city_full_refresh(city: str) -> None:
+    if _enqueue_collector_refresh_request(city, "full"):
+        return
+    _start_city_full_stale_refresh(city)
+
+
+def _build_initializing_city_payload(city: str, *, detail_depth: str) -> Dict[str, Any]:
+    city_meta = legacy_routes.CITY_REGISTRY.get(city, {}) or {}
+    city_info = legacy_routes.CITIES.get(city, {}) or {}
+    risk = legacy_routes.CITY_RISK_PROFILES.get(city, {}) or {}
+    return {
+        "city": city,
+        "name": city,
+        "display_name": str(city_meta.get("display_name") or city_meta.get("name") or city.title()),
+        "detail_depth": detail_depth,
+        "status": "initializing",
+        "stale": True,
+        "stale_reason": "collector_refresh_queued",
+        "lat": city_info.get("lat"),
+        "lon": city_info.get("lon"),
+        "temp_symbol": "°F" if city_info.get("f") else "°C",
+        "risk": {
+            "level": risk.get("risk_level", "low"),
+            "emoji": risk.get("risk_emoji", "🟢"),
+            "airport": risk.get("airport_name", ""),
+            "icao": risk.get("icao", ""),
+            "distance_km": risk.get("distance_km", 0),
+            "warning": risk.get("warning", ""),
+        },
+        "current": {
+            "temp": None,
+            "source_code": None,
+            "settlement_source": None,
+            "settlement_source_label": None,
+            "obs_time": None,
+            "freshness": {
+                "freshness_status": "missing",
+                "freshness_reason": "collector_refresh_queued",
+            },
+            "observation_status": "initializing",
+        },
+        "airport_current": {},
+        "airport_primary": {},
+        "canonical_temperature": None,
+        "deb": {"prediction": None},
+        "probabilities": {"mu": None, "distribution": []},
+        "hourly": {"times": [], "temps": []},
+        "multi_model_daily": {},
+    }
+
+
+def _queue_and_build_initializing_city_payload(city: str, *, kind: str) -> Dict[str, Any]:
+    _enqueue_collector_refresh_request(city, kind, reason="cold_start")
+    return _build_initializing_city_payload(city, detail_depth=kind)
+
+
 async def _get_or_start_city_force_refresh_task(
    key: str,
    refresh_factory: Callable[[], Awaitable[Dict[str, Any]]],
@@ -158,16 +285,37 @@ async def _refresh_city_payload_with_stale_timeout(
    kind: str,
    refresh_factory: Callable[[], Awaitable[Dict[str, Any]]],
 ) -> Dict[str, Any]:
+    cached_before_refresh = await _get_cached_city_payload(city, kind)
+    if not cached_before_refresh:
+        canonical_payload = await _get_canonical_city_payload(city, detail_depth=kind)
+        if canonical_payload:
+            _enqueue_collector_refresh_request(city, kind, reason="force_refresh")
+            logger.warning(
+                "city force refresh returning canonical latest without sync refresh city={} kind={}",
+                city,
+                kind,
+            )
+            return canonical_payload
+        return _queue_and_build_initializing_city_payload(city, kind=kind)
+
    task, started = await _get_or_start_city_force_refresh_task(f"{kind}:{city}", refresh_factory)
    if not started:
-        cached_payload = await _get_cached_city_payload(city, kind)
-        if cached_payload:
+        if cached_before_refresh:
            logger.warning(
                "city force refresh already running city={} kind={}; returning stale cache",
                city,
                kind,
            )
-            return await _overlay_cached_wunderground(city, cached_payload)
+            return await _overlay_cached_wunderground(city, cached_before_refresh)
+    canonical_payload = await _get_canonical_city_payload(city, detail_depth=kind)
+    if canonical_payload:
+        _enqueue_collector_refresh_request(city, kind, reason="force_refresh")
+        logger.warning(
+            "city force refresh returning canonical latest while refresh runs city={} kind={}",
+            city,
+            kind,
+        )
+        return canonical_payload
    timeout_sec = _city_force_refresh_timeout_sec()
    try:
        return await asyncio.wait_for(asyncio.shield(task), timeout=timeout_sec)
@@ -384,9 +532,17 @@ async def _get_city_full_data(city: str, *, force_refresh: bool) -> Dict[str, An
            if payload:
                _start_city_full_stale_refresh(city)
                return await _overlay_cached_wunderground(city, payload)
-            return await _refresh_city_full_data(city, False)
+            canonical_payload = await _get_canonical_city_payload(city, detail_depth="full")
+            if canonical_payload:
+                _request_city_full_refresh(city)
+                return canonical_payload
+            return _queue_and_build_initializing_city_payload(city, kind="full")
        return await _overlay_cached_wunderground(city, payload)
-    return await _refresh_city_full_data(city, False)
+    canonical_payload = await _get_canonical_city_payload(city, detail_depth="full")
+    if canonical_payload:
+        _request_city_full_refresh(city)
+        return canonical_payload
+    return _queue_and_build_initializing_city_payload(city, kind="full")


 async def _get_city_chart_data(city: str, *, force_refresh: bool) -> Dict[str, Any]:
@@ -714,9 +870,13 @@ async def get_city_detail_payload(
                if payload:
                    _start_city_cache_stale_refresh(city, "panel", legacy_routes._refresh_city_panel_cache)
                    return await _overlay_cached_wunderground(city, payload)
-                return await run_in_threadpool(legacy_routes._refresh_city_panel_cache, city, False)
+                return _queue_and_build_initializing_city_payload(city, kind="panel")
            return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
-        return await run_in_threadpool(legacy_routes._refresh_city_panel_cache, city, False)
+        canonical_payload = await _get_canonical_city_payload(city, detail_depth="panel")
+        if canonical_payload:
+            _request_city_cache_refresh(city, "panel", legacy_routes._refresh_city_panel_cache)
+            return canonical_payload
+        return _queue_and_build_initializing_city_payload(city, kind="panel")
    if detail_mode == "nearby":
        if force_refresh:
            return await _refresh_city_cache_with_stale_timeout(
@@ -731,9 +891,17 @@ async def get_city_detail_payload(
                if payload:
                    _start_city_cache_stale_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
                    return await _overlay_cached_wunderground(city, payload)
-                return await run_in_threadpool(legacy_routes._refresh_city_nearby_cache, city, False)
+                canonical_payload = await _get_canonical_city_payload(city, detail_depth="nearby")
+                if canonical_payload:
+                    _request_city_cache_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
+                    return canonical_payload
+                return _queue_and_build_initializing_city_payload(city, kind="nearby")
            return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
-        return await run_in_threadpool(legacy_routes._refresh_city_nearby_cache, city, False)
+        canonical_payload = await _get_canonical_city_payload(city, detail_depth="nearby")
+        if canonical_payload:
+            _request_city_cache_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
+            return canonical_payload
+        return _queue_and_build_initializing_city_payload(city, kind="nearby")
    if detail_mode == "market":
        if force_refresh:
            return await _refresh_city_cache_with_stale_timeout(
@@ -748,9 +916,17 @@ async def get_city_detail_payload(
                if payload:
                    _start_city_cache_stale_refresh(city, "market", legacy_routes._refresh_city_market_cache)
                    return await _overlay_cached_wunderground(city, payload)
-                return await run_in_threadpool(legacy_routes._refresh_city_market_cache, city, False)
+                canonical_payload = await _get_canonical_city_payload(city, detail_depth="market")
+                if canonical_payload:
+                    _request_city_cache_refresh(city, "market", legacy_routes._refresh_city_market_cache)
+                    return canonical_payload
+                return _queue_and_build_initializing_city_payload(city, kind="market")
            return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
-        return await run_in_threadpool(legacy_routes._refresh_city_market_cache, city, False)
+        canonical_payload = await _get_canonical_city_payload(city, detail_depth="market")
+        if canonical_payload:
+            _request_city_cache_refresh(city, "market", legacy_routes._refresh_city_market_cache)
+            return canonical_payload
+        return _queue_and_build_initializing_city_payload(city, kind="market")
    return await run_in_threadpool(legacy_routes._analyze, city, force_refresh, False, detail_mode)


@@ -774,9 +950,13 @@ async def get_city_summary_payload(
            if payload:
                _start_city_cache_stale_refresh(city, "summary", legacy_routes._refresh_city_summary_cache)
                return await _overlay_cached_wunderground(city, payload)
-            return await run_in_threadpool(legacy_routes._refresh_city_summary_cache, city, False)
+            return _queue_and_build_initializing_city_payload(city, kind="summary")
        return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
-    return await run_in_threadpool(legacy_routes._refresh_city_summary_cache, city, False)
+    canonical_payload = await _get_canonical_city_payload(city, detail_depth="summary")
+    if canonical_payload:
+        _request_city_cache_refresh(city, "summary", legacy_routes._refresh_city_summary_cache)
+        return canonical_payload
+    return _queue_and_build_initializing_city_payload(city, kind="summary")


 async def get_city_detail_aggregate_payload(
@@ -803,6 +983,8 @@ async def get_city_detail_aggregate_payload(
            "full_data",
            lambda: _get_city_full_data(city, force_refresh=force_refresh),
        )
+        if isinstance(data, dict) and data.get("status") == "initializing":
+            return data

        return await timer.measure_async(
            "detail_payload",