Queue stale weather refreshes from user paths

src/bot/analysis/city_analysis_service.py

@@ -1,13 +1,136 @@
 from __future__ import annotations
 
-from typing import Any
+from datetime import datetime, timezone, timedelta
+from typing import Any, Dict, Optional
+
+from src.database.db_manager import DBManager
+from src.data_collection.city_registry import CITY_REGISTRY
+from web.services.canonical_temperature import build_city_weather_from_canonical
+
+
+def _safe_float(value: Any) -> Optional[float]:
+    try:
+        if value is None or value == "":
+            return None
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _today_for_city(city_name: str) -> str:
+    meta = CITY_REGISTRY.get(str(city_name or "").strip().lower()) or {}
+    offset = int(meta.get("tz_offset") or 0)
+    return datetime.now(timezone.utc).astimezone(timezone(timedelta(seconds=offset))).strftime("%Y-%m-%d")
+
+
+def _first_forecast_high(payload: Dict[str, Any], city_name: str) -> Optional[float]:
+    deb = payload.get("deb") if isinstance(payload.get("deb"), dict) else {}
+    value = _safe_float(deb.get("prediction"))
+    if value is not None:
+        return value
+
+    local_date = str(payload.get("local_date") or _today_for_city(city_name)).strip()
+    daily = payload.get("multi_model_daily")
+    if isinstance(daily, dict):
+        dates = [local_date] if local_date in daily else sorted(daily.keys())
+        for date_key in dates:
+            entry = daily.get(date_key)
+            if not isinstance(entry, dict):
+                continue
+            daily_deb = entry.get("deb") if isinstance(entry.get("deb"), dict) else {}
+            value = _safe_float(daily_deb.get("prediction"))
+            if value is not None:
+                return value
+            models = entry.get("models") if isinstance(entry.get("models"), dict) else {}
+            values = sorted(
+                value
+                for value in (_safe_float(item) for item in models.values())
+                if value is not None
+            )
+            if values:
+                return values[len(values) // 2]
+
+    current = payload.get("current") if isinstance(payload.get("current"), dict) else {}
+    for key in ("max_so_far", "max_temp_so_far", "temp"):
+        value = _safe_float(current.get(key))
+        if value is not None:
+            return value
+    return None
+
+
+def _city_cache_payload_to_weather_data(city_name: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+    city_key = str(city_name or "").strip().lower()
+    meta = CITY_REGISTRY.get(city_key) or {}
+    offset = int(meta.get("tz_offset") or 0)
+    local_date = str(payload.get("local_date") or _today_for_city(city_key)).strip()
+    current = payload.get("current") if isinstance(payload.get("current"), dict) else {}
+    airport_current = (
+        payload.get("airport_current")
+        if isinstance(payload.get("airport_current"), dict)
+        else current
+    )
+    current_temp = _safe_float(current.get("temp"))
+    if current_temp is None:
+        current_temp = _safe_float(airport_current.get("temp"))
+    max_so_far = _safe_float(current.get("max_so_far"))
+    if max_so_far is None:
+        max_so_far = _safe_float(current.get("max_temp_so_far"))
+    if max_so_far is None:
+        max_so_far = _safe_float(airport_current.get("max_so_far"))
+    if max_so_far is None:
+        max_so_far = _safe_float(airport_current.get("max_temp_so_far"))
+
+    forecast_high = _first_forecast_high(payload, city_key)
+    dates = [local_date] if local_date else []
+    highs = [forecast_high] if forecast_high is not None else []
+    observed_at = str(
+        current.get("observed_at")
+        or current.get("observation_time")
+        or airport_current.get("observation_time")
+        or airport_current.get("obs_time")
+        or ""
+    ).strip()
+    metar_current = {
+        "temp": current_temp,
+        "max_temp_so_far": max_so_far,
+        "max_temp_time": current.get("max_time") or current.get("max_temp_time"),
+        "wind_speed_kt": current.get("wind_speed_kt") or airport_current.get("wind_speed_kt"),
+        "wind_dir": current.get("wind_dir") or airport_current.get("wind_dir"),
+        "visibility_mi": current.get("visibility_mi") or airport_current.get("visibility_mi"),
+        "humidity": current.get("humidity") or airport_current.get("humidity"),
+        "clouds": current.get("clouds") or airport_current.get("clouds") or [],
+        "wx_desc": current.get("wx_desc") or airport_current.get("wx_desc"),
+    }
+    weather_data = {
+        "open-meteo": {
+            "utc_offset": offset,
+            "current": {"local_time": payload.get("local_time")},
+            "daily": {
+                "time": dates,
+                "temperature_2m_max": highs,
+            },
+        },
+        "metar": {
+            "observation_time": observed_at,
+            "current": metar_current,
+        },
+        "settlement_current": {
+            "observation_time": observed_at,
+            "current": dict(metar_current),
+        },
+        "amos": payload.get("amos") if isinstance(payload.get("amos"), dict) else {},
+        "deb": payload.get("deb") if isinstance(payload.get("deb"), dict) else {},
+        "multi_model": payload.get("multi_model") if isinstance(payload.get("multi_model"), dict) else {},
+    }
+    return weather_data
 
 
 class CityAnalysisService:
     """City analysis adapter with lazy imports to trim cold startup."""
 
-    def __init__(self, weather: Any):
+    def __init__(self, weather: Any, cache_db: Optional[Any] = None):
         self.weather = weather
+        self.cache_db = cache_db or DBManager()
 
     def resolve_city(self, city_input: str):
         from src.analysis.city_query_service import resolve_city_name
@@ -19,12 +142,11 @@ class CityAnalysisService:
         if not coords:
             raise ValueError(f"未找到城市坐标: {city_name}")
 
-        weather_data = self.weather.fetch_all_sources(
-            city_name,
-            lat=coords["lat"],
-            lon=coords["lon"],
-            force_refresh=True,
-        )
+        payload = self._load_cached_city_payload(city_name)
+        if not payload:
+            self._enqueue_refresh(city_name)
+            raise RuntimeError("城市天气缓存正在初始化，请稍后重试")
+        weather_data = _city_cache_payload_to_weather_data(city_name, payload)
 
         from src.analysis.city_query_service import build_city_query_report
 
@@ -34,3 +156,36 @@ class CityAnalysisService:
             city_query_cost=city_query_cost,
         )
 
+    def _load_cached_city_payload(self, city_name: str) -> Optional[Dict[str, Any]]:
+        city_key = str(city_name or "").strip().lower()
+        getter = getattr(self.cache_db, "get_city_cache", None)
+        if callable(getter):
+            for kind in ("panel", "full", "summary"):
+                entry = getter(kind, city_key)
+                payload = entry.get("payload") if isinstance(entry, dict) else None
+                if isinstance(payload, dict):
+                    return payload
+
+        canonical_getter = getattr(self.cache_db, "get_canonical_temperature", None)
+        canonical_entry = canonical_getter(city_key) if callable(canonical_getter) else None
+        canonical = (
+            canonical_entry.get("payload")
+            if isinstance(canonical_entry, dict) and isinstance(canonical_entry.get("payload"), dict)
+            else canonical_entry
+        )
+        return (
+            build_city_weather_from_canonical(city_key, canonical)
+            if isinstance(canonical, dict)
+            else None
+        )
+
+    def _enqueue_refresh(self, city_name: str) -> None:
+        enqueue = getattr(self.cache_db, "enqueue_observation_refresh_request", None)
+        if not callable(enqueue):
+            return
+        enqueue(
+            city=str(city_name or "").strip().lower(),
+            kind="panel",
+            priority="high",
+            reason="bot_city_report",
+        )

src/utils/daily_weather_report.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import os
-import re
 import threading
 import time
 from datetime import datetime
@@ -20,9 +19,11 @@ except Exception:
 
     ZoneInfo = None  # type: ignore[assignment]
 
+from src.database.db_manager import DBManager
 from src.data_collection.city_registry import CITY_REGISTRY
 from src.data_collection.weather_sources import WeatherDataCollector
 from src.utils.telegram_i18n import copy_text, normalize_push_language, telegram_push_language
+from web.services.canonical_temperature import build_city_weather_from_canonical
 
 TARGET_CITIES: List[str] = [
     "beijing",
@@ -46,18 +47,7 @@ CITY_NAME_ZH: Dict[str, str] = {
     "qingdao": "青岛",
 }
 
-# weather.com.cn city codes
-CMA_CITY_CODES: Dict[str, str] = {
-    "beijing": "101010100",
-    "shanghai": "101020100",
-    "guangzhou": "101280101",
-    "chengdu": "101270101",
-    "chongqing": "101040100",
-    "wuhan": "101200101",
-    "qingdao": "101120201",
-}
-
-_CMA_FORECAST_URL = "http://www.weather.com.cn/weather/{code}.shtml"
+_DAILY_REPORT_DB = DBManager()
 
 
 def _env_bool(name: str, default: bool) -> bool:
@@ -82,85 +72,119 @@ def _daily_report_language() -> str:
     )
 
 
-def _fetch_cma_forecast(city_key: str) -> Optional[Dict[str, Any]]:
-    """Scrape today's forecast from weather.com.cn (CMA)."""
-    code = CMA_CITY_CODES.get(city_key)
-    if not code:
-        return None
-
-    url = _CMA_FORECAST_URL.format(code=code)
+def _safe_float(value: Any) -> Optional[float]:
     try:
-        resp = httpx.get(
-            url,
-            headers={
-                "User-Agent": (
-                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                    "AppleWebKit/537.36 (KHTML, like Gecko) "
-                    "Chrome/120.0.0.0 Safari/537.36"
-                ),
-            },
-            timeout=httpx.Timeout(timeout=10.0, connect=5.0, read=10.0),
-            follow_redirects=True,
+        if value is None or value == "":
+            return None
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _enqueue_daily_report_refresh(city_key: str) -> None:
+    enqueue = getattr(_DAILY_REPORT_DB, "enqueue_observation_refresh_request", None)
+    if not callable(enqueue):
+        return
+    try:
+        enqueue(
+            city=city_key,
+            kind="panel",
+            priority="normal",
+            reason="daily_weather_report",
         )
-        resp.raise_for_status()
-        html = resp.text
     except Exception as exc:
-        logger.warning(
-            "daily_weather_report: CMA fetch failed for {}: {}", city_key, exc
+        logger.debug("daily_weather_report: refresh enqueue skipped city={}: {}", city_key, exc)
+
+
+def _load_cached_city_payload(city_key: str) -> Optional[Dict[str, Any]]:
+    getter = getattr(_DAILY_REPORT_DB, "get_city_cache", None)
+    if callable(getter):
+        for kind in ("panel", "full", "summary"):
+            try:
+                entry = getter(kind, city_key)
+            except Exception as exc:
+                logger.debug(
+                    "daily_weather_report: city cache read skipped city={} kind={}: {}",
+                    city_key,
+                    kind,
+                    exc,
+                )
+                continue
+            payload = entry.get("payload") if isinstance(entry, dict) else None
+            if isinstance(payload, dict):
+                return payload
+
+    canonical_getter = getattr(_DAILY_REPORT_DB, "get_canonical_temperature", None)
+    if callable(canonical_getter):
+        try:
+            canonical_entry = canonical_getter(city_key)
+        except Exception as exc:
+            logger.debug("daily_weather_report: canonical read skipped city={}: {}", city_key, exc)
+            canonical_entry = None
+        canonical = (
+            canonical_entry.get("payload")
+            if isinstance(canonical_entry, dict) and isinstance(canonical_entry.get("payload"), dict)
+            else canonical_entry
         )
-        return None
+        payload = (
+            build_city_weather_from_canonical(city_key, canonical)
+            if isinstance(canonical, dict)
+            else None
+        )
+        if payload:
+            return payload
 
-    # Parse today's weather block from the 7-day forecast page.
-    # The HTML structure has entries like:
-    #   <p class="wea">晴转多云</p>
-    #   <p class="tem"><span>25℃</span> / <i>19℃</i></p>
-    # We target the first occurrence (today).
-
-    weather = _extract_first(html, r'<p[^>]*class="wea"[^>]*>([^<]+)</p>')
-    tem_text = _extract_first(html, r'<p[^>]*class="tem"[^>]*>(.+?)</p>')
-
-    high_str: Optional[str] = None
-    low_str: Optional[str] = None
-
-    if tem_text:
-        # Strip all HTML tags, then find numbers followed by degree
-        clean = re.sub(r"<[^>]+>", " ", tem_text)
-        nums = re.findall(r"(-?\d+)\s*(?:℃|°C|°c|°)?", clean)
-        if len(nums) >= 1:
-            high_str = nums[0]
-        if len(nums) >= 2:
-            low_str = nums[1]
-
-    if not weather and not high_str:
-        return None
-
-    result: Dict[str, Any] = {"source": "cma"}
-    if weather:
-        result["weather"] = weather.strip()
-    if high_str:
-        try:
-            result["forecast_high"] = float(high_str)
-        except (TypeError, ValueError):
-            result["forecast_high"] = None
-    if low_str:
-        try:
-            result["forecast_low"] = float(low_str)
-        except (TypeError, ValueError):
-            result["forecast_low"] = None
-
-    logger.info(
-        "daily_weather_report: CMA parsed {} weather={} high={} low={}",
-        city_key,
-        result.get("weather"),
-        result.get("forecast_high"),
-        result.get("forecast_low"),
-    )
-    return result
+    _enqueue_daily_report_refresh(city_key)
+    return None
 
 
-def _extract_first(html: str, pattern: str) -> Optional[str]:
-    m = re.search(pattern, html, re.IGNORECASE | re.DOTALL)
-    return m.group(1) if m else None
+def _daily_report_forecast_high(payload: Dict[str, Any]) -> Optional[float]:
+    deb = payload.get("deb") if isinstance(payload.get("deb"), dict) else {}
+    value = _safe_float(deb.get("prediction"))
+    if value is not None:
+        return value
+
+    local_date = str(payload.get("local_date") or "").strip()
+    multi_model_daily = payload.get("multi_model_daily")
+    if isinstance(multi_model_daily, dict):
+        dates = [local_date] if local_date in multi_model_daily else sorted(multi_model_daily.keys())
+        for date_key in dates:
+            entry = multi_model_daily.get(date_key)
+            if not isinstance(entry, dict):
+                continue
+            daily_deb = entry.get("deb") if isinstance(entry.get("deb"), dict) else {}
+            value = _safe_float(daily_deb.get("prediction"))
+            if value is not None:
+                return value
+            models = entry.get("models") if isinstance(entry.get("models"), dict) else {}
+            values = sorted(
+                value
+                for value in (_safe_float(item) for item in models.values())
+                if value is not None
+            )
+            if values:
+                return values[len(values) // 2]
+
+    current = payload.get("current") if isinstance(payload.get("current"), dict) else {}
+    for key in ("max_so_far", "max_temp_so_far", "temp"):
+        value = _safe_float(current.get(key))
+        if value is not None:
+            return value
+    return None
+
+
+def _daily_report_weather_text(payload: Dict[str, Any]) -> str:
+    current = payload.get("current") if isinstance(payload.get("current"), dict) else {}
+    for value in (
+        payload.get("weather"),
+        payload.get("weather_desc"),
+        current.get("wx_desc"),
+        current.get("weather"),
+    ):
+        text = str(value or "").strip()
+        if text:
+            return text
+    return "?"
 
 
 def _fetch_city_data(
@@ -170,82 +194,32 @@ def _fetch_city_data(
     info = CITY_REGISTRY.get(city_key)
     name_en = str((info or {}).get("name") or city_key.title())
 
-    # 1. Try CMA first for weather description
-    cma = _fetch_cma_forecast(city_key)
-
-    # 2. Get Open-Meteo data for temperature fallback
-    om_high = None
-    if info:
-        try:
-            results = collector.fetch_all_sources(
-                city_key,
-                lat=info["lat"],
-                lon=info["lon"],
-                include_taf=False,
-                include_ensemble=False,
-                include_multi_model=False,
-            )
-            if isinstance(results, dict):
-                om = results.get("open-meteo", {})
-                daily = om.get("daily", {}) if isinstance(om, dict) else {}
-                daily_highs = daily.get("temperature_2m_max", []) or []
-                om_high = daily_highs[0] if daily_highs else None
-        except Exception as exc:
-            logger.warning(
-                "daily_weather_report: OM fetch failed for {}: {}", city_key, exc
-            )
-
-    # Use CMA weather + CMA high, fall back to OM high
-    weather: Optional[str] = None
-    forecast_high: Optional[float] = None
-
-    if cma:
-        weather = cma.get("weather")
-        forecast_high = cma.get("forecast_high")
-    if forecast_high is None:
-        forecast_high = om_high
-
-    if not weather and not forecast_high:
+    payload = _load_cached_city_payload(city_key)
+    if not payload:
         return None
 
+    forecast_high = _daily_report_forecast_high(payload)
+    if forecast_high is None:
+        _enqueue_daily_report_refresh(city_key)
+        return None
+    weather = _daily_report_weather_text(payload)
+
     logger.info(
-        "daily_weather_report: {} weather={} high={} (cma_ok={})",
+        "daily_weather_report: {} weather={} high={} cache_only=true",
         city_key,
-        weather or "?",
+        weather,
         forecast_high,
-        bool(cma and cma.get("weather")),
     )
 
     return {
         "city": city_key,
         "name": name,
         "name_en": name_en,
-        "weather": weather or "?",
+        "weather": weather,
         "forecast_high": forecast_high,
     }
 
 
-def _wmo_to_weather(code: Any) -> str:
-    """Translate WMO weather code to Chinese (fallback only)."""
-    try:
-        c = int(code or 0)
-    except (TypeError, ValueError):
-        return "未知"
-    if c == 0:
-        return "晴"
-    if 1 <= c <= 3:
-        return "多云"
-    if c in (45, 48):
-        return "雾"
-    if 51 <= c <= 67:
-        return "雨"
-    if 71 <= c <= 86:
-        return "雪"
-    if 95 <= c <= 99:
-        return "雷暴"
-    return "阴"
-
-
 def _build_ai_prompt(
     cities_data: List[Dict[str, Any]],
     report_date: str,