Add Wunderground as Shenzhen settlement source

src/data_collection/city_registry.py

@@ -391,6 +391,9 @@ CITY_REGISTRY = {
         "lat": 22.6393,
         "lon": 113.8107,
         "icao": "ZGSZ",
+        "settlement_source": "wunderground",
+        "settlement_url": "https://www.wunderground.com/weather/cn/shenzhen/ZGSZ",
+        "settlement_station_label": "Shenzhen Bao'an Intl Airport Station",
         "tz_offset": 28800,
         "use_fahrenheit": False,
         "is_major": True,

src/data_collection/settlement_sources.py

@@ -183,6 +183,49 @@ class SettlementSourceMixin:
             )
             return self._sort_temp_points(points)
 
+    def _update_official_today_obs(
+        self,
+        *,
+        source_code: str,
+        station_code: str,
+        obs_iso: Optional[str],
+        current_temp: Optional[float],
+        utc_offset_seconds: int,
+    ) -> List[Dict[str, Any]]:
+        if not obs_iso or current_temp is None:
+            return []
+
+        try:
+            obs_dt = datetime.fromisoformat(str(obs_iso).replace("Z", "+00:00"))
+        except Exception:
+            return []
+        if obs_dt.tzinfo is None:
+            obs_dt = obs_dt.replace(tzinfo=timezone.utc)
+        local_tz = timezone(timedelta(seconds=int(utc_offset_seconds or 0)))
+        local_dt = obs_dt.astimezone(local_tz)
+        date_str = local_dt.strftime("%Y-%m-%d")
+        time_str = local_dt.strftime("%H:%M")
+        mode = get_state_storage_mode()
+        if mode not in {STATE_STORAGE_DUAL, STATE_STORAGE_SQLITE}:
+            return [{"time": time_str, "temp": round(float(current_temp), 1)}]
+
+        lock = self._get_settlement_series_lock()
+        with lock:
+            _official_intraday_repo.upsert_point(
+                source_code=source_code,
+                station_code=station_code,
+                target_date=date_str,
+                observation_time=time_str,
+                value=round(float(current_temp), 1),
+                payload={"time": time_str, "temp": round(float(current_temp), 1)},
+            )
+            points = _official_intraday_repo.load_points(
+                source_code=source_code,
+                station_code=station_code,
+                target_date=date_str,
+            )
+            return self._sort_temp_points(points)
+
     def fetch_hko_settlement_current(self) -> Optional[Dict[str, Any]]:
         cache_key = "hko:hong_kong"
         cached = self._get_settlement_cache(cache_key)
@@ -537,6 +580,22 @@ class SettlementSourceMixin:
 
     def fetch_settlement_current(self, city: str) -> Optional[Dict[str, Any]]:
         normalized = str(city or "").strip().lower()
+        try:
+            from src.data_collection.city_registry import CITY_REGISTRY
+
+            city_meta = CITY_REGISTRY.get(normalized) or {}
+            settlement_source = str(city_meta.get("settlement_source") or "").strip().lower()
+            if settlement_source == "wunderground":
+                settlement_url = str(city_meta.get("settlement_url") or "").strip()
+                if settlement_url:
+                    return self.fetch_wunderground_settlement_current(
+                        normalized,
+                        url=settlement_url,
+                        station_label=str(city_meta.get("settlement_station_label") or "").strip() or None,
+                        icao=str(city_meta.get("icao") or "").strip() or None,
+                    )
+        except Exception as exc:
+            logger.warning(f"Wunderground settlement dispatch failed city={city}: {exc}")
         if normalized == "hong kong":
             return self.fetch_hko_settlement_current()
         if normalized == "taipei":

src/data_collection/weather_sources.py

@@ -10,9 +10,10 @@ from src.data_collection.settlement_sources import SettlementSourceMixin
 from src.data_collection.metar_sources import MetarSourceMixin
 from src.data_collection.mgm_sources import MgmSourceMixin
 from src.data_collection.nws_open_meteo_sources import NwsOpenMeteoSourceMixin
+from src.data_collection.wunderground_sources import WundergroundSourceMixin
 
 
-class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, NwsOpenMeteoSourceMixin):
+class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, NwsOpenMeteoSourceMixin, WundergroundSourceMixin):
     """
     Multi-source weather data collector
 

src/data_collection/wunderground_sources.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from loguru import logger
+from src.data_collection.city_registry import CITY_REGISTRY
+
+
+class WundergroundSourceMixin:
+    _WU_PAGE_TTL_SEC = 180
+
+    def _fetch_wunderground_page(self, url: str) -> Optional[str]:
+        cache_key = f"wu:page:{url}"
+        cached = self._get_settlement_cache(cache_key)
+        if isinstance(cached, dict):
+            html = str(cached.get("html") or "")
+            if html:
+                return html
+
+        try:
+            response = self.session.get(
+                url,
+                headers={
+                    "User-Agent": "Mozilla/5.0",
+                    "Referer": url,
+                },
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            html = str(response.text or "")
+            if not html:
+                return None
+            ttl_backup = getattr(self, "settlement_cache_ttl_sec", self._WU_PAGE_TTL_SEC)
+            try:
+                self.settlement_cache_ttl_sec = self._WU_PAGE_TTL_SEC
+                self._set_settlement_cache(cache_key, {"html": html})
+            finally:
+                self.settlement_cache_ttl_sec = ttl_backup
+            return html
+        except Exception as exc:
+            logger.warning(f"Wunderground page fetch failed url={url}: {exc}")
+            return None
+
+    @staticmethod
+    def _wu_extract_station_name(html: str, fallback_icao: str) -> Optional[str]:
+        pattern = re.compile(
+            r'</lib-display-unit>\s*([^<]+?)\s*</a>',
+            re.IGNORECASE,
+        )
+        for match in pattern.finditer(html):
+            candidate = re.sub(r"\s+", " ", str(match.group(1) or "")).strip()
+            if fallback_icao.lower() in candidate.lower() or "station" in candidate.lower():
+                return candidate
+        return None
+
+    @staticmethod
+    def _wu_extract_station_temperature(
+        html: str,
+        *,
+        station_name: Optional[str],
+    ) -> tuple[Optional[float], Optional[str]]:
+        station_anchor = station_name or "Station"
+        station_pos = html.find(station_anchor)
+        if station_pos < 0:
+            station_pos = html.lower().find("station-name")
+        if station_pos < 0:
+            return None, None
+
+        window_start = max(0, station_pos - 1800)
+        window = html[window_start:station_pos]
+        temp_match = re.search(
+            r'wu-value[^>]*>\s*(-?\d+(?:\.\d+)?)\s*</span>.*?<span[^>]*>\s*([CF])\s*</span>',
+            window,
+            re.IGNORECASE | re.DOTALL,
+        )
+        if not temp_match:
+            return None, None
+
+        try:
+            value = float(temp_match.group(1))
+        except Exception:
+            return None, None
+        unit = str(temp_match.group(2) or "").upper().strip() or None
+        return value, unit
+
+    @staticmethod
+    def _wu_to_celsius(value: Optional[float], unit: Optional[str]) -> Optional[float]:
+        if value is None:
+            return None
+        normalized = str(unit or "").upper().strip()
+        if normalized == "F":
+            return round((float(value) - 32.0) * 5.0 / 9.0, 1)
+        return round(float(value), 1)
+
+    def fetch_wunderground_settlement_current(
+        self,
+        city: str,
+        *,
+        url: str,
+        station_label: Optional[str] = None,
+        icao: Optional[str] = None,
+    ) -> Optional[Dict[str, Any]]:
+        normalized_city = str(city or "").strip().lower()
+        cache_key = f"wu:settlement:{normalized_city}"
+        cached = self._get_settlement_cache(cache_key)
+        if cached:
+            return cached
+
+        html = self._fetch_wunderground_page(url)
+        if not html:
+            return None
+
+        fallback_icao = str(icao or "").strip()
+        station_name = station_label or self._wu_extract_station_name(html, fallback_icao)
+        display_temp, display_unit = self._wu_extract_station_temperature(
+            html,
+            station_name=station_name,
+        )
+        temp_c = self._wu_to_celsius(display_temp, display_unit)
+        if temp_c is None:
+            logger.warning(f"Wunderground temperature parse failed city={city} url={url}")
+            return None
+
+        city_meta = CITY_REGISTRY.get(normalized_city) or {}
+        utc_offset_seconds = int(city_meta.get("tz_offset") or 0)
+        obs_iso = datetime.now(timezone.utc).isoformat()
+        today_obs = self._update_official_today_obs(
+            source_code="wunderground",
+            station_code=fallback_icao or normalized_city,
+            obs_iso=obs_iso,
+            current_temp=temp_c,
+            utc_offset_seconds=utc_offset_seconds,
+        )
+        max_so_far = None
+        max_temp_time = None
+        today_low = None
+        if today_obs:
+            hottest = max(today_obs, key=lambda item: float(item.get("temp") or -999))
+            coldest = min(today_obs, key=lambda item: float(item.get("temp") or 999))
+            max_so_far = self._wu_to_celsius(float(hottest.get("temp")), "C")
+            today_low = self._wu_to_celsius(float(coldest.get("temp")), "C")
+            max_temp_time = str(hottest.get("time") or "").strip() or None
+
+        payload: Dict[str, Any] = {
+            "source": "wunderground",
+            "source_label": "Wunderground",
+            "station_code": fallback_icao or None,
+            "station_name": station_name or fallback_icao or str(city or "").title(),
+            "observation_time": obs_iso,
+            "source_url": url,
+            "current": {
+                "temp": temp_c,
+                "display_temp": display_temp,
+                "display_unit": display_unit,
+                "max_temp_so_far": max_so_far,
+                "max_temp_time": max_temp_time,
+                "today_low": today_low,
+                "humidity": None,
+                "wind_speed_kt": None,
+                "wind_dir": None,
+            },
+            "today_obs": today_obs,
+            "unit": "celsius",
+        }
+        self._set_settlement_cache(cache_key, payload)
+        return payload