Aurora→Denver 重命名 + 接入 MADIS HFMETAR 5 分钟数据源

frontend/lib/dashboard-official-sources.ts

@@ -258,7 +258,7 @@ const CITY_SPECIFIC_SOURCES: Record<string, OfficialSourceLink[]> = {
       kind: "metar",
     },
   ],
-  aurora: [
+  denver: [
     {
       label: "NWS Denver/Boulder",
       href: "https://www.weather.gov/bou/",

frontend/lib/scan-market-focus.ts

@@ -60,7 +60,6 @@ const CITY_REGION_FALLBACK: Record<string, MarketRegionKey> = {
   toronto: "americas",
   "los angeles": "americas",
   "san francisco": "americas",
-  aurora: "americas",
   denver: "americas",
   austin: "americas",
   houston: "americas",

src/data_collection/city_registry.py

@@ -383,7 +383,7 @@ CITY_REGISTRY = {
         "distance_km": 20.0,
         "warning": "海湾冷空气和平流雾常压制机场升温，SFO 与市区体感差异可快速放大。",
     },
-    "aurora": {
+    "denver": {
         "name": "Denver",
         "display_name": "Denver",
         "lat": 39.7017,
@@ -825,7 +825,7 @@ ALIASES = {
     "nyc": "new york", "ny": "new york", "chi": "chicago", 
     "la": "los angeles", "lax": "los angeles", "losangeles": "los angeles",
     "sf": "san francisco", "sfo": "san francisco", "sanfrancisco": "san francisco",
-    "aur": "aurora", "denver": "aurora", "buckley": "aurora", "kbkf": "aurora",
+    "aur": "denver", "denver": "denver", "buckley": "denver", "kbkf": "denver", "aurora": "denver",
     "aus": "austin", "kaus": "austin",
     "hou": "houston", "khou": "houston", "hobby": "houston",
     "cdmx": "mexico city", "mexicocity": "mexico city", "mmmx": "mexico city",
@@ -862,8 +862,8 @@ ALIASES = {
     "纽约": "new york",
     "洛杉矶": "los angeles",
     "旧金山": "san francisco",
-    "奥罗拉": "aurora",
+    "奥罗拉": "denver",
-    "丹佛": "aurora",
+    "丹佛": "denver",
     "奥斯汀": "austin",
     "休斯顿": "houston",
     "墨西哥城": "mexico city",

src/data_collection/city_time.py

@@ -15,7 +15,7 @@ CITY_TIME_ZONES = {
     "amsterdam": "Europe/Amsterdam",
     "ankara": "Europe/Istanbul",
     "atlanta": "America/New_York",
-    "aurora": "America/Denver",
+    "denver": "America/Denver",
     "austin": "America/Chicago",
     "beijing": "Asia/Shanghai",
     "buenos aires": "America/Argentina/Buenos_Aires",

src/data_collection/madis_sources.py

@@ -0,0 +1,195 @@
+"""MADIS HFMETAR 5-minute real-time data source.
+
+Fetches NetCDF files from NOAA MADIS public archive.
+HFMETAR data updates every 5 minutes (12 values/hour/station).
+Public anonymous access — no API key required.
+
+URL: https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/
+"""
+
+from __future__ import annotations
+
+import gzip
+import time
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional, List
+
+from loguru import logger
+
+from src.utils.metrics import record_source_call
+
+MADIS_HFMETAR_URL = (
+    "https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/"
+)
+
+
+class MadisSourceMixin:
+    """Mixin that adds MADIS HFMETAR 5-min data to WeatherDataCollector."""
+
+    def _madis_http_get(self, url: str) -> bytes:
+        getter = getattr(self, "_http_get", None)
+        if callable(getter):
+            resp = getter(url)
+            return resp.content if hasattr(resp, "content") else resp
+        resp = self.session.get(url, timeout=self.timeout)
+        resp.raise_for_status()
+        return resp.content
+
+    def _madis_latest_file(self) -> Optional[str]:
+        """Find the most recent HFMETAR NetCDF file."""
+        try:
+            # The MADIS HFMETAR directory listing returns HTML
+            html = self._madis_http_get(MADIS_HFMETAR_URL).decode("utf-8", errors="replace")
+            import re
+            # Match file names like 20260514_1200.gz or 202605141200.gz
+            matches = re.findall(r'(\d{8}[_ ]?\d{4})\.gz', html)
+            if not matches:
+                logger.warning("MADIS: no HFMETAR files found in directory listing")
+                return None
+            # Sort by timestamp descending
+            matches.sort(reverse=True)
+            latest = matches[0].replace(" ", "_")
+            return f"{latest}.gz"
+        except Exception as exc:
+            logger.warning("MADIS: failed to list HFMETAR files: {}", exc)
+            return None
+
+    def _madis_parse_hfmetar(
+        self, nc_bytes: bytes, fname: str
+    ) -> List[Dict[str, Any]]:
+        """Parse a MADIS HFMETAR NetCDF file and return per-station observations."""
+        try:
+            from netCDF4 import Dataset
+            nc = Dataset(fname, memory=nc_bytes)
+        except ImportError:
+            logger.error("netCDF4 not installed; MADIS data unavailable")
+            return []
+        except Exception as exc:
+            logger.warning("MADIS: netCDF4 open failed: {}", exc)
+            return []
+
+        results: List[Dict[str, Any]] = []
+        try:
+            # MADIS HFMETAR uses these variable names
+            icaos = [str(s).strip() for s in nc.variables.get("icaoId", [])[:]]
+            temps = nc.variables.get("temperature", None)  # in Celsius
+            dewpts = nc.variables.get("dewpoint", None)
+            winds = nc.variables.get("windSpeed", None)
+            pressures = nc.variables.get("seaLevelPress", None)
+            obs_times = nc.variables.get("observationTime", None)
+
+            n = len(icaos)
+            for i in range(n):
+                icao = str(icaos[i]).strip().upper()
+                if not icao or icao == "0":
+                    continue
+
+                temp_c = None
+                if temps is not None:
+                    try:
+                        v = float(temps[i])
+                        if -90 < v < 60:
+                            temp_c = round(v, 1)
+                    except (ValueError, IndexError):
+                        pass
+                if temp_c is None:
+                    continue  # skip stations without temperature
+
+                dewp_c = None
+                if dewpts is not None:
+                    try:
+                        v = float(dewpts[i])
+                        if -90 < v < 60:
+                            dewp_c = round(v, 1)
+                    except (ValueError, IndexError):
+                        pass
+
+                wind_kt = None
+                if winds is not None:
+                    try:
+                        w = float(winds[i])
+                        if w >= 0:
+                            wind_kt = round(w * 1.94384, 1)
+                    except (ValueError, IndexError):
+                        pass
+
+                pressure_hpa = None
+                if pressures is not None:
+                    try:
+                        p = float(pressures[i])
+                        if 800 < p < 1100:
+                            pressure_hpa = round(p, 1)
+                    except (ValueError, IndexError):
+                        pass
+
+                obs_time = ""
+                if obs_times is not None:
+                    try:
+                        ts = str(obs_times[i]).strip()
+                        # Try to parse as epoch or ISO
+                        if ts and ts != "0":
+                            try:
+                                epoch = int(float(ts))
+                                dt = datetime.fromtimestamp(epoch, tz=timezone.utc)
+                                obs_time = dt.isoformat()
+                            except (ValueError, OverflowError):
+                                obs_time = ts
+                    except (ValueError, IndexError):
+                        pass
+
+                results.append({
+                    "icao": icao,
+                    "temp_c": temp_c,
+                    "dewp_c": dewp_c,
+                    "wind_kt": wind_kt,
+                    "pressure_hpa": pressure_hpa,
+                    "obs_time": obs_time,
+                    "source": "madis_hfmetar",
+                })
+
+        except Exception as exc:
+            logger.warning("MADIS: HFMETAR parse error: {}", exc)
+        finally:
+            try:
+                nc.close()
+            except Exception:
+                pass
+
+        return results
+
+    def fetch_madis_hfmetar(self) -> List[Dict[str, Any]]:
+        """Fetch latest MADIS HFMETAR data and return parsed observations."""
+        started = time.perf_counter()
+
+        fname = self._madis_latest_file()
+        if not fname:
+            record_source_call("madis_hfmetar", "current", "no_file",
+                               (time.perf_counter() - started) * 1000.0)
+            return []
+
+        try:
+            url = f"{MADIS_HFMETAR_URL}{fname}"
+            raw = self._madis_http_get(url)
+
+            # Decompress gzip
+            try:
+                nc_bytes = gzip.decompress(raw)
+            except gzip.BadGzipFile:
+                # Maybe not compressed
+                nc_bytes = raw
+
+            results = self._madis_parse_hfmetar(nc_bytes, fname)
+            if results:
+                logger.info("MADIS HFMETAR: {} stations from {}", len(results), fname)
+                record_source_call("madis_hfmetar", "current", "success",
+                                   (time.perf_counter() - started) * 1000.0)
+            else:
+                record_source_call("madis_hfmetar", "current", "empty",
+                                   (time.perf_counter() - started) * 1000.0)
+            return results
+
+        except Exception as exc:
+            logger.warning("MADIS HFMETAR fetch failed: {}", exc)
+            record_source_call("madis_hfmetar", "current", "error",
+                               (time.perf_counter() - started) * 1000.0)
+            return []

src/data_collection/weather_sources.py

@@ -3,7 +3,7 @@ import httpx
 import re
 import threading
 import time
-from typing import Optional, Dict, List
+from typing import Any, Optional, Dict, List
 from datetime import datetime, timedelta
 from loguru import logger
 from src.data_collection.open_meteo_cache import OpenMeteoCacheMixin
@@ -18,10 +18,11 @@ from src.data_collection.amos_station_sources import AmosStationSourceMixin
 from src.data_collection.fmi_sources import FmiSourceMixin
 from src.data_collection.knmi_sources import KnmiSourceMixin
 from src.data_collection.hko_obs_sources import HkoObsSourceMixin
+from src.data_collection.madis_sources import MadisSourceMixin
 from src.database.db_manager import DBManager
 
 
-class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NmcSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin):
+class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NmcSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin):
     """
     Multi-source weather data collector
 
@@ -49,7 +50,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
         "new york": ["KLGA", "KJFK", "KEWR", "KTEB", "KHPN"],
         "los angeles": ["KLAX", "KBUR", "KLGB", "KSNA", "KVNY"],
         "san francisco": ["KSFO", "KOAK", "KSJC", "KHAF"],
-        "aurora": ["KBKF", "KDEN", "KAPA", "KBJC"],
+        "denver": ["KBKF", "KDEN", "KAPA", "KBJC"],
         "austin": ["KAUS", "KEDC", "KSAT"],
         "houston": ["KHOU", "KIAH", "KSGR", "KCXO"],
         "mexico city": ["MMMX", "MMSM", "MMTO"],
@@ -107,7 +108,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
         "new york's central park",
         "portland",
         "denver",
-        "aurora",
         "austin",
         "san diego",
         "detroit",
@@ -238,6 +238,12 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
             os.getenv("HKO_OBS_CACHE_TTL_SEC", "60")
         )
         self._hko_obs_cache: Dict[str, Dict] = {}
+        self.madis_cache_ttl_sec = int(
+            os.getenv("MADIS_CACHE_TTL_SEC", "300")  # 5 min match update rate
+        )
+        self._madis_cache: Optional[List[Dict[str, Any]]] = None
+        self._madis_cache_ts: float = 0.0
+        self._madis_cache_lock = threading.Lock()
         self._hko_obs_cache_lock = threading.Lock()
         self.cwa_open_data_auth = (
             os.getenv("CWA_OPEN_DATA_AUTH")
@@ -1052,6 +1058,47 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
         except Exception as exc:
             logger.warning("AMOS attach failed city={}: {}", city_lower, exc)
 
+    def _attach_madis_hfmetar_data(
+        self, results: Dict, city_lower: str
+    ) -> None:
+        """Fetch MADIS HFMETAR data and attach matching US station observations."""
+        # Only run for US cities (ICAO starts with K)
+        city_meta = self.CITY_REGISTRY.get(city_lower) or {}
+        icao = str(city_meta.get("icao") or "").strip().upper()
+        if not icao.startswith("K"):
+            return
+
+        now_ts = time.time()
+        with self._madis_cache_lock:
+            if (self._madis_cache is not None
+                    and now_ts - self._madis_cache_ts < self.madis_cache_ttl_sec):
+                obs_list = self._madis_cache
+            else:
+                obs_list = self.fetch_madis_hfmetar()
+                self._madis_cache = obs_list
+                self._madis_cache_ts = now_ts
+
+        if not obs_list:
+            return
+
+        # Find this city's station in the MADIS results
+        for obs in obs_list:
+            if obs["icao"] == icao:
+                try:
+                    DBManager().append_airport_obs(
+                        icao=icao,
+                        city=city_lower,
+                        temp_c=obs["temp_c"],
+                        wind_kt=obs.get("wind_kt"),
+                        pressure_hpa=obs.get("pressure_hpa"),
+                        obs_time=obs["obs_time"] or datetime.now().isoformat(),
+                    )
+                except Exception:
+                    logger.exception(
+                        "airport_obs_log append failed for madis city={}", city_lower
+                    )
+                break
+
     def _attach_russia_official_nearby(
         self, results: Dict, city_lower: str, use_fahrenheit: bool
     ) -> None:
@@ -1177,6 +1224,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
                     include_nearby=include_nearby,
                 )
                 self._attach_korean_amos_data(results, city_lower, use_fahrenheit)
+                self._attach_madis_hfmetar_data(results, city_lower)
                 if include_nearby:
                     self._attach_china_official_nearby(results, city_lower, use_fahrenheit)
                     self._attach_japan_official_nearby(results, city_lower, use_fahrenheit)
@@ -1222,6 +1270,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
                     include_nearby=include_nearby,
                 )
                 self._attach_korean_amos_data(results, city_lower, use_fahrenheit)
+                self._attach_madis_hfmetar_data(results, city_lower)
                 if include_nearby:
                     self._attach_china_official_nearby(results, city_lower, use_fahrenheit)
                     self._attach_japan_official_nearby(results, city_lower, use_fahrenheit)