From 6c4f43fba66f7db6c8e0e37d6f2c369d746c8a5a Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Thu, 14 May 2026 01:09:17 +0800 Subject: [PATCH] =?UTF-8?q?Aurora=E2=86=92Denver=20=E9=87=8D=E5=91=BD?= =?UTF-8?q?=E5=90=8D=20+=20=E6=8E=A5=E5=85=A5=20MADIS=20HFMETAR=205=20?= =?UTF-8?q?=E5=88=86=E9=92=9F=E6=95=B0=E6=8D=AE=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/lib/dashboard-official-sources.ts | 2 +- frontend/lib/scan-market-focus.ts | 1 - src/data_collection/city_registry.py | 8 +- src/data_collection/city_time.py | 2 +- src/data_collection/madis_sources.py | 195 +++++++++++++++++++++ src/data_collection/weather_sources.py | 57 +++++- 6 files changed, 254 insertions(+), 11 deletions(-) create mode 100644 src/data_collection/madis_sources.py diff --git a/frontend/lib/dashboard-official-sources.ts b/frontend/lib/dashboard-official-sources.ts index a429266f..53076966 100644 --- a/frontend/lib/dashboard-official-sources.ts +++ b/frontend/lib/dashboard-official-sources.ts @@ -258,7 +258,7 @@ const CITY_SPECIFIC_SOURCES: Record = { kind: "metar", }, ], - aurora: [ + denver: [ { label: "NWS Denver/Boulder", href: "https://www.weather.gov/bou/", diff --git a/frontend/lib/scan-market-focus.ts b/frontend/lib/scan-market-focus.ts index f6af5bf2..b2db2427 100644 --- a/frontend/lib/scan-market-focus.ts +++ b/frontend/lib/scan-market-focus.ts @@ -60,7 +60,6 @@ const CITY_REGION_FALLBACK: Record = { toronto: "americas", "los angeles": "americas", "san francisco": "americas", - aurora: "americas", denver: "americas", austin: "americas", houston: "americas", diff --git a/src/data_collection/city_registry.py b/src/data_collection/city_registry.py index d2bbb2a3..9b249b2e 100644 --- a/src/data_collection/city_registry.py +++ b/src/data_collection/city_registry.py @@ -383,7 +383,7 @@ CITY_REGISTRY = { "distance_km": 20.0, "warning": "海湾冷空气和平流雾常压制机场升温，SFO 与市区体感差异可快速放大。", }, - "aurora": { + "denver": { "name": "Denver", "display_name": "Denver", "lat": 39.7017, @@ -825,7 +825,7 @@ ALIASES = { "nyc": "new york", "ny": "new york", "chi": "chicago", "la": "los angeles", "lax": "los angeles", "losangeles": "los angeles", "sf": "san francisco", "sfo": "san francisco", "sanfrancisco": "san francisco", - "aur": "aurora", "denver": "aurora", "buckley": "aurora", "kbkf": "aurora", + "aur": "denver", "denver": "denver", "buckley": "denver", "kbkf": "denver", "aurora": "denver", "aus": "austin", "kaus": "austin", "hou": "houston", "khou": "houston", "hobby": "houston", "cdmx": "mexico city", "mexicocity": "mexico city", "mmmx": "mexico city", @@ -862,8 +862,8 @@ ALIASES = { "纽约": "new york", "洛杉矶": "los angeles", "旧金山": "san francisco", - "奥罗拉": "aurora", - "丹佛": "aurora", + "奥罗拉": "denver", + "丹佛": "denver", "奥斯汀": "austin", "休斯顿": "houston", "墨西哥城": "mexico city", diff --git a/src/data_collection/city_time.py b/src/data_collection/city_time.py index 0cbb04f5..ac1a9bc3 100644 --- a/src/data_collection/city_time.py +++ b/src/data_collection/city_time.py @@ -15,7 +15,7 @@ CITY_TIME_ZONES = { "amsterdam": "Europe/Amsterdam", "ankara": "Europe/Istanbul", "atlanta": "America/New_York", - "aurora": "America/Denver", + "denver": "America/Denver", "austin": "America/Chicago", "beijing": "Asia/Shanghai", "buenos aires": "America/Argentina/Buenos_Aires", diff --git a/src/data_collection/madis_sources.py b/src/data_collection/madis_sources.py new file mode 100644 index 00000000..2a93f60d --- /dev/null +++ b/src/data_collection/madis_sources.py @@ -0,0 +1,195 @@ +"""MADIS HFMETAR 5-minute real-time data source. + +Fetches NetCDF files from NOAA MADIS public archive. +HFMETAR data updates every 5 minutes (12 values/hour/station). +Public anonymous access — no API key required. + +URL: https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/ +""" + +from __future__ import annotations + +import gzip +import time +from datetime import datetime, timezone +from typing import Any, Dict, Optional, List + +from loguru import logger + +from src.utils.metrics import record_source_call + +MADIS_HFMETAR_URL = ( + "https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/" +) + + +class MadisSourceMixin: + """Mixin that adds MADIS HFMETAR 5-min data to WeatherDataCollector.""" + + def _madis_http_get(self, url: str) -> bytes: + getter = getattr(self, "_http_get", None) + if callable(getter): + resp = getter(url) + return resp.content if hasattr(resp, "content") else resp + resp = self.session.get(url, timeout=self.timeout) + resp.raise_for_status() + return resp.content + + def _madis_latest_file(self) -> Optional[str]: + """Find the most recent HFMETAR NetCDF file.""" + try: + # The MADIS HFMETAR directory listing returns HTML + html = self._madis_http_get(MADIS_HFMETAR_URL).decode("utf-8", errors="replace") + import re + # Match file names like 20260514_1200.gz or 202605141200.gz + matches = re.findall(r'(\d{8}[_ ]?\d{4})\.gz', html) + if not matches: + logger.warning("MADIS: no HFMETAR files found in directory listing") + return None + # Sort by timestamp descending + matches.sort(reverse=True) + latest = matches[0].replace(" ", "_") + return f"{latest}.gz" + except Exception as exc: + logger.warning("MADIS: failed to list HFMETAR files: {}", exc) + return None + + def _madis_parse_hfmetar( + self, nc_bytes: bytes, fname: str + ) -> List[Dict[str, Any]]: + """Parse a MADIS HFMETAR NetCDF file and return per-station observations.""" + try: + from netCDF4 import Dataset + nc = Dataset(fname, memory=nc_bytes) + except ImportError: + logger.error("netCDF4 not installed; MADIS data unavailable") + return [] + except Exception as exc: + logger.warning("MADIS: netCDF4 open failed: {}", exc) + return [] + + results: List[Dict[str, Any]] = [] + try: + # MADIS HFMETAR uses these variable names + icaos = [str(s).strip() for s in nc.variables.get("icaoId", [])[:]] + temps = nc.variables.get("temperature", None) # in Celsius + dewpts = nc.variables.get("dewpoint", None) + winds = nc.variables.get("windSpeed", None) + pressures = nc.variables.get("seaLevelPress", None) + obs_times = nc.variables.get("observationTime", None) + + n = len(icaos) + for i in range(n): + icao = str(icaos[i]).strip().upper() + if not icao or icao == "0": + continue + + temp_c = None + if temps is not None: + try: + v = float(temps[i]) + if -90 < v < 60: + temp_c = round(v, 1) + except (ValueError, IndexError): + pass + if temp_c is None: + continue # skip stations without temperature + + dewp_c = None + if dewpts is not None: + try: + v = float(dewpts[i]) + if -90 < v < 60: + dewp_c = round(v, 1) + except (ValueError, IndexError): + pass + + wind_kt = None + if winds is not None: + try: + w = float(winds[i]) + if w >= 0: + wind_kt = round(w * 1.94384, 1) + except (ValueError, IndexError): + pass + + pressure_hpa = None + if pressures is not None: + try: + p = float(pressures[i]) + if 800 < p < 1100: + pressure_hpa = round(p, 1) + except (ValueError, IndexError): + pass + + obs_time = "" + if obs_times is not None: + try: + ts = str(obs_times[i]).strip() + # Try to parse as epoch or ISO + if ts and ts != "0": + try: + epoch = int(float(ts)) + dt = datetime.fromtimestamp(epoch, tz=timezone.utc) + obs_time = dt.isoformat() + except (ValueError, OverflowError): + obs_time = ts + except (ValueError, IndexError): + pass + + results.append({ + "icao": icao, + "temp_c": temp_c, + "dewp_c": dewp_c, + "wind_kt": wind_kt, + "pressure_hpa": pressure_hpa, + "obs_time": obs_time, + "source": "madis_hfmetar", + }) + + except Exception as exc: + logger.warning("MADIS: HFMETAR parse error: {}", exc) + finally: + try: + nc.close() + except Exception: + pass + + return results + + def fetch_madis_hfmetar(self) -> List[Dict[str, Any]]: + """Fetch latest MADIS HFMETAR data and return parsed observations.""" + started = time.perf_counter() + + fname = self._madis_latest_file() + if not fname: + record_source_call("madis_hfmetar", "current", "no_file", + (time.perf_counter() - started) * 1000.0) + return [] + + try: + url = f"{MADIS_HFMETAR_URL}{fname}" + raw = self._madis_http_get(url) + + # Decompress gzip + try: + nc_bytes = gzip.decompress(raw) + except gzip.BadGzipFile: + # Maybe not compressed + nc_bytes = raw + + results = self._madis_parse_hfmetar(nc_bytes, fname) + if results: + logger.info("MADIS HFMETAR: {} stations from {}", len(results), fname) + record_source_call("madis_hfmetar", "current", "success", + (time.perf_counter() - started) * 1000.0) + else: + record_source_call("madis_hfmetar", "current", "empty", + (time.perf_counter() - started) * 1000.0) + return results + + except Exception as exc: + logger.warning("MADIS HFMETAR fetch failed: {}", exc) + record_source_call("madis_hfmetar", "current", "error", + (time.perf_counter() - started) * 1000.0) + return [] diff --git a/src/data_collection/weather_sources.py b/src/data_collection/weather_sources.py index 632f1e67..6b5f8751 100644 --- a/src/data_collection/weather_sources.py +++ b/src/data_collection/weather_sources.py @@ -3,7 +3,7 @@ import httpx import re import threading import time -from typing import Optional, Dict, List +from typing import Any, Optional, Dict, List from datetime import datetime, timedelta from loguru import logger from src.data_collection.open_meteo_cache import OpenMeteoCacheMixin @@ -18,10 +18,11 @@ from src.data_collection.amos_station_sources import AmosStationSourceMixin from src.data_collection.fmi_sources import FmiSourceMixin from src.data_collection.knmi_sources import KnmiSourceMixin from src.data_collection.hko_obs_sources import HkoObsSourceMixin +from src.data_collection.madis_sources import MadisSourceMixin from src.database.db_manager import DBManager -class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NmcSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin): +class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NmcSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin): """ Multi-source weather data collector @@ -49,7 +50,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour "new york": ["KLGA", "KJFK", "KEWR", "KTEB", "KHPN"], "los angeles": ["KLAX", "KBUR", "KLGB", "KSNA", "KVNY"], "san francisco": ["KSFO", "KOAK", "KSJC", "KHAF"], - "aurora": ["KBKF", "KDEN", "KAPA", "KBJC"], + "denver": ["KBKF", "KDEN", "KAPA", "KBJC"], "austin": ["KAUS", "KEDC", "KSAT"], "houston": ["KHOU", "KIAH", "KSGR", "KCXO"], "mexico city": ["MMMX", "MMSM", "MMTO"], @@ -107,7 +108,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour "new york's central park", "portland", "denver", - "aurora", "austin", "san diego", "detroit", @@ -238,6 +238,12 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour os.getenv("HKO_OBS_CACHE_TTL_SEC", "60") ) self._hko_obs_cache: Dict[str, Dict] = {} + self.madis_cache_ttl_sec = int( + os.getenv("MADIS_CACHE_TTL_SEC", "300") # 5 min match update rate + ) + self._madis_cache: Optional[List[Dict[str, Any]]] = None + self._madis_cache_ts: float = 0.0 + self._madis_cache_lock = threading.Lock() self._hko_obs_cache_lock = threading.Lock() self.cwa_open_data_auth = ( os.getenv("CWA_OPEN_DATA_AUTH") @@ -1052,6 +1058,47 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour except Exception as exc: logger.warning("AMOS attach failed city={}: {}", city_lower, exc) + def _attach_madis_hfmetar_data( + self, results: Dict, city_lower: str + ) -> None: + """Fetch MADIS HFMETAR data and attach matching US station observations.""" + # Only run for US cities (ICAO starts with K) + city_meta = self.CITY_REGISTRY.get(city_lower) or {} + icao = str(city_meta.get("icao") or "").strip().upper() + if not icao.startswith("K"): + return + + now_ts = time.time() + with self._madis_cache_lock: + if (self._madis_cache is not None + and now_ts - self._madis_cache_ts < self.madis_cache_ttl_sec): + obs_list = self._madis_cache + else: + obs_list = self.fetch_madis_hfmetar() + self._madis_cache = obs_list + self._madis_cache_ts = now_ts + + if not obs_list: + return + + # Find this city's station in the MADIS results + for obs in obs_list: + if obs["icao"] == icao: + try: + DBManager().append_airport_obs( + icao=icao, + city=city_lower, + temp_c=obs["temp_c"], + wind_kt=obs.get("wind_kt"), + pressure_hpa=obs.get("pressure_hpa"), + obs_time=obs["obs_time"] or datetime.now().isoformat(), + ) + except Exception: + logger.exception( + "airport_obs_log append failed for madis city={}", city_lower + ) + break + def _attach_russia_official_nearby( self, results: Dict, city_lower: str, use_fahrenheit: bool ) -> None: @@ -1177,6 +1224,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour include_nearby=include_nearby, ) self._attach_korean_amos_data(results, city_lower, use_fahrenheit) + self._attach_madis_hfmetar_data(results, city_lower) if include_nearby: self._attach_china_official_nearby(results, city_lower, use_fahrenheit) self._attach_japan_official_nearby(results, city_lower, use_fahrenheit) @@ -1222,6 +1270,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour include_nearby=include_nearby, ) self._attach_korean_amos_data(results, city_lower, use_fahrenheit) + self._attach_madis_hfmetar_data(results, city_lower) if include_nearby: self._attach_china_official_nearby(results, city_lower, use_fahrenheit) self._attach_japan_official_nearby(results, city_lower, use_fahrenheit)