Aurora→Denver 重命名 + 接入 MADIS HFMETAR 5 分钟数据源
This commit is contained in:
2569718930@qq.com
@@ -258,7 +258,7 @@ const CITY_SPECIFIC_SOURCES: Record<string, OfficialSourceLink[]> = {
|
||||
kind: "metar",
|
||||
},
|
||||
],
|
||||
aurora: [
|
||||
denver: [
|
||||
{
|
||||
label: "NWS Denver/Boulder",
|
||||
href: "https://www.weather.gov/bou/",
|
||||
|
||||
@@ -60,7 +60,6 @@ const CITY_REGION_FALLBACK: Record<string, MarketRegionKey> = {
|
||||
toronto: "americas",
|
||||
"los angeles": "americas",
|
||||
"san francisco": "americas",
|
||||
aurora: "americas",
|
||||
denver: "americas",
|
||||
austin: "americas",
|
||||
houston: "americas",
|
||||
|
||||
@@ -383,7 +383,7 @@ CITY_REGISTRY = {
|
||||
"distance_km": 20.0,
|
||||
"warning": "海湾冷空气和平流雾常压制机场升温，SFO 与市区体感差异可快速放大。",
|
||||
},
|
||||
"aurora": {
|
||||
"denver": {
|
||||
"name": "Denver",
|
||||
"display_name": "Denver",
|
||||
"lat": 39.7017,
|
||||
@@ -825,7 +825,7 @@ ALIASES = {
|
||||
"nyc": "new york", "ny": "new york", "chi": "chicago",
|
||||
"la": "los angeles", "lax": "los angeles", "losangeles": "los angeles",
|
||||
"sf": "san francisco", "sfo": "san francisco", "sanfrancisco": "san francisco",
|
||||
"aur": "aurora", "denver": "aurora", "buckley": "aurora", "kbkf": "aurora",
|
||||
"aur": "denver", "denver": "denver", "buckley": "denver", "kbkf": "denver", "aurora": "denver",
|
||||
"aus": "austin", "kaus": "austin",
|
||||
"hou": "houston", "khou": "houston", "hobby": "houston",
|
||||
"cdmx": "mexico city", "mexicocity": "mexico city", "mmmx": "mexico city",
|
||||
@@ -862,8 +862,8 @@ ALIASES = {
|
||||
"纽约": "new york",
|
||||
"洛杉矶": "los angeles",
|
||||
"旧金山": "san francisco",
|
||||
"奥罗拉": "aurora",
|
||||
"丹佛": "aurora",
|
||||
"奥罗拉": "denver",
|
||||
"丹佛": "denver",
|
||||
"奥斯汀": "austin",
|
||||
"休斯顿": "houston",
|
||||
"墨西哥城": "mexico city",
|
||||
|
||||
@@ -15,7 +15,7 @@ CITY_TIME_ZONES = {
|
||||
"amsterdam": "Europe/Amsterdam",
|
||||
"ankara": "Europe/Istanbul",
|
||||
"atlanta": "America/New_York",
|
||||
"aurora": "America/Denver",
|
||||
"denver": "America/Denver",
|
||||
"austin": "America/Chicago",
|
||||
"beijing": "Asia/Shanghai",
|
||||
"buenos aires": "America/Argentina/Buenos_Aires",
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
"""MADIS HFMETAR 5-minute real-time data source.
|
||||
|
||||
Fetches NetCDF files from NOAA MADIS public archive.
|
||||
HFMETAR data updates every 5 minutes (12 values/hour/station).
|
||||
Public anonymous access — no API key required.
|
||||
|
||||
URL: https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Optional, List
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from src.utils.metrics import record_source_call
|
||||
|
||||
MADIS_HFMETAR_URL = (
|
||||
"https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/"
|
||||
)
|
||||
|
||||
|
||||
class MadisSourceMixin:
|
||||
"""Mixin that adds MADIS HFMETAR 5-min data to WeatherDataCollector."""
|
||||
|
||||
def _madis_http_get(self, url: str) -> bytes:
|
||||
getter = getattr(self, "_http_get", None)
|
||||
if callable(getter):
|
||||
resp = getter(url)
|
||||
return resp.content if hasattr(resp, "content") else resp
|
||||
resp = self.session.get(url, timeout=self.timeout)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
def _madis_latest_file(self) -> Optional[str]:
|
||||
"""Find the most recent HFMETAR NetCDF file."""
|
||||
try:
|
||||
# The MADIS HFMETAR directory listing returns HTML
|
||||
html = self._madis_http_get(MADIS_HFMETAR_URL).decode("utf-8", errors="replace")
|
||||
import re
|
||||
# Match file names like 20260514_1200.gz or 202605141200.gz
|
||||
matches = re.findall(r'(\d{8}[_ ]?\d{4})\.gz', html)
|
||||
if not matches:
|
||||
logger.warning("MADIS: no HFMETAR files found in directory listing")
|
||||
return None
|
||||
# Sort by timestamp descending
|
||||
matches.sort(reverse=True)
|
||||
latest = matches[0].replace(" ", "_")
|
||||
return f"{latest}.gz"
|
||||
except Exception as exc:
|
||||
logger.warning("MADIS: failed to list HFMETAR files: {}", exc)
|
||||
return None
|
||||
|
||||
def _madis_parse_hfmetar(
|
||||
self, nc_bytes: bytes, fname: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Parse a MADIS HFMETAR NetCDF file and return per-station observations."""
|
||||
try:
|
||||
from netCDF4 import Dataset
|
||||
nc = Dataset(fname, memory=nc_bytes)
|
||||
except ImportError:
|
||||
logger.error("netCDF4 not installed; MADIS data unavailable")
|
||||
return []
|
||||
except Exception as exc:
|
||||
logger.warning("MADIS: netCDF4 open failed: {}", exc)
|
||||
return []
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
try:
|
||||
# MADIS HFMETAR uses these variable names
|
||||
icaos = [str(s).strip() for s in nc.variables.get("icaoId", [])[:]]
|
||||
temps = nc.variables.get("temperature", None) # in Celsius
|
||||
dewpts = nc.variables.get("dewpoint", None)
|
||||
winds = nc.variables.get("windSpeed", None)
|
||||
pressures = nc.variables.get("seaLevelPress", None)
|
||||
obs_times = nc.variables.get("observationTime", None)
|
||||
|
||||
n = len(icaos)
|
||||
for i in range(n):
|
||||
icao = str(icaos[i]).strip().upper()
|
||||
if not icao or icao == "0":
|
||||
continue
|
||||
|
||||
temp_c = None
|
||||
if temps is not None:
|
||||
try:
|
||||
v = float(temps[i])
|
||||
if -90 < v < 60:
|
||||
temp_c = round(v, 1)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
if temp_c is None:
|
||||
continue # skip stations without temperature
|
||||
|
||||
dewp_c = None
|
||||
if dewpts is not None:
|
||||
try:
|
||||
v = float(dewpts[i])
|
||||
if -90 < v < 60:
|
||||
dewp_c = round(v, 1)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
wind_kt = None
|
||||
if winds is not None:
|
||||
try:
|
||||
w = float(winds[i])
|
||||
if w >= 0:
|
||||
wind_kt = round(w * 1.94384, 1)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
pressure_hpa = None
|
||||
if pressures is not None:
|
||||
try:
|
||||
p = float(pressures[i])
|
||||
if 800 < p < 1100:
|
||||
pressure_hpa = round(p, 1)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
obs_time = ""
|
||||
if obs_times is not None:
|
||||
try:
|
||||
ts = str(obs_times[i]).strip()
|
||||
# Try to parse as epoch or ISO
|
||||
if ts and ts != "0":
|
||||
try:
|
||||
epoch = int(float(ts))
|
||||
dt = datetime.fromtimestamp(epoch, tz=timezone.utc)
|
||||
obs_time = dt.isoformat()
|
||||
except (ValueError, OverflowError):
|
||||
obs_time = ts
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
results.append({
|
||||
"icao": icao,
|
||||
"temp_c": temp_c,
|
||||
"dewp_c": dewp_c,
|
||||
"wind_kt": wind_kt,
|
||||
"pressure_hpa": pressure_hpa,
|
||||
"obs_time": obs_time,
|
||||
"source": "madis_hfmetar",
|
||||
})
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("MADIS: HFMETAR parse error: {}", exc)
|
||||
finally:
|
||||
try:
|
||||
nc.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
def fetch_madis_hfmetar(self) -> List[Dict[str, Any]]:
|
||||
"""Fetch latest MADIS HFMETAR data and return parsed observations."""
|
||||
started = time.perf_counter()
|
||||
|
||||
fname = self._madis_latest_file()
|
||||
if not fname:
|
||||
record_source_call("madis_hfmetar", "current", "no_file",
|
||||
(time.perf_counter() - started) * 1000.0)
|
||||
return []
|
||||
|
||||
try:
|
||||
url = f"{MADIS_HFMETAR_URL}{fname}"
|
||||
raw = self._madis_http_get(url)
|
||||
|
||||
# Decompress gzip
|
||||
try:
|
||||
nc_bytes = gzip.decompress(raw)
|
||||
except gzip.BadGzipFile:
|
||||
# Maybe not compressed
|
||||
nc_bytes = raw
|
||||
|
||||
results = self._madis_parse_hfmetar(nc_bytes, fname)
|
||||
if results:
|
||||
logger.info("MADIS HFMETAR: {} stations from {}", len(results), fname)
|
||||
record_source_call("madis_hfmetar", "current", "success",
|
||||
(time.perf_counter() - started) * 1000.0)
|
||||
else:
|
||||
record_source_call("madis_hfmetar", "current", "empty",
|
||||
(time.perf_counter() - started) * 1000.0)
|
||||
return results
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("MADIS HFMETAR fetch failed: {}", exc)
|
||||
record_source_call("madis_hfmetar", "current", "error",
|
||||
(time.perf_counter() - started) * 1000.0)
|
||||
return []
|
||||
@@ -3,7 +3,7 @@ import httpx
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional, Dict, List
|
||||
from typing import Any, Optional, Dict, List
|
||||
from datetime import datetime, timedelta
|
||||
from loguru import logger
|
||||
from src.data_collection.open_meteo_cache import OpenMeteoCacheMixin
|
||||
@@ -18,10 +18,11 @@ from src.data_collection.amos_station_sources import AmosStationSourceMixin
|
||||
from src.data_collection.fmi_sources import FmiSourceMixin
|
||||
from src.data_collection.knmi_sources import KnmiSourceMixin
|
||||
from src.data_collection.hko_obs_sources import HkoObsSourceMixin
|
||||
from src.data_collection.madis_sources import MadisSourceMixin
|
||||
from src.database.db_manager import DBManager
|
||||
|
||||
|
||||
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NmcSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin):
|
||||
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NmcSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin):
|
||||
"""
|
||||
Multi-source weather data collector
|
||||
|
||||
@@ -49,7 +50,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
"new york": ["KLGA", "KJFK", "KEWR", "KTEB", "KHPN"],
|
||||
"los angeles": ["KLAX", "KBUR", "KLGB", "KSNA", "KVNY"],
|
||||
"san francisco": ["KSFO", "KOAK", "KSJC", "KHAF"],
|
||||
"aurora": ["KBKF", "KDEN", "KAPA", "KBJC"],
|
||||
"denver": ["KBKF", "KDEN", "KAPA", "KBJC"],
|
||||
"austin": ["KAUS", "KEDC", "KSAT"],
|
||||
"houston": ["KHOU", "KIAH", "KSGR", "KCXO"],
|
||||
"mexico city": ["MMMX", "MMSM", "MMTO"],
|
||||
@@ -107,7 +108,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
"new york's central park",
|
||||
"portland",
|
||||
"denver",
|
||||
"aurora",
|
||||
"austin",
|
||||
"san diego",
|
||||
"detroit",
|
||||
@@ -238,6 +238,12 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
os.getenv("HKO_OBS_CACHE_TTL_SEC", "60")
|
||||
)
|
||||
self._hko_obs_cache: Dict[str, Dict] = {}
|
||||
self.madis_cache_ttl_sec = int(
|
||||
os.getenv("MADIS_CACHE_TTL_SEC", "300") # 5 min match update rate
|
||||
)
|
||||
self._madis_cache: Optional[List[Dict[str, Any]]] = None
|
||||
self._madis_cache_ts: float = 0.0
|
||||
self._madis_cache_lock = threading.Lock()
|
||||
self._hko_obs_cache_lock = threading.Lock()
|
||||
self.cwa_open_data_auth = (
|
||||
os.getenv("CWA_OPEN_DATA_AUTH")
|
||||
@@ -1052,6 +1058,47 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
except Exception as exc:
|
||||
logger.warning("AMOS attach failed city={}: {}", city_lower, exc)
|
||||
|
||||
def _attach_madis_hfmetar_data(
|
||||
self, results: Dict, city_lower: str
|
||||
) -> None:
|
||||
"""Fetch MADIS HFMETAR data and attach matching US station observations."""
|
||||
# Only run for US cities (ICAO starts with K)
|
||||
city_meta = self.CITY_REGISTRY.get(city_lower) or {}
|
||||
icao = str(city_meta.get("icao") or "").strip().upper()
|
||||
if not icao.startswith("K"):
|
||||
return
|
||||
|
||||
now_ts = time.time()
|
||||
with self._madis_cache_lock:
|
||||
if (self._madis_cache is not None
|
||||
and now_ts - self._madis_cache_ts < self.madis_cache_ttl_sec):
|
||||
obs_list = self._madis_cache
|
||||
else:
|
||||
obs_list = self.fetch_madis_hfmetar()
|
||||
self._madis_cache = obs_list
|
||||
self._madis_cache_ts = now_ts
|
||||
|
||||
if not obs_list:
|
||||
return
|
||||
|
||||
# Find this city's station in the MADIS results
|
||||
for obs in obs_list:
|
||||
if obs["icao"] == icao:
|
||||
try:
|
||||
DBManager().append_airport_obs(
|
||||
icao=icao,
|
||||
city=city_lower,
|
||||
temp_c=obs["temp_c"],
|
||||
wind_kt=obs.get("wind_kt"),
|
||||
pressure_hpa=obs.get("pressure_hpa"),
|
||||
obs_time=obs["obs_time"] or datetime.now().isoformat(),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"airport_obs_log append failed for madis city={}", city_lower
|
||||
)
|
||||
break
|
||||
|
||||
def _attach_russia_official_nearby(
|
||||
self, results: Dict, city_lower: str, use_fahrenheit: bool
|
||||
) -> None:
|
||||
@@ -1177,6 +1224,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
include_nearby=include_nearby,
|
||||
)
|
||||
self._attach_korean_amos_data(results, city_lower, use_fahrenheit)
|
||||
self._attach_madis_hfmetar_data(results, city_lower)
|
||||
if include_nearby:
|
||||
self._attach_china_official_nearby(results, city_lower, use_fahrenheit)
|
||||
self._attach_japan_official_nearby(results, city_lower, use_fahrenheit)
|
||||
@@ -1222,6 +1270,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
include_nearby=include_nearby,
|
||||
)
|
||||
self._attach_korean_amos_data(results, city_lower, use_fahrenheit)
|
||||
self._attach_madis_hfmetar_data(results, city_lower)
|
||||
if include_nearby:
|
||||
self._attach_china_official_nearby(results, city_lower, use_fahrenheit)
|
||||
self._attach_japan_official_nearby(results, city_lower, use_fahrenheit)
|
||||
|
||||
Reference in New Issue
Block a user