gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add observation collector and source gate

Browse Source
This commit is contained in:
2569718930@qq.com
2026-06-05 14:38:58 +08:00
parent c3fd0582ad
commit 2af1cf9cd8
27 changed files with 1112 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/data_collection/amos_station_sources.py
+21
View File
@@ -15,6 +15,7 @@ from typing import Any, Dict, Optional
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
AMOS_BASE_URL = os.getenv("AMOS_BASE_URL", "").strip() or "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do"
@@ -469,6 +470,26 @@ class AmosStationSourceMixin:
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
city_key = str(city or "").strip().lower()
interval_sec = max(
30,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_AMOS_SEC", "60") or "60"),
)
return run_observation_source(
"amos",
city_key,
interval_sec,
lambda: self._fetch_amos_official_current_uncached(
city_key,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_amos_official_current_uncached(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
"""Fetch AMOS runway-level observations for Seoul or Busan.
src/data_collection/amsc_awos_sources.py
+22
View File
@@ -19,6 +19,7 @@ from urllib.request import Request, urlopen
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
from src.utils.runtime_secrets import get_runtime_secret
@@ -343,6 +344,27 @@ class AmscAwosSourceMixin:
city_key: str,
*,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
normalized_city = str(city_key or "").strip().lower()
interval_sec = max(
60,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_AMSC_SEC", "180") or "180"),
)
return run_observation_source(
"amsc_awos",
normalized_city,
interval_sec,
lambda: self._fetch_amsc_awos_current_uncached(
normalized_city,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_amsc_awos_current_uncached(
self,
city_key: str,
*,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
del use_fahrenheit # AMSC reports Celsius; project UI converts elsewhere if needed.
normalized_city = str(city_key or "").strip().lower()
src/data_collection/cowin_sources.py
+21
View File
@@ -16,6 +16,7 @@ from urllib.parse import urlencode
import requests
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
COWIN_BASE_URL = os.getenv("COWIN_BASE_URL", "").strip() or "https://cowin.hku.hk"
@@ -94,6 +95,26 @@ class CowinSourceMixin:
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
city_key = str(city or "").strip().lower()
interval_sec = max(
30,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_COWIN_SEC", "60") or "60"),
)
return run_observation_source(
"cowin_obs",
city_key,
interval_sec,
lambda: self._fetch_cowin_obs_current_uncached(
city_key,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_cowin_obs_current_uncached(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
started = time.perf_counter()
city_key = str(city or "").strip().lower()
src/data_collection/hko_obs_sources.py
+21
View File
@@ -17,6 +17,7 @@ from typing import Any, Dict, Optional
from loguru import logger
# pyrefly: ignore [missing-import]
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
HKO_BASE_URL = os.getenv("HKO_BASE_URL", "").strip() or "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
@@ -67,6 +68,26 @@ class HkoObsSourceMixin:
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
city_key = (city or "").strip().lower()
interval_sec = max(
60,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_HKO_SEC", "600") or "600"),
)
return run_observation_source(
"hko_obs",
city_key,
interval_sec,
lambda: self._fetch_hko_obs_current_uncached(
city_key,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_hko_obs_current_uncached(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
started = time.perf_counter()
city_key = (city or "").strip().lower()
src/data_collection/madis_sources.py
+15
View File
@@ -10,12 +10,14 @@ URL: https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/netCDF/
from __future__ import annotations
import gzip
import os
import time
from datetime import datetime, timezone
from typing import Any, Dict, Optional, List
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
MADIS_HFMETAR_URL = (
@@ -177,6 +179,19 @@ class MadisSourceMixin:
def fetch_madis_hfmetar(self) -> List[Dict[str, Any]]:
"""Fetch latest MADIS HFMETAR data and return parsed observations."""
interval_sec = max(
60,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC", "300") or "300"),
)
return run_observation_source(
"madis_hfmetar",
"global",
interval_sec,
self._fetch_madis_hfmetar_uncached,
) or []
def _fetch_madis_hfmetar_uncached(self) -> List[Dict[str, Any]]:
"""Fetch latest MADIS HFMETAR data without source-level gate."""
started = time.perf_counter()
fname = self._madis_latest_file()
src/data_collection/nws_open_meteo_sources.py
+50 -2
View File
@@ -34,6 +34,54 @@ OPEN_METEO_MULTI_MODEL_SPECS: Dict[str, Dict[str, Any]] = {
"resolution_km": None,
"horizon": "16d",
},
"gfs_global": {
"label": "GFS Global",
"provider": "NOAA",
"model": "GFS Global 0.11°/0.25°",
"tier": "global",
"resolution_km": 13,
"horizon": "16d",
},
"ncep_hrrr_conus": {
"label": "HRRR",
"provider": "NOAA",
"model": "HRRR CONUS",
"tier": "short_range_north_america",
"resolution_km": 3,
"horizon": "18h/48h",
},
"ncep_nbm_conus": {
"label": "NBM",
"provider": "NOAA",
"model": "National Blend of Models CONUS",
"tier": "regional_north_america",
"resolution_km": 2.5,
"horizon": "11d",
},
"ncep_nam_conus": {
"label": "NAM",
"provider": "NOAA",
"model": "NAM CONUS",
"tier": "short_range_north_america",
"resolution_km": 3,
"horizon": "60h",
},
"ncep_gfs_graphcast025": {
"label": "GFS GraphCast",
"provider": "Google/NOAA",
"model": "GFS GraphCast 0.25°",
"tier": "ai_global",
"resolution_km": 25,
"horizon": "16d",
},
"ncep_aigfs025": {
"label": "AI-GFS",
"provider": "NOAA",
"model": "AIGFS 0.25°",
"tier": "ai_global",
"resolution_km": 25,
"horizon": "16d",
},
"icon_seamless": {
"label": "ICON",
"provider": "DWD",
@@ -784,7 +832,7 @@ class NwsOpenMeteoSourceMixin:
模型列表:
- ECMWF IFS / AIFS
- GFS (美国 NOAA)
- GFS / HRRR / NBM / NAM / AI-GFS (美国 NOAA)
- ICON Seamless / ICON-EU / ICON-D2 (德国气象局 DWD)
- GEM Seamless / GDPS / RDPS / HRDPS (加拿大 ECCC)
- JMA (日本气象厅)
@@ -893,7 +941,7 @@ class NwsOpenMeteoSourceMixin:
"model_keys": model_keys,
"dates": dates,
"unit": "fahrenheit" if use_fahrenheit else "celsius",
"attribution": "Open-Meteo forecast model API; underlying models from ECMWF, DWD, ECCC, NOAA and JMA.",
"attribution": "Open-Meteo forecast model API; underlying models from ECMWF, DWD, ECCC, NOAA/NCEP, Google and JMA.",
}
with self._multi_model_cache_lock:
previous = self._multi_model_cache.get(cache_key)
src/data_collection/observation_source_gate.py
+147
View File
@@ -0,0 +1,147 @@
"""Shared singleflight and cooldown guard for high-frequency observation sources."""
from __future__ import annotations
import os
import threading
import time
from dataclasses import dataclass
from typing import Any, Callable, Dict, Hashable, Optional, Tuple, TypeVar
from loguru import logger
from src.database.db_manager import DBManager
T = TypeVar("T")
ObservationSourceKey = Tuple[str, str]
@dataclass
class _SourceState:
result: Any = None
has_result: bool = False
cooldown_until_ts: float = 0.0
inflight: Optional[threading.Event] = None
error: Optional[BaseException] = None
_GATE_LOCK = threading.Lock()
_SOURCE_STATES: Dict[ObservationSourceKey, _SourceState] = {}
def _env_bool(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
def _normalize_part(value: Any) -> str:
return str(value or "").strip().lower()
def _state_for(key: ObservationSourceKey) -> _SourceState:
state = _SOURCE_STATES.get(key)
if state is None:
state = _SourceState()
_SOURCE_STATES[key] = state
return state
def _acquire_cross_process_cooldown(key: ObservationSourceKey, ttl_sec: int) -> bool:
if not _env_bool("POLYWEATHER_OBSERVATION_SOURCE_DB_LOCK_ENABLED", True):
return True
cache_key = f"observation-source:{key[0]}:{key[1]}"
try:
owner = DBManager().acquire_cache_refresh_lock(
cache_key,
ttl_sec=max(15, int(ttl_sec or 60)),
)
except Exception as exc:
logger.debug("observation source DB cooldown skipped key={}: {}", cache_key, exc)
return True
return bool(owner)
def run_observation_source(
source: Hashable,
city_or_scope: Hashable,
interval_sec: int,
fetcher: Callable[[], T],
*,
failure_cooldown_sec: int = 30,
) -> Optional[T]:
"""Run a source fetch once per source/city interval and share in-flight work.
The in-process gate returns the previous result during cooldown. The SQLite
lock is intentionally left to expire instead of being released so multiple
service processes also observe a coarse source cooldown.
"""
if not _env_bool("POLYWEATHER_OBSERVATION_SOURCE_GATE_ENABLED", True):
return fetcher()
source_key = _normalize_part(source)
scope_key = _normalize_part(city_or_scope)
if not source_key or not scope_key:
return fetcher()
interval = max(1, int(interval_sec or 60))
key: ObservationSourceKey = (source_key, scope_key)
while True:
wait_event: Optional[threading.Event] = None
now_ts = time.time()
with _GATE_LOCK:
state = _state_for(key)
if now_ts < state.cooldown_until_ts:
return state.result if state.has_result else None
if state.inflight is None:
event = threading.Event()
state.inflight = event
state.error = None
break
wait_event = state.inflight
if wait_event is not None:
wait_event.wait(timeout=max(5.0, float(interval)))
with _GATE_LOCK:
state = _state_for(key)
if state.error is not None:
raise state.error
if state.has_result:
return state.result
if state.inflight is None:
continue
return None
owner_event = event
try:
if not _acquire_cross_process_cooldown(key, interval):
with _GATE_LOCK:
state = _state_for(key)
return state.result if state.has_result else None
result = fetcher()
with _GATE_LOCK:
state = _state_for(key)
state.result = result
state.has_result = True
state.cooldown_until_ts = time.time() + interval
state.error = None
return result
except BaseException as exc:
with _GATE_LOCK:
state = _state_for(key)
state.error = exc
state.cooldown_until_ts = time.time() + max(1, int(failure_cooldown_sec or 30))
raise
finally:
with _GATE_LOCK:
state = _state_for(key)
if state.inflight is owner_event:
state.inflight = None
owner_event.set()
def reset_observation_source_gate_for_tests() -> None:
with _GATE_LOCK:
_SOURCE_STATES.clear()
src/data_collection/weather_sources.py
+8 -7
View File
@@ -183,8 +183,8 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
)
self.open_meteo_multi_model_cache_ttl_sec = min(self.open_meteo_multi_model_cache_ttl_sec, MODEL_CACHE_TTL_SEC)
self.multi_model_cache_version = str(
os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_VERSION", "v3")
).strip() or "v3"
os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_VERSION", "v4")
).strip() or "v4"
self._open_meteo_cache: Dict[str, Dict] = {}
self._ensemble_cache: Dict[str, Dict] = {}
self._multi_model_cache: Dict[str, Dict] = {}
@@ -263,8 +263,9 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
self._madis_cache_lock = threading.Lock()
self._hko_obs_cache_lock = threading.Lock()
self.cowin_obs_cache_ttl_sec = int(
os.getenv("COWIN_OBS_CACHE_TTL_SEC", "120")
os.getenv("COWIN_OBS_CACHE_TTL_SEC", str(OBSERVATION_REFRESH_SEC))
)
self.cowin_obs_cache_ttl_sec = min(self.cowin_obs_cache_ttl_sec, OBSERVATION_REFRESH_SEC)
self._cowin_obs_cache: Dict[str, Dict] = {}
self._cowin_obs_cache_lock = threading.Lock()
self.cwa_open_data_auth = (
@@ -1732,10 +1733,10 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
self._attach_wunderground_historical(results, city_lower, use_fahrenheit)
if lat and lon:
# When force_refresh_observations_only is set (airport push loop),
# skip the OM fetch entirely if cached data exists — the 60 s cycle
# must not hammer the Open-Meteo API. Stale model data is fine;
# the loop only needs fresh METAR / AMOS observations.
# When force_refresh_observations_only is set by an explicit
# observation refresh caller, skip the OM fetch entirely if cached
# data exists. Stale model data is fine; the caller only needs
# fresh METAR / AMOS observations.
om_from_cache_only = force_refresh_observations_only
if om_from_cache_only:
self._maybe_reload_open_meteo_disk_cache()