gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add observation collector and source gate

Browse Source
This commit is contained in:
2569718930@qq.com
2026-06-05 14:38:58 +08:00
parent c3fd0582ad
commit 2af1cf9cd8
27 changed files with 1112 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/data_collection/amos_station_sources.py
+21
View File
@@ -15,6 +15,7 @@ from typing import Any, Dict, Optional
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
AMOS_BASE_URL = os.getenv("AMOS_BASE_URL", "").strip() or "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do"
@@ -469,6 +470,26 @@ class AmosStationSourceMixin:
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
city_key = str(city or "").strip().lower()
interval_sec = max(
30,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_AMOS_SEC", "60") or "60"),
)
return run_observation_source(
"amos",
city_key,
interval_sec,
lambda: self._fetch_amos_official_current_uncached(
city_key,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_amos_official_current_uncached(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
"""Fetch AMOS runway-level observations for Seoul or Busan.
src/data_collection/amsc_awos_sources.py
+22
View File
@@ -19,6 +19,7 @@ from urllib.request import Request, urlopen
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
from src.utils.runtime_secrets import get_runtime_secret
@@ -343,6 +344,27 @@ class AmscAwosSourceMixin:
city_key: str,
*,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
normalized_city = str(city_key or "").strip().lower()
interval_sec = max(
60,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_AMSC_SEC", "180") or "180"),
)
return run_observation_source(
"amsc_awos",
normalized_city,
interval_sec,
lambda: self._fetch_amsc_awos_current_uncached(
normalized_city,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_amsc_awos_current_uncached(
self,
city_key: str,
*,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
del use_fahrenheit # AMSC reports Celsius; project UI converts elsewhere if needed.
normalized_city = str(city_key or "").strip().lower()
src/data_collection/cowin_sources.py
+21
View File
@@ -16,6 +16,7 @@ from urllib.parse import urlencode
import requests
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
COWIN_BASE_URL = os.getenv("COWIN_BASE_URL", "").strip() or "https://cowin.hku.hk"
@@ -94,6 +95,26 @@ class CowinSourceMixin:
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
city_key = str(city or "").strip().lower()
interval_sec = max(
30,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_COWIN_SEC", "60") or "60"),
)
return run_observation_source(
"cowin_obs",
city_key,
interval_sec,
lambda: self._fetch_cowin_obs_current_uncached(
city_key,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_cowin_obs_current_uncached(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
started = time.perf_counter()
city_key = str(city or "").strip().lower()
src/data_collection/hko_obs_sources.py
+21
View File
@@ -17,6 +17,7 @@ from typing import Any, Dict, Optional
from loguru import logger
# pyrefly: ignore [missing-import]
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
HKO_BASE_URL = os.getenv("HKO_BASE_URL", "").strip() or "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
@@ -67,6 +68,26 @@ class HkoObsSourceMixin:
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
city_key = (city or "").strip().lower()
interval_sec = max(
60,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_HKO_SEC", "600") or "600"),
)
return run_observation_source(
"hko_obs",
city_key,
interval_sec,
lambda: self._fetch_hko_obs_current_uncached(
city_key,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_hko_obs_current_uncached(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
started = time.perf_counter()
city_key = (city or "").strip().lower()
src/data_collection/madis_sources.py
+15
View File
@@ -10,12 +10,14 @@ URL: https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/netCDF/
from __future__ import annotations
import gzip
import os
import time
from datetime import datetime, timezone
from typing import Any, Dict, Optional, List
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
MADIS_HFMETAR_URL = (
@@ -177,6 +179,19 @@ class MadisSourceMixin:
def fetch_madis_hfmetar(self) -> List[Dict[str, Any]]:
"""Fetch latest MADIS HFMETAR data and return parsed observations."""
interval_sec = max(
60,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC", "300") or "300"),
)
return run_observation_source(
"madis_hfmetar",
"global",
interval_sec,
self._fetch_madis_hfmetar_uncached,
) or []
def _fetch_madis_hfmetar_uncached(self) -> List[Dict[str, Any]]:
"""Fetch latest MADIS HFMETAR data without source-level gate."""
started = time.perf_counter()
fname = self._madis_latest_file()
src/data_collection/nws_open_meteo_sources.py
+50 -2
View File
@@ -34,6 +34,54 @@ OPEN_METEO_MULTI_MODEL_SPECS: Dict[str, Dict[str, Any]] = {
"resolution_km": None,
"horizon": "16d",
},
"gfs_global": {
"label": "GFS Global",
"provider": "NOAA",
"model": "GFS Global 0.11°/0.25°",
"tier": "global",
"resolution_km": 13,
"horizon": "16d",
},
"ncep_hrrr_conus": {
"label": "HRRR",
"provider": "NOAA",
"model": "HRRR CONUS",
"tier": "short_range_north_america",
"resolution_km": 3,
"horizon": "18h/48h",
},
"ncep_nbm_conus": {
"label": "NBM",
"provider": "NOAA",
"model": "National Blend of Models CONUS",
"tier": "regional_north_america",
"resolution_km": 2.5,
"horizon": "11d",
},
"ncep_nam_conus": {
"label": "NAM",
"provider": "NOAA",
"model": "NAM CONUS",
"tier": "short_range_north_america",
"resolution_km": 3,
"horizon": "60h",
},
"ncep_gfs_graphcast025": {
"label": "GFS GraphCast",
"provider": "Google/NOAA",
"model": "GFS GraphCast 0.25°",
"tier": "ai_global",
"resolution_km": 25,
"horizon": "16d",
},
"ncep_aigfs025": {
"label": "AI-GFS",
"provider": "NOAA",
"model": "AIGFS 0.25°",
"tier": "ai_global",
"resolution_km": 25,
"horizon": "16d",
},
"icon_seamless": {
"label": "ICON",
"provider": "DWD",
@@ -784,7 +832,7 @@ class NwsOpenMeteoSourceMixin:
模型列表:
- ECMWF IFS / AIFS
- GFS (美国 NOAA)
- GFS / HRRR / NBM / NAM / AI-GFS (美国 NOAA)
- ICON Seamless / ICON-EU / ICON-D2 (德国气象局 DWD)
- GEM Seamless / GDPS / RDPS / HRDPS (加拿大 ECCC)
- JMA (日本气象厅)
@@ -893,7 +941,7 @@ class NwsOpenMeteoSourceMixin:
"model_keys": model_keys,
"dates": dates,
"unit": "fahrenheit" if use_fahrenheit else "celsius",
"attribution": "Open-Meteo forecast model API; underlying models from ECMWF, DWD, ECCC, NOAA and JMA.",
"attribution": "Open-Meteo forecast model API; underlying models from ECMWF, DWD, ECCC, NOAA/NCEP, Google and JMA.",
}
with self._multi_model_cache_lock:
previous = self._multi_model_cache.get(cache_key)
src/data_collection/observation_source_gate.py
+147
View File
@@ -0,0 +1,147 @@
"""Shared singleflight and cooldown guard for high-frequency observation sources."""
from __future__ import annotations
import os
import threading
import time
from dataclasses import dataclass
from typing import Any, Callable, Dict, Hashable, Optional, Tuple, TypeVar
from loguru import logger
from src.database.db_manager import DBManager
T = TypeVar("T")
ObservationSourceKey = Tuple[str, str]
@dataclass
class _SourceState:
result: Any = None
has_result: bool = False
cooldown_until_ts: float = 0.0
inflight: Optional[threading.Event] = None
error: Optional[BaseException] = None
_GATE_LOCK = threading.Lock()
_SOURCE_STATES: Dict[ObservationSourceKey, _SourceState] = {}
def _env_bool(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
def _normalize_part(value: Any) -> str:
return str(value or "").strip().lower()
def _state_for(key: ObservationSourceKey) -> _SourceState:
state = _SOURCE_STATES.get(key)
if state is None:
state = _SourceState()
_SOURCE_STATES[key] = state
return state
def _acquire_cross_process_cooldown(key: ObservationSourceKey, ttl_sec: int) -> bool:
if not _env_bool("POLYWEATHER_OBSERVATION_SOURCE_DB_LOCK_ENABLED", True):
return True
cache_key = f"observation-source:{key[0]}:{key[1]}"
try:
owner = DBManager().acquire_cache_refresh_lock(
cache_key,
ttl_sec=max(15, int(ttl_sec or 60)),
)
except Exception as exc:
logger.debug("observation source DB cooldown skipped key={}: {}", cache_key, exc)
return True
return bool(owner)
def run_observation_source(
source: Hashable,
city_or_scope: Hashable,
interval_sec: int,
fetcher: Callable[[], T],
*,
failure_cooldown_sec: int = 30,
) -> Optional[T]:
"""Run a source fetch once per source/city interval and share in-flight work.
The in-process gate returns the previous result during cooldown. The SQLite
lock is intentionally left to expire instead of being released so multiple
service processes also observe a coarse source cooldown.
"""
if not _env_bool("POLYWEATHER_OBSERVATION_SOURCE_GATE_ENABLED", True):
return fetcher()
source_key = _normalize_part(source)
scope_key = _normalize_part(city_or_scope)
if not source_key or not scope_key:
return fetcher()
interval = max(1, int(interval_sec or 60))
key: ObservationSourceKey = (source_key, scope_key)
while True:
wait_event: Optional[threading.Event] = None
now_ts = time.time()
with _GATE_LOCK:
state = _state_for(key)
if now_ts < state.cooldown_until_ts:
return state.result if state.has_result else None
if state.inflight is None:
event = threading.Event()
state.inflight = event
state.error = None
break
wait_event = state.inflight
if wait_event is not None:
wait_event.wait(timeout=max(5.0, float(interval)))
with _GATE_LOCK:
state = _state_for(key)
if state.error is not None:
raise state.error
if state.has_result:
return state.result
if state.inflight is None:
continue
return None
owner_event = event
try:
if not _acquire_cross_process_cooldown(key, interval):
with _GATE_LOCK:
state = _state_for(key)
return state.result if state.has_result else None
result = fetcher()
with _GATE_LOCK:
state = _state_for(key)
state.result = result
state.has_result = True
state.cooldown_until_ts = time.time() + interval
state.error = None
return result
except BaseException as exc:
with _GATE_LOCK:
state = _state_for(key)
state.error = exc
state.cooldown_until_ts = time.time() + max(1, int(failure_cooldown_sec or 30))
raise
finally:
with _GATE_LOCK:
state = _state_for(key)
if state.inflight is owner_event:
state.inflight = None
owner_event.set()
def reset_observation_source_gate_for_tests() -> None:
with _GATE_LOCK:
_SOURCE_STATES.clear()
src/data_collection/weather_sources.py
+8 -7
View File
@@ -183,8 +183,8 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
)
self.open_meteo_multi_model_cache_ttl_sec = min(self.open_meteo_multi_model_cache_ttl_sec, MODEL_CACHE_TTL_SEC)
self.multi_model_cache_version = str(
os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_VERSION", "v3")
).strip() or "v3"
os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_VERSION", "v4")
).strip() or "v4"
self._open_meteo_cache: Dict[str, Dict] = {}
self._ensemble_cache: Dict[str, Dict] = {}
self._multi_model_cache: Dict[str, Dict] = {}
@@ -263,8 +263,9 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
self._madis_cache_lock = threading.Lock()
self._hko_obs_cache_lock = threading.Lock()
self.cowin_obs_cache_ttl_sec = int(
os.getenv("COWIN_OBS_CACHE_TTL_SEC", "120")
os.getenv("COWIN_OBS_CACHE_TTL_SEC", str(OBSERVATION_REFRESH_SEC))
)
self.cowin_obs_cache_ttl_sec = min(self.cowin_obs_cache_ttl_sec, OBSERVATION_REFRESH_SEC)
self._cowin_obs_cache: Dict[str, Dict] = {}
self._cowin_obs_cache_lock = threading.Lock()
self.cwa_open_data_auth = (
@@ -1732,10 +1733,10 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
self._attach_wunderground_historical(results, city_lower, use_fahrenheit)
if lat and lon:
# When force_refresh_observations_only is set (airport push loop),
# skip the OM fetch entirely if cached data exists — the 60 s cycle
# must not hammer the Open-Meteo API. Stale model data is fine;
# the loop only needs fresh METAR / AMOS observations.
# When force_refresh_observations_only is set by an explicit
# observation refresh caller, skip the OM fetch entirely if cached
# data exists. Stale model data is fine; the caller only needs
# fresh METAR / AMOS observations.
om_from_cache_only = force_refresh_observations_only
if om_from_cache_only:
self._maybe_reload_open_meteo_disk_cache()
src/utils/telegram_push.py
+107 -16
View File
@@ -1373,18 +1373,89 @@ def _get_airport_daily_high(city_weather: Dict[str, Any]):
return max_so_far, max_time
# Per-city push interval — unified to 60s, obs_time dedup prevents spam
# Per-city push interval. The loop wakes every minute, but Telegram should
# read recent city cache instead of acting as an upstream observation producer.
_AIRPORT_PUSH_INTERVAL = {
"seoul": 60, "busan": 60, "tokyo": 60, "ankara": 60,
"helsinki": 60, "amsterdam": 60, "istanbul": 60, "paris": 60,
"hong kong": 60, "shenzhen": 60, "singapore": 60, "taipei": 60,
"beijing": 60, "shanghai": 60, "guangzhou": 60, "qingdao": 60,
"chengdu": 60, "chongqing": 60, "wuhan": 60,
"new york": 60, "los angeles": 60, "chicago": 60, "denver": 60,
"atlanta": 60, "miami": 60, "san francisco": 60, "houston": 60,
"dallas": 60, "austin": 60, "seattle": 60,
"tel aviv": 60,
city: 600 for city in HIGH_FREQ_AIRPORT_CITIES
}
_AIRPORT_PUSH_INTERVAL.update({
"seoul": 60,
"busan": 60,
"beijing": 180,
"shanghai": 180,
"guangzhou": 180,
"qingdao": 180,
"chengdu": 180,
"chongqing": 180,
"wuhan": 180,
})
def _airport_push_cache_max_age_sec(city: str) -> int:
interval = int(_AIRPORT_PUSH_INTERVAL.get((city or "").strip().lower(), 600) or 600)
return max(90, interval * 2)
def _read_cached_airport_city_weather(city: str, max_age_sec: Optional[int] = None) -> Optional[Dict[str, Any]]:
"""Read web city cache for Telegram without triggering collection."""
normalized_city = (city or "").strip().lower()
if not normalized_city:
return None
max_age = int(max_age_sec if max_age_sec is not None else _airport_push_cache_max_age_sec(normalized_city))
now_ts = time.time()
stale_candidate: Optional[Tuple[float, Dict[str, Any]]] = None
try:
db = DBManager()
for kind in ("full", "panel"):
entry = db.get_city_cache(kind, normalized_city)
if not isinstance(entry, dict):
continue
updated_at_ts = float(entry.get("updated_at_ts") or 0.0)
payload = entry.get("payload")
if updated_at_ts <= 0 or not isinstance(payload, dict):
continue
age_sec = now_ts - updated_at_ts
if age_sec > max_age:
logger.debug(
"airport push cache stale city={} kind={} age_sec={} max_age_sec={}",
normalized_city,
kind,
round(age_sec, 1),
max_age,
)
if stale_candidate is None or updated_at_ts > stale_candidate[0]:
stale_candidate = (updated_at_ts, dict(payload))
continue
logger.debug(
"airport push cache hit city={} kind={} age_sec={}",
normalized_city,
kind,
round(max(0.0, age_sec), 1),
)
return dict(payload)
except Exception as exc:
logger.debug("airport push city cache read failed city={}: {}", normalized_city, exc)
if stale_candidate is not None:
logger.debug("airport push using stale cache city={}", normalized_city)
return stale_candidate[1]
return None
def _load_airport_city_weather_for_push(city: str) -> Dict[str, Any]:
cached = _read_cached_airport_city_weather(city)
if cached is not None:
return cached
from web.app import _analyze # lazy import - only the bot process needs it
return _analyze(
city,
force_refresh=False,
force_refresh_observations_only=False,
detail_mode="panel",
)
# Per-city temperature window threshold (°C below DEB predicted high)
# Continental airports: wider window (temp rises steadily over land)
# Maritime airports: narrower (sea breeze moderates temp)
@@ -1467,12 +1538,10 @@ def _process_airport_city(
if now_ts - last_city_ts < city_interval:
return None
from web.app import _analyze # lazy import — only the bot process needs it
city_weather: Dict[str, Any] = {}
deb_pred: Optional[float] = None
try:
city_weather = _analyze(city, force_refresh_observations_only=True)
city_weather = _load_airport_city_weather_for_push(city)
deb_raw = (city_weather.get("deb") or {}).get("prediction")
if deb_raw is not None:
deb_pred = float(deb_raw)
@@ -1564,7 +1633,7 @@ def _process_airport_city(
and now_ts - last_city_ts > 540):
time.sleep(4)
try:
city_weather = _analyze(city, force_refresh_observations_only=True)
city_weather = _load_airport_city_weather_for_push(city)
deb_raw2 = (city_weather.get("deb") or {}).get("prediction")
if deb_raw2 is not None:
deb_pred = float(deb_raw2)
@@ -1620,6 +1689,21 @@ def _process_airport_city(
return None
def _due_airport_cities(
cities: Set[str],
now_ts: int,
last_by_city: Dict[str, Any],
) -> List[str]:
due: List[str] = []
for city in sorted(cities):
last_city = last_by_city.get(city) or {}
last_city_ts = int(last_city.get("ts") or 0)
city_interval = _AIRPORT_PUSH_INTERVAL.get(city, 600)
if now_ts - last_city_ts >= city_interval:
due.append(city)
return due
def _run_high_freq_airport_cycle(
bot: Any,
config: Dict[str, Any],
@@ -1630,9 +1714,16 @@ def _run_high_freq_airport_cycle(
now_ts = int(time.time())
last_by_city = state.setdefault("last_by_city", {})
max_workers = max(1, min(4, _env_int("TELEGRAM_AIRPORT_PUSH_MAX_WORKERS", 1)))
logger.info("airport cycle tick cities={} max_workers={}", len(HIGH_FREQ_AIRPORT_CITIES), max_workers)
cities = _due_airport_cities(HIGH_FREQ_AIRPORT_CITIES, now_ts, last_by_city)
logger.info(
"airport cycle tick cities={} due={} max_workers={}",
len(HIGH_FREQ_AIRPORT_CITIES),
len(cities),
max_workers,
)
if not cities:
return False
cities = sorted(HIGH_FREQ_AIRPORT_CITIES)
pool = _get_airport_executor(max_workers)
futures = {
pool.submit(