"""Independent high-frequency observation collector for the web runtime.""" from __future__ import annotations import os import threading import time from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from typing import Any, Callable, Iterable, List, Optional, Sequence, Tuple from loguru import logger from src.data_collection.amos_station_sources import AMOS_AIRPORT_CODES from src.data_collection.amsc_awos_sources import AMSC_AWOS_AIRPORTS from src.data_collection.city_registry import CITY_REGISTRY from src.data_collection.hko_obs_sources import HKO_STATIONS from src.database.runtime_state import ObservationCollectorStatusRepository def _env_bool(name: str, default: bool) -> bool: raw = os.getenv(name) if raw is None: return default return raw.strip().lower() in {"1", "true", "yes", "on"} def _env_int(name: str, default: int) -> int: raw = os.getenv(name) if raw is None: return default try: return int(raw) except Exception: return default def _normalized_cities(cities: Iterable[str]) -> Tuple[str, ...]: return tuple(sorted({str(city or "").strip().lower() for city in cities if str(city or "").strip()})) @dataclass(frozen=True) class ObservationSourceProfile: source: str cities: Tuple[str, ...] interval_sec: int class ObservationCollector: def __init__( self, *, weather: Any, profiles: Sequence[ObservationSourceProfile], cache_refresher: Optional[Callable[[str], Any]] = None, status_recorder: Optional[ObservationCollectorStatusRepository] = None, async_cache_refresh: Optional[bool] = None, cache_refresh_workers: Optional[int] = None, ) -> None: self.weather = weather self.profiles = list(profiles) self.cache_refresher = cache_refresher self.status_recorder = status_recorder self._last_run_ts: dict[tuple[str, str], float] = {} self._lock = threading.Lock() self._cache_refresh_lock = threading.Lock() self._cache_refresh_inflight: set[str] = set() self._cache_refresh_async = ( _env_bool("POLYWEATHER_OBSERVATION_COLLECTOR_CACHE_REFRESH_ASYNC", True) if async_cache_refresh is None else bool(async_cache_refresh) ) worker_count = max( 1, min( 4, int( cache_refresh_workers if cache_refresh_workers is not None else _env_int("POLYWEATHER_OBSERVATION_COLLECTOR_CACHE_REFRESH_WORKERS", 1) ), ), ) self._cache_refresh_executor: Optional[ThreadPoolExecutor] = ( ThreadPoolExecutor(max_workers=worker_count) if callable(cache_refresher) and self._cache_refresh_async else None ) def run_due_once(self, *, now_ts: Optional[float] = None) -> int: now = float(time.time() if now_ts is None else now_ts) due: List[tuple[ObservationSourceProfile, str]] = [] with self._lock: for profile in self.profiles: interval = max(1, int(profile.interval_sec or 60)) for city in profile.cities: key = (profile.source, city) last_ts = float(self._last_run_ts.get(key) or 0.0) if now - last_ts >= interval: self._last_run_ts[key] = now due.append((profile, city)) completed = 0 for profile, city in due: started_wall = time.time() started_ts = now if now_ts is not None else started_wall ok = False error: Optional[str] = None try: ok = self._collect_city_source(profile.source, city) if ok: completed += 1 self._refresh_city_cache(city) else: error = "no_results" except Exception as exc: error = str(exc) or exc.__class__.__name__ logger.warning( "observation collector source failed source={} city={}: {}", profile.source, city, exc, ) finally: completed_ts = started_ts + max(0.0, time.time() - started_wall) self._record_source_status( profile=profile, city=city, due_ts=now, started_ts=started_ts, completed_ts=completed_ts, ok=ok, error=error, ) return completed def _record_source_status( self, *, profile: ObservationSourceProfile, city: str, due_ts: float, started_ts: float, completed_ts: float, ok: bool, error: Optional[str], ) -> None: if not self.status_recorder: return try: self.status_recorder.record_result( source=profile.source, city=city, interval_sec=profile.interval_sec, due_ts=due_ts, started_ts=started_ts, completed_ts=completed_ts, ok=ok, error=error, ) except Exception as exc: logger.warning( "observation collector status write failed source={} city={}: {}", profile.source, city, exc, ) def _collect_city_source(self, source: str, city: str) -> bool: normalized_source = str(source or "").strip().lower() normalized_city = str(city or "").strip().lower() if not normalized_source or not normalized_city: return False use_fahrenheit = bool(self.weather._uses_fahrenheit(normalized_city)) results: dict[str, Any] = {} if normalized_source == "amsc_awos": self.weather._attach_china_amsc_awos_data(results, normalized_city, use_fahrenheit) elif normalized_source == "amos": self.weather._attach_korean_amos_data(results, normalized_city, use_fahrenheit) elif normalized_source == "madis_hfmetar": self.weather._attach_madis_hfmetar_data(results, normalized_city, use_fahrenheit) elif normalized_source == "hko_obs": self.weather._attach_hko_obs_official_nearby(results, normalized_city, use_fahrenheit) elif normalized_source == "cowin_obs": self.weather._attach_cowin_official_nearby(results, normalized_city, use_fahrenheit) else: logger.debug("observation collector skipped unknown source={}", normalized_source) return False return bool(results) def _refresh_city_cache(self, city: str) -> None: if not callable(self.cache_refresher): return normalized_city = str(city or "").strip().lower() if not normalized_city: return if self._cache_refresh_executor is None: self._refresh_city_cache_inline(normalized_city) return with self._cache_refresh_lock: if normalized_city in self._cache_refresh_inflight: return self._cache_refresh_inflight.add(normalized_city) try: self._cache_refresh_executor.submit(self._refresh_city_cache_task, normalized_city) except Exception as exc: with self._cache_refresh_lock: self._cache_refresh_inflight.discard(normalized_city) logger.warning( "observation collector cache refresh queue failed city={}: {}", normalized_city, exc, ) def _refresh_city_cache_task(self, city: str) -> None: try: self._refresh_city_cache_inline(city) finally: with self._cache_refresh_lock: self._cache_refresh_inflight.discard(city) def _refresh_city_cache_inline(self, city: str) -> None: try: self.cache_refresher(city) except Exception as exc: logger.warning("observation collector cache refresh failed city={}: {}", city, exc) def close(self) -> None: if self._cache_refresh_executor is not None: self._cache_refresh_executor.shutdown(wait=False) def build_observation_source_profiles() -> List[ObservationSourceProfile]: us_madis_cities = [ city for city, meta in CITY_REGISTRY.items() if str((meta or {}).get("icao") or "").strip().upper().startswith("K") ] return [ ObservationSourceProfile( source="amos", cities=_normalized_cities(AMOS_AIRPORT_CODES.keys()), interval_sec=max(30, _env_int("POLYWEATHER_OBSERVATION_COLLECTOR_AMOS_SEC", 60)), ), ObservationSourceProfile( source="amsc_awos", cities=_normalized_cities(AMSC_AWOS_AIRPORTS.keys()), interval_sec=max(60, _env_int("POLYWEATHER_OBSERVATION_COLLECTOR_AMSC_SEC", 180)), ), ObservationSourceProfile( source="madis_hfmetar", cities=_normalized_cities(us_madis_cities), interval_sec=max(60, _env_int("POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC", 300)), ), ObservationSourceProfile( source="cowin_obs", cities=("hong kong",), interval_sec=max(30, _env_int("POLYWEATHER_OBSERVATION_COLLECTOR_COWIN_SEC", 60)), ), ObservationSourceProfile( source="hko_obs", cities=_normalized_cities(HKO_STATIONS.keys()), interval_sec=max(60, _env_int("POLYWEATHER_OBSERVATION_COLLECTOR_HKO_SEC", 600)), ), ] _COLLECTOR_THREAD: Optional[threading.Thread] = None _COLLECTOR_LOCK = threading.Lock() def start_observation_collector_loop( *, weather: Any, cache_refresher: Optional[Callable[[str], Any]] = None, profiles: Optional[Sequence[ObservationSourceProfile]] = None, status_recorder: Optional[ObservationCollectorStatusRepository] = None, ) -> Optional[threading.Thread]: if not _env_bool("POLYWEATHER_OBSERVATION_COLLECTOR_ENABLED", True): return None tick_sec = max(5, _env_int("POLYWEATHER_OBSERVATION_COLLECTOR_TICK_SEC", 30)) initial_delay_sec = max( 0, _env_int("POLYWEATHER_OBSERVATION_COLLECTOR_INITIAL_DELAY_SEC", 5), ) selected_profiles = list(profiles or build_observation_source_profiles()) collector = ObservationCollector( weather=weather, profiles=selected_profiles, cache_refresher=cache_refresher, status_recorder=status_recorder or ObservationCollectorStatusRepository(), ) global _COLLECTOR_THREAD with _COLLECTOR_LOCK: if _COLLECTOR_THREAD is not None and _COLLECTOR_THREAD.is_alive(): return _COLLECTOR_THREAD def _runner() -> None: logger.info( "observation collector started profiles={} tick_sec={}", len(selected_profiles), tick_sec, ) if initial_delay_sec: time.sleep(initial_delay_sec) while True: started = time.time() collector.run_due_once(now_ts=started) elapsed = time.time() - started time.sleep(max(1.0, tick_sec - elapsed)) _COLLECTOR_THREAD = threading.Thread( target=_runner, name="observation-collector", daemon=True, ) _COLLECTOR_THREAD.start() return _COLLECTOR_THREAD