Harden Open-Meteo rate limiting

src/data_collection/nws_open_meteo_sources.py

@@ -351,8 +351,9 @@ class NwsOpenMeteoSourceMixin:
         now_ts = time.time()
         # ── 429 冷却期检查（所有 Open-Meteo 端点共享）─────────────────
         with self._open_meteo_rl_lock:
-            if now_ts < self._open_meteo_rate_limit_until:
-                remaining = int(self._open_meteo_rate_limit_until - now_ts)
+            cooldown_until = self._refresh_open_meteo_rate_limit_until()
+            if now_ts < cooldown_until:
+                remaining = int(cooldown_until - now_ts)
                 logger.debug(f"Open-Meteo 冷却期中，跳过请求，还需 {remaining}s")
                 with self._open_meteo_cache_lock:
                         stale = self._open_meteo_cache.get(cache_key)
@@ -494,7 +495,7 @@ class NwsOpenMeteoSourceMixin:
                 )
                 # 设置全局冷却期，避免短时内重复触发 429
                 with self._open_meteo_rl_lock:
-                    self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
+                    self._set_open_meteo_rate_limit_until(time.time() + cooldown_to_use, reason="forecast_429")
                     logger.warning(f"Open-Meteo 触发限流，设置 {cooldown_to_use}s 冷却期")
             else:
                 logger.error(f"Open-Meteo forecast failed: {e}")
@@ -527,8 +528,9 @@ class NwsOpenMeteoSourceMixin:
         now_ts = time.time()
         # ── 429 冷却期检查（所有 Open-Meteo 端点共享）─────────────────
         with self._open_meteo_rl_lock:
-            if now_ts < self._open_meteo_rate_limit_until:
-                remaining = int(self._open_meteo_rate_limit_until - now_ts)
+            cooldown_until = self._refresh_open_meteo_rate_limit_until()
+            if now_ts < cooldown_until:
+                remaining = int(cooldown_until - now_ts)
                 logger.debug(f"Open-Meteo Ensemble 冷却期中，跳过请求，还需 {remaining}s")
                 with self._ensemble_cache_lock:
                     stale = self._ensemble_cache.get(cache_key)
@@ -647,7 +649,7 @@ class NwsOpenMeteoSourceMixin:
                     f"Ensemble API rate limited (429), fallback to cache if available: lat={lat}, lon={lon}"
                 )
                 with self._open_meteo_rl_lock:
-                    self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
+                    self._set_open_meteo_rate_limit_until(time.time() + cooldown_to_use, reason="ensemble_429")
             else:
                 logger.warning(f"Ensemble API 请求失败: {e}")
             with self._ensemble_cache_lock:
@@ -690,8 +692,9 @@ class NwsOpenMeteoSourceMixin:
         now_ts = time.time()
         # ── 429 冷却期检查（所有 Open-Meteo 端点共享）─────────────────
         with self._open_meteo_rl_lock:
-            if now_ts < self._open_meteo_rate_limit_until:
-                remaining = int(self._open_meteo_rate_limit_until - now_ts)
+            cooldown_until = self._refresh_open_meteo_rate_limit_until()
+            if now_ts < cooldown_until:
+                remaining = int(cooldown_until - now_ts)
                 logger.debug(f"Open-Meteo Multi-model 冷却期中，跳过请求，还需 {remaining}s")
                 with self._multi_model_cache_lock:
                     stale = self._multi_model_cache.get(cache_key)
@@ -808,7 +811,7 @@ class NwsOpenMeteoSourceMixin:
                     f"Multi-model API rate limited (429), fallback to cache if available: lat={lat}, lon={lon}"
                 )
                 with self._open_meteo_rl_lock:
-                    self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
+                    self._set_open_meteo_rate_limit_until(time.time() + cooldown_to_use, reason="multi_model_429")
             else:
                 logger.warning(f"Multi-model API 请求失败: {e}")
             with self._multi_model_cache_lock:

src/data_collection/open_meteo_cache.py

@@ -7,11 +7,13 @@ import time
 from loguru import logger
 from src.database.runtime_state import (
     OpenMeteoCacheRepository,
+    OpenMeteoRateLimitRepository,
     STATE_STORAGE_SQLITE,
     get_state_storage_mode,
 )
 
 _open_meteo_cache_repo = OpenMeteoCacheRepository()
+_open_meteo_rate_limit_repo = OpenMeteoRateLimitRepository()
 
 
 class OpenMeteoCacheMixin:
@@ -182,3 +184,26 @@ class OpenMeteoCacheMixin:
                 time.sleep(wait_for)
                 now_ts = time.time()
             self._open_meteo_last_call_ts = now_ts
+
+    def _refresh_open_meteo_rate_limit_until(self) -> float:
+        """Load the cross-process Open-Meteo cooldown marker from runtime state."""
+        if get_state_storage_mode() != STATE_STORAGE_SQLITE:
+            return getattr(self, "_open_meteo_rate_limit_until", 0.0)
+        try:
+            persisted_until = _open_meteo_rate_limit_repo.load_until()
+        except Exception as exc:
+            logger.debug(f"Open-Meteo cooldown load skipped: {exc}")
+            return getattr(self, "_open_meteo_rate_limit_until", 0.0)
+        if persisted_until > getattr(self, "_open_meteo_rate_limit_until", 0.0):
+            self._open_meteo_rate_limit_until = persisted_until
+        return self._open_meteo_rate_limit_until
+
+    def _set_open_meteo_rate_limit_until(self, rate_limit_until: float, *, reason: str = "") -> None:
+        """Persist the shared Open-Meteo cooldown marker for all workers."""
+        self._open_meteo_rate_limit_until = float(rate_limit_until)
+        if get_state_storage_mode() != STATE_STORAGE_SQLITE:
+            return
+        try:
+            _open_meteo_rate_limit_repo.set_until(rate_limit_until, reason=reason)
+        except Exception as exc:
+            logger.debug(f"Open-Meteo cooldown persist skipped: {exc}")

src/data_collection/weather_sources.py

@@ -1316,7 +1316,11 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
                 lat=lat,
                 lon=lon,
                 use_fahrenheit=use_fahrenheit,
-                keep_model_caches=force_refresh_observations_only,
+                # Force-refresh is usually a UI/API freshness request for live
+                # observations.  Do not evict longer-lived Open-Meteo forecast
+                # caches by default: one accidental all-city refresh can
+                # otherwise cold-start the VPS into Open-Meteo 429s.
+                keep_model_caches=True,
             )
         self._log_temperature_unit(city, use_fahrenheit)
         self._attach_settlement_sources(results, city_lower)

src/database/runtime_state.py

@@ -193,6 +193,16 @@ class RuntimeStateDB:
             conn.execute(
                 "CREATE INDEX IF NOT EXISTS idx_open_meteo_cache_expires ON open_meteo_cache_store(source_kind, expires_at)"
             )
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS open_meteo_rate_limit_state (
+                    state_key TEXT PRIMARY KEY,
+                    rate_limit_until REAL NOT NULL,
+                    reason TEXT,
+                    updated_at REAL NOT NULL
+                )
+                """
+            )
             conn.execute(
                 """
                 CREATE TABLE IF NOT EXISTS official_intraday_observations_store (
@@ -1001,6 +1011,43 @@ class OpenMeteoCacheRepository:
             return 0.0
 
 
+class OpenMeteoRateLimitRepository:
+    def __init__(self, db: Optional[RuntimeStateDB] = None):
+        self.db = db or RuntimeStateDB.instance()
+
+    def set_until(self, rate_limit_until: float, *, reason: str = "") -> None:
+        with self.db.connect() as conn:
+            conn.execute(
+                """
+                INSERT INTO open_meteo_rate_limit_state (
+                    state_key, rate_limit_until, reason, updated_at
+                ) VALUES ('global', ?, ?, ?)
+                ON CONFLICT(state_key) DO UPDATE SET
+                    rate_limit_until = excluded.rate_limit_until,
+                    reason = excluded.reason,
+                    updated_at = excluded.updated_at
+                """,
+                (float(rate_limit_until), str(reason or ""), time.time()),
+            )
+            conn.commit()
+
+    def load_until(self) -> float:
+        with self.db.connect() as conn:
+            row = conn.execute(
+                """
+                SELECT rate_limit_until
+                FROM open_meteo_rate_limit_state
+                WHERE state_key = 'global'
+                """
+            ).fetchone()
+        if not row:
+            return 0.0
+        try:
+            return float(row["rate_limit_until"] or 0.0)
+        except Exception:
+            return 0.0
+
+
 class OfficialIntradayObservationRepository:
     def __init__(self, db: Optional[RuntimeStateDB] = None):
         self.db = db or RuntimeStateDB.instance()