From a6e7d6682d16202c8924f31e3f55639315c3b9ad Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Tue, 19 May 2026 16:57:00 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A7=BB=E9=99=A4=E4=BF=84=E7=BD=97=E6=96=AF?= =?UTF-8?q?=20pogodaiklimat=20=E6=95=B0=E6=8D=AE=E6=BA=90=EF=BC=9A?= =?UTF-8?q?=E5=91=A8=E8=BE=B9=E8=A7=82=E6=B5=8B=E7=AB=99=E4=B8=8D=E9=9C=80?= =?UTF-8?q?=E8=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 1 - src/data_collection/country_networks.py | 2 - src/data_collection/russia_station_sources.py | 365 ------------------ src/data_collection/weather_sources.py | 30 +- tests/test_russia_station_sources.py | 61 --- 5 files changed, 1 insertion(+), 458 deletions(-) delete mode 100644 src/data_collection/russia_station_sources.py delete mode 100644 tests/test_russia_station_sources.py diff --git a/.env.example b/.env.example index e294e8ff..5bdaa1fe 100644 --- a/.env.example +++ b/.env.example @@ -76,7 +76,6 @@ JMA_AMEDAS_CACHE_TTL_SEC=120 # FMI_BASE_URL=https://opendata.fmi.fi/wfs # HKO_BASE_URL=https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather # SINGAPORE_MSS_BASE_URL=https://api.data.gov.sg/v1/environment/air-temperature -# RUSSIA_POGODAIKLIMAT_BASE_URL=https://www.pogodaiklimat.ru ######################################## # 4) Auth / entitlement diff --git a/src/data_collection/country_networks.py b/src/data_collection/country_networks.py index 451d9145..b095b3e4 100644 --- a/src/data_collection/country_networks.py +++ b/src/data_collection/country_networks.py @@ -560,7 +560,6 @@ def _kma_rows(raw: Dict[str, Any], city: str) -> List[Dict[str, Any]]: return out -def _ru_rows(raw: Dict[str, Any], city: str) -> List[Dict[str, Any]]: rows = raw.get("ru_official_nearby") or [] out: List[Dict[str, Any]] = [] for row in rows: @@ -574,7 +573,6 @@ def _ru_rows(raw: Dict[str, Any], city: str) -> List[Dict[str, Any]]: lat=row.get("lat"), lon=row.get("lon"), obs_time=row.get("obs_time"), - source_code="ru_station_web", source_label="Russia station web", is_official=True, is_airport_station=_bool(row.get("is_airport_station")), diff --git a/src/data_collection/russia_station_sources.py b/src/data_collection/russia_station_sources.py deleted file mode 100644 index beb82aa2..00000000 --- a/src/data_collection/russia_station_sources.py +++ /dev/null @@ -1,365 +0,0 @@ -from __future__ import annotations - -import html -import math -import re -import os -import time -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional - -from loguru import logger - -from src.utils.metrics import record_source_call - - -RUSSIA_MOSCOW_STATIONS: Dict[str, Dict[str, Any]] = { - "27524": { - "station_code": "27524", - "station_label": "Vnukovo", - "lat": 55.5870, - "lon": 37.2500, - }, - "27500": { - "station_code": "27500", - "station_label": "Tolstopaltsevo", - "lat": 55.5900, - "lon": 37.1940, - }, - "27518": { - "station_code": "27518", - "station_label": "Podmoskovnaya", - "lat": 55.7084, - "lon": 37.1823, - }, - "27515": { - "station_code": "27515", - "station_label": "Nemchinovka", - "lat": 55.7065, - "lon": 37.3719, - }, - "27504": { - "station_code": "27504", - "station_label": "Moscow (Butovo)", - "lat": 55.5780, - "lon": 37.5541, - }, - "27614": { - "station_code": "27614", - "station_label": "Mikhailovskoye", - "lat": 55.3671, - "lon": 37.2033, - }, - "27601": { - "station_code": "27601", - "station_label": "Krasnogorsk", - "lat": 55.8069, - "lon": 37.3446, - }, - "27416": { - "station_code": "27416", - "station_label": "Moscow (Strogino)", - "lat": 55.7976, - "lon": 37.3982, - }, - "27605": { - "station_code": "27605", - "station_label": "Moscow (Balchug)", - "lat": 55.7455, - "lon": 37.6300, - }, - "27619": { - "station_code": "27619", - "station_label": "Moscow (Tushino)", - "lat": 55.8783, - "lon": 37.4367, - }, - "27621": { - "station_code": "27621", - "station_label": "Gorki Leninskie", - "lat": 55.5079, - "lon": 37.7755, - }, -} - -RUSSIA_MOSCOW_MAP_STATION_LIMIT = 10 - - -class RussiaStationSourceMixin: - def _ru_http_get_text(self, url: str) -> str: - getter = getattr(self, "_http_get", None) - if callable(getter): - response = getter(url) - else: - response = self.session.get(url, timeout=self.timeout) - response.raise_for_status() - return response.text - - @staticmethod - def _ru_safe_float(value: Any) -> Optional[float]: - try: - if value in (None, "", "-", "—"): - return None - text = str(value).strip().replace(",", ".") - return float(text) - except Exception: - return None - - @staticmethod - def _ru_distance_km( - lat1: Optional[float], - lon1: Optional[float], - lat2: Optional[float], - lon2: Optional[float], - ) -> Optional[float]: - if None in (lat1, lon1, lat2, lon2): - return None - try: - r = 6371.0 - d_lat = math.radians(float(lat2) - float(lat1)) - d_lon = math.radians(float(lon2) - float(lon1)) - a = ( - math.sin(d_lat / 2) ** 2 - + math.cos(math.radians(float(lat1))) - * math.cos(math.radians(float(lat2))) - * math.sin(d_lon / 2) ** 2 - ) - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) - return round(r * c, 2) - except Exception: - return None - - @staticmethod - def _ru_clean_cell(cell_html: str) -> str: - text = re.sub(r"<[^>]+>", " ", str(cell_html or "")) - text = html.unescape(text) - text = text.replace("\xa0", " ") - text = re.sub(r"\s+", " ", text) - return text.strip() - - @classmethod - def _ru_parse_table_rows(cls, table_html: str) -> List[List[str]]: - rows: List[List[str]] = [] - for row_html in re.findall(r"]*>(.*?)", table_html, flags=re.S | re.I): - cells = re.findall(r"]*>(.*?)", row_html, flags=re.S | re.I) - cleaned = [cls._ru_clean_cell(cell) for cell in cells] - if cleaned: - rows.append(cleaned) - return rows - - @staticmethod - def _ru_build_obs_time(hour_text: str, day_month_text: str) -> Optional[str]: - hour_match = re.search(r"(\d{1,2})", str(hour_text or "")) - day_match = re.search(r"(\d{1,2})\.(\d{1,2})", str(day_month_text or "")) - if not hour_match or not day_match: - return None - hour = int(hour_match.group(1)) - day = int(day_match.group(1)) - month = int(day_match.group(2)) - now_utc = datetime.now(timezone.utc) - year = now_utc.year - try: - candidate = datetime(year, month, day, hour, 0, tzinfo=timezone.utc) - except ValueError: - return None - if candidate > now_utc and (candidate - now_utc).days > 40: - candidate = datetime(year - 1, month, day, hour, 0, tzinfo=timezone.utc) - return candidate.isoformat() - - def _ru_parse_station_current_from_weather_html(self, html_text: str) -> Optional[Dict[str, Any]]: - tables = re.findall(r"]*>(.*?)", html_text, flags=re.S | re.I) - if len(tables) < 2: - return None - time_rows = self._ru_parse_table_rows(tables[0]) - data_rows = self._ru_parse_table_rows(tables[1]) - if len(time_rows) < 2 or len(data_rows) < 2: - return None - - pair_count = min(len(time_rows), len(data_rows)) - 1 - for idx in range(pair_count - 1, -1, -1): - time_row = time_rows[idx + 1] - data_row = data_rows[idx + 1] - if len(time_row) < 2 or len(data_row) < 6: - continue - temp_c = self._ru_safe_float(data_row[5]) - if temp_c is None: - continue - obs_time = self._ru_build_obs_time(time_row[0], time_row[1]) - return { - "temp_c": round(temp_c, 1), - "obs_time": obs_time, - "raw_hour": time_row[0], - "raw_day_month": time_row[1], - } - return None - - def _ru_cached_station_current( - self, - station_code: str, - station_meta: Dict[str, Any], - use_fahrenheit: bool = False, - ) -> Optional[Dict[str, Any]]: - cache_key = f"{station_code}:{use_fahrenheit}" - now_ts = time.time() - with self._ru_station_cache_lock: - cached = self._ru_station_cache.get(cache_key) - if cached and now_ts - cached["t"] < self.ru_station_cache_ttl_sec: - return cached["d"] - - started = time.perf_counter() - try: - _ru_base = os.getenv('RUSSIA_POGODAIKLIMAT_BASE_URL', '').strip() or 'https://www.pogodaiklimat.ru' - url = f"{_ru_base}/weather.php?id={station_code}" - html_text = self._ru_http_get_text(url) - parsed = self._ru_parse_station_current_from_weather_html(html_text) - if not parsed: - record_source_call( - "ru_station_web", - "current", - "empty", - (time.perf_counter() - started) * 1000.0, - ) - return None - obs_time = parsed.get("obs_time") - max_stale_sec = max( - 0, - int(getattr(self, "ru_station_max_stale_sec", 4 * 3600)), - ) - if obs_time and max_stale_sec > 0: - try: - obs_dt = datetime.fromisoformat(str(obs_time)) - obs_age_sec = (datetime.now(timezone.utc) - obs_dt).total_seconds() - if obs_age_sec > max_stale_sec: - logger.info( - "Russia station web row is stale station={} obs_time={} age_hours={:.1f}", - station_code, - obs_time, - obs_age_sec / 3600.0, - ) - record_source_call( - "ru_station_web", - "current", - "stale_row", - (time.perf_counter() - started) * 1000.0, - ) - return None - except Exception: - pass - temp_c = parsed.get("temp_c") - if temp_c is None: - record_source_call( - "ru_station_web", - "current", - "no_temperature", - (time.perf_counter() - started) * 1000.0, - ) - return None - temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1) - result = { - "station_code": station_code, - "station_label": station_meta.get("station_label") or f"RU {station_code}", - "name": station_meta.get("station_label") or f"RU {station_code}", - "lat": station_meta.get("lat"), - "lon": station_meta.get("lon"), - "temp": temp, - "obs_time": parsed.get("obs_time"), - "source": "ru_station_web", - "source_label": "Russia station web", - "source_code": "ru_station_web", - "is_official": True, - "is_airport_station": station_code == "27524", - "is_settlement_anchor": False, - "page_url": f"{_ru_base}/weather.php?id={station_code}", - } - with self._ru_station_cache_lock: - self._ru_station_cache[cache_key] = {"d": result, "t": now_ts} - record_source_call( - "ru_station_web", - "current", - "success", - (time.perf_counter() - started) * 1000.0, - ) - return result - except Exception as exc: - logger.warning("Russia station web fetch failed station={} error={}", station_code, exc) - with self._ru_station_cache_lock: - stale = self._ru_station_cache.get(cache_key) - if stale: - record_source_call( - "ru_station_web", - "current", - "stale_cache", - (time.perf_counter() - started) * 1000.0, - ) - return stale["d"] - record_source_call( - "ru_station_web", - "current", - "error", - (time.perf_counter() - started) * 1000.0, - ) - return None - - def fetch_russia_moscow_official_nearby( - self, - city: str, - use_fahrenheit: bool = False, - ) -> List[Dict[str, Any]]: - started = time.perf_counter() - city_key = str(city or "").strip().lower() - if city_key != "moscow": - record_source_call( - "ru_station_web", - "nearby", - "unsupported_city", - (time.perf_counter() - started) * 1000.0, - ) - return [] - - city_meta = self.CITY_REGISTRY.get(city_key) or {} - anchor_lat = self._ru_safe_float(city_meta.get("lat")) - anchor_lon = self._ru_safe_float(city_meta.get("lon")) - rows: List[Dict[str, Any]] = [] - try: - for station_code, station_meta in RUSSIA_MOSCOW_STATIONS.items(): - current = self._ru_cached_station_current( - station_code, - station_meta, - use_fahrenheit=use_fahrenheit, - ) - if not current: - continue - row = dict(current) - row["distance_km"] = self._ru_distance_km( - anchor_lat, - anchor_lon, - self._ru_safe_float(row.get("lat")), - self._ru_safe_float(row.get("lon")), - ) - row["icao"] = station_code - row["istNo"] = station_code - rows.append(row) - rows.sort( - key=lambda item: ( - item.get("distance_km") is None, - item.get("distance_km") if item.get("distance_km") is not None else 9999, - item.get("station_label") or "", - ) - ) - trimmed = rows[:RUSSIA_MOSCOW_MAP_STATION_LIMIT] - record_source_call( - "ru_station_web", - "nearby", - "success" if trimmed else "empty", - (time.perf_counter() - started) * 1000.0, - ) - return trimmed - except Exception as exc: - logger.warning("Russia station nearby fetch failed city={} error={}", city_key, exc) - record_source_call( - "ru_station_web", - "nearby", - "error", - (time.perf_counter() - started) * 1000.0, - ) - return [] diff --git a/src/data_collection/weather_sources.py b/src/data_collection/weather_sources.py index dacc5065..6c6b206c 100644 --- a/src/data_collection/weather_sources.py +++ b/src/data_collection/weather_sources.py @@ -11,7 +11,6 @@ from src.data_collection.settlement_sources import SettlementSourceMixin from src.data_collection.metar_sources import MetarSourceMixin from src.data_collection.mgm_sources import MgmSourceMixin from src.data_collection.jma_amedas_sources import JmaAmedasSourceMixin -from src.data_collection.russia_station_sources import RussiaStationSourceMixin from src.data_collection.nws_open_meteo_sources import NwsOpenMeteoSourceMixin from src.data_collection.amos_station_sources import AmosStationSourceMixin from src.data_collection.amsc_awos_sources import AmscAwosSourceMixin @@ -23,7 +22,7 @@ from src.data_collection.singapore_mss_sources import SingaporeMssSourceMixin from src.database.db_manager import DBManager -class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin): +class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin): """ Multi-source weather data collector @@ -208,14 +207,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour ) self._jma_cache: Dict[str, Dict] = {} self._jma_cache_lock = threading.Lock() - self.ru_station_cache_ttl_sec = int( - os.getenv("RU_STATION_CACHE_TTL_SEC", "300") - ) - self.ru_station_max_stale_sec = int( - os.getenv("RU_STATION_MAX_STALE_SEC", str(4 * 3600)) - ) - self._ru_station_cache: Dict[str, Dict] = {} - self._ru_station_cache_lock = threading.Lock() self.settlement_cache_ttl_sec = int( os.getenv("SETTLEMENT_SOURCE_CACHE_TTL_SEC", "120") ) @@ -767,8 +758,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour self._knmi_cache.pop(f"knmi:{normalized}:{use_fahrenheit}", None) with self._hko_obs_cache_lock: self._hko_obs_cache.pop(f"hko_obs:{normalized}:{use_fahrenheit}", None) - with self._ru_station_cache_lock: - self._ru_station_cache.pop(f"{normalized}:{use_fahrenheit}", None) with self._settlement_cache_lock: city_meta = self.CITY_REGISTRY.get(normalized) or {} settlement_source = str(city_meta.get("settlement_source") or "").strip().lower() @@ -1195,21 +1184,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour "airport_obs_log append failed for singapore_mss city={}", city_lower ) - def _attach_russia_official_nearby( - self, results: Dict, city_lower: str, use_fahrenheit: bool - ) -> None: - if city_lower != "moscow": - return - official_rows = self.fetch_russia_moscow_official_nearby( - city_lower, use_fahrenheit=use_fahrenheit - ) - if not official_rows: - return - # Pogodaiklimat station rows are SYNOP/archive-style reference observations, - # not realtime enough for the map. Keep them out of official_nearby/mgm_nearby - # so Moscow uses the live METAR cluster for nearby map temperatures. - results["ru_reference_nearby"] = official_rows - def _attach_warsaw_official_nearby( self, results: Dict, use_fahrenheit: bool ) -> None: @@ -1375,7 +1349,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour self._attach_knmi_official_nearby(results, city_lower, use_fahrenheit) self._attach_hko_obs_official_nearby(results, city_lower, use_fahrenheit) self._attach_cwa_settlement_nearby(results, city_lower, use_fahrenheit) - self._attach_russia_official_nearby(results, city_lower, use_fahrenheit) if city_lower == "warsaw": self._attach_warsaw_official_nearby(results, use_fahrenheit) self._attach_global_nearby_cluster( @@ -1424,7 +1397,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour self._attach_knmi_official_nearby(results, city_lower, use_fahrenheit) self._attach_hko_obs_official_nearby(results, city_lower, use_fahrenheit) self._attach_cwa_settlement_nearby(results, city_lower, use_fahrenheit) - self._attach_russia_official_nearby(results, city_lower, use_fahrenheit) if city_lower == "warsaw": self._attach_warsaw_official_nearby(results, use_fahrenheit) self._attach_global_nearby_cluster( diff --git a/tests/test_russia_station_sources.py b/tests/test_russia_station_sources.py deleted file mode 100644 index c8451e71..00000000 --- a/tests/test_russia_station_sources.py +++ /dev/null @@ -1,61 +0,0 @@ -from src.data_collection.russia_station_sources import ( - RUSSIA_MOSCOW_MAP_STATION_LIMIT, - RUSSIA_MOSCOW_STATIONS, - RussiaStationSourceMixin, -) - - -def test_moscow_station_registry_includes_vnukovo_nearby_ring(): - assert "27524" in RUSSIA_MOSCOW_STATIONS - assert "27500" in RUSSIA_MOSCOW_STATIONS - assert RUSSIA_MOSCOW_STATIONS["27500"]["station_label"] == "Tolstopaltsevo" - assert RUSSIA_MOSCOW_MAP_STATION_LIMIT == 10 - - -def test_fetch_moscow_official_nearby_keeps_nearest_ten_station_rows(): - class FakeRussiaSource(RussiaStationSourceMixin): - CITY_REGISTRY = { - "moscow": { - "lat": 55.5915, - "lon": 37.2615, - } - } - - def _ru_cached_station_current(self, station_code, station_meta, use_fahrenheit=False): - return { - "station_code": station_code, - "station_label": station_meta["station_label"], - "name": station_meta["station_label"], - "lat": station_meta["lat"], - "lon": station_meta["lon"], - "temp": 10.0, - "source_code": "ru_station_web", - } - - rows = FakeRussiaSource().fetch_russia_moscow_official_nearby("moscow") - - assert len(rows) == RUSSIA_MOSCOW_MAP_STATION_LIMIT - assert rows[0]["station_code"] == "27524" - assert rows[1]["station_code"] == "27500" - assert all(row["distance_km"] is not None for row in rows) - - -def test_russia_weather_archive_parser_uses_latest_valid_row(): - html = """ - - - - -
hourdate
0911.04
2117.04
- - - - -
abcdcloudtemp
xxxxx+4.2
xxxxx+5.8
- """ - - parsed = RussiaStationSourceMixin()._ru_parse_station_current_from_weather_html(html) - - assert parsed["temp_c"] == 5.8 - assert parsed["raw_hour"] == "21" - assert parsed["raw_day_month"] == "17.04"