diff --git a/.env.example b/.env.example index 2affa2d1..e294e8ff 100644 --- a/.env.example +++ b/.env.example @@ -68,8 +68,6 @@ JMA_AMEDAS_CACHE_TTL_SEC=120 # These are kept in .env to avoid exposing competitive data-source discovery # work on the public GitHub repository. Leave empty to use built-in defaults. # AMSC_AWOS_BASE_URL=https://www.amsc.net.cn/gateway/api/saas/rest/amc/AwosController/getWindPlate -# NMC_FORECAST_BASE_URL=https://m.nmc.cn/publish/forecast -# NMC_REALTIME_BASE_URL=https://www.nmc.cn/rest/real # KMA_BASE_URL=https://www.weather.go.kr # AMOS_BASE_URL=https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do # JMA_AMEDAS_BASE_URL=https://www.jma.go.jp diff --git a/src/data_collection/country_networks.py b/src/data_collection/country_networks.py index 386af9e6..451d9145 100644 --- a/src/data_collection/country_networks.py +++ b/src/data_collection/country_networks.py @@ -508,41 +508,6 @@ def _metar_cluster_rows(raw: Dict[str, Any]) -> List[Dict[str, Any]]: return out -def _nmc_rows(raw: Dict[str, Any], city: str) -> List[Dict[str, Any]]: - rows = raw.get("nmc_official_nearby") or [] - city_offset = get_city_utc_offset_seconds(city) - out: List[Dict[str, Any]] = [] - for row in rows: - if not isinstance(row, dict): - continue - out.append( - _normalize_station_row( - station_code=row.get("icao") or row.get("istNo"), - station_label=row.get("name"), - temp=row.get("temp"), - lat=row.get("lat"), - lon=row.get("lon"), - obs_time=row.get("obs_time"), - source_code="nmc", - source_label="NMC", - is_official=True, - is_airport_station=False, - is_settlement_anchor=False, - extra={ - "obs_time_utc_offset_seconds": city_offset, - "page_url": row.get("page_url"), - "humidity": _safe_float(row.get("humidity")), - "rain": _safe_float(row.get("rain")), - "airpressure": _safe_float(row.get("airpressure")), - "wx_desc": row.get("wx_desc"), - "wind_direction_text": row.get("wind_direction_text"), - "wind_power_text": row.get("wind_power_text"), - }, - ) - ) - return out - - def _jma_rows(raw: Dict[str, Any], city: str) -> List[Dict[str, Any]]: rows = raw.get("jma_official_nearby") or [] out: List[Dict[str, Any]] = [] @@ -830,22 +795,16 @@ class TurkeyMgmNetworkProvider(CountryNetworkProvider): class ChinaCmaNetworkProvider(CountryNetworkProvider): def __init__(self) -> None: - super().__init__("china_cma", "CMA/NMC") + super().__init__("china_cma", "CMA") def official_nearby_current(self, city: str, raw: Dict[str, Any]) -> List[Dict[str, Any]]: - rows = _nmc_rows(raw, city) - if rows: - return rows return _metar_cluster_rows(raw) def official_network_status(self, city: str, raw: Dict[str, Any]) -> Dict[str, Any]: rows = self.official_nearby_current(city, raw) - has_nmc = bool(_nmc_rows(raw, city)) return { "provider_code": self.provider_code, "provider_label": self.provider_label, - "available": has_nmc, - "mode": "official_active" if has_nmc else ("fallback_metar_cluster" if rows else "reference_only"), "row_count": len(rows), } diff --git a/src/data_collection/nmc_sources.py b/src/data_collection/nmc_sources.py deleted file mode 100644 index c0f46e47..00000000 --- a/src/data_collection/nmc_sources.py +++ /dev/null @@ -1,211 +0,0 @@ -from __future__ import annotations - -import os -import re -import time -from datetime import datetime -from typing import Any, Dict, List, Optional - -from loguru import logger - -from src.utils.metrics import record_source_call - -_NMC_FORECAST_BASE = os.getenv("NMC_FORECAST_BASE_URL", "").strip() -_NMC_REALTIME_BASE = os.getenv("NMC_REALTIME_BASE_URL", "").strip() - -NMC_CITY_REFERENCES: Dict[str, Dict[str, Any]] = { - "shanghai": { - "region_label": "浦东", - "page_url": f"{_NMC_FORECAST_BASE}/ASH/pudong.html" if _NMC_FORECAST_BASE else "", - "station_code": "atcMf", - }, - "beijing": { - "region_label": "顺义", - "page_url": f"{_NMC_FORECAST_BASE}/ABJ/shunyi.html" if _NMC_FORECAST_BASE else "", - "station_code": "MKoqG", - }, - "chongqing": { - "region_label": "渝北", - "page_url": f"{_NMC_FORECAST_BASE}/ACQ/yubei.html" if _NMC_FORECAST_BASE else "", - "station_code": "xFVYU", - }, - "chengdu": { - "region_label": "双流", - "page_url": f"{_NMC_FORECAST_BASE}/ASC/shuangliu.html" if _NMC_FORECAST_BASE else "", - "station_code": "grFhZ", - }, - "wuhan": { - "region_label": "武汉", - "page_url": f"{_NMC_FORECAST_BASE}/AHB/wuhan.html" if _NMC_FORECAST_BASE else "", - "station_code": "bSpCz", - }, - "shenzhen": { - "region_label": "深圳", - "page_url": f"{_NMC_FORECAST_BASE}/AGD/shenzuo.html" if _NMC_FORECAST_BASE else "", - "station_code": "AhpEU", - }, -} - - -class NmcSourceMixin: - def _nmc_http_get(self, url: str): - getter = getattr(self, "_http_get", None) - if callable(getter): - return getter(url) - return self.session.get(url, timeout=self.timeout) - - def _nmc_http_get_json(self, url: str): - getter = getattr(self, "_http_get_json", None) - if callable(getter): - return getter(url) - response = self.session.get(url, timeout=self.timeout) - response.raise_for_status() - return response.json() - - @staticmethod - def _nmc_optional_text(value: Any) -> Optional[str]: - text = str(value or "").strip() - if text in ("", "9999"): - return None - return text - - @staticmethod - def _nmc_optional_float(value: Any) -> Optional[float]: - if value in (None, "", "9999", 9999, 9999.0): - return None - try: - return float(value) - except Exception: - return None - - def _resolve_nmc_station_code(self, city: str) -> Optional[str]: - city_key = str(city or "").strip().lower() - meta = NMC_CITY_REFERENCES.get(city_key) or {} - station_code = str(meta.get("station_code") or "").strip() - if station_code: - return station_code - - page_url = str(meta.get("page_url") or "").strip() - if not page_url: - return None - - try: - resp = self._nmc_http_get(page_url) - resp.raise_for_status() - match = re.search( - r"renderWeatherRealPanel\('([^']+)',\s*'([^']+)'\)", - resp.text, - ) - if not match: - return None - station_code = str(match.group(1) or "").strip() - if station_code: - meta["station_code"] = station_code - return station_code - except Exception as exc: - logger.warning("NMC station code resolve failed city={} error={}", city_key, exc) - return None - - def fetch_nmc_region_current( - self, - city: str, - use_fahrenheit: bool = False, - ) -> Optional[Dict[str, Any]]: - started = time.perf_counter() - city_key = str(city or "").strip().lower() - meta = NMC_CITY_REFERENCES.get(city_key) or {} - if not meta: - record_source_call("nmc", "current", "unsupported_city", (time.perf_counter() - started) * 1000.0) - return None - - cache_key = f"{city_key}:{use_fahrenheit}" - now_ts = time.time() - with self._nmc_cache_lock: - cached = self._nmc_cache.get(cache_key) - if cached and now_ts - cached["t"] < self.nmc_cache_ttl_sec: - record_source_call("nmc", "current", "cache_hit", (time.perf_counter() - started) * 1000.0) - return cached["d"] - - station_code = self._resolve_nmc_station_code(city_key) - if not station_code: - record_source_call("nmc", "current", "missing_station_code", (time.perf_counter() - started) * 1000.0) - return None - - try: - url = f"{_NMC_REALTIME_BASE}/{station_code}" - payload = self._nmc_http_get_json(url) - if not isinstance(payload, dict) or not isinstance(payload.get("weather"), dict): - record_source_call("nmc", "current", "empty", (time.perf_counter() - started) * 1000.0) - return None - - weather = payload.get("weather") or {} - temp_c = weather.get("temperature") - if temp_c in (None, "", "9999"): - record_source_call("nmc", "current", "no_temperature", (time.perf_counter() - started) * 1000.0) - return None - temp_c = float(temp_c) - temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1) - - station = payload.get("station") or {} - result = { - "source": "nmc", - "timestamp": datetime.utcnow().isoformat(), - "station_code": station_code, - "station_name": station.get("city") or meta.get("region_label") or city_key.title(), - "page_url": meta.get("page_url"), - "publish_time": payload.get("publish_time"), - "current": { - "temp": temp, - "humidity": self._nmc_optional_float(weather.get("humidity")), - "rain": self._nmc_optional_float(weather.get("rain")), - "airpressure": self._nmc_optional_float(weather.get("airpressure")), - "wx_desc": weather.get("info"), - "wind_direction": self._nmc_optional_text((payload.get("wind") or {}).get("direct")), - "wind_power": self._nmc_optional_text((payload.get("wind") or {}).get("power")), - }, - } - with self._nmc_cache_lock: - self._nmc_cache[cache_key] = {"d": result, "t": now_ts} - record_source_call("nmc", "current", "success", (time.perf_counter() - started) * 1000.0) - return result - except Exception as exc: - logger.warning("NMC current fetch failed city={} code={} error={}", city_key, station_code, exc) - with self._nmc_cache_lock: - stale = self._nmc_cache.get(cache_key) - if stale: - record_source_call("nmc", "current", "stale_cache", (time.perf_counter() - started) * 1000.0) - return stale["d"] - record_source_call("nmc", "current", "error", (time.perf_counter() - started) * 1000.0) - return None - - def fetch_nmc_official_nearby( - self, - city: str, - use_fahrenheit: bool = False, - ) -> List[Dict[str, Any]]: - current = self.fetch_nmc_region_current(city, use_fahrenheit=use_fahrenheit) - if not current: - return [] - meta = NMC_CITY_REFERENCES.get(str(city or "").strip().lower()) or {} - city_meta = self.CITY_REGISTRY.get(str(city or "").strip().lower()) or {} - return [ - { - "name": f"{meta.get('region_label') or current.get('station_name')}区域实况 (NMC)", - "station_label": f"{meta.get('region_label') or current.get('station_name')}区域实况 (NMC)", - "lat": city_meta.get("lat"), - "lon": city_meta.get("lon"), - "temp": current.get("current", {}).get("temp"), - "istNo": current.get("station_code"), - "icao": current.get("station_code"), - "source": "nmc", - "source_label": "NMC", - "obs_time": current.get("publish_time"), - "page_url": current.get("page_url"), - "humidity": current.get("current", {}).get("humidity"), - "rain": current.get("current", {}).get("rain"), - "airpressure": current.get("current", {}).get("airpressure"), - "wx_desc": current.get("current", {}).get("wx_desc"), - "wind_direction_text": current.get("current", {}).get("wind_direction"), - "wind_power_text": current.get("current", {}).get("wind_power"), - } - ] diff --git a/src/data_collection/weather_sources.py b/src/data_collection/weather_sources.py index 78de5ee6..dacc5065 100644 --- a/src/data_collection/weather_sources.py +++ b/src/data_collection/weather_sources.py @@ -12,7 +12,6 @@ from src.data_collection.metar_sources import MetarSourceMixin from src.data_collection.mgm_sources import MgmSourceMixin from src.data_collection.jma_amedas_sources import JmaAmedasSourceMixin from src.data_collection.russia_station_sources import RussiaStationSourceMixin -from src.data_collection.nmc_sources import NmcSourceMixin from src.data_collection.nws_open_meteo_sources import NwsOpenMeteoSourceMixin from src.data_collection.amos_station_sources import AmosStationSourceMixin from src.data_collection.amsc_awos_sources import AmscAwosSourceMixin @@ -24,7 +23,7 @@ from src.data_collection.singapore_mss_sources import SingaporeMssSourceMixin from src.database.db_manager import DBManager -class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NmcSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin): +class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin): """ Multi-source weather data collector @@ -35,7 +34,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour - AMSC AWOS (China mainland runway-point airport sensors) - NWS (US National Weather Service) - MGM (Turkish Meteorological Service) - - JMA / NMC / HKO / CWA (country official networks) + - JMA / HKO / CWA (country official networks) - Polymarket (weather derivative markets) """ @@ -204,11 +203,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour ) self._taf_cache: Dict[str, Dict] = {} self._taf_cache_lock = threading.Lock() - self.nmc_cache_ttl_sec = int( - os.getenv("NMC_CACHE_TTL_SEC", "300") - ) - self._nmc_cache: Dict[str, Dict] = {} - self._nmc_cache_lock = threading.Lock() self.jma_cache_ttl_sec = int( os.getenv("JMA_AMEDAS_CACHE_TTL_SEC", "120") ) @@ -773,8 +767,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour self._knmi_cache.pop(f"knmi:{normalized}:{use_fahrenheit}", None) with self._hko_obs_cache_lock: self._hko_obs_cache.pop(f"hko_obs:{normalized}:{use_fahrenheit}", None) - with self._nmc_cache_lock: - self._nmc_cache.pop(f"{normalized}:{use_fahrenheit}", None) with self._ru_station_cache_lock: self._ru_station_cache.pop(f"{normalized}:{use_fahrenheit}", None) with self._settlement_cache_lock: @@ -907,24 +899,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour def _attach_china_official_nearby( self, results: Dict, city_lower: str, use_fahrenheit: bool ) -> None: - if city_lower not in { - "beijing", - "chengdu", - "chongqing", - "shanghai", - "shenzhen", - "wuhan", - }: - return - official_rows = self.fetch_nmc_official_nearby( - city_lower, use_fahrenheit=use_fahrenheit - ) - if not official_rows: - return - results["nmc_official_nearby"] = official_rows - if "mgm_nearby" not in results: - results["mgm_nearby"] = official_rows - results["nearby_source"] = "nmc" + return def _attach_japan_official_nearby( self, results: Dict, city_lower: str, use_fahrenheit: bool diff --git a/tests/test_nmc_sources.py b/tests/test_nmc_sources.py deleted file mode 100644 index 3240cc30..00000000 --- a/tests/test_nmc_sources.py +++ /dev/null @@ -1,104 +0,0 @@ -import threading - -from src.data_collection.nmc_sources import NmcSourceMixin - - -class _DummyResponse: - def __init__(self, *, text="", payload=None, status_code=200): - self.text = text - self._payload = payload - self.status_code = status_code - - def raise_for_status(self): - if self.status_code >= 400: - raise RuntimeError(f"HTTP {self.status_code}") - - def json(self): - return self._payload - - -class _DummySession: - def __init__(self, mapping): - self.mapping = mapping - - def get(self, url, timeout=None): - if url not in self.mapping: - raise AssertionError(f"unexpected url {url}") - return self.mapping[url] - - -class _DummyCollector(NmcSourceMixin): - CITY_REGISTRY = { - "shanghai": {"lat": 31.1434, "lon": 121.8052}, - } - - def __init__(self, mapping): - self.session = _DummySession(mapping) - self.timeout = 5 - self.nmc_cache_ttl_sec = 300 - self._nmc_cache = {} - self._nmc_cache_lock = threading.Lock() - - -def test_fetch_nmc_region_current_parses_rest_payload(monkeypatch): - import src.data_collection.nmc_sources as _nmc - monkeypatch.setattr(_nmc, "_NMC_REALTIME_BASE", "https://www.nmc.cn/rest/real") - collector = _DummyCollector( - { - "https://www.nmc.cn/rest/real/atcMf": _DummyResponse( - payload={ - "station": {"code": "atcMf", "city": "浦东"}, - "publish_time": "2026-04-06 06:50", - "weather": { - "temperature": 17.9, - "humidity": 83.0, - "rain": 0.0, - "airpressure": 9999.0, - "info": "多云", - }, - "wind": {"direct": "东北风", "power": "3级"}, - } - ) - } - ) - - out = collector.fetch_nmc_region_current("shanghai") - - assert out is not None - assert out["source"] == "nmc" - assert out["station_code"] == "atcMf" - assert out["current"]["temp"] == 17.9 - assert out["current"]["humidity"] == 83.0 - assert out["current"]["airpressure"] is None - - -def test_fetch_nmc_official_nearby_returns_normalized_row(monkeypatch): - import src.data_collection.nmc_sources as _nmc - monkeypatch.setattr(_nmc, "_NMC_REALTIME_BASE", "https://www.nmc.cn/rest/real") - collector = _DummyCollector( - { - "https://www.nmc.cn/rest/real/atcMf": _DummyResponse( - payload={ - "station": {"code": "atcMf", "city": "浦东"}, - "publish_time": "2026-04-06 06:50", - "weather": { - "temperature": 17.9, - "humidity": 83.0, - "rain": 0.0, - "airpressure": 9999.0, - "info": "多云", - }, - "wind": {"direct": "东北风", "power": "3级"}, - } - ) - } - ) - - rows = collector.fetch_nmc_official_nearby("shanghai") - - assert len(rows) == 1 - assert rows[0]["source"] == "nmc" - assert rows[0]["temp"] == 17.9 - assert rows[0]["name"] == "浦东区域实况 (NMC)" - assert rows[0]["station_label"] == "浦东区域实况 (NMC)" - assert rows[0]["lat"] == 31.1434 diff --git a/web/analysis_service.py b/web/analysis_service.py index 30ad6c91..427ae0ce 100644 --- a/web/analysis_service.py +++ b/web/analysis_service.py @@ -27,7 +27,6 @@ from src.analysis.settlement_rounding import apply_city_settlement from src.data_collection.country_networks import build_country_network_snapshot from src.data_collection.city_registry import ALIASES, CITY_REGISTRY from src.data_collection.city_time import get_city_utc_offset_seconds -from src.data_collection.nmc_sources import NMC_CITY_REFERENCES from src.database.runtime_state import IntradayPathSnapshotRepository from web.services.city_payloads import ( build_city_detail_payload as _city_payload_detail, @@ -126,18 +125,7 @@ def _format_observation_time_local(value: Any, utc_offset: int) -> str: def _fetch_nmc_current_fallback(city: str, *, use_fahrenheit: bool) -> Dict[str, Any]: - city_key = str(city or "").strip().lower() - if city_key not in NMC_CITY_REFERENCES: - return {} - try: - payload = _weather.fetch_nmc_region_current( - city_key, - use_fahrenheit=use_fahrenheit, - ) - return payload if isinstance(payload, dict) else {} - except Exception as exc: - logger.debug("NMC current fallback failed city={}: {}", city_key, exc) - return {} + return {} def _is_plausible_city_temp(city: str, value: Any, unit: str = "°C") -> bool: @@ -263,13 +251,6 @@ _OBSERVATION_SOURCE_PROFILES: Dict[str, Dict[str, Any]] = { "expected_grace_sec": 900, "stale_after_sec": 3600, }, - "nmc": { - "label": "NMC", - "native_update_interval_sec": 3600, - "fresh_window_sec": 3600, - "expected_grace_sec": 1800, - "stale_after_sec": 7200, - }, } @@ -293,8 +274,6 @@ def _canonical_observation_source_code(value: Any) -> str: return "mgm" if "noaa" in raw: return "noaa" - if "nmc" in raw: - return "nmc" if "wunderground" in raw or raw == "wu": return "wunderground" return raw diff --git a/web/core.py b/web/core.py index b6b73657..fefb6f20 100644 --- a/web/core.py +++ b/web/core.py @@ -425,7 +425,6 @@ def _cache_summary() -> Dict[str, Any]: open_meteo_multi_model_entries = len(getattr(_weather, "_multi_model_cache", {}) or {}) metar_entries = len(getattr(_weather, "_metar_cache", {}) or {}) taf_entries = len(getattr(_weather, "_taf_cache", {}) or {}) - nmc_entries = len(getattr(_weather, "_nmc_cache", {}) or {}) settlement_entries = len(getattr(_weather, "_settlement_cache", {}) or {}) gauge_set("polyweather_api_cache_entries", len(_cache)) @@ -434,7 +433,6 @@ def _cache_summary() -> Dict[str, Any]: gauge_set("polyweather_open_meteo_multi_model_cache_entries", open_meteo_multi_model_entries) gauge_set("polyweather_metar_cache_entries", metar_entries) gauge_set("polyweather_taf_cache_entries", taf_entries) - gauge_set("polyweather_nmc_cache_entries", nmc_entries) gauge_set("polyweather_settlement_cache_entries", settlement_entries) return { "api_cache_entries": len(_cache), @@ -443,7 +441,6 @@ def _cache_summary() -> Dict[str, Any]: "open_meteo_multi_model_entries": open_meteo_multi_model_entries, "metar_entries": metar_entries, "taf_entries": taf_entries, - "nmc_entries": nmc_entries, "settlement_entries": settlement_entries, "analysis": get_analysis_cache_stats(), }