移除 NMC(中国国家气象中心)数据源:无实际使用价值

This commit is contained in:
2569718930@qq.com
2026-05-19 16:20:48 +08:00
parent a7be2adbbb
commit 3ca5b71299
7 changed files with 5 additions and 412 deletions
-2
View File
@@ -68,8 +68,6 @@ JMA_AMEDAS_CACHE_TTL_SEC=120
# These are kept in .env to avoid exposing competitive data-source discovery
# work on the public GitHub repository. Leave empty to use built-in defaults.
# AMSC_AWOS_BASE_URL=https://www.amsc.net.cn/gateway/api/saas/rest/amc/AwosController/getWindPlate
# NMC_FORECAST_BASE_URL=https://m.nmc.cn/publish/forecast
# NMC_REALTIME_BASE_URL=https://www.nmc.cn/rest/real
# KMA_BASE_URL=https://www.weather.go.kr
# AMOS_BASE_URL=https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do
# JMA_AMEDAS_BASE_URL=https://www.jma.go.jp
+1 -42
View File
@@ -508,41 +508,6 @@ def _metar_cluster_rows(raw: Dict[str, Any]) -> List[Dict[str, Any]]:
return out
def _nmc_rows(raw: Dict[str, Any], city: str) -> List[Dict[str, Any]]:
rows = raw.get("nmc_official_nearby") or []
city_offset = get_city_utc_offset_seconds(city)
out: List[Dict[str, Any]] = []
for row in rows:
if not isinstance(row, dict):
continue
out.append(
_normalize_station_row(
station_code=row.get("icao") or row.get("istNo"),
station_label=row.get("name"),
temp=row.get("temp"),
lat=row.get("lat"),
lon=row.get("lon"),
obs_time=row.get("obs_time"),
source_code="nmc",
source_label="NMC",
is_official=True,
is_airport_station=False,
is_settlement_anchor=False,
extra={
"obs_time_utc_offset_seconds": city_offset,
"page_url": row.get("page_url"),
"humidity": _safe_float(row.get("humidity")),
"rain": _safe_float(row.get("rain")),
"airpressure": _safe_float(row.get("airpressure")),
"wx_desc": row.get("wx_desc"),
"wind_direction_text": row.get("wind_direction_text"),
"wind_power_text": row.get("wind_power_text"),
},
)
)
return out
def _jma_rows(raw: Dict[str, Any], city: str) -> List[Dict[str, Any]]:
rows = raw.get("jma_official_nearby") or []
out: List[Dict[str, Any]] = []
@@ -830,22 +795,16 @@ class TurkeyMgmNetworkProvider(CountryNetworkProvider):
class ChinaCmaNetworkProvider(CountryNetworkProvider):
def __init__(self) -> None:
super().__init__("china_cma", "CMA/NMC")
super().__init__("china_cma", "CMA")
def official_nearby_current(self, city: str, raw: Dict[str, Any]) -> List[Dict[str, Any]]:
rows = _nmc_rows(raw, city)
if rows:
return rows
return _metar_cluster_rows(raw)
def official_network_status(self, city: str, raw: Dict[str, Any]) -> Dict[str, Any]:
rows = self.official_nearby_current(city, raw)
has_nmc = bool(_nmc_rows(raw, city))
return {
"provider_code": self.provider_code,
"provider_label": self.provider_label,
"available": has_nmc,
"mode": "official_active" if has_nmc else ("fallback_metar_cluster" if rows else "reference_only"),
"row_count": len(rows),
}
-211
View File
@@ -1,211 +0,0 @@
from __future__ import annotations
import os
import re
import time
from datetime import datetime
from typing import Any, Dict, List, Optional
from loguru import logger
from src.utils.metrics import record_source_call
_NMC_FORECAST_BASE = os.getenv("NMC_FORECAST_BASE_URL", "").strip()
_NMC_REALTIME_BASE = os.getenv("NMC_REALTIME_BASE_URL", "").strip()
NMC_CITY_REFERENCES: Dict[str, Dict[str, Any]] = {
"shanghai": {
"region_label": "浦东",
"page_url": f"{_NMC_FORECAST_BASE}/ASH/pudong.html" if _NMC_FORECAST_BASE else "",
"station_code": "atcMf",
},
"beijing": {
"region_label": "顺义",
"page_url": f"{_NMC_FORECAST_BASE}/ABJ/shunyi.html" if _NMC_FORECAST_BASE else "",
"station_code": "MKoqG",
},
"chongqing": {
"region_label": "渝北",
"page_url": f"{_NMC_FORECAST_BASE}/ACQ/yubei.html" if _NMC_FORECAST_BASE else "",
"station_code": "xFVYU",
},
"chengdu": {
"region_label": "双流",
"page_url": f"{_NMC_FORECAST_BASE}/ASC/shuangliu.html" if _NMC_FORECAST_BASE else "",
"station_code": "grFhZ",
},
"wuhan": {
"region_label": "武汉",
"page_url": f"{_NMC_FORECAST_BASE}/AHB/wuhan.html" if _NMC_FORECAST_BASE else "",
"station_code": "bSpCz",
},
"shenzhen": {
"region_label": "深圳",
"page_url": f"{_NMC_FORECAST_BASE}/AGD/shenzuo.html" if _NMC_FORECAST_BASE else "",
"station_code": "AhpEU",
},
}
class NmcSourceMixin:
def _nmc_http_get(self, url: str):
getter = getattr(self, "_http_get", None)
if callable(getter):
return getter(url)
return self.session.get(url, timeout=self.timeout)
def _nmc_http_get_json(self, url: str):
getter = getattr(self, "_http_get_json", None)
if callable(getter):
return getter(url)
response = self.session.get(url, timeout=self.timeout)
response.raise_for_status()
return response.json()
@staticmethod
def _nmc_optional_text(value: Any) -> Optional[str]:
text = str(value or "").strip()
if text in ("", "9999"):
return None
return text
@staticmethod
def _nmc_optional_float(value: Any) -> Optional[float]:
if value in (None, "", "9999", 9999, 9999.0):
return None
try:
return float(value)
except Exception:
return None
def _resolve_nmc_station_code(self, city: str) -> Optional[str]:
city_key = str(city or "").strip().lower()
meta = NMC_CITY_REFERENCES.get(city_key) or {}
station_code = str(meta.get("station_code") or "").strip()
if station_code:
return station_code
page_url = str(meta.get("page_url") or "").strip()
if not page_url:
return None
try:
resp = self._nmc_http_get(page_url)
resp.raise_for_status()
match = re.search(
r"renderWeatherRealPanel\('([^']+)',\s*'([^']+)'\)",
resp.text,
)
if not match:
return None
station_code = str(match.group(1) or "").strip()
if station_code:
meta["station_code"] = station_code
return station_code
except Exception as exc:
logger.warning("NMC station code resolve failed city={} error={}", city_key, exc)
return None
def fetch_nmc_region_current(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
started = time.perf_counter()
city_key = str(city or "").strip().lower()
meta = NMC_CITY_REFERENCES.get(city_key) or {}
if not meta:
record_source_call("nmc", "current", "unsupported_city", (time.perf_counter() - started) * 1000.0)
return None
cache_key = f"{city_key}:{use_fahrenheit}"
now_ts = time.time()
with self._nmc_cache_lock:
cached = self._nmc_cache.get(cache_key)
if cached and now_ts - cached["t"] < self.nmc_cache_ttl_sec:
record_source_call("nmc", "current", "cache_hit", (time.perf_counter() - started) * 1000.0)
return cached["d"]
station_code = self._resolve_nmc_station_code(city_key)
if not station_code:
record_source_call("nmc", "current", "missing_station_code", (time.perf_counter() - started) * 1000.0)
return None
try:
url = f"{_NMC_REALTIME_BASE}/{station_code}"
payload = self._nmc_http_get_json(url)
if not isinstance(payload, dict) or not isinstance(payload.get("weather"), dict):
record_source_call("nmc", "current", "empty", (time.perf_counter() - started) * 1000.0)
return None
weather = payload.get("weather") or {}
temp_c = weather.get("temperature")
if temp_c in (None, "", "9999"):
record_source_call("nmc", "current", "no_temperature", (time.perf_counter() - started) * 1000.0)
return None
temp_c = float(temp_c)
temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1)
station = payload.get("station") or {}
result = {
"source": "nmc",
"timestamp": datetime.utcnow().isoformat(),
"station_code": station_code,
"station_name": station.get("city") or meta.get("region_label") or city_key.title(),
"page_url": meta.get("page_url"),
"publish_time": payload.get("publish_time"),
"current": {
"temp": temp,
"humidity": self._nmc_optional_float(weather.get("humidity")),
"rain": self._nmc_optional_float(weather.get("rain")),
"airpressure": self._nmc_optional_float(weather.get("airpressure")),
"wx_desc": weather.get("info"),
"wind_direction": self._nmc_optional_text((payload.get("wind") or {}).get("direct")),
"wind_power": self._nmc_optional_text((payload.get("wind") or {}).get("power")),
},
}
with self._nmc_cache_lock:
self._nmc_cache[cache_key] = {"d": result, "t": now_ts}
record_source_call("nmc", "current", "success", (time.perf_counter() - started) * 1000.0)
return result
except Exception as exc:
logger.warning("NMC current fetch failed city={} code={} error={}", city_key, station_code, exc)
with self._nmc_cache_lock:
stale = self._nmc_cache.get(cache_key)
if stale:
record_source_call("nmc", "current", "stale_cache", (time.perf_counter() - started) * 1000.0)
return stale["d"]
record_source_call("nmc", "current", "error", (time.perf_counter() - started) * 1000.0)
return None
def fetch_nmc_official_nearby(
self,
city: str,
use_fahrenheit: bool = False,
) -> List[Dict[str, Any]]:
current = self.fetch_nmc_region_current(city, use_fahrenheit=use_fahrenheit)
if not current:
return []
meta = NMC_CITY_REFERENCES.get(str(city or "").strip().lower()) or {}
city_meta = self.CITY_REGISTRY.get(str(city or "").strip().lower()) or {}
return [
{
"name": f"{meta.get('region_label') or current.get('station_name')}区域实况 (NMC)",
"station_label": f"{meta.get('region_label') or current.get('station_name')}区域实况 (NMC)",
"lat": city_meta.get("lat"),
"lon": city_meta.get("lon"),
"temp": current.get("current", {}).get("temp"),
"istNo": current.get("station_code"),
"icao": current.get("station_code"),
"source": "nmc",
"source_label": "NMC",
"obs_time": current.get("publish_time"),
"page_url": current.get("page_url"),
"humidity": current.get("current", {}).get("humidity"),
"rain": current.get("current", {}).get("rain"),
"airpressure": current.get("current", {}).get("airpressure"),
"wx_desc": current.get("current", {}).get("wx_desc"),
"wind_direction_text": current.get("current", {}).get("wind_direction"),
"wind_power_text": current.get("current", {}).get("wind_power"),
}
]
+3 -28
View File
@@ -12,7 +12,6 @@ from src.data_collection.metar_sources import MetarSourceMixin
from src.data_collection.mgm_sources import MgmSourceMixin
from src.data_collection.jma_amedas_sources import JmaAmedasSourceMixin
from src.data_collection.russia_station_sources import RussiaStationSourceMixin
from src.data_collection.nmc_sources import NmcSourceMixin
from src.data_collection.nws_open_meteo_sources import NwsOpenMeteoSourceMixin
from src.data_collection.amos_station_sources import AmosStationSourceMixin
from src.data_collection.amsc_awos_sources import AmscAwosSourceMixin
@@ -24,7 +23,7 @@ from src.data_collection.singapore_mss_sources import SingaporeMssSourceMixin
from src.database.db_manager import DBManager
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NmcSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin):
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, RussiaStationSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin):
"""
Multi-source weather data collector
@@ -35,7 +34,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
- AMSC AWOS (China mainland runway-point airport sensors)
- NWS (US National Weather Service)
- MGM (Turkish Meteorological Service)
- JMA / NMC / HKO / CWA (country official networks)
- JMA / HKO / CWA (country official networks)
- Polymarket (weather derivative markets)
"""
@@ -204,11 +203,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
)
self._taf_cache: Dict[str, Dict] = {}
self._taf_cache_lock = threading.Lock()
self.nmc_cache_ttl_sec = int(
os.getenv("NMC_CACHE_TTL_SEC", "300")
)
self._nmc_cache: Dict[str, Dict] = {}
self._nmc_cache_lock = threading.Lock()
self.jma_cache_ttl_sec = int(
os.getenv("JMA_AMEDAS_CACHE_TTL_SEC", "120")
)
@@ -773,8 +767,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
self._knmi_cache.pop(f"knmi:{normalized}:{use_fahrenheit}", None)
with self._hko_obs_cache_lock:
self._hko_obs_cache.pop(f"hko_obs:{normalized}:{use_fahrenheit}", None)
with self._nmc_cache_lock:
self._nmc_cache.pop(f"{normalized}:{use_fahrenheit}", None)
with self._ru_station_cache_lock:
self._ru_station_cache.pop(f"{normalized}:{use_fahrenheit}", None)
with self._settlement_cache_lock:
@@ -907,24 +899,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
def _attach_china_official_nearby(
self, results: Dict, city_lower: str, use_fahrenheit: bool
) -> None:
if city_lower not in {
"beijing",
"chengdu",
"chongqing",
"shanghai",
"shenzhen",
"wuhan",
}:
return
official_rows = self.fetch_nmc_official_nearby(
city_lower, use_fahrenheit=use_fahrenheit
)
if not official_rows:
return
results["nmc_official_nearby"] = official_rows
if "mgm_nearby" not in results:
results["mgm_nearby"] = official_rows
results["nearby_source"] = "nmc"
return
def _attach_japan_official_nearby(
self, results: Dict, city_lower: str, use_fahrenheit: bool
-104
View File
@@ -1,104 +0,0 @@
import threading
from src.data_collection.nmc_sources import NmcSourceMixin
class _DummyResponse:
def __init__(self, *, text="", payload=None, status_code=200):
self.text = text
self._payload = payload
self.status_code = status_code
def raise_for_status(self):
if self.status_code >= 400:
raise RuntimeError(f"HTTP {self.status_code}")
def json(self):
return self._payload
class _DummySession:
def __init__(self, mapping):
self.mapping = mapping
def get(self, url, timeout=None):
if url not in self.mapping:
raise AssertionError(f"unexpected url {url}")
return self.mapping[url]
class _DummyCollector(NmcSourceMixin):
CITY_REGISTRY = {
"shanghai": {"lat": 31.1434, "lon": 121.8052},
}
def __init__(self, mapping):
self.session = _DummySession(mapping)
self.timeout = 5
self.nmc_cache_ttl_sec = 300
self._nmc_cache = {}
self._nmc_cache_lock = threading.Lock()
def test_fetch_nmc_region_current_parses_rest_payload(monkeypatch):
import src.data_collection.nmc_sources as _nmc
monkeypatch.setattr(_nmc, "_NMC_REALTIME_BASE", "https://www.nmc.cn/rest/real")
collector = _DummyCollector(
{
"https://www.nmc.cn/rest/real/atcMf": _DummyResponse(
payload={
"station": {"code": "atcMf", "city": "浦东"},
"publish_time": "2026-04-06 06:50",
"weather": {
"temperature": 17.9,
"humidity": 83.0,
"rain": 0.0,
"airpressure": 9999.0,
"info": "多云",
},
"wind": {"direct": "东北风", "power": "3级"},
}
)
}
)
out = collector.fetch_nmc_region_current("shanghai")
assert out is not None
assert out["source"] == "nmc"
assert out["station_code"] == "atcMf"
assert out["current"]["temp"] == 17.9
assert out["current"]["humidity"] == 83.0
assert out["current"]["airpressure"] is None
def test_fetch_nmc_official_nearby_returns_normalized_row(monkeypatch):
import src.data_collection.nmc_sources as _nmc
monkeypatch.setattr(_nmc, "_NMC_REALTIME_BASE", "https://www.nmc.cn/rest/real")
collector = _DummyCollector(
{
"https://www.nmc.cn/rest/real/atcMf": _DummyResponse(
payload={
"station": {"code": "atcMf", "city": "浦东"},
"publish_time": "2026-04-06 06:50",
"weather": {
"temperature": 17.9,
"humidity": 83.0,
"rain": 0.0,
"airpressure": 9999.0,
"info": "多云",
},
"wind": {"direct": "东北风", "power": "3级"},
}
)
}
)
rows = collector.fetch_nmc_official_nearby("shanghai")
assert len(rows) == 1
assert rows[0]["source"] == "nmc"
assert rows[0]["temp"] == 17.9
assert rows[0]["name"] == "浦东区域实况 (NMC)"
assert rows[0]["station_label"] == "浦东区域实况 (NMC)"
assert rows[0]["lat"] == 31.1434
+1 -22
View File
@@ -27,7 +27,6 @@ from src.analysis.settlement_rounding import apply_city_settlement
from src.data_collection.country_networks import build_country_network_snapshot
from src.data_collection.city_registry import ALIASES, CITY_REGISTRY
from src.data_collection.city_time import get_city_utc_offset_seconds
from src.data_collection.nmc_sources import NMC_CITY_REFERENCES
from src.database.runtime_state import IntradayPathSnapshotRepository
from web.services.city_payloads import (
build_city_detail_payload as _city_payload_detail,
@@ -126,18 +125,7 @@ def _format_observation_time_local(value: Any, utc_offset: int) -> str:
def _fetch_nmc_current_fallback(city: str, *, use_fahrenheit: bool) -> Dict[str, Any]:
city_key = str(city or "").strip().lower()
if city_key not in NMC_CITY_REFERENCES:
return {}
try:
payload = _weather.fetch_nmc_region_current(
city_key,
use_fahrenheit=use_fahrenheit,
)
return payload if isinstance(payload, dict) else {}
except Exception as exc:
logger.debug("NMC current fallback failed city={}: {}", city_key, exc)
return {}
return {}
def _is_plausible_city_temp(city: str, value: Any, unit: str = "°C") -> bool:
@@ -263,13 +251,6 @@ _OBSERVATION_SOURCE_PROFILES: Dict[str, Dict[str, Any]] = {
"expected_grace_sec": 900,
"stale_after_sec": 3600,
},
"nmc": {
"label": "NMC",
"native_update_interval_sec": 3600,
"fresh_window_sec": 3600,
"expected_grace_sec": 1800,
"stale_after_sec": 7200,
},
}
@@ -293,8 +274,6 @@ def _canonical_observation_source_code(value: Any) -> str:
return "mgm"
if "noaa" in raw:
return "noaa"
if "nmc" in raw:
return "nmc"
if "wunderground" in raw or raw == "wu":
return "wunderground"
return raw
-3
View File
@@ -425,7 +425,6 @@ def _cache_summary() -> Dict[str, Any]:
open_meteo_multi_model_entries = len(getattr(_weather, "_multi_model_cache", {}) or {})
metar_entries = len(getattr(_weather, "_metar_cache", {}) or {})
taf_entries = len(getattr(_weather, "_taf_cache", {}) or {})
nmc_entries = len(getattr(_weather, "_nmc_cache", {}) or {})
settlement_entries = len(getattr(_weather, "_settlement_cache", {}) or {})
gauge_set("polyweather_api_cache_entries", len(_cache))
@@ -434,7 +433,6 @@ def _cache_summary() -> Dict[str, Any]:
gauge_set("polyweather_open_meteo_multi_model_cache_entries", open_meteo_multi_model_entries)
gauge_set("polyweather_metar_cache_entries", metar_entries)
gauge_set("polyweather_taf_cache_entries", taf_entries)
gauge_set("polyweather_nmc_cache_entries", nmc_entries)
gauge_set("polyweather_settlement_cache_entries", settlement_entries)
return {
"api_cache_entries": len(_cache),
@@ -443,7 +441,6 @@ def _cache_summary() -> Dict[str, Any]:
"open_meteo_multi_model_entries": open_meteo_multi_model_entries,
"metar_entries": metar_entries,
"taf_entries": taf_entries,
"nmc_entries": nmc_entries,
"settlement_entries": settlement_entries,
"analysis": get_analysis_cache_stats(),
}