Add Wunderground as Shenzhen settlement source

This commit is contained in:
2569718930@qq.com
2026-03-25 17:00:36 +08:00
parent b1ce479909
commit 2b0e6a6f67
10 changed files with 244 additions and 8 deletions
@@ -117,3 +117,5 @@
{"city": "test_city", "timestamp": "2026-03-04 14:00", "date": "2026-03-04", "temp_symbol": "°C", "raw_mu": 29.85, "raw_sigma": 1.09375, "deb_prediction": null, "ensemble": {"p10": 27.0, "median": 29.5, "p90": 31.0}, "multi_model": {"Open-Meteo": 30.0}, "max_so_far": 29.5, "peak_status": "in_window", "prob_snapshot": [{"v": 30, "p": 0.565}, {"v": 31, "p": 0.341}, {"v": 32, "p": 0.094}], "shadow_prob_snapshot": [{"v": 30, "p": 0.565}, {"v": 31, "p": 0.341}, {"v": 32, "p": 0.094}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320132525", "calibration_source": "artifacts\\probability_calibration\\default.json", "calibrated_mu": 29.85, "calibrated_sigma": 1.09375}
{"city": "test_city", "timestamp": "2026-03-04 14:30", "date": "2026-03-04", "temp_symbol": "°C", "raw_mu": 29.7, "raw_sigma": 1.09375, "deb_prediction": null, "ensemble": {"p10": 27.0, "median": 29.0, "p90": 31.0}, "multi_model": {"Open-Meteo": 30.0}, "max_so_far": 28.0, "peak_status": "in_window", "prob_snapshot": [{"v": 30, "p": 0.35}, {"v": 29, "p": 0.299}, {"v": 31, "p": 0.187}, {"v": 28, "p": 0.117}], "shadow_prob_snapshot": [{"v": 30, "p": 0.35}, {"v": 29, "p": 0.299}, {"v": 31, "p": 0.187}, {"v": 28, "p": 0.117}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320132525", "calibration_source": "artifacts\\probability_calibration\\default.json", "calibrated_mu": 29.7, "calibrated_sigma": 1.09375}
{"city": "hong kong", "timestamp": "2026-03-23T21:10:00+08:00", "date": "2026-03-23", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.6806250000000001, "deb_prediction": 25.2, "ensemble": {"p10": 26.3, "median": 26.4, "p90": 26.6}, "multi_model": {"Open-Meteo": 24.8, "HKO(港天文)": 27.0, "ECMWF": 25.4, "GFS": 25.1, "ICON": 24.8, "GEM": 25.3, "JMA": 23.6}, "max_so_far": 27.4, "peak_status": "past", "prob_snapshot": [{"v": 27, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null}
{"city": "shenzhen", "timestamp": "2026-03-25T08:43:15.528748+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.18016764322916676, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 29.0, "peak_status": "past", "prob_snapshot": [{"v": 29, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null}
{"city": "shenzhen", "timestamp": "2026-03-25T08:57:11.783182+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": 26.7, "raw_sigma": 0.18016764322916676, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 26.7, "peak_status": "past", "prob_snapshot": [{"v": 27, "p": 1.0}], "shadow_prob_snapshot": [{"v": 27, "p": 1.0}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320132525", "calibration_source": "artifacts\\probability_calibration\\default.json", "calibrated_mu": 26.7, "calibrated_sigma": 0.24322631835937514}
+6 -6
View File
@@ -188,7 +188,7 @@ export const DOCS_PAGES: DocsPage[] = [
title: "什么叫机场端压温风险偏高",
blocks: [
{ type: "paragraph", text: "它的意思不是整座城市一定更冷,而是作为结算依据的机场站点,在峰值窗口里更可能因为云、阵雨或雷暴扰动,冲不到本来可能达到的更高温度。" },
{ type: "callout", tone: "warning", title: "重点区别", text: "TAF 负责告诉你机场侧未来几个小时会不会出现压温扰动,不直接等于结算温度本身。结算仍然看 METAR、HKO、MGM、NOAA RCTP 等实际结算源。" },
{ type: "callout", tone: "warning", title: "重点区别", text: "TAF 负责告诉你机场侧未来几个小时会不会出现压温扰动,不直接等于结算温度本身。结算仍然看 METAR、HKO、MGM、NOAA RCTP、Wunderground 指定站点等实际结算源。" },
],
},
],
@@ -216,7 +216,7 @@ export const DOCS_PAGES: DocsPage[] = [
title: "What airport-side suppression risk means",
blocks: [
{ type: "paragraph", text: "It does not mean the entire city must run cooler. It means the airport station used for settlement is more likely to get capped by clouds, showers, or thunderstorm disruption during the peak window and fail to reach the next warmer bucket." },
{ type: "callout", tone: "warning", title: "Important distinction", text: "TAF explains whether the airport side may face suppressive weather over the next few hours. Settlement still comes from the actual settlement source such as METAR, HKO, MGM, or NOAA RCTP." },
{ type: "callout", tone: "warning", title: "Important distinction", text: "TAF explains whether the airport side may face suppressive weather over the next few hours. Settlement still comes from the actual settlement source such as METAR, HKO, MGM, NOAA RCTP, or a designated Wunderground station." },
],
},
],
@@ -242,14 +242,14 @@ export const DOCS_PAGES: DocsPage[] = [
id: "city-rules",
title: "当前主要口径",
blocks: [
{ type: "bullets", items: ["多数欧美机场市场:按机场 METAR 或机场主站实况结算。", "香港:按香港天文台 HKO 主口径,不接机场 TAF 作为主结算逻辑。", "台北:按 NOAA RCTP(台湾桃园国际机场)最终完成质控后的最高整度摄氏值结算,机场观测和市区体感不可混用。", "Ankara:结算主站以 LTAC / Esenboğa 为准,同时保留 Turkish MGM 作为领先结构参考。"] },
{ type: "bullets", items: ["多数欧美机场市场:按机场 METAR 或机场主站实况结算。", "香港:按香港天文台 HKO 主口径,不接机场 TAF 作为主结算逻辑。", "台北:按 NOAA RCTP(台湾桃园国际机场)最终完成质控后的最高整度摄氏值结算,机场观测和市区体感不可混用。", "Shenzhen:当前按 Wunderground 指定站点(Shenzhen Bao'an Intl Airport Station)镜像结算页主值。", "Ankara:结算主站以 LTAC / Esenboğa 为准,同时保留 Turkish MGM 作为领先结构参考。"] },
],
},
{
id: "common-mistakes",
title: "最常见的误解",
blocks: [
{ type: "bullets", items: ["TAF 不是结算源,它只告诉你机场未来有没有压温扰动。", "市场按机场结算时,城区更热不代表市场就该结到更高温桶。", "香港和台北不能简单套用机场 TAF / METAR 主链逻辑。"] },
{ type: "bullets", items: ["TAF 不是结算源,它只告诉你机场未来有没有压温扰动。", "市场按机场结算时,城区更热不代表市场就该结到更高温桶。", "如果结算页直接看 Wunderground 站点主值,就应优先镜像该站点,而不是擅自回退到其他站。", "香港和台北不能简单套用机场 TAF / METAR 主链逻辑。"] },
],
},
],
@@ -269,14 +269,14 @@ export const DOCS_PAGES: DocsPage[] = [
id: "city-rules",
title: "Current primary rules",
blocks: [
{ type: "bullets", items: ["Most airport-linked Western markets: settle on airport METAR or the airport primary observing site.", "Hong Kong: settles on HKO, not on airport TAF as the main settlement logic.", "Taipei: settles against NOAA RCTP using the finalized highest rounded whole-degree Celsius reading; airport observations and downtown feel should not be mixed.", "Ankara: settlement centers on LTAC / Esenboğa, with Turkish MGM retained as a leading-structure reference."] },
{ type: "bullets", items: ["Most airport-linked Western markets: settle on airport METAR or the airport primary observing site.", "Hong Kong: settles on HKO, not on airport TAF as the main settlement logic.", "Taipei: settles against NOAA RCTP using the finalized highest rounded whole-degree Celsius reading; airport observations and downtown feel should not be mixed.", "Shenzhen: currently mirrors the designated Wunderground station value for Shenzhen Bao'an Intl Airport Station.", "Ankara: settlement centers on LTAC / Esenboğa, with Turkish MGM retained as a leading-structure reference."] },
],
},
{
id: "common-mistakes",
title: "Common mistakes",
blocks: [
{ type: "bullets", items: ["TAF is not a settlement source. It only tells you whether airport-side suppressive weather may appear.", "If the market settles on an airport site, a hotter downtown feel does not automatically justify a warmer settlement bucket.", "Hong Kong and Taipei should not be forced into the generic airport TAF / METAR chain."] },
{ type: "bullets", items: ["TAF is not a settlement source. It only tells you whether airport-side suppressive weather may appear.", "If the market settles on an airport site, a hotter downtown feel does not automatically justify a warmer settlement bucket.", "If the settlement page itself references a Wunderground station, the product should mirror that station first instead of silently switching to another source.", "Hong Kong and Taipei should not be forced into the generic airport TAF / METAR chain."] },
],
},
],
+1
View File
@@ -39,6 +39,7 @@ def _resolve_settlement_source(city_meta: Dict[str, Any]) -> Tuple[str, str]:
"cwa": "CWA",
"noaa": "NOAA",
"mgm": "MGM",
"wunderground": "Wunderground",
}
return source, source_label_map.get(source, source.upper())
+1
View File
@@ -30,6 +30,7 @@ SETTLEMENT_SOURCE_LABELS = {
"cwa": "CWA",
"noaa": "NOAA",
"mgm": "MGM",
"wunderground": "Wunderground",
}
_CLOUD_RANK_LABELS = {
+3
View File
@@ -391,6 +391,9 @@ CITY_REGISTRY = {
"lat": 22.6393,
"lon": 113.8107,
"icao": "ZGSZ",
"settlement_source": "wunderground",
"settlement_url": "https://www.wunderground.com/weather/cn/shenzhen/ZGSZ",
"settlement_station_label": "Shenzhen Bao'an Intl Airport Station",
"tz_offset": 28800,
"use_fahrenheit": False,
"is_major": True,
+59
View File
@@ -183,6 +183,49 @@ class SettlementSourceMixin:
)
return self._sort_temp_points(points)
def _update_official_today_obs(
self,
*,
source_code: str,
station_code: str,
obs_iso: Optional[str],
current_temp: Optional[float],
utc_offset_seconds: int,
) -> List[Dict[str, Any]]:
if not obs_iso or current_temp is None:
return []
try:
obs_dt = datetime.fromisoformat(str(obs_iso).replace("Z", "+00:00"))
except Exception:
return []
if obs_dt.tzinfo is None:
obs_dt = obs_dt.replace(tzinfo=timezone.utc)
local_tz = timezone(timedelta(seconds=int(utc_offset_seconds or 0)))
local_dt = obs_dt.astimezone(local_tz)
date_str = local_dt.strftime("%Y-%m-%d")
time_str = local_dt.strftime("%H:%M")
mode = get_state_storage_mode()
if mode not in {STATE_STORAGE_DUAL, STATE_STORAGE_SQLITE}:
return [{"time": time_str, "temp": round(float(current_temp), 1)}]
lock = self._get_settlement_series_lock()
with lock:
_official_intraday_repo.upsert_point(
source_code=source_code,
station_code=station_code,
target_date=date_str,
observation_time=time_str,
value=round(float(current_temp), 1),
payload={"time": time_str, "temp": round(float(current_temp), 1)},
)
points = _official_intraday_repo.load_points(
source_code=source_code,
station_code=station_code,
target_date=date_str,
)
return self._sort_temp_points(points)
def fetch_hko_settlement_current(self) -> Optional[Dict[str, Any]]:
cache_key = "hko:hong_kong"
cached = self._get_settlement_cache(cache_key)
@@ -537,6 +580,22 @@ class SettlementSourceMixin:
def fetch_settlement_current(self, city: str) -> Optional[Dict[str, Any]]:
normalized = str(city or "").strip().lower()
try:
from src.data_collection.city_registry import CITY_REGISTRY
city_meta = CITY_REGISTRY.get(normalized) or {}
settlement_source = str(city_meta.get("settlement_source") or "").strip().lower()
if settlement_source == "wunderground":
settlement_url = str(city_meta.get("settlement_url") or "").strip()
if settlement_url:
return self.fetch_wunderground_settlement_current(
normalized,
url=settlement_url,
station_label=str(city_meta.get("settlement_station_label") or "").strip() or None,
icao=str(city_meta.get("icao") or "").strip() or None,
)
except Exception as exc:
logger.warning(f"Wunderground settlement dispatch failed city={city}: {exc}")
if normalized == "hong kong":
return self.fetch_hko_settlement_current()
if normalized == "taipei":
+2 -1
View File
@@ -10,9 +10,10 @@ from src.data_collection.settlement_sources import SettlementSourceMixin
from src.data_collection.metar_sources import MetarSourceMixin
from src.data_collection.mgm_sources import MgmSourceMixin
from src.data_collection.nws_open_meteo_sources import NwsOpenMeteoSourceMixin
from src.data_collection.wunderground_sources import WundergroundSourceMixin
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, NwsOpenMeteoSourceMixin):
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, NwsOpenMeteoSourceMixin, WundergroundSourceMixin):
"""
Multi-source weather data collector
+168
View File
@@ -0,0 +1,168 @@
from __future__ import annotations
import re
from datetime import datetime, timezone
from typing import Any, Dict, Optional
from loguru import logger
from src.data_collection.city_registry import CITY_REGISTRY
class WundergroundSourceMixin:
_WU_PAGE_TTL_SEC = 180
def _fetch_wunderground_page(self, url: str) -> Optional[str]:
cache_key = f"wu:page:{url}"
cached = self._get_settlement_cache(cache_key)
if isinstance(cached, dict):
html = str(cached.get("html") or "")
if html:
return html
try:
response = self.session.get(
url,
headers={
"User-Agent": "Mozilla/5.0",
"Referer": url,
},
timeout=self.timeout,
)
response.raise_for_status()
html = str(response.text or "")
if not html:
return None
ttl_backup = getattr(self, "settlement_cache_ttl_sec", self._WU_PAGE_TTL_SEC)
try:
self.settlement_cache_ttl_sec = self._WU_PAGE_TTL_SEC
self._set_settlement_cache(cache_key, {"html": html})
finally:
self.settlement_cache_ttl_sec = ttl_backup
return html
except Exception as exc:
logger.warning(f"Wunderground page fetch failed url={url}: {exc}")
return None
@staticmethod
def _wu_extract_station_name(html: str, fallback_icao: str) -> Optional[str]:
pattern = re.compile(
r'</lib-display-unit>\s*([^<]+?)\s*</a>',
re.IGNORECASE,
)
for match in pattern.finditer(html):
candidate = re.sub(r"\s+", " ", str(match.group(1) or "")).strip()
if fallback_icao.lower() in candidate.lower() or "station" in candidate.lower():
return candidate
return None
@staticmethod
def _wu_extract_station_temperature(
html: str,
*,
station_name: Optional[str],
) -> tuple[Optional[float], Optional[str]]:
station_anchor = station_name or "Station"
station_pos = html.find(station_anchor)
if station_pos < 0:
station_pos = html.lower().find("station-name")
if station_pos < 0:
return None, None
window_start = max(0, station_pos - 1800)
window = html[window_start:station_pos]
temp_match = re.search(
r'wu-value[^>]*>\s*(-?\d+(?:\.\d+)?)\s*</span>.*?<span[^>]*>\s*([CF])\s*</span>',
window,
re.IGNORECASE | re.DOTALL,
)
if not temp_match:
return None, None
try:
value = float(temp_match.group(1))
except Exception:
return None, None
unit = str(temp_match.group(2) or "").upper().strip() or None
return value, unit
@staticmethod
def _wu_to_celsius(value: Optional[float], unit: Optional[str]) -> Optional[float]:
if value is None:
return None
normalized = str(unit or "").upper().strip()
if normalized == "F":
return round((float(value) - 32.0) * 5.0 / 9.0, 1)
return round(float(value), 1)
def fetch_wunderground_settlement_current(
self,
city: str,
*,
url: str,
station_label: Optional[str] = None,
icao: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
normalized_city = str(city or "").strip().lower()
cache_key = f"wu:settlement:{normalized_city}"
cached = self._get_settlement_cache(cache_key)
if cached:
return cached
html = self._fetch_wunderground_page(url)
if not html:
return None
fallback_icao = str(icao or "").strip()
station_name = station_label or self._wu_extract_station_name(html, fallback_icao)
display_temp, display_unit = self._wu_extract_station_temperature(
html,
station_name=station_name,
)
temp_c = self._wu_to_celsius(display_temp, display_unit)
if temp_c is None:
logger.warning(f"Wunderground temperature parse failed city={city} url={url}")
return None
city_meta = CITY_REGISTRY.get(normalized_city) or {}
utc_offset_seconds = int(city_meta.get("tz_offset") or 0)
obs_iso = datetime.now(timezone.utc).isoformat()
today_obs = self._update_official_today_obs(
source_code="wunderground",
station_code=fallback_icao or normalized_city,
obs_iso=obs_iso,
current_temp=temp_c,
utc_offset_seconds=utc_offset_seconds,
)
max_so_far = None
max_temp_time = None
today_low = None
if today_obs:
hottest = max(today_obs, key=lambda item: float(item.get("temp") or -999))
coldest = min(today_obs, key=lambda item: float(item.get("temp") or 999))
max_so_far = self._wu_to_celsius(float(hottest.get("temp")), "C")
today_low = self._wu_to_celsius(float(coldest.get("temp")), "C")
max_temp_time = str(hottest.get("time") or "").strip() or None
payload: Dict[str, Any] = {
"source": "wunderground",
"source_label": "Wunderground",
"station_code": fallback_icao or None,
"station_name": station_name or fallback_icao or str(city or "").title(),
"observation_time": obs_iso,
"source_url": url,
"current": {
"temp": temp_c,
"display_temp": display_temp,
"display_unit": display_unit,
"max_temp_so_far": max_so_far,
"max_temp_time": max_temp_time,
"today_low": today_low,
"humidity": None,
"wind_speed_kt": None,
"wind_dir": None,
},
"today_obs": today_obs,
"unit": "celsius",
}
self._set_settlement_cache(cache_key, payload)
return payload
+1 -1
View File
@@ -665,7 +665,7 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]:
mc = metar.get("current", {}) if metar else {}
mg_cur = mgm.get("current", {}) if mgm else {}
sc_cur = settlement_current.get("current", {}) if settlement_current else {}
use_settlement_current = settlement_source in {"hko", "cwa", "noaa"} and bool(sc_cur)
use_settlement_current = settlement_source in {"hko", "cwa", "noaa", "wunderground"} and bool(sc_cur)
primary_current = sc_cur if use_settlement_current else mc
cur_temp = _sf(primary_current.get("temp"))
if cur_temp is None:
+1
View File
@@ -76,6 +76,7 @@ SETTLEMENT_SOURCE_LABELS: Dict[str, str] = {
"cwa": "CWA",
"noaa": "NOAA",
"mgm": "MGM",
"wunderground": "Wunderground",
}
_cache: Dict[str, Dict] = {}