diff --git a/data/probability_training_snapshots.jsonl b/data/probability_training_snapshots.jsonl index 3c1c98f5..4452491e 100644 --- a/data/probability_training_snapshots.jsonl +++ b/data/probability_training_snapshots.jsonl @@ -117,3 +117,5 @@ {"city": "test_city", "timestamp": "2026-03-04 14:00", "date": "2026-03-04", "temp_symbol": "°C", "raw_mu": 29.85, "raw_sigma": 1.09375, "deb_prediction": null, "ensemble": {"p10": 27.0, "median": 29.5, "p90": 31.0}, "multi_model": {"Open-Meteo": 30.0}, "max_so_far": 29.5, "peak_status": "in_window", "prob_snapshot": [{"v": 30, "p": 0.565}, {"v": 31, "p": 0.341}, {"v": 32, "p": 0.094}], "shadow_prob_snapshot": [{"v": 30, "p": 0.565}, {"v": 31, "p": 0.341}, {"v": 32, "p": 0.094}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320132525", "calibration_source": "artifacts\\probability_calibration\\default.json", "calibrated_mu": 29.85, "calibrated_sigma": 1.09375} {"city": "test_city", "timestamp": "2026-03-04 14:30", "date": "2026-03-04", "temp_symbol": "°C", "raw_mu": 29.7, "raw_sigma": 1.09375, "deb_prediction": null, "ensemble": {"p10": 27.0, "median": 29.0, "p90": 31.0}, "multi_model": {"Open-Meteo": 30.0}, "max_so_far": 28.0, "peak_status": "in_window", "prob_snapshot": [{"v": 30, "p": 0.35}, {"v": 29, "p": 0.299}, {"v": 31, "p": 0.187}, {"v": 28, "p": 0.117}], "shadow_prob_snapshot": [{"v": 30, "p": 0.35}, {"v": 29, "p": 0.299}, {"v": 31, "p": 0.187}, {"v": 28, "p": 0.117}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320132525", "calibration_source": "artifacts\\probability_calibration\\default.json", "calibrated_mu": 29.7, "calibrated_sigma": 1.09375} {"city": "hong kong", "timestamp": "2026-03-23T21:10:00+08:00", "date": "2026-03-23", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.6806250000000001, "deb_prediction": 25.2, "ensemble": {"p10": 26.3, "median": 26.4, "p90": 26.6}, "multi_model": {"Open-Meteo": 24.8, "HKO(港天文)": 27.0, "ECMWF": 25.4, "GFS": 25.1, "ICON": 24.8, "GEM": 25.3, "JMA": 23.6}, "max_so_far": 27.4, "peak_status": "past", "prob_snapshot": [{"v": 27, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null} +{"city": "shenzhen", "timestamp": "2026-03-25T08:43:15.528748+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.18016764322916676, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 29.0, "peak_status": "past", "prob_snapshot": [{"v": 29, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null} +{"city": "shenzhen", "timestamp": "2026-03-25T08:57:11.783182+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": 26.7, "raw_sigma": 0.18016764322916676, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 26.7, "peak_status": "past", "prob_snapshot": [{"v": 27, "p": 1.0}], "shadow_prob_snapshot": [{"v": 27, "p": 1.0}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320132525", "calibration_source": "artifacts\\probability_calibration\\default.json", "calibrated_mu": 26.7, "calibrated_sigma": 0.24322631835937514} diff --git a/frontend/content/docs/docs.ts b/frontend/content/docs/docs.ts index 9db2b1aa..d3928d15 100644 --- a/frontend/content/docs/docs.ts +++ b/frontend/content/docs/docs.ts @@ -188,7 +188,7 @@ export const DOCS_PAGES: DocsPage[] = [ title: "什么叫机场端压温风险偏高", blocks: [ { type: "paragraph", text: "它的意思不是整座城市一定更冷，而是作为结算依据的机场站点，在峰值窗口里更可能因为云、阵雨或雷暴扰动，冲不到本来可能达到的更高温度。" }, - { type: "callout", tone: "warning", title: "重点区别", text: "TAF 负责告诉你机场侧未来几个小时会不会出现压温扰动，不直接等于结算温度本身。结算仍然看 METAR、HKO、MGM、NOAA RCTP 等实际结算源。" }, + { type: "callout", tone: "warning", title: "重点区别", text: "TAF 负责告诉你机场侧未来几个小时会不会出现压温扰动，不直接等于结算温度本身。结算仍然看 METAR、HKO、MGM、NOAA RCTP、Wunderground 指定站点等实际结算源。" }, ], }, ], @@ -216,7 +216,7 @@ export const DOCS_PAGES: DocsPage[] = [ title: "What airport-side suppression risk means", blocks: [ { type: "paragraph", text: "It does not mean the entire city must run cooler. It means the airport station used for settlement is more likely to get capped by clouds, showers, or thunderstorm disruption during the peak window and fail to reach the next warmer bucket." }, - { type: "callout", tone: "warning", title: "Important distinction", text: "TAF explains whether the airport side may face suppressive weather over the next few hours. Settlement still comes from the actual settlement source such as METAR, HKO, MGM, or NOAA RCTP." }, + { type: "callout", tone: "warning", title: "Important distinction", text: "TAF explains whether the airport side may face suppressive weather over the next few hours. Settlement still comes from the actual settlement source such as METAR, HKO, MGM, NOAA RCTP, or a designated Wunderground station." }, ], }, ], @@ -242,14 +242,14 @@ export const DOCS_PAGES: DocsPage[] = [ id: "city-rules", title: "当前主要口径", blocks: [ - { type: "bullets", items: ["多数欧美机场市场：按机场 METAR 或机场主站实况结算。", "香港：按香港天文台 HKO 主口径，不接机场 TAF 作为主结算逻辑。", "台北：按 NOAA RCTP（台湾桃园国际机场）最终完成质控后的最高整度摄氏值结算，机场观测和市区体感不可混用。", "Ankara：结算主站以 LTAC / Esenboğa 为准，同时保留 Turkish MGM 作为领先结构参考。"] }, + { type: "bullets", items: ["多数欧美机场市场：按机场 METAR 或机场主站实况结算。", "香港：按香港天文台 HKO 主口径，不接机场 TAF 作为主结算逻辑。", "台北：按 NOAA RCTP（台湾桃园国际机场）最终完成质控后的最高整度摄氏值结算，机场观测和市区体感不可混用。", "Shenzhen：当前按 Wunderground 指定站点（Shenzhen Bao'an Intl Airport Station）镜像结算页主值。", "Ankara：结算主站以 LTAC / Esenboğa 为准，同时保留 Turkish MGM 作为领先结构参考。"] }, ], }, { id: "common-mistakes", title: "最常见的误解", blocks: [ - { type: "bullets", items: ["TAF 不是结算源，它只告诉你机场未来有没有压温扰动。", "市场按机场结算时，城区更热不代表市场就该结到更高温桶。", "香港和台北不能简单套用机场 TAF / METAR 主链逻辑。"] }, + { type: "bullets", items: ["TAF 不是结算源，它只告诉你机场未来有没有压温扰动。", "市场按机场结算时，城区更热不代表市场就该结到更高温桶。", "如果结算页直接看 Wunderground 站点主值，就应优先镜像该站点，而不是擅自回退到其他站。", "香港和台北不能简单套用机场 TAF / METAR 主链逻辑。"] }, ], }, ], @@ -269,14 +269,14 @@ export const DOCS_PAGES: DocsPage[] = [ id: "city-rules", title: "Current primary rules", blocks: [ - { type: "bullets", items: ["Most airport-linked Western markets: settle on airport METAR or the airport primary observing site.", "Hong Kong: settles on HKO, not on airport TAF as the main settlement logic.", "Taipei: settles against NOAA RCTP using the finalized highest rounded whole-degree Celsius reading; airport observations and downtown feel should not be mixed.", "Ankara: settlement centers on LTAC / Esenboğa, with Turkish MGM retained as a leading-structure reference."] }, + { type: "bullets", items: ["Most airport-linked Western markets: settle on airport METAR or the airport primary observing site.", "Hong Kong: settles on HKO, not on airport TAF as the main settlement logic.", "Taipei: settles against NOAA RCTP using the finalized highest rounded whole-degree Celsius reading; airport observations and downtown feel should not be mixed.", "Shenzhen: currently mirrors the designated Wunderground station value for Shenzhen Bao'an Intl Airport Station.", "Ankara: settlement centers on LTAC / Esenboğa, with Turkish MGM retained as a leading-structure reference."] }, ], }, { id: "common-mistakes", title: "Common mistakes", blocks: [ - { type: "bullets", items: ["TAF is not a settlement source. It only tells you whether airport-side suppressive weather may appear.", "If the market settles on an airport site, a hotter downtown feel does not automatically justify a warmer settlement bucket.", "Hong Kong and Taipei should not be forced into the generic airport TAF / METAR chain."] }, + { type: "bullets", items: ["TAF is not a settlement source. It only tells you whether airport-side suppressive weather may appear.", "If the market settles on an airport site, a hotter downtown feel does not automatically justify a warmer settlement bucket.", "If the settlement page itself references a Wunderground station, the product should mirror that station first instead of silently switching to another source.", "Hong Kong and Taipei should not be forced into the generic airport TAF / METAR chain."] }, ], }, ], diff --git a/src/analysis/city_query_service.py b/src/analysis/city_query_service.py index fe4ae36e..3cddf5d1 100644 --- a/src/analysis/city_query_service.py +++ b/src/analysis/city_query_service.py @@ -39,6 +39,7 @@ def _resolve_settlement_source(city_meta: Dict[str, Any]) -> Tuple[str, str]: "cwa": "CWA", "noaa": "NOAA", "mgm": "MGM", + "wunderground": "Wunderground", } return source, source_label_map.get(source, source.upper()) diff --git a/src/analysis/trend_engine.py b/src/analysis/trend_engine.py index f7a5b19f..e38325eb 100644 --- a/src/analysis/trend_engine.py +++ b/src/analysis/trend_engine.py @@ -30,6 +30,7 @@ SETTLEMENT_SOURCE_LABELS = { "cwa": "CWA", "noaa": "NOAA", "mgm": "MGM", + "wunderground": "Wunderground", } _CLOUD_RANK_LABELS = { diff --git a/src/data_collection/city_registry.py b/src/data_collection/city_registry.py index afe083b2..70f02c2a 100644 --- a/src/data_collection/city_registry.py +++ b/src/data_collection/city_registry.py @@ -391,6 +391,9 @@ CITY_REGISTRY = { "lat": 22.6393, "lon": 113.8107, "icao": "ZGSZ", + "settlement_source": "wunderground", + "settlement_url": "https://www.wunderground.com/weather/cn/shenzhen/ZGSZ", + "settlement_station_label": "Shenzhen Bao'an Intl Airport Station", "tz_offset": 28800, "use_fahrenheit": False, "is_major": True, diff --git a/src/data_collection/settlement_sources.py b/src/data_collection/settlement_sources.py index d00e1129..63d41b9b 100644 --- a/src/data_collection/settlement_sources.py +++ b/src/data_collection/settlement_sources.py @@ -183,6 +183,49 @@ class SettlementSourceMixin: ) return self._sort_temp_points(points) + def _update_official_today_obs( + self, + *, + source_code: str, + station_code: str, + obs_iso: Optional[str], + current_temp: Optional[float], + utc_offset_seconds: int, + ) -> List[Dict[str, Any]]: + if not obs_iso or current_temp is None: + return [] + + try: + obs_dt = datetime.fromisoformat(str(obs_iso).replace("Z", "+00:00")) + except Exception: + return [] + if obs_dt.tzinfo is None: + obs_dt = obs_dt.replace(tzinfo=timezone.utc) + local_tz = timezone(timedelta(seconds=int(utc_offset_seconds or 0))) + local_dt = obs_dt.astimezone(local_tz) + date_str = local_dt.strftime("%Y-%m-%d") + time_str = local_dt.strftime("%H:%M") + mode = get_state_storage_mode() + if mode not in {STATE_STORAGE_DUAL, STATE_STORAGE_SQLITE}: + return [{"time": time_str, "temp": round(float(current_temp), 1)}] + + lock = self._get_settlement_series_lock() + with lock: + _official_intraday_repo.upsert_point( + source_code=source_code, + station_code=station_code, + target_date=date_str, + observation_time=time_str, + value=round(float(current_temp), 1), + payload={"time": time_str, "temp": round(float(current_temp), 1)}, + ) + points = _official_intraday_repo.load_points( + source_code=source_code, + station_code=station_code, + target_date=date_str, + ) + return self._sort_temp_points(points) + def fetch_hko_settlement_current(self) -> Optional[Dict[str, Any]]: cache_key = "hko:hong_kong" cached = self._get_settlement_cache(cache_key) @@ -537,6 +580,22 @@ class SettlementSourceMixin: def fetch_settlement_current(self, city: str) -> Optional[Dict[str, Any]]: normalized = str(city or "").strip().lower() + try: + from src.data_collection.city_registry import CITY_REGISTRY + + city_meta = CITY_REGISTRY.get(normalized) or {} + settlement_source = str(city_meta.get("settlement_source") or "").strip().lower() + if settlement_source == "wunderground": + settlement_url = str(city_meta.get("settlement_url") or "").strip() + if settlement_url: + return self.fetch_wunderground_settlement_current( + normalized, + url=settlement_url, + station_label=str(city_meta.get("settlement_station_label") or "").strip() or None, + icao=str(city_meta.get("icao") or "").strip() or None, + ) + except Exception as exc: + logger.warning(f"Wunderground settlement dispatch failed city={city}: {exc}") if normalized == "hong kong": return self.fetch_hko_settlement_current() if normalized == "taipei": diff --git a/src/data_collection/weather_sources.py b/src/data_collection/weather_sources.py index 3e12ed2f..e5fbbe1c 100644 --- a/src/data_collection/weather_sources.py +++ b/src/data_collection/weather_sources.py @@ -10,9 +10,10 @@ from src.data_collection.settlement_sources import SettlementSourceMixin from src.data_collection.metar_sources import MetarSourceMixin from src.data_collection.mgm_sources import MgmSourceMixin from src.data_collection.nws_open_meteo_sources import NwsOpenMeteoSourceMixin +from src.data_collection.wunderground_sources import WundergroundSourceMixin -class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, NwsOpenMeteoSourceMixin): +class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, NwsOpenMeteoSourceMixin, WundergroundSourceMixin): """ Multi-source weather data collector diff --git a/src/data_collection/wunderground_sources.py b/src/data_collection/wunderground_sources.py new file mode 100644 index 00000000..d456b115 --- /dev/null +++ b/src/data_collection/wunderground_sources.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import re +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from loguru import logger +from src.data_collection.city_registry import CITY_REGISTRY + + +class WundergroundSourceMixin: + _WU_PAGE_TTL_SEC = 180 + + def _fetch_wunderground_page(self, url: str) -> Optional[str]: + cache_key = f"wu:page:{url}" + cached = self._get_settlement_cache(cache_key) + if isinstance(cached, dict): + html = str(cached.get("html") or "") + if html: + return html + + try: + response = self.session.get( + url, + headers={ + "User-Agent": "Mozilla/5.0", + "Referer": url, + }, + timeout=self.timeout, + ) + response.raise_for_status() + html = str(response.text or "") + if not html: + return None + ttl_backup = getattr(self, "settlement_cache_ttl_sec", self._WU_PAGE_TTL_SEC) + try: + self.settlement_cache_ttl_sec = self._WU_PAGE_TTL_SEC + self._set_settlement_cache(cache_key, {"html": html}) + finally: + self.settlement_cache_ttl_sec = ttl_backup + return html + except Exception as exc: + logger.warning(f"Wunderground page fetch failed url={url}: {exc}") + return None + + @staticmethod + def _wu_extract_station_name(html: str, fallback_icao: str) -> Optional[str]: + pattern = re.compile( + r'\s*([^<]+?)\s*', + re.IGNORECASE, + ) + for match in pattern.finditer(html): + candidate = re.sub(r"\s+", " ", str(match.group(1) or "")).strip() + if fallback_icao.lower() in candidate.lower() or "station" in candidate.lower(): + return candidate + return None + + @staticmethod + def _wu_extract_station_temperature( + html: str, + *, + station_name: Optional[str], + ) -> tuple[Optional[float], Optional[str]]: + station_anchor = station_name or "Station" + station_pos = html.find(station_anchor) + if station_pos < 0: + station_pos = html.lower().find("station-name") + if station_pos < 0: + return None, None + + window_start = max(0, station_pos - 1800) + window = html[window_start:station_pos] + temp_match = re.search( + r'wu-value[^>]*>\s*(-?\d+(?:\.\d+)?)\s*.*?]*>\s*([CF])\s*', + window, + re.IGNORECASE | re.DOTALL, + ) + if not temp_match: + return None, None + + try: + value = float(temp_match.group(1)) + except Exception: + return None, None + unit = str(temp_match.group(2) or "").upper().strip() or None + return value, unit + + @staticmethod + def _wu_to_celsius(value: Optional[float], unit: Optional[str]) -> Optional[float]: + if value is None: + return None + normalized = str(unit or "").upper().strip() + if normalized == "F": + return round((float(value) - 32.0) * 5.0 / 9.0, 1) + return round(float(value), 1) + + def fetch_wunderground_settlement_current( + self, + city: str, + *, + url: str, + station_label: Optional[str] = None, + icao: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + normalized_city = str(city or "").strip().lower() + cache_key = f"wu:settlement:{normalized_city}" + cached = self._get_settlement_cache(cache_key) + if cached: + return cached + + html = self._fetch_wunderground_page(url) + if not html: + return None + + fallback_icao = str(icao or "").strip() + station_name = station_label or self._wu_extract_station_name(html, fallback_icao) + display_temp, display_unit = self._wu_extract_station_temperature( + html, + station_name=station_name, + ) + temp_c = self._wu_to_celsius(display_temp, display_unit) + if temp_c is None: + logger.warning(f"Wunderground temperature parse failed city={city} url={url}") + return None + + city_meta = CITY_REGISTRY.get(normalized_city) or {} + utc_offset_seconds = int(city_meta.get("tz_offset") or 0) + obs_iso = datetime.now(timezone.utc).isoformat() + today_obs = self._update_official_today_obs( + source_code="wunderground", + station_code=fallback_icao or normalized_city, + obs_iso=obs_iso, + current_temp=temp_c, + utc_offset_seconds=utc_offset_seconds, + ) + max_so_far = None + max_temp_time = None + today_low = None + if today_obs: + hottest = max(today_obs, key=lambda item: float(item.get("temp") or -999)) + coldest = min(today_obs, key=lambda item: float(item.get("temp") or 999)) + max_so_far = self._wu_to_celsius(float(hottest.get("temp")), "C") + today_low = self._wu_to_celsius(float(coldest.get("temp")), "C") + max_temp_time = str(hottest.get("time") or "").strip() or None + + payload: Dict[str, Any] = { + "source": "wunderground", + "source_label": "Wunderground", + "station_code": fallback_icao or None, + "station_name": station_name or fallback_icao or str(city or "").title(), + "observation_time": obs_iso, + "source_url": url, + "current": { + "temp": temp_c, + "display_temp": display_temp, + "display_unit": display_unit, + "max_temp_so_far": max_so_far, + "max_temp_time": max_temp_time, + "today_low": today_low, + "humidity": None, + "wind_speed_kt": None, + "wind_dir": None, + }, + "today_obs": today_obs, + "unit": "celsius", + } + self._set_settlement_cache(cache_key, payload) + return payload diff --git a/web/analysis_service.py b/web/analysis_service.py index 7d8c47c7..c60884c7 100644 --- a/web/analysis_service.py +++ b/web/analysis_service.py @@ -665,7 +665,7 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]: mc = metar.get("current", {}) if metar else {} mg_cur = mgm.get("current", {}) if mgm else {} sc_cur = settlement_current.get("current", {}) if settlement_current else {} - use_settlement_current = settlement_source in {"hko", "cwa", "noaa"} and bool(sc_cur) + use_settlement_current = settlement_source in {"hko", "cwa", "noaa", "wunderground"} and bool(sc_cur) primary_current = sc_cur if use_settlement_current else mc cur_temp = _sf(primary_current.get("temp")) if cur_temp is None: diff --git a/web/core.py b/web/core.py index a6e0bed7..adc9b20e 100644 --- a/web/core.py +++ b/web/core.py @@ -76,6 +76,7 @@ SETTLEMENT_SOURCE_LABELS: Dict[str, str] = { "cwa": "CWA", "noaa": "NOAA", "mgm": "MGM", + "wunderground": "Wunderground", } _cache: Dict[str, Dict] = {}