From 7589fd74bf99973da72a33d702b5a2d434cebe93 Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Wed, 25 Mar 2026 18:03:56 +0800 Subject: [PATCH] feat: Implement Wunderground data integration, enhance dashboard weather data processing, and add future forecast modal. --- data/probability_training_snapshots.jsonl | 1 + .../dashboard/FutureForecastModal.tsx | 54 +++++ frontend/lib/dashboard-types.ts | 20 ++ frontend/lib/dashboard-utils.ts | 32 +++ src/data_collection/wunderground_sources.py | 208 ++++++++++++++++-- web/analysis_service.py | 48 +++- 6 files changed, 331 insertions(+), 32 deletions(-) diff --git a/data/probability_training_snapshots.jsonl b/data/probability_training_snapshots.jsonl index 4452491e..a3eae55f 100644 --- a/data/probability_training_snapshots.jsonl +++ b/data/probability_training_snapshots.jsonl @@ -119,3 +119,4 @@ {"city": "hong kong", "timestamp": "2026-03-23T21:10:00+08:00", "date": "2026-03-23", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.6806250000000001, "deb_prediction": 25.2, "ensemble": {"p10": 26.3, "median": 26.4, "p90": 26.6}, "multi_model": {"Open-Meteo": 24.8, "HKO(港天文)": 27.0, "ECMWF": 25.4, "GFS": 25.1, "ICON": 24.8, "GEM": 25.3, "JMA": 23.6}, "max_so_far": 27.4, "peak_status": "past", "prob_snapshot": [{"v": 27, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null} {"city": "shenzhen", "timestamp": "2026-03-25T08:43:15.528748+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.18016764322916676, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 29.0, "peak_status": "past", "prob_snapshot": [{"v": 29, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null} {"city": "shenzhen", "timestamp": "2026-03-25T08:57:11.783182+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": 26.7, "raw_sigma": 0.18016764322916676, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 26.7, "peak_status": "past", "prob_snapshot": [{"v": 27, "p": 1.0}], "shadow_prob_snapshot": [{"v": 27, "p": 1.0}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320132525", "calibration_source": "artifacts\\probability_calibration\\default.json", "calibrated_mu": 26.7, "calibrated_sigma": 0.24322631835937514} +{"city": "shenzhen", "timestamp": "2026-03-25T09:32:32+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.16637912326388898, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 28.9, "peak_status": "past", "prob_snapshot": [{"v": 29, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null} diff --git a/frontend/components/dashboard/FutureForecastModal.tsx b/frontend/components/dashboard/FutureForecastModal.tsx index 508064fe..a2d01340 100644 --- a/frontend/components/dashboard/FutureForecastModal.tsx +++ b/frontend/components/dashboard/FutureForecastModal.tsx @@ -256,6 +256,24 @@ function DailyTemperatureChart({ dateStr }: { dateStr: string }) { pointRadius: 5, }); + if ( + todayChartData.datasets.airportMetarPoints?.some((value) => value != null) + ) { + datasets.push({ + backgroundColor: "#60a5fa", + borderColor: "#60a5fa", + borderWidth: 1, + data: todayChartData.datasets.airportMetarPoints, + fill: false, + label: + locale === "en-US" ? "Airport METAR" : "机场 METAR", + order: 0, + pointHoverRadius: 6, + pointRadius: 4, + showLine: false, + }); + } + if (todayChartData.datasets.mgmPoints.some((value) => value != null)) { datasets.push({ backgroundColor: "#facc15", @@ -759,6 +777,22 @@ export function FutureForecastModal() { : risk.airport ? `${risk.airport}${risk.icao ? ` (${risk.icao})` : ""}` : "--"; + const airportCurrentText = + detail.airport_current?.temp != null + ? `${detail.airport_current.temp}${detail.temp_symbol}${ + detail.airport_current?.obs_time + ? ` @${detail.airport_current.obs_time}` + : "" + }` + : "--"; + const airportMaxText = + detail.airport_current?.max_so_far != null + ? `${detail.airport_current.max_so_far}${detail.temp_symbol}${ + detail.airport_current?.max_temp_time + ? ` @${detail.airport_current.max_temp_time}` + : "" + }` + : "--"; const displayedUpperAirSummary = marketAwareUpperAirCue?.summary || view.front.upperAirSummary; const displayedUpperAirMetrics = (view.front.upperAirMetrics || []).map( @@ -1061,6 +1095,26 @@ export function FutureForecastModal() { {settlementProfileValue} + {settlementSourceCode === "wunderground" ? ( +
+ + {locale === "en-US" + ? "Airport METAR" + : "机场 METAR"} + + {airportCurrentText} +
+ ) : null} + {settlementSourceCode === "wunderground" ? ( +
+ + {locale === "en-US" + ? "Airport high" + : "机场目前最高温"} + + {airportMaxText} +
+ ) : null}
{t("section.distance")} diff --git a/frontend/lib/dashboard-types.ts b/frontend/lib/dashboard-types.ts index 8016cb0d..7ecbae88 100644 --- a/frontend/lib/dashboard-types.ts +++ b/frontend/lib/dashboard-types.ts @@ -66,6 +66,25 @@ export interface CurrentConditions { dewpoint?: number | null; } +export interface AirportCurrentConditions { + temp: number | null; + obs_time: string | null; + max_so_far?: number | null; + max_temp_time?: string | null; + obs_age_min?: number | null; + report_time?: string | null; + receipt_time?: string | null; + obs_time_epoch?: number | null; + wind_speed_kt?: number | null; + wind_dir?: number | null; + humidity?: number | null; + cloud_desc?: string | null; + visibility_mi?: number | null; + wx_desc?: string | null; + raw_metar?: string | null; + source_label?: string | null; +} + export interface NearbyStation { name?: string; icao?: string; @@ -271,6 +290,7 @@ export interface CityDetail { local_date: string; risk: DashboardRisk; current: CurrentConditions; + airport_current?: AirportCurrentConditions; mgm?: MgmData; mgm_nearby?: NearbyStation[]; nearby_source?: string; diff --git a/frontend/lib/dashboard-utils.ts b/frontend/lib/dashboard-utils.ts index c5c8dbfa..97909ab6 100644 --- a/frontend/lib/dashboard-utils.ts +++ b/frontend/lib/dashboard-utils.ts @@ -284,6 +284,10 @@ export function getTemperatureChartData( ? metarObservationSource : officialObservationSource : metarObservationSource; + const airportMetarSource = + settlementSource && observationCode === "wunderground" + ? metarObservationSource + : []; const metarFallbackTag = (() => { const icao = String(detail.risk?.icao || "").trim().toUpperCase(); if (!icao) return "METAR"; @@ -314,6 +318,20 @@ export function getTemperatureChartData( existing == null ? temp : Math.max(Number(existing), Number(temp)); } }); + const airportMetarPoints = new Array(times.length).fill(null); + airportMetarSource.forEach((item) => { + const parts = String(item.time || "").split(":"); + const hour = Number.parseInt(parts[0], 10); + if (Number.isNaN(hour)) return; + const key = `${String(hour).padStart(2, "0")}:00`; + const index = times.indexOf(key); + const temp = item.temp ?? null; + if (index >= 0 && temp != null) { + const existing = airportMetarPoints[index]; + airportMetarPoints[index] = + existing == null ? temp : Math.max(Number(existing), Number(temp)); + } + }); const mgmPoints = new Array(times.length).fill(null); if (detail.mgm?.temp != null && detail.mgm?.time) { @@ -344,6 +362,7 @@ export function getTemperatureChartData( const allValues = [ ...debTemps.filter((value) => value != null), ...metarPoints.filter((value) => value != null), + ...airportMetarPoints.filter((value) => value != null), ...mgmPoints.filter((value) => value != null), ...mgmHourlyPoints.filter((value) => value != null), ] as number[]; @@ -414,6 +433,18 @@ export function getTemperatureChartData( .join(" -> "); legendParts.push(`${observationDisplayTag}: ${recentText}`); } + if (airportMetarSource.length > 0) { + const airportRecentText = [...airportMetarSource] + .slice(-4) + .reverse() + .map((item) => `${item.temp}${detail.temp_symbol}@${item.time}`) + .join(" -> "); + legendParts.push( + isEnglish(locale) + ? `${metarFallbackTag}: ${airportRecentText}` + : `${metarFallbackTag}: ${airportRecentText}`, + ); + } if (shouldUseMetarFallback) { legendParts.push( isEnglish(locale) @@ -453,6 +484,7 @@ export function getTemperatureChartData( return { datasets: { + airportMetarPoints, debFuture, debPast, hasMgmHourly, diff --git a/src/data_collection/wunderground_sources.py b/src/data_collection/wunderground_sources.py index d456b115..85d14b2c 100644 --- a/src/data_collection/wunderground_sources.py +++ b/src/data_collection/wunderground_sources.py @@ -1,7 +1,8 @@ from __future__ import annotations +import json import re -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Dict, Optional from loguru import logger @@ -43,6 +44,65 @@ class WundergroundSourceMixin: logger.warning(f"Wunderground page fetch failed url={url}: {exc}") return None + @staticmethod + def _wu_extract_app_state(html: str) -> Optional[Dict[str, Any]]: + match = re.search( + r'', + html, + re.IGNORECASE | re.DOTALL, + ) + if not match: + return None + try: + payload = json.loads(str(match.group(1) or "")) + return payload if isinstance(payload, dict) else None + except Exception: + return None + + @staticmethod + def _wu_extract_station_history_url(html: str) -> Optional[str]: + match = re.search( + r']*class="station-name"[^>]*>\s*]+href="([^"]*?/history/daily/[^"]+)"', + html, + re.IGNORECASE | re.DOTALL, + ) + if not match: + return None + href = str(match.group(1) or "").strip() + if not href: + return None + if href.startswith("http://") or href.startswith("https://"): + return href + return f"https://www.wunderground.com{href}" + + @staticmethod + def _wu_extract_station_id_from_history_url(url: Optional[str]) -> Optional[str]: + text = str(url or "").strip() + if not text: + return None + match = re.search(r"/history/daily/(?:[^/]+/){1,3}([^/]+)/date/", text, re.IGNORECASE) + return str(match.group(1) or "").strip() or None if match else None + + @staticmethod + def _wu_extract_station_name_from_history_url(url: Optional[str]) -> Optional[str]: + text = str(url or "").strip() + if not text: + return None + match = re.search(r"/history/daily/(?:[^/]+/){1,3}([^/]+)/date/", text, re.IGNORECASE) + return str(match.group(1) or "").strip() or None if match else None + + @staticmethod + def _safe_float(value: Any) -> Optional[float]: + if value is None: + return None + text = str(value).strip() + if not text: + return None + try: + return float(text) + except Exception: + return None + @staticmethod def _wu_extract_station_name(html: str, fallback_icao: str) -> Optional[str]: pattern = re.compile( @@ -94,6 +154,44 @@ class WundergroundSourceMixin: return round((float(value) - 32.0) * 5.0 / 9.0, 1) return round(float(value), 1) + @staticmethod + def _wu_parse_observation_iso(raw: Optional[str], utc_offset_seconds: int) -> Optional[str]: + text = str(raw or "").strip() + if not text: + return None + try: + parsed = datetime.strptime(text, "%Y-%m-%dT%H:%M:%S%z") + return parsed.astimezone(timezone.utc).isoformat() + except Exception: + pass + try: + parsed = datetime.strptime(text, "%Y-%m-%d %H:%M:%S") + parsed = parsed.replace(tzinfo=timezone(timedelta(seconds=utc_offset_seconds))) + return parsed.astimezone(timezone.utc).isoformat() + except Exception: + return None + + @staticmethod + def _wu_find_current_observation_block( + app_state: Dict[str, Any], + *, + icao: Optional[str], + ) -> Optional[Dict[str, Any]]: + normalized_icao = str(icao or "").strip().upper() + fallback_block: Optional[Dict[str, Any]] = None + for value in app_state.values(): + if not isinstance(value, dict): + continue + url = str(value.get("u") or "") + body = value.get("b") + if not isinstance(body, dict) or "observations/current" not in url: + continue + if normalized_icao and f"icaoCode={normalized_icao}" in url: + return body + if fallback_block is None: + fallback_block = body + return fallback_block + def fetch_wunderground_settlement_current( self, city: str, @@ -112,44 +210,108 @@ class WundergroundSourceMixin: if not html: return None - fallback_icao = str(icao or "").strip() - station_name = station_label or self._wu_extract_station_name(html, fallback_icao) - display_temp, display_unit = self._wu_extract_station_temperature( - html, - station_name=station_name, + city_meta = CITY_REGISTRY.get(normalized_city) or {} + utc_offset_seconds = int(city_meta.get("tz_offset") or 0) + fallback_icao = str(icao or "").strip().upper() + history_url = self._wu_extract_station_history_url(html) + station_id = self._wu_extract_station_id_from_history_url(history_url) + station_name = ( + station_label + or self._wu_extract_station_name(html, fallback_icao) + or self._wu_extract_station_name_from_history_url(history_url) + or fallback_icao + or str(city or "").title() ) - temp_c = self._wu_to_celsius(display_temp, display_unit) + + app_state = self._wu_extract_app_state(html) or {} + current_block = self._wu_find_current_observation_block(app_state, icao=fallback_icao) + + display_temp = None + display_unit = "F" + obs_iso = None + humidity = None + wind_speed_kt = None + wind_dir = None + wx_phrase = None + dewpoint_c = None + official_high_c = None + official_low_c = None + + if current_block: + display_temp = self._safe_float(current_block.get("temperature")) + obs_iso = self._wu_parse_observation_iso( + current_block.get("validTimeLocal"), + utc_offset_seconds, + ) + humidity = self._safe_float(current_block.get("relativeHumidity")) + wind_speed_kt = self._safe_float(current_block.get("windSpeed")) + wind_dir = str(current_block.get("windDirectionCardinal") or "").strip() or None + wx_phrase = str(current_block.get("wxPhraseLong") or "").strip() or None + dewpoint_c = self._wu_to_celsius( + self._safe_float(current_block.get("temperatureDewPoint")), + "F", + ) + official_high_c = self._wu_to_celsius( + self._safe_float( + current_block.get("temperatureMaxSince7Am") + or current_block.get("temperatureMax24Hour") + ), + "F", + ) + official_low_c = self._wu_to_celsius( + self._safe_float(current_block.get("temperatureMin24Hour")), + "F", + ) + + if display_temp is None: + display_temp, display_unit = self._wu_extract_station_temperature( + html, + station_name=station_name, + ) + temp_c = self._wu_to_celsius(self._safe_float(display_temp), display_unit) if temp_c is None: logger.warning(f"Wunderground temperature parse failed city={city} url={url}") return None - city_meta = CITY_REGISTRY.get(normalized_city) or {} - utc_offset_seconds = int(city_meta.get("tz_offset") or 0) - obs_iso = datetime.now(timezone.utc).isoformat() + obs_iso = obs_iso or datetime.now(timezone.utc).isoformat() today_obs = self._update_official_today_obs( source_code="wunderground", - station_code=fallback_icao or normalized_city, + station_code=station_id or fallback_icao or normalized_city, obs_iso=obs_iso, current_temp=temp_c, utc_offset_seconds=utc_offset_seconds, ) - max_so_far = None - max_temp_time = None - today_low = None + + sampled_max = None + sampled_max_time = None + sampled_low = None if today_obs: hottest = max(today_obs, key=lambda item: float(item.get("temp") or -999)) coldest = min(today_obs, key=lambda item: float(item.get("temp") or 999)) - max_so_far = self._wu_to_celsius(float(hottest.get("temp")), "C") - today_low = self._wu_to_celsius(float(coldest.get("temp")), "C") - max_temp_time = str(hottest.get("time") or "").strip() or None + sampled_max = self._wu_to_celsius(float(hottest.get("temp")), "C") + sampled_max_time = str(hottest.get("time") or "").strip() or None + sampled_low = self._wu_to_celsius(float(coldest.get("temp")), "C") + + max_so_far = official_high_c if official_high_c is not None else sampled_max + today_low = official_low_c if official_low_c is not None else sampled_low + max_temp_time = sampled_max_time + if max_so_far is not None and abs(float(max_so_far) - float(temp_c)) < 0.05: + try: + local_obs = datetime.fromisoformat(obs_iso.replace("Z", "+00:00")).astimezone( + timezone(timedelta(seconds=utc_offset_seconds)) + ) + max_temp_time = local_obs.strftime("%H:%M") + except Exception: + pass payload: Dict[str, Any] = { "source": "wunderground", "source_label": "Wunderground", - "station_code": fallback_icao or None, - "station_name": station_name or fallback_icao or str(city or "").title(), + "station_code": station_id or fallback_icao or None, + "station_name": station_name, "observation_time": obs_iso, "source_url": url, + "history_url": history_url, "current": { "temp": temp_c, "display_temp": display_temp, @@ -157,9 +319,11 @@ class WundergroundSourceMixin: "max_temp_so_far": max_so_far, "max_temp_time": max_temp_time, "today_low": today_low, - "humidity": None, - "wind_speed_kt": None, - "wind_dir": None, + "humidity": humidity, + "wind_speed_kt": wind_speed_kt, + "wind_dir": wind_dir, + "cloud_desc": wx_phrase, + "dewpoint": dewpoint_c, }, "today_obs": today_obs, "unit": "celsius", diff --git a/web/analysis_service.py b/web/analysis_service.py index c60884c7..381fa95c 100644 --- a/web/analysis_service.py +++ b/web/analysis_service.py @@ -760,6 +760,15 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]: for t, v in (metar.get("today_obs", []) if metar else []) ] metar_recent_obs_payload = metar.get("recent_obs", []) if metar else [] + airport_max_so_far = None + airport_max_temp_time = None + for point in metar_today_obs_payload: + value = _sf(point.get("temp")) if isinstance(point, dict) else None + if value is None: + continue + if airport_max_so_far is None or value >= airport_max_so_far: + airport_max_so_far = value + airport_max_temp_time = str(point.get("time") or "") or None # ── 3. Local time parsing ── local_time_full = om.get("current", {}).get("local_time", "") @@ -1272,10 +1281,10 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]: "settlement_source": settlement_source, "settlement_source_label": settlement_source_label, "obs_time": obs_time_str, - "obs_age_min": metar_age_min, - "report_time": metar.get("report_time") if metar else None, - "receipt_time": metar.get("receipt_time") if metar else None, - "obs_time_epoch": metar.get("obs_time_epoch") if metar else None, + "obs_age_min": None if use_settlement_current else metar_age_min, + "report_time": primary_current.get("report_time"), + "receipt_time": primary_current.get("receipt_time"), + "obs_time_epoch": primary_current.get("obs_time_epoch"), "wind_speed_kt": _sf(primary_current.get("wind_speed_kt")), "wind_dir": _sf(primary_current.get("wind_dir")), "humidity": _sf(primary_current.get("humidity")), @@ -1287,6 +1296,24 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]: "wx_desc": primary_current.get("wx_desc"), "raw_metar": primary_current.get("raw_metar"), }, + "airport_current": { + "temp": _sf(mc.get("temp")), + "obs_time": metar.get("obs_time"), + "max_so_far": airport_max_so_far, + "max_temp_time": airport_max_temp_time, + "obs_age_min": metar_age_min, + "report_time": metar.get("report_time") if metar else None, + "receipt_time": metar.get("receipt_time") if metar else None, + "obs_time_epoch": metar.get("obs_time_epoch") if metar else None, + "wind_speed_kt": _sf(mc.get("wind_speed_kt")), + "wind_dir": _sf(mc.get("wind_dir")), + "humidity": _sf(mc.get("humidity")), + "cloud_desc": metar.get("cloud_desc") if metar else None, + "visibility_mi": _sf(mc.get("visibility_mi")), + "wx_desc": mc.get("wx_desc"), + "raw_metar": mc.get("raw_metar"), + "source_label": "METAR", + }, "mgm": mgm_data, "mgm_nearby": raw.get("mgm_nearby", []), "nearby_source": raw.get("nearby_source") or ("mgm" if city.lower() == "ankara" else "metar_cluster"), @@ -1489,12 +1516,12 @@ def _build_city_detail_payload( "official": { "available": bool(data.get("current", {}).get("temp") is not None), "metar": { - "observation_time": data.get("current", {}).get("obs_time"), - "obs_age_min": data.get("current", {}).get("obs_age_min"), - "report_time": data.get("current", {}).get("report_time"), - "receipt_time": data.get("current", {}).get("receipt_time"), - "raw_metar": data.get("current", {}).get("raw_metar"), - "current": data.get("current"), + "observation_time": data.get("airport_current", {}).get("obs_time"), + "obs_age_min": data.get("airport_current", {}).get("obs_age_min"), + "report_time": data.get("airport_current", {}).get("report_time"), + "receipt_time": data.get("airport_current", {}).get("receipt_time"), + "raw_metar": data.get("airport_current", {}).get("raw_metar"), + "current": data.get("airport_current") or {}, }, "taf": data.get("taf") or {}, "weather_gov": {}, @@ -1521,6 +1548,7 @@ def _build_city_detail_payload( "taf": data.get("taf") or {}, "market_scan": market_scan, "risk": data.get("risk"), + "airport_current": data.get("airport_current") or {}, "nearby_source": data.get("nearby_source") or ("mgm" if data.get("name") == "ankara" else "metar_cluster"), "ai_analysis": data.get("ai_analysis") or "", "errors": {},