diff --git a/data/probability_training_snapshots.jsonl b/data/probability_training_snapshots.jsonl index a3eae55f..6c5a7304 100644 --- a/data/probability_training_snapshots.jsonl +++ b/data/probability_training_snapshots.jsonl @@ -120,3 +120,4 @@ {"city": "shenzhen", "timestamp": "2026-03-25T08:43:15.528748+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.18016764322916676, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 29.0, "peak_status": "past", "prob_snapshot": [{"v": 29, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null} {"city": "shenzhen", "timestamp": "2026-03-25T08:57:11.783182+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": 26.7, "raw_sigma": 0.18016764322916676, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 26.7, "peak_status": "past", "prob_snapshot": [{"v": 27, "p": 1.0}], "shadow_prob_snapshot": [{"v": 27, "p": 1.0}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320132525", "calibration_source": "artifacts\\probability_calibration\\default.json", "calibrated_mu": 26.7, "calibrated_sigma": 0.24322631835937514} {"city": "shenzhen", "timestamp": "2026-03-25T09:32:32+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.16637912326388898, "deb_prediction": 28.1, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.6, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.6, "GEM": 30.7, "JMA": 25.5}, "max_so_far": 28.9, "peak_status": "past", "prob_snapshot": [{"v": 29, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null} +{"city": "shenzhen", "timestamp": "2026-03-25T10:02:35+00:00", "date": "2026-03-25", "temp_symbol": "°C", "raw_mu": null, "raw_sigma": 0.15911458333333342, "deb_prediction": 28.2, "ensemble": {"p10": 30.5, "median": 31.4, "p90": 31.8}, "multi_model": {"Open-Meteo": 26.5, "ECMWF": 28.8, "GFS": 30.3, "ICON": 26.5, "GEM": 30.7, "JMA": 26.2}, "max_so_far": 28.9, "peak_status": "past", "prob_snapshot": [{"v": 28, "p": 1.0}], "shadow_prob_snapshot": [], "probability_engine": "legacy", "probability_mode": "legacy", "calibration_version": null, "calibration_source": null, "calibrated_mu": null, "calibrated_sigma": null} diff --git a/src/analysis/settlement_rounding.py b/src/analysis/settlement_rounding.py index bed93047..866f405f 100644 --- a/src/analysis/settlement_rounding.py +++ b/src/analysis/settlement_rounding.py @@ -30,7 +30,7 @@ def is_exact_settlement_city(city: str) -> bool: def apply_city_settlement(city: str, value: Optional[Number]) -> Optional[int]: """ 根据城市返回最终的结算值： - - 香港/台北: 向下取整 (e.g. 28.9 -> 28) + - 香港: 向下取整 (e.g. 28.9 -> 28) - 其他: WU 规则四舍五入 """ if value is None: diff --git a/src/data_collection/wunderground_sources.py b/src/data_collection/wunderground_sources.py index 85d14b2c..60e65d4c 100644 --- a/src/data_collection/wunderground_sources.py +++ b/src/data_collection/wunderground_sources.py @@ -44,6 +44,40 @@ class WundergroundSourceMixin: logger.warning(f"Wunderground page fetch failed url={url}: {exc}") return None + def _fetch_wunderground_history_page(self, url: str) -> Optional[str]: + if not url: + return None + cache_key = f"wu:history:{url}" + cached = self._get_settlement_cache(cache_key) + if isinstance(cached, dict): + html = str(cached.get("html") or "") + if html: + return html + + try: + response = self.session.get( + url, + headers={ + "User-Agent": "Mozilla/5.0", + "Referer": url, + }, + timeout=self.timeout, + ) + response.raise_for_status() + html = str(response.text or "") + if not html: + return None + ttl_backup = getattr(self, "settlement_cache_ttl_sec", self._WU_PAGE_TTL_SEC) + try: + self.settlement_cache_ttl_sec = self._WU_PAGE_TTL_SEC + self._set_settlement_cache(cache_key, {"html": html}) + finally: + self.settlement_cache_ttl_sec = ttl_backup + return html + except Exception as exc: + logger.warning(f"Wunderground history page fetch failed url={url}: {exc}") + return None + @staticmethod def _wu_extract_app_state(html: str) -> Optional[Dict[str, Any]]: match = re.search( @@ -159,6 +193,109 @@ class WundergroundSourceMixin: text = str(raw or "").strip() if not text: return None + + @staticmethod + def _wu_parse_history_time(raw: Any, utc_offset_seconds: int) -> Optional[str]: + if raw is None: + return None + if isinstance(raw, (int, float)): + try: + return datetime.fromtimestamp(float(raw), timezone.utc).isoformat() + except Exception: + return None + text = str(raw).strip() + if not text: + return None + for fmt in ( + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M", + ): + try: + parsed = datetime.strptime(text, fmt) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone(timedelta(seconds=utc_offset_seconds))) + return parsed.astimezone(timezone.utc).isoformat() + except Exception: + continue + return None + + @classmethod + def _wu_observation_temp_c(cls, obs: Dict[str, Any]) -> Optional[float]: + if not isinstance(obs, dict): + return None + metric = obs.get("metric") + if isinstance(metric, dict): + for key in ("temp", "temperature", "tempAvg", "temperatureAvg"): + value = cls._safe_float(metric.get(key)) + if value is not None: + return round(float(value), 1) + imperial = obs.get("imperial") + if isinstance(imperial, dict): + for key in ("temp", "temperature", "tempAvg", "temperatureAvg"): + value = cls._safe_float(imperial.get(key)) + if value is not None: + return cls._wu_to_celsius(value, "F") + for key in ("temp", "temperature"): + value = cls._safe_float(obs.get(key)) + if value is not None: + return round(float(value), 1) + return None + + @classmethod + def _wu_observation_time_iso(cls, obs: Dict[str, Any], utc_offset_seconds: int) -> Optional[str]: + if not isinstance(obs, dict): + return None + for key in ("validTimeLocal", "obsTimeLocal", "observationTime", "timestamp", "validTimeUtc", "epoch"): + value = obs.get(key) + parsed = cls._wu_parse_history_time(value, utc_offset_seconds) + if parsed: + return parsed + return None + + @classmethod + def _wu_iter_lists(cls, node: Any): + if isinstance(node, dict): + for value in node.values(): + yield from cls._wu_iter_lists(value) + elif isinstance(node, list): + yield node + for item in node: + yield from cls._wu_iter_lists(item) + + @classmethod + def _wu_extract_history_observations( + cls, + app_state: Dict[str, Any], + *, + utc_offset_seconds: int, + ) -> list[dict[str, Any]]: + best: list[dict[str, Any]] = [] + for candidate in cls._wu_iter_lists(app_state): + if not candidate or not all(isinstance(item, dict) for item in candidate): + continue + parsed: list[dict[str, Any]] = [] + seen_times: set[str] = set() + for item in candidate: + temp_c = cls._wu_observation_temp_c(item) + time_iso = cls._wu_observation_time_iso(item, utc_offset_seconds) + if temp_c is None or not time_iso: + continue + try: + local_dt = datetime.fromisoformat(time_iso.replace("Z", "+00:00")).astimezone( + timezone(timedelta(seconds=utc_offset_seconds)) + ) + except Exception: + continue + hhmm = local_dt.strftime("%H:%M") + if hhmm in seen_times: + continue + seen_times.add(hhmm) + parsed.append({"time": hhmm, "temp": round(float(temp_c), 1)}) + if len(parsed) > len(best): + best = parsed + return best try: parsed = datetime.strptime(text, "%Y-%m-%dT%H:%M:%S%z") return parsed.astimezone(timezone.utc).isoformat() @@ -274,13 +411,48 @@ class WundergroundSourceMixin: return None obs_iso = obs_iso or datetime.now(timezone.utc).isoformat() - today_obs = self._update_official_today_obs( - source_code="wunderground", - station_code=station_id or fallback_icao or normalized_city, - obs_iso=obs_iso, - current_temp=temp_c, - utc_offset_seconds=utc_offset_seconds, + today_obs: list[dict[str, Any]] = [] + history_html = self._fetch_wunderground_history_page(history_url) if history_url else None + history_state = self._wu_extract_app_state(history_html or "") if history_html else None + history_obs = ( + self._wu_extract_history_observations( + history_state or {}, + utc_offset_seconds=utc_offset_seconds, + ) + if isinstance(history_state, dict) + else [] ) + if history_obs: + today_obs = history_obs + for point in history_obs: + point_time = str(point.get("time") or "").strip() + point_temp = self._safe_float(point.get("temp")) + if not point_time or point_temp is None: + continue + try: + local_dt = datetime.strptime(point_time, "%H:%M").replace( + year=datetime.now(timezone(timedelta(seconds=utc_offset_seconds))).year, + month=datetime.now(timezone(timedelta(seconds=utc_offset_seconds))).month, + day=datetime.now(timezone(timedelta(seconds=utc_offset_seconds))).day, + tzinfo=timezone(timedelta(seconds=utc_offset_seconds)), + ) + self._update_official_today_obs( + source_code="wunderground", + station_code=station_id or fallback_icao or normalized_city, + obs_iso=local_dt.astimezone(timezone.utc).isoformat(), + current_temp=point_temp, + utc_offset_seconds=utc_offset_seconds, + ) + except Exception: + continue + else: + today_obs = self._update_official_today_obs( + source_code="wunderground", + station_code=station_id or fallback_icao or normalized_city, + obs_iso=obs_iso, + current_temp=temp_c, + utc_offset_seconds=utc_offset_seconds, + ) sampled_max = None sampled_max_time = None diff --git a/web/analysis_service.py b/web/analysis_service.py index 381fa95c..c1fc6006 100644 --- a/web/analysis_service.py +++ b/web/analysis_service.py @@ -689,7 +689,9 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]: if max_temp_time == "": max_temp_time = None - wu_settle = apply_city_settlement(city.lower(), max_so_far) if max_so_far is not None else None + raw_settlement_max = max_so_far + wu_settle = apply_city_settlement(city.lower(), raw_settlement_max) if raw_settlement_max is not None else None + display_settlement_max = wu_settle if settlement_source == "wunderground" and wu_settle is not None else raw_settlement_max # Observation time → local obs_time_str = "" @@ -1275,8 +1277,9 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]: }, "current": { "temp": cur_temp, - "max_so_far": max_so_far, + "max_so_far": display_settlement_max, "max_temp_time": max_temp_time, + "raw_max_so_far": raw_settlement_max, "wu_settlement": wu_settle, "settlement_source": settlement_source, "settlement_source_label": settlement_source_label,