From fef3097c36fbb6305f1520b82961407afe42097d Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Wed, 18 Mar 2026 00:20:04 +0800 Subject: [PATCH] Remove RP5 multi-model data --- .env.example | 3 - .../dashboard/FutureForecastModal.tsx | 66 ++++ scripts/rp5_scrape_forecast.py | 51 --- src/data_collection/rp5_scraper.py | 342 ------------------ src/data_collection/weather_sources.py | 218 ----------- 5 files changed, 66 insertions(+), 614 deletions(-) delete mode 100644 scripts/rp5_scrape_forecast.py delete mode 100644 src/data_collection/rp5_scraper.py diff --git a/.env.example b/.env.example index 7baaf75f..1a116fc9 100644 --- a/.env.example +++ b/.env.example @@ -26,9 +26,6 @@ OPEN_METEO_CACHE_TTL_SEC=7200 OPEN_METEO_ENSEMBLE_CACHE_TTL_SEC=7200 OPEN_METEO_MULTI_MODEL_CACHE_TTL_SEC=7200 OPEN_METEO_MULTI_MODEL_CACHE_VERSION=v2 -# RP5 public page scrape (for extra model in multi-model forecasts) -RP5_MULTI_MODEL_ENABLED=true -RP5_HTTP_TIMEOUT_SEC=20 # Proxy Setting (optional) HTTPS_PROXY=http://127.0.0.1:7890 diff --git a/frontend/components/dashboard/FutureForecastModal.tsx b/frontend/components/dashboard/FutureForecastModal.tsx index 6374cf04..303b0994 100644 --- a/frontend/components/dashboard/FutureForecastModal.tsx +++ b/frontend/components/dashboard/FutureForecastModal.tsx @@ -25,6 +25,7 @@ import { } from "@/components/dashboard/PanelSections"; import { getFutureModalView, + parseAiAnalysis, getTemperatureChartData, getWeatherSummary, } from "@/lib/dashboard-utils"; @@ -531,6 +532,8 @@ export function FutureForecastModal() { detail.current?.raw_metar, detail.current?.visibility_mi, ); + const ai = parseAiAnalysis(detail.ai_analysis); + const risk = detail.risk || {}; return (
+
+
+
+

+ {locale === "en-US" ? "City Risk Profile" : "城市风险档案"} +

+
+ {!risk.airport ? ( + + {t("section.noRiskProfile")} + + ) : ( + <> +
+ + {t("section.airport")} + + + {risk.airport} + {risk.icao ? ` (${risk.icao})` : ""} + +
+
+ + {t("section.distance")} + + {risk.distance_km ?? "--"}km +
+ {risk.warning ? ( +
+ + {t("section.note")} + + {risk.warning} +
+ ) : null} + + )} +
+
+
+

{t("future.ai")}

+
+ {!ai.summary && ai.bullets.length === 0 ? ( + + {t("future.noAi")} + + ) : ( + <> + {ai.summary ? ( +
{ai.summary}
+ ) : null} + {ai.bullets.length > 0 ? ( +
    + {ai.bullets.map((item) => ( +
  • {item}
    • + ))} +
+ ) : null} + + )} +
+
diff --git a/scripts/rp5_scrape_forecast.py b/scripts/rp5_scrape_forecast.py deleted file mode 100644 index d8a58e54..00000000 --- a/scripts/rp5_scrape_forecast.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -import sys - -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if ROOT not in sys.path: - sys.path.insert(0, ROOT) - -from src.data_collection.rp5_scraper import scrape_rp5_forecast - - -def main() -> int: - parser = argparse.ArgumentParser( - description="Scrape public RP5 forecast page and output JSON.", - ) - parser.add_argument( - "--url", - required=True, - help="RP5 city weather URL, e.g. https://rp5.am/Weather_in_Ankara%2C_Esenboga_%28airport%29", - ) - parser.add_argument( - "--timeout", - type=int, - default=20, - help="HTTP timeout seconds (default: 20)", - ) - parser.add_argument( - "--out", - default="", - help="Optional output file path. If empty, print to stdout.", - ) - args = parser.parse_args() - - data = scrape_rp5_forecast(args.url, timeout_sec=args.timeout) - payload = json.dumps(data, ensure_ascii=False, indent=2) - - if args.out: - with open(args.out, "w", encoding="utf-8") as f: - f.write(payload + "\n") - print(f"saved: {args.out}") - else: - print(payload) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) - diff --git a/src/data_collection/rp5_scraper.py b/src/data_collection/rp5_scraper.py deleted file mode 100644 index 060855ed..00000000 --- a/src/data_collection/rp5_scraper.py +++ /dev/null @@ -1,342 +0,0 @@ -from __future__ import annotations - -import re -import unicodedata -from datetime import datetime, timezone -from html import unescape -from typing import Any, Dict, List, Optional -from urllib.parse import quote, unquote, urljoin - -import requests - -DEFAULT_TIMEOUT_SEC = 20 -DEFAULT_UA = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/124.0.0.0 Safari/537.36" -) - -_TABLE_BY_ID_RE = r'(?is)]*\bid=["\']{table_id}["\'][^>]*>.*?' -_ROW_RE = re.compile(r"(?is)]*>.*?") -_CELL_RE = re.compile(r"(?is)<(td|th)\b([^>]*)>(.*?)") -_TITLE_RE = re.compile(r"(?is)(.*?)") -_COLSPAN_RE = re.compile(r'(?is)\bcolspan\s*=\s*["\']?(\d+)') -_SPACE_RE = re.compile(r"\s+") -_TAG_RE = re.compile(r"(?is)<[^>]+>") -_NUM_RE = re.compile(r"[+-]?\d+(?:\.\d+)?") - -RP5_BASE_URL = "https://rp5.am" -RP5_CITY_URL_OVERRIDES: Dict[str, str] = { - # City pages that do not resolve correctly via simple /Weather_in_ - "london": "https://rp5.am/Weather_in_London%2C_St._James%27s_Park", - "paris": "https://rp5.am/Weather_in_Paris,_France", - "toronto": "https://rp5.am/Weather_in_Toronto,_Canada", - "new york": "https://rp5.am/Weather_in_New_York,_USA", - "warsaw": "https://rp5.am/Weather_in_Warsaw%2C_Okecie_%28airport%29", - "dallas": "https://rp5.am/Weather_in_Dallas%2C_Love_Field_%28airport%29", - "miami": "https://rp5.am/Weather_in_Miami_%28airport%29%2C_Florida", - "atlanta": "https://rp5.am/Weather_in_Atlanta%2C_Georgia", - "sao paulo": "https://rp5.am/Weather_in_Sao_Paulo", - "hong kong": "https://rp5.am/Weather_in_Hong_Kong_%28airport%29", - "singapore": "https://rp5.am/Weather_in_Singapore_%28airport%29", - "shanghai": "https://rp5.am/Weather_in_Shanghai_Pudong_(airport)", - "madrid": "https://rp5.am/Weather_in_Madrid,_Barajas_(airport)", -} - - -def _strip_html(raw: str) -> str: - text = re.sub(r"(?is)<(script|style)\b[^>]*>.*?", " ", raw) - text = _TAG_RE.sub(" ", text) - text = unescape(text).replace("\xa0", " ") - return _SPACE_RE.sub(" ", text).strip() - - -def _extract_title(html: str) -> str: - match = _TITLE_RE.search(html) - if not match: - return "" - return _strip_html(match.group(1)) - - -def _extract_table_html(html: str, table_id: str) -> str: - pattern = re.compile(_TABLE_BY_ID_RE.format(table_id=re.escape(table_id))) - match = pattern.search(html) - return match.group(0) if match else "" - - -def _parse_cells(row_html: str) -> List[Dict[str, Any]]: - cells: List[Dict[str, Any]] = [] - for match in _CELL_RE.finditer(row_html): - attrs = match.group(2) or "" - inner = match.group(3) or "" - colspan_match = _COLSPAN_RE.search(attrs) - colspan = int(colspan_match.group(1)) if colspan_match else 1 - text = _strip_html(inner) - cells.append({"text": text, "colspan": max(1, colspan)}) - return cells - - -def _parse_table(table_html: str) -> List[List[Dict[str, Any]]]: - rows: List[List[Dict[str, Any]]] = [] - for row_match in _ROW_RE.finditer(table_html): - row_cells = _parse_cells(row_match.group(0)) - if row_cells: - rows.append(row_cells) - return rows - - -def _expand_row_values(cells: List[Dict[str, Any]]) -> List[str]: - expanded: List[str] = [] - for cell in cells: - expanded.extend([cell.get("text", "")] * int(cell.get("colspan") or 1)) - return expanded - - -def _to_float_first(text: str) -> Optional[float]: - if not text: - return None - values = _NUM_RE.findall(text) - if not values: - return None - try: - return float(values[0]) - except Exception: - return None - - -def _to_float_last(text: str) -> Optional[float]: - if not text: - return None - values = _NUM_RE.findall(text) - if not values: - return None - try: - return float(values[-1]) - except Exception: - return None - - -def _find_row(rows: List[List[Dict[str, Any]]], prefix: str) -> List[Dict[str, Any]]: - low = prefix.lower() - for row in rows: - if not row: - continue - label = str(row[0].get("text") or "").strip().lower() - if label.startswith(low): - return row - return [] - - -def _build_timeseries(rows: List[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: - if len(rows) < 2: - return [] - - header_row = rows[0] - local_row = _find_row(rows, "Local time") - if not header_row or not local_row: - return [] - - day_slots = _expand_row_values(header_row) - time_slots = _expand_row_values(local_row[1:]) - slot_count = min(len(day_slots), len(time_slots)) - if slot_count <= 0: - return [] - - day_slots = day_slots[:slot_count] - time_slots = time_slots[:slot_count] - - temp_row = _find_row(rows, "Temperature") - pressure_row = _find_row(rows, "Pressure") - wind_speed_row = _find_row(rows, "Wind: speed") - wind_dir_row = _find_row(rows, "direction") - humidity_row = _find_row(rows, "Humidity") - precip_row = _find_row(rows, "Precipitation") - - temp_values = _expand_row_values(temp_row[1:])[:slot_count] if temp_row else [""] * slot_count - pressure_values = _expand_row_values(pressure_row[1:])[:slot_count] if pressure_row else [""] * slot_count - wind_speed_values = _expand_row_values(wind_speed_row[1:])[:slot_count] if wind_speed_row else [""] * slot_count - wind_dir_values = _expand_row_values(wind_dir_row[1:])[:slot_count] if wind_dir_row else [""] * slot_count - humidity_values = _expand_row_values(humidity_row[1:])[:slot_count] if humidity_row else [""] * slot_count - precip_values = _expand_row_values(precip_row[1:])[:slot_count] if precip_row else [""] * slot_count - - out: List[Dict[str, Any]] = [] - for idx in range(slot_count): - entry = { - "day_label": day_slots[idx], - "local_time": time_slots[idx], - "temp_c": _to_float_first(temp_values[idx]), - "pressure_hpa": _to_float_last(pressure_values[idx]), - "wind_mps": _to_float_first(wind_speed_values[idx]), - "wind_dir": (wind_dir_values[idx] or "").strip() or None, - "humidity_pct": _to_float_first(humidity_values[idx]), - "precip_mm": _to_float_first(precip_values[idx]), - } - out.append(entry) - return out - - -def _extract_summary(html: str) -> str: - block = re.search( - r"(?is)]*>\s*Today we expect.*?", - html, - ) - if block: - return _strip_html(block.group(0)) - - idx = html.find("Today we expect") - if idx < 0: - return "" - chunk = html[idx : idx + 1400] - text = _strip_html(chunk) - match = re.search(r"Today we expect.*?(?:Tomorrow:.*?$)", text, re.I) - if match: - return match.group(0).strip() - return "" - - -def _extract_weather_links(html: str) -> List[str]: - links = re.findall(r'(?is)href=["\'](/Weather_in_[^"\']+)["\']', html) - seen = set() - out = [] - for link in links: - if link in seen: - continue - seen.add(link) - out.append(link) - return out - - -def _normalize_for_match(value: str) -> str: - text = unquote(str(value or "")).lower() - text = unicodedata.normalize("NFKD", text) - text = "".join(ch for ch in text if not unicodedata.combining(ch)) - text = re.sub(r"[^a-z0-9]+", " ", text) - return re.sub(r"\s+", " ", text).strip() - - -def _score_weather_link(link: str, city_hint: str) -> int: - low_raw = unquote(str(link or "")).lower() - low = _normalize_for_match(link) - city_norm = _normalize_for_match(city_hint) - words = [w for w in city_norm.split() if len(w) >= 3] - matched = sum(1 for w in words if w in low) - exact_phrase = bool(city_norm and city_norm in low) - starts_with_city = bool(city_norm and low.startswith(f"weather in {city_norm}")) - tokens = set(low.split()) - has_airport = "airport" in tokens - - if words and matched == 0: - return -300 - - score = 0 - score += matched * 30 - if exact_phrase: - score += 20 - if starts_with_city: - score += 20 - if has_airport: - score += 35 if matched > 0 else -60 - if "_region" in low_raw: - score -= 20 - if "_district" in low_raw: - score -= 25 - if "_county" in low_raw or "_province" in low_raw: - score -= 15 - for bad in [ - "weather_in_the_world", - "weather_in_russia", - "weather_in_ukraine", - "weather_in_belarus", - "weather_in_lithuania", - ]: - if bad in low_raw: - score -= 120 - return score - - -def build_rp5_city_url_candidates(city_name: str, city_key: str = "") -> List[str]: - key = str(city_key or city_name).strip().lower() - name = str(city_name or city_key).strip() - out: List[str] = [] - - if key in RP5_CITY_URL_OVERRIDES: - out.append(RP5_CITY_URL_OVERRIDES[key]) - - if name: - tokens = [name.replace(" ", "_"), name] - for token in tokens: - out.append(f"{RP5_BASE_URL}/Weather_in_{quote(token)}") - - # Keep order, deduplicate. - seen = set() - unique: List[str] = [] - for item in out: - if item in seen: - continue - seen.add(item) - unique.append(item) - return unique - - -def scrape_rp5_forecast( - url: str, - timeout_sec: int = DEFAULT_TIMEOUT_SEC, - city_hint: str = "", - max_hops: int = 2, -) -> Dict[str, Any]: - sess = requests.Session() - headers = { - "User-Agent": DEFAULT_UA, - "Accept-Language": "en-US,en;q=0.9", - } - visited = set() - current_url = url - last_payload: Dict[str, Any] = {} - - for _ in range(max(1, int(max_hops))): - if current_url in visited: - break - visited.add(current_url) - - resp = sess.get(current_url, timeout=timeout_sec, headers=headers) - resp.raise_for_status() - html = resp.text - - table_html = _extract_table_html(html, "forecastTable") - rows = _parse_table(table_html) if table_html else [] - points = _build_timeseries(rows) - - last_payload = { - "source": "rp5_html", - "url": url, - "resolved_url": resp.url, - "fetched_at_utc": datetime.now(timezone.utc).isoformat(), - "title": _extract_title(html), - "summary": _extract_summary(html), - "points": points, - } - if points: - return last_payload - - links = _extract_weather_links(html) - if not links: - break - ranked = sorted( - links, - key=lambda item: _score_weather_link(item, city_hint), - reverse=True, - ) - best = ranked[0] - if _score_weather_link(best, city_hint) < 1: - break - current_url = urljoin(resp.url, best) - - return last_payload or { - "source": "rp5_html", - "url": url, - "resolved_url": url, - "fetched_at_utc": datetime.now(timezone.utc).isoformat(), - "title": "", - "summary": "", - "points": [], - } diff --git a/src/data_collection/weather_sources.py b/src/data_collection/weather_sources.py index 6b6afcfc..c85ab921 100644 --- a/src/data_collection/weather_sources.py +++ b/src/data_collection/weather_sources.py @@ -7,10 +7,6 @@ import threading from typing import Optional, Dict, List, Any from datetime import datetime, timedelta, timezone from loguru import logger -from src.data_collection.rp5_scraper import ( - build_rp5_city_url_candidates, - scrape_rp5_forecast, -) class WeatherDataCollector: @@ -74,10 +70,6 @@ class WeatherDataCollector: self.multi_model_cache_version = str( os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_VERSION", "v2") ).strip() or "v2" - self.rp5_multi_model_enabled = str( - os.getenv("RP5_MULTI_MODEL_ENABLED", "true") - ).strip().lower() in {"1", "true", "yes", "on"} - self.rp5_timeout_sec = max(5, int(os.getenv("RP5_HTTP_TIMEOUT_SEC", "20"))) self._open_meteo_cache: Dict[str, Dict] = {} self._ensemble_cache: Dict[str, Dict] = {} self._multi_model_cache: Dict[str, Dict] = {} @@ -1696,208 +1688,6 @@ class WeatherDataCollector: return fallback return None - def _merge_rp5_into_daily_forecasts( - self, - city: str, - dates: List[str], - daily_forecasts: Dict[str, Dict[str, float]], - use_fahrenheit: bool, - ) -> Optional[Dict[str, Any]]: - """ - Scrape RP5 public forecast page and merge as an extra model line ("RP5"). - """ - if not self.rp5_multi_model_enabled: - return None - city_key = str(city or "").strip().lower() - if not city_key: - return None - - city_meta = self.CITY_REGISTRY.get(city_key, {}) - city_name = str(city_meta.get("name") or city).strip() - candidates = build_rp5_city_url_candidates(city_name=city_name, city_key=city_key) - if not candidates: - return None - - payload: Dict[str, Any] = {} - points: List[Dict[str, Any]] = [] - for url in candidates: - try: - probe = scrape_rp5_forecast( - url=url, - timeout_sec=self.rp5_timeout_sec, - city_hint=city_name, - max_hops=3, - ) - except Exception as exc: - logger.debug(f"RP5 scrape failed city={city_name} url={url}: {exc}") - continue - p = probe.get("points") - if isinstance(p, list) and p: - payload = probe - points = p - break - - if not points: - return None - - tz_offset_sec = int(city_meta.get("tz_offset") or 0) - city_tz = timezone(timedelta(seconds=tz_offset_sec)) - city_now = datetime.now(city_tz) - - month_map = { - "jan": 1, - "january": 1, - "feb": 2, - "february": 2, - "mar": 3, - "march": 3, - "apr": 4, - "april": 4, - "may": 5, - "jun": 6, - "june": 6, - "jul": 7, - "july": 7, - "aug": 8, - "august": 8, - "sep": 9, - "sept": 9, - "september": 9, - "oct": 10, - "october": 10, - "nov": 11, - "november": 11, - "dec": 12, - "december": 12, - } - - def _parse_rp5_day_to_date_key(day_label: str) -> Optional[str]: - text = str(day_label or "").strip() - if not text: - return None - low = text.lower() - today_local = city_now.date() - if low.startswith("today"): - return today_local.isoformat() - if low.startswith("tomorrow"): - return (today_local + timedelta(days=1)).isoformat() - - match = re.search(r"\b([A-Za-z]+)\s+(\d{1,2})\b", text) - if not match: - return None - month_token = match.group(1).strip().lower().rstrip(".") - day_token = int(match.group(2)) - month = month_map.get(month_token) - if month is None: - return None - - # RP5 day label omits year; choose the closest year around "now". - candidates: List[datetime] = [] - for y in [today_local.year - 1, today_local.year, today_local.year + 1]: - try: - candidates.append(datetime(y, month, day_token, tzinfo=city_tz)) - except ValueError: - continue - if not candidates: - return None - chosen = min( - candidates, - key=lambda d: abs((d.date() - today_local).days), - ) - return chosen.date().isoformat() - - by_date: Dict[str, float] = {} - ordered_date_keys: List[str] = [] - by_label: Dict[str, float] = {} - ordered_labels: List[str] = [] - for row in points: - if not isinstance(row, dict): - continue - label = str(row.get("day_label") or "").strip() - if not label: - continue - temp_c = row.get("temp_c") - if temp_c is None: - continue - try: - temp_c_val = float(temp_c) - except Exception: - continue - - date_key = _parse_rp5_day_to_date_key(label) - if date_key: - if date_key not in by_date: - by_date[date_key] = temp_c_val - ordered_date_keys.append(date_key) - elif temp_c_val > by_date[date_key]: - by_date[date_key] = temp_c_val - - # Legacy label aggregation fallback (for non-English day labels etc.). - if label not in by_label: - by_label[label] = temp_c_val - ordered_labels.append(label) - elif temp_c_val > by_label[label]: - by_label[label] = temp_c_val - - if not ordered_labels and not ordered_date_keys: - return None - - mapped_count = 0 - mapped_labels: List[str] = [] - mapped_values: List[float] = [by_label[k] for k in ordered_labels] - - # Preferred path: map by actual calendar date, avoids day-shift bugs near midnight. - if dates and by_date: - for date_key in dates: - value_c = by_date.get(date_key) - if value_c is None: - continue - value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c - day_bucket = daily_forecasts.setdefault(date_key, {}) - day_bucket["RP5"] = round(value, 1) - mapped_count += 1 - mapped_labels.append(date_key) - elif dates: - mapped_count = min(len(dates), len(mapped_values)) - for idx in range(mapped_count): - date_key = dates[idx] - value_c = mapped_values[idx] - value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c - day_bucket = daily_forecasts.setdefault(date_key, {}) - day_bucket["RP5"] = round(value, 1) - mapped_labels.append(ordered_labels[idx]) - - if not dates and mapped_values: - # Fallback: if Open-Meteo dates missing unexpectedly, create today bucket. - today_key = datetime.now(timezone.utc).strftime("%Y-%m-%d") - value_c = mapped_values[0] - value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c - daily_forecasts.setdefault(today_key, {})["RP5"] = round(value, 1) - mapped_count = 1 - mapped_labels.append(today_key) - - if dates and mapped_count == 0: - logger.info( - "RP5 no date-overlap city={} rp5_dates={} om_dates={}", - city_name, - ordered_date_keys[:6], - dates[:6], - ) - - logger.info( - "RP5 merged city={} mapped_days={} url={}", - city_name, - mapped_count if dates else 1, - payload.get("resolved_url") or payload.get("url") or "", - ) - return { - "source": "rp5_html", - "url": payload.get("url"), - "resolved_url": payload.get("resolved_url"), - "summary": payload.get("summary"), - "mapped_labels": mapped_labels[:mapped_count] if dates else mapped_labels[:1], - } - def fetch_multi_model( self, lat: float, @@ -1992,13 +1782,6 @@ class WeatherDataCollector: if day_data: daily_forecasts[date_str] = day_data - rp5_meta = self._merge_rp5_into_daily_forecasts( - city=city, - dates=dates, - daily_forecasts=daily_forecasts, - use_fahrenheit=use_fahrenheit, - ) - if not daily_forecasts: logger.warning("Multi-model: 无有效模型数据") return None @@ -2017,7 +1800,6 @@ class WeatherDataCollector: "forecasts": forecasts, # 今天 {"ECMWF": 12.3, "GFS": 11.8, ...} (向后兼容) "daily_forecasts": daily_forecasts, # 按天 {"2026-02-23": {...}, "2026-02-24": {...}} "dates": dates, - "rp5": rp5_meta or {}, "unit": "fahrenheit" if use_fahrenheit else "celsius", } with self._multi_model_cache_lock: