gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove RP5 multi-model data

Browse Source
This commit is contained in:
2569718930@qq.com
2026-03-18 00:20:04 +08:00
parent 5057892255
commit fef3097c36
5 changed files with 66 additions and 614 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
-3
View File
@@ -26,9 +26,6 @@ OPEN_METEO_CACHE_TTL_SEC=7200
OPEN_METEO_ENSEMBLE_CACHE_TTL_SEC=7200
OPEN_METEO_MULTI_MODEL_CACHE_TTL_SEC=7200
OPEN_METEO_MULTI_MODEL_CACHE_VERSION=v2
# RP5 public page scrape (for extra model in multi-model forecasts)
RP5_MULTI_MODEL_ENABLED=true
RP5_HTTP_TIMEOUT_SEC=20
# Proxy Setting (optional)
HTTPS_PROXY=http://127.0.0.1:7890
frontend/components/dashboard/FutureForecastModal.tsx
+66
View File
@@ -25,6 +25,7 @@ import {
} from "@/components/dashboard/PanelSections";
import {
getFutureModalView,
parseAiAnalysis,
getTemperatureChartData,
getWeatherSummary,
} from "@/lib/dashboard-utils";
@@ -531,6 +532,8 @@ export function FutureForecastModal() {
detail.current?.raw_metar,
detail.current?.visibility_mi,
);
const ai = parseAiAnalysis(detail.ai_analysis);
const risk = detail.risk || {};
return (
<div
@@ -880,7 +883,70 @@ export function FutureForecastModal() {
))}
</div>
</section>
</div>
<div className="future-modal-grid">
<section className="future-modal-section">
<h3>
{locale === "en-US" ? "City Risk Profile" : "城市风险档案"}
</h3>
<div className="risk-info">
{!risk.airport ? (
<span style={{ color: "var(--text-muted)" }}>
{t("section.noRiskProfile")}
</span>
) : (
<>
<div className="risk-row">
<span className="risk-label">
{t("section.airport")}
</span>
<span>
{risk.airport}
{risk.icao ? ` (${risk.icao})` : ""}
</span>
</div>
<div className="risk-row">
<span className="risk-label">
{t("section.distance")}
</span>
<span>{risk.distance_km ?? "--"}km</span>
</div>
{risk.warning ? (
<div className="risk-row">
<span className="risk-label">
{t("section.note")}
</span>
<span>{risk.warning}</span>
</div>
) : null}
</>
)}
</div>
</section>
<section className="future-modal-section">
<h3>{t("future.ai")}</h3>
<div className="ai-box">
{!ai.summary && ai.bullets.length === 0 ? (
<span className="ai-placeholder">
{t("future.noAi")}
</span>
) : (
<>
{ai.summary ? (
<div className="ai-summary">{ai.summary}</div>
) : null}
{ai.bullets.length > 0 ? (
<ul className="ai-list">
{ai.bullets.map((item) => (
<li key={item}>{item}</li>
))}
</ul>
) : null}
</>
)}
</div>
</section>
</div>
</main>
</div>
scripts/rp5_scrape_forecast.py
-51
View File
@@ -1,51 +0,0 @@
from __future__ import annotations
import argparse
import json
import os
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if ROOT not in sys.path:
sys.path.insert(0, ROOT)
from src.data_collection.rp5_scraper import scrape_rp5_forecast
def main() -> int:
parser = argparse.ArgumentParser(
description="Scrape public RP5 forecast page and output JSON.",
)
parser.add_argument(
"--url",
required=True,
help="RP5 city weather URL, e.g. https://rp5.am/Weather_in_Ankara%2C_Esenboga_%28airport%29",
)
parser.add_argument(
"--timeout",
type=int,
default=20,
help="HTTP timeout seconds (default: 20)",
)
parser.add_argument(
"--out",
default="",
help="Optional output file path. If empty, print to stdout.",
)
args = parser.parse_args()
data = scrape_rp5_forecast(args.url, timeout_sec=args.timeout)
payload = json.dumps(data, ensure_ascii=False, indent=2)
if args.out:
with open(args.out, "w", encoding="utf-8") as f:
f.write(payload + "\n")
print(f"saved: {args.out}")
else:
print(payload)
return 0
if __name__ == "__main__":
raise SystemExit(main())
src/data_collection/rp5_scraper.py
-342
View File
@@ -1,342 +0,0 @@
from __future__ import annotations
import re
import unicodedata
from datetime import datetime, timezone
from html import unescape
from typing import Any, Dict, List, Optional
from urllib.parse import quote, unquote, urljoin
import requests
DEFAULT_TIMEOUT_SEC = 20
DEFAULT_UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
)
_TABLE_BY_ID_RE = r'(?is)<table\b[^>]*\bid=["\']{table_id}["\'][^>]*>.*?</table>'
_ROW_RE = re.compile(r"(?is)<tr\b[^>]*>.*?</tr>")
_CELL_RE = re.compile(r"(?is)<(td|th)\b([^>]*)>(.*?)</\1>")
_TITLE_RE = re.compile(r"(?is)<title>(.*?)</title>")
_COLSPAN_RE = re.compile(r'(?is)\bcolspan\s*=\s*["\']?(\d+)')
_SPACE_RE = re.compile(r"\s+")
_TAG_RE = re.compile(r"(?is)<[^>]+>")
_NUM_RE = re.compile(r"[+-]?\d+(?:\.\d+)?")
RP5_BASE_URL = "https://rp5.am"
RP5_CITY_URL_OVERRIDES: Dict[str, str] = {
# City pages that do not resolve correctly via simple /Weather_in_<City>
"london": "https://rp5.am/Weather_in_London%2C_St._James%27s_Park",
"paris": "https://rp5.am/Weather_in_Paris,_France",
"toronto": "https://rp5.am/Weather_in_Toronto,_Canada",
"new york": "https://rp5.am/Weather_in_New_York,_USA",
"warsaw": "https://rp5.am/Weather_in_Warsaw%2C_Okecie_%28airport%29",
"dallas": "https://rp5.am/Weather_in_Dallas%2C_Love_Field_%28airport%29",
"miami": "https://rp5.am/Weather_in_Miami_%28airport%29%2C_Florida",
"atlanta": "https://rp5.am/Weather_in_Atlanta%2C_Georgia",
"sao paulo": "https://rp5.am/Weather_in_Sao_Paulo",
"hong kong": "https://rp5.am/Weather_in_Hong_Kong_%28airport%29",
"singapore": "https://rp5.am/Weather_in_Singapore_%28airport%29",
"shanghai": "https://rp5.am/Weather_in_Shanghai_Pudong_(airport)",
"madrid": "https://rp5.am/Weather_in_Madrid,_Barajas_(airport)",
}
def _strip_html(raw: str) -> str:
text = re.sub(r"(?is)<(script|style)\b[^>]*>.*?</\1>", " ", raw)
text = _TAG_RE.sub(" ", text)
text = unescape(text).replace("\xa0", " ")
return _SPACE_RE.sub(" ", text).strip()
def _extract_title(html: str) -> str:
match = _TITLE_RE.search(html)
if not match:
return ""
return _strip_html(match.group(1))
def _extract_table_html(html: str, table_id: str) -> str:
pattern = re.compile(_TABLE_BY_ID_RE.format(table_id=re.escape(table_id)))
match = pattern.search(html)
return match.group(0) if match else ""
def _parse_cells(row_html: str) -> List[Dict[str, Any]]:
cells: List[Dict[str, Any]] = []
for match in _CELL_RE.finditer(row_html):
attrs = match.group(2) or ""
inner = match.group(3) or ""
colspan_match = _COLSPAN_RE.search(attrs)
colspan = int(colspan_match.group(1)) if colspan_match else 1
text = _strip_html(inner)
cells.append({"text": text, "colspan": max(1, colspan)})
return cells
def _parse_table(table_html: str) -> List[List[Dict[str, Any]]]:
rows: List[List[Dict[str, Any]]] = []
for row_match in _ROW_RE.finditer(table_html):
row_cells = _parse_cells(row_match.group(0))
if row_cells:
rows.append(row_cells)
return rows
def _expand_row_values(cells: List[Dict[str, Any]]) -> List[str]:
expanded: List[str] = []
for cell in cells:
expanded.extend([cell.get("text", "")] * int(cell.get("colspan") or 1))
return expanded
def _to_float_first(text: str) -> Optional[float]:
if not text:
return None
values = _NUM_RE.findall(text)
if not values:
return None
try:
return float(values[0])
except Exception:
return None
def _to_float_last(text: str) -> Optional[float]:
if not text:
return None
values = _NUM_RE.findall(text)
if not values:
return None
try:
return float(values[-1])
except Exception:
return None
def _find_row(rows: List[List[Dict[str, Any]]], prefix: str) -> List[Dict[str, Any]]:
low = prefix.lower()
for row in rows:
if not row:
continue
label = str(row[0].get("text") or "").strip().lower()
if label.startswith(low):
return row
return []
def _build_timeseries(rows: List[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
if len(rows) < 2:
return []
header_row = rows[0]
local_row = _find_row(rows, "Local time")
if not header_row or not local_row:
return []
day_slots = _expand_row_values(header_row)
time_slots = _expand_row_values(local_row[1:])
slot_count = min(len(day_slots), len(time_slots))
if slot_count <= 0:
return []
day_slots = day_slots[:slot_count]
time_slots = time_slots[:slot_count]
temp_row = _find_row(rows, "Temperature")
pressure_row = _find_row(rows, "Pressure")
wind_speed_row = _find_row(rows, "Wind: speed")
wind_dir_row = _find_row(rows, "direction")
humidity_row = _find_row(rows, "Humidity")
precip_row = _find_row(rows, "Precipitation")
temp_values = _expand_row_values(temp_row[1:])[:slot_count] if temp_row else [""] * slot_count
pressure_values = _expand_row_values(pressure_row[1:])[:slot_count] if pressure_row else [""] * slot_count
wind_speed_values = _expand_row_values(wind_speed_row[1:])[:slot_count] if wind_speed_row else [""] * slot_count
wind_dir_values = _expand_row_values(wind_dir_row[1:])[:slot_count] if wind_dir_row else [""] * slot_count
humidity_values = _expand_row_values(humidity_row[1:])[:slot_count] if humidity_row else [""] * slot_count
precip_values = _expand_row_values(precip_row[1:])[:slot_count] if precip_row else [""] * slot_count
out: List[Dict[str, Any]] = []
for idx in range(slot_count):
entry = {
"day_label": day_slots[idx],
"local_time": time_slots[idx],
"temp_c": _to_float_first(temp_values[idx]),
"pressure_hpa": _to_float_last(pressure_values[idx]),
"wind_mps": _to_float_first(wind_speed_values[idx]),
"wind_dir": (wind_dir_values[idx] or "").strip() or None,
"humidity_pct": _to_float_first(humidity_values[idx]),
"precip_mm": _to_float_first(precip_values[idx]),
}
out.append(entry)
return out
def _extract_summary(html: str) -> str:
block = re.search(
r"(?is)<b\b[^>]*>\s*Today we expect.*?</b>",
html,
)
if block:
return _strip_html(block.group(0))
idx = html.find("Today we expect")
if idx < 0:
return ""
chunk = html[idx : idx + 1400]
text = _strip_html(chunk)
match = re.search(r"Today we expect.*?(?:Tomorrow:.*?$)", text, re.I)
if match:
return match.group(0).strip()
return ""
def _extract_weather_links(html: str) -> List[str]:
links = re.findall(r'(?is)href=["\'](/Weather_in_[^"\']+)["\']', html)
seen = set()
out = []
for link in links:
if link in seen:
continue
seen.add(link)
out.append(link)
return out
def _normalize_for_match(value: str) -> str:
text = unquote(str(value or "")).lower()
text = unicodedata.normalize("NFKD", text)
text = "".join(ch for ch in text if not unicodedata.combining(ch))
text = re.sub(r"[^a-z0-9]+", " ", text)
return re.sub(r"\s+", " ", text).strip()
def _score_weather_link(link: str, city_hint: str) -> int:
low_raw = unquote(str(link or "")).lower()
low = _normalize_for_match(link)
city_norm = _normalize_for_match(city_hint)
words = [w for w in city_norm.split() if len(w) >= 3]
matched = sum(1 for w in words if w in low)
exact_phrase = bool(city_norm and city_norm in low)
starts_with_city = bool(city_norm and low.startswith(f"weather in {city_norm}"))
tokens = set(low.split())
has_airport = "airport" in tokens
if words and matched == 0:
return -300
score = 0
score += matched * 30
if exact_phrase:
score += 20
if starts_with_city:
score += 20
if has_airport:
score += 35 if matched > 0 else -60
if "_region" in low_raw:
score -= 20
if "_district" in low_raw:
score -= 25
if "_county" in low_raw or "_province" in low_raw:
score -= 15
for bad in [
"weather_in_the_world",
"weather_in_russia",
"weather_in_ukraine",
"weather_in_belarus",
"weather_in_lithuania",
]:
if bad in low_raw:
score -= 120
return score
def build_rp5_city_url_candidates(city_name: str, city_key: str = "") -> List[str]:
key = str(city_key or city_name).strip().lower()
name = str(city_name or city_key).strip()
out: List[str] = []
if key in RP5_CITY_URL_OVERRIDES:
out.append(RP5_CITY_URL_OVERRIDES[key])
if name:
tokens = [name.replace(" ", "_"), name]
for token in tokens:
out.append(f"{RP5_BASE_URL}/Weather_in_{quote(token)}")
# Keep order, deduplicate.
seen = set()
unique: List[str] = []
for item in out:
if item in seen:
continue
seen.add(item)
unique.append(item)
return unique
def scrape_rp5_forecast(
url: str,
timeout_sec: int = DEFAULT_TIMEOUT_SEC,
city_hint: str = "",
max_hops: int = 2,
) -> Dict[str, Any]:
sess = requests.Session()
headers = {
"User-Agent": DEFAULT_UA,
"Accept-Language": "en-US,en;q=0.9",
}
visited = set()
current_url = url
last_payload: Dict[str, Any] = {}
for _ in range(max(1, int(max_hops))):
if current_url in visited:
break
visited.add(current_url)
resp = sess.get(current_url, timeout=timeout_sec, headers=headers)
resp.raise_for_status()
html = resp.text
table_html = _extract_table_html(html, "forecastTable")
rows = _parse_table(table_html) if table_html else []
points = _build_timeseries(rows)
last_payload = {
"source": "rp5_html",
"url": url,
"resolved_url": resp.url,
"fetched_at_utc": datetime.now(timezone.utc).isoformat(),
"title": _extract_title(html),
"summary": _extract_summary(html),
"points": points,
}
if points:
return last_payload
links = _extract_weather_links(html)
if not links:
break
ranked = sorted(
links,
key=lambda item: _score_weather_link(item, city_hint),
reverse=True,
)
best = ranked[0]
if _score_weather_link(best, city_hint) < 1:
break
current_url = urljoin(resp.url, best)
return last_payload or {
"source": "rp5_html",
"url": url,
"resolved_url": url,
"fetched_at_utc": datetime.now(timezone.utc).isoformat(),
"title": "",
"summary": "",
"points": [],
}
src/data_collection/weather_sources.py
-218
View File
@@ -7,10 +7,6 @@ import threading
from typing import Optional, Dict, List, Any
from datetime import datetime, timedelta, timezone
from loguru import logger
from src.data_collection.rp5_scraper import (
build_rp5_city_url_candidates,
scrape_rp5_forecast,
)
class WeatherDataCollector:
@@ -74,10 +70,6 @@ class WeatherDataCollector:
self.multi_model_cache_version = str(
os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_VERSION", "v2")
).strip() or "v2"
self.rp5_multi_model_enabled = str(
os.getenv("RP5_MULTI_MODEL_ENABLED", "true")
).strip().lower() in {"1", "true", "yes", "on"}
self.rp5_timeout_sec = max(5, int(os.getenv("RP5_HTTP_TIMEOUT_SEC", "20")))
self._open_meteo_cache: Dict[str, Dict] = {}
self._ensemble_cache: Dict[str, Dict] = {}
self._multi_model_cache: Dict[str, Dict] = {}
@@ -1696,208 +1688,6 @@ class WeatherDataCollector:
return fallback
return None
def _merge_rp5_into_daily_forecasts(
self,
city: str,
dates: List[str],
daily_forecasts: Dict[str, Dict[str, float]],
use_fahrenheit: bool,
) -> Optional[Dict[str, Any]]:
"""
Scrape RP5 public forecast page and merge as an extra model line ("RP5").
"""
if not self.rp5_multi_model_enabled:
return None
city_key = str(city or "").strip().lower()
if not city_key:
return None
city_meta = self.CITY_REGISTRY.get(city_key, {})
city_name = str(city_meta.get("name") or city).strip()
candidates = build_rp5_city_url_candidates(city_name=city_name, city_key=city_key)
if not candidates:
return None
payload: Dict[str, Any] = {}
points: List[Dict[str, Any]] = []
for url in candidates:
try:
probe = scrape_rp5_forecast(
url=url,
timeout_sec=self.rp5_timeout_sec,
city_hint=city_name,
max_hops=3,
)
except Exception as exc:
logger.debug(f"RP5 scrape failed city={city_name} url={url}: {exc}")
continue
p = probe.get("points")
if isinstance(p, list) and p:
payload = probe
points = p
break
if not points:
return None
tz_offset_sec = int(city_meta.get("tz_offset") or 0)
city_tz = timezone(timedelta(seconds=tz_offset_sec))
city_now = datetime.now(city_tz)
month_map = {
"jan": 1,
"january": 1,
"feb": 2,
"february": 2,
"mar": 3,
"march": 3,
"apr": 4,
"april": 4,
"may": 5,
"jun": 6,
"june": 6,
"jul": 7,
"july": 7,
"aug": 8,
"august": 8,
"sep": 9,
"sept": 9,
"september": 9,
"oct": 10,
"october": 10,
"nov": 11,
"november": 11,
"dec": 12,
"december": 12,
}
def _parse_rp5_day_to_date_key(day_label: str) -> Optional[str]:
text = str(day_label or "").strip()
if not text:
return None
low = text.lower()
today_local = city_now.date()
if low.startswith("today"):
return today_local.isoformat()
if low.startswith("tomorrow"):
return (today_local + timedelta(days=1)).isoformat()
match = re.search(r"\b([A-Za-z]+)\s+(\d{1,2})\b", text)
if not match:
return None
month_token = match.group(1).strip().lower().rstrip(".")
day_token = int(match.group(2))
month = month_map.get(month_token)
if month is None:
return None
# RP5 day label omits year; choose the closest year around "now".
candidates: List[datetime] = []
for y in [today_local.year - 1, today_local.year, today_local.year + 1]:
try:
candidates.append(datetime(y, month, day_token, tzinfo=city_tz))
except ValueError:
continue
if not candidates:
return None
chosen = min(
candidates,
key=lambda d: abs((d.date() - today_local).days),
)
return chosen.date().isoformat()
by_date: Dict[str, float] = {}
ordered_date_keys: List[str] = []
by_label: Dict[str, float] = {}
ordered_labels: List[str] = []
for row in points:
if not isinstance(row, dict):
continue
label = str(row.get("day_label") or "").strip()
if not label:
continue
temp_c = row.get("temp_c")
if temp_c is None:
continue
try:
temp_c_val = float(temp_c)
except Exception:
continue
date_key = _parse_rp5_day_to_date_key(label)
if date_key:
if date_key not in by_date:
by_date[date_key] = temp_c_val
ordered_date_keys.append(date_key)
elif temp_c_val > by_date[date_key]:
by_date[date_key] = temp_c_val
# Legacy label aggregation fallback (for non-English day labels etc.).
if label not in by_label:
by_label[label] = temp_c_val
ordered_labels.append(label)
elif temp_c_val > by_label[label]:
by_label[label] = temp_c_val
if not ordered_labels and not ordered_date_keys:
return None
mapped_count = 0
mapped_labels: List[str] = []
mapped_values: List[float] = [by_label[k] for k in ordered_labels]
# Preferred path: map by actual calendar date, avoids day-shift bugs near midnight.
if dates and by_date:
for date_key in dates:
value_c = by_date.get(date_key)
if value_c is None:
continue
value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c
day_bucket = daily_forecasts.setdefault(date_key, {})
day_bucket["RP5"] = round(value, 1)
mapped_count += 1
mapped_labels.append(date_key)
elif dates:
mapped_count = min(len(dates), len(mapped_values))
for idx in range(mapped_count):
date_key = dates[idx]
value_c = mapped_values[idx]
value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c
day_bucket = daily_forecasts.setdefault(date_key, {})
day_bucket["RP5"] = round(value, 1)
mapped_labels.append(ordered_labels[idx])
if not dates and mapped_values:
# Fallback: if Open-Meteo dates missing unexpectedly, create today bucket.
today_key = datetime.now(timezone.utc).strftime("%Y-%m-%d")
value_c = mapped_values[0]
value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c
daily_forecasts.setdefault(today_key, {})["RP5"] = round(value, 1)
mapped_count = 1
mapped_labels.append(today_key)
if dates and mapped_count == 0:
logger.info(
"RP5 no date-overlap city={} rp5_dates={} om_dates={}",
city_name,
ordered_date_keys[:6],
dates[:6],
)
logger.info(
"RP5 merged city={} mapped_days={} url={}",
city_name,
mapped_count if dates else 1,
payload.get("resolved_url") or payload.get("url") or "",
)
return {
"source": "rp5_html",
"url": payload.get("url"),
"resolved_url": payload.get("resolved_url"),
"summary": payload.get("summary"),
"mapped_labels": mapped_labels[:mapped_count] if dates else mapped_labels[:1],
}
def fetch_multi_model(
self,
lat: float,
@@ -1992,13 +1782,6 @@ class WeatherDataCollector:
if day_data:
daily_forecasts[date_str] = day_data
rp5_meta = self._merge_rp5_into_daily_forecasts(
city=city,
dates=dates,
daily_forecasts=daily_forecasts,
use_fahrenheit=use_fahrenheit,
)
if not daily_forecasts:
logger.warning("Multi-model: 无有效模型数据")
return None
@@ -2017,7 +1800,6 @@ class WeatherDataCollector:
"forecasts": forecasts, # 今天 {"ECMWF": 12.3, "GFS": 11.8, ...} (向后兼容)
"daily_forecasts": daily_forecasts, # 按天 {"2026-02-23": {...}, "2026-02-24": {...}}
"dates": dates,
"rp5": rp5_meta or {},
"unit": "fahrenheit" if use_fahrenheit else "celsius",
}
with self._multi_model_cache_lock: