Remove RP5 multi-model data
This commit is contained in:
2569718930@qq.com
@@ -7,10 +7,6 @@ import threading
|
||||
from typing import Optional, Dict, List, Any
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from loguru import logger
|
||||
from src.data_collection.rp5_scraper import (
|
||||
build_rp5_city_url_candidates,
|
||||
scrape_rp5_forecast,
|
||||
)
|
||||
|
||||
|
||||
class WeatherDataCollector:
|
||||
@@ -74,10 +70,6 @@ class WeatherDataCollector:
|
||||
self.multi_model_cache_version = str(
|
||||
os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_VERSION", "v2")
|
||||
).strip() or "v2"
|
||||
self.rp5_multi_model_enabled = str(
|
||||
os.getenv("RP5_MULTI_MODEL_ENABLED", "true")
|
||||
).strip().lower() in {"1", "true", "yes", "on"}
|
||||
self.rp5_timeout_sec = max(5, int(os.getenv("RP5_HTTP_TIMEOUT_SEC", "20")))
|
||||
self._open_meteo_cache: Dict[str, Dict] = {}
|
||||
self._ensemble_cache: Dict[str, Dict] = {}
|
||||
self._multi_model_cache: Dict[str, Dict] = {}
|
||||
@@ -1696,208 +1688,6 @@ class WeatherDataCollector:
|
||||
return fallback
|
||||
return None
|
||||
|
||||
def _merge_rp5_into_daily_forecasts(
|
||||
self,
|
||||
city: str,
|
||||
dates: List[str],
|
||||
daily_forecasts: Dict[str, Dict[str, float]],
|
||||
use_fahrenheit: bool,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Scrape RP5 public forecast page and merge as an extra model line ("RP5").
|
||||
"""
|
||||
if not self.rp5_multi_model_enabled:
|
||||
return None
|
||||
city_key = str(city or "").strip().lower()
|
||||
if not city_key:
|
||||
return None
|
||||
|
||||
city_meta = self.CITY_REGISTRY.get(city_key, {})
|
||||
city_name = str(city_meta.get("name") or city).strip()
|
||||
candidates = build_rp5_city_url_candidates(city_name=city_name, city_key=city_key)
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
payload: Dict[str, Any] = {}
|
||||
points: List[Dict[str, Any]] = []
|
||||
for url in candidates:
|
||||
try:
|
||||
probe = scrape_rp5_forecast(
|
||||
url=url,
|
||||
timeout_sec=self.rp5_timeout_sec,
|
||||
city_hint=city_name,
|
||||
max_hops=3,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(f"RP5 scrape failed city={city_name} url={url}: {exc}")
|
||||
continue
|
||||
p = probe.get("points")
|
||||
if isinstance(p, list) and p:
|
||||
payload = probe
|
||||
points = p
|
||||
break
|
||||
|
||||
if not points:
|
||||
return None
|
||||
|
||||
tz_offset_sec = int(city_meta.get("tz_offset") or 0)
|
||||
city_tz = timezone(timedelta(seconds=tz_offset_sec))
|
||||
city_now = datetime.now(city_tz)
|
||||
|
||||
month_map = {
|
||||
"jan": 1,
|
||||
"january": 1,
|
||||
"feb": 2,
|
||||
"february": 2,
|
||||
"mar": 3,
|
||||
"march": 3,
|
||||
"apr": 4,
|
||||
"april": 4,
|
||||
"may": 5,
|
||||
"jun": 6,
|
||||
"june": 6,
|
||||
"jul": 7,
|
||||
"july": 7,
|
||||
"aug": 8,
|
||||
"august": 8,
|
||||
"sep": 9,
|
||||
"sept": 9,
|
||||
"september": 9,
|
||||
"oct": 10,
|
||||
"october": 10,
|
||||
"nov": 11,
|
||||
"november": 11,
|
||||
"dec": 12,
|
||||
"december": 12,
|
||||
}
|
||||
|
||||
def _parse_rp5_day_to_date_key(day_label: str) -> Optional[str]:
|
||||
text = str(day_label or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
low = text.lower()
|
||||
today_local = city_now.date()
|
||||
if low.startswith("today"):
|
||||
return today_local.isoformat()
|
||||
if low.startswith("tomorrow"):
|
||||
return (today_local + timedelta(days=1)).isoformat()
|
||||
|
||||
match = re.search(r"\b([A-Za-z]+)\s+(\d{1,2})\b", text)
|
||||
if not match:
|
||||
return None
|
||||
month_token = match.group(1).strip().lower().rstrip(".")
|
||||
day_token = int(match.group(2))
|
||||
month = month_map.get(month_token)
|
||||
if month is None:
|
||||
return None
|
||||
|
||||
# RP5 day label omits year; choose the closest year around "now".
|
||||
candidates: List[datetime] = []
|
||||
for y in [today_local.year - 1, today_local.year, today_local.year + 1]:
|
||||
try:
|
||||
candidates.append(datetime(y, month, day_token, tzinfo=city_tz))
|
||||
except ValueError:
|
||||
continue
|
||||
if not candidates:
|
||||
return None
|
||||
chosen = min(
|
||||
candidates,
|
||||
key=lambda d: abs((d.date() - today_local).days),
|
||||
)
|
||||
return chosen.date().isoformat()
|
||||
|
||||
by_date: Dict[str, float] = {}
|
||||
ordered_date_keys: List[str] = []
|
||||
by_label: Dict[str, float] = {}
|
||||
ordered_labels: List[str] = []
|
||||
for row in points:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
label = str(row.get("day_label") or "").strip()
|
||||
if not label:
|
||||
continue
|
||||
temp_c = row.get("temp_c")
|
||||
if temp_c is None:
|
||||
continue
|
||||
try:
|
||||
temp_c_val = float(temp_c)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
date_key = _parse_rp5_day_to_date_key(label)
|
||||
if date_key:
|
||||
if date_key not in by_date:
|
||||
by_date[date_key] = temp_c_val
|
||||
ordered_date_keys.append(date_key)
|
||||
elif temp_c_val > by_date[date_key]:
|
||||
by_date[date_key] = temp_c_val
|
||||
|
||||
# Legacy label aggregation fallback (for non-English day labels etc.).
|
||||
if label not in by_label:
|
||||
by_label[label] = temp_c_val
|
||||
ordered_labels.append(label)
|
||||
elif temp_c_val > by_label[label]:
|
||||
by_label[label] = temp_c_val
|
||||
|
||||
if not ordered_labels and not ordered_date_keys:
|
||||
return None
|
||||
|
||||
mapped_count = 0
|
||||
mapped_labels: List[str] = []
|
||||
mapped_values: List[float] = [by_label[k] for k in ordered_labels]
|
||||
|
||||
# Preferred path: map by actual calendar date, avoids day-shift bugs near midnight.
|
||||
if dates and by_date:
|
||||
for date_key in dates:
|
||||
value_c = by_date.get(date_key)
|
||||
if value_c is None:
|
||||
continue
|
||||
value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c
|
||||
day_bucket = daily_forecasts.setdefault(date_key, {})
|
||||
day_bucket["RP5"] = round(value, 1)
|
||||
mapped_count += 1
|
||||
mapped_labels.append(date_key)
|
||||
elif dates:
|
||||
mapped_count = min(len(dates), len(mapped_values))
|
||||
for idx in range(mapped_count):
|
||||
date_key = dates[idx]
|
||||
value_c = mapped_values[idx]
|
||||
value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c
|
||||
day_bucket = daily_forecasts.setdefault(date_key, {})
|
||||
day_bucket["RP5"] = round(value, 1)
|
||||
mapped_labels.append(ordered_labels[idx])
|
||||
|
||||
if not dates and mapped_values:
|
||||
# Fallback: if Open-Meteo dates missing unexpectedly, create today bucket.
|
||||
today_key = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
value_c = mapped_values[0]
|
||||
value = value_c * 9 / 5 + 32 if use_fahrenheit else value_c
|
||||
daily_forecasts.setdefault(today_key, {})["RP5"] = round(value, 1)
|
||||
mapped_count = 1
|
||||
mapped_labels.append(today_key)
|
||||
|
||||
if dates and mapped_count == 0:
|
||||
logger.info(
|
||||
"RP5 no date-overlap city={} rp5_dates={} om_dates={}",
|
||||
city_name,
|
||||
ordered_date_keys[:6],
|
||||
dates[:6],
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"RP5 merged city={} mapped_days={} url={}",
|
||||
city_name,
|
||||
mapped_count if dates else 1,
|
||||
payload.get("resolved_url") or payload.get("url") or "",
|
||||
)
|
||||
return {
|
||||
"source": "rp5_html",
|
||||
"url": payload.get("url"),
|
||||
"resolved_url": payload.get("resolved_url"),
|
||||
"summary": payload.get("summary"),
|
||||
"mapped_labels": mapped_labels[:mapped_count] if dates else mapped_labels[:1],
|
||||
}
|
||||
|
||||
def fetch_multi_model(
|
||||
self,
|
||||
lat: float,
|
||||
@@ -1992,13 +1782,6 @@ class WeatherDataCollector:
|
||||
if day_data:
|
||||
daily_forecasts[date_str] = day_data
|
||||
|
||||
rp5_meta = self._merge_rp5_into_daily_forecasts(
|
||||
city=city,
|
||||
dates=dates,
|
||||
daily_forecasts=daily_forecasts,
|
||||
use_fahrenheit=use_fahrenheit,
|
||||
)
|
||||
|
||||
if not daily_forecasts:
|
||||
logger.warning("Multi-model: 无有效模型数据")
|
||||
return None
|
||||
@@ -2017,7 +1800,6 @@ class WeatherDataCollector:
|
||||
"forecasts": forecasts, # 今天 {"ECMWF": 12.3, "GFS": 11.8, ...} (向后兼容)
|
||||
"daily_forecasts": daily_forecasts, # 按天 {"2026-02-23": {...}, "2026-02-24": {...}}
|
||||
"dates": dates,
|
||||
"rp5": rp5_meta or {},
|
||||
"unit": "fahrenheit" if use_fahrenheit else "celsius",
|
||||
}
|
||||
with self._multi_model_cache_lock:
|
||||
|
||||
Reference in New Issue
Block a user