Fix stale multi-model DEB inputs
This commit is contained in:
@@ -20,6 +20,7 @@ from src.analysis.deb_hourly_consensus import build_deb_hourly_consensus_path
|
|||||||
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
|
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
|
||||||
from src.data_collection.city_registry import CITY_REGISTRY
|
from src.data_collection.city_registry import CITY_REGISTRY
|
||||||
from src.data_collection.city_risk_profiles import get_city_risk_profile
|
from src.data_collection.city_risk_profiles import get_city_risk_profile
|
||||||
|
from src.data_collection.multi_model_freshness import multi_model_forecasts_for_local_date
|
||||||
|
|
||||||
SETTLEMENT_SOURCE_LABELS = {
|
SETTLEMENT_SOURCE_LABELS = {
|
||||||
"metar": "METAR",
|
"metar": "METAR",
|
||||||
@@ -464,11 +465,6 @@ def analyze_weather_trend(
|
|||||||
if weather_data.get("cwa_forecast") is not None:
|
if weather_data.get("cwa_forecast") is not None:
|
||||||
current_forecasts["CWA(台气象)"] = _sf(weather_data.get("cwa_forecast"))
|
current_forecasts["CWA(台气象)"] = _sf(weather_data.get("cwa_forecast"))
|
||||||
|
|
||||||
mm_forecasts = weather_data.get("multi_model", {}).get("forecasts", {})
|
|
||||||
for m_name, m_val in mm_forecasts.items():
|
|
||||||
if m_val is not None and not _is_excluded_model_name(m_name):
|
|
||||||
current_forecasts[m_name] = _sf(m_val)
|
|
||||||
|
|
||||||
forecast_highs = [h for h in current_forecasts.values() if h is not None]
|
forecast_highs = [h for h in current_forecasts.values() if h is not None]
|
||||||
forecast_high = max(forecast_highs) if forecast_highs else None
|
forecast_high = max(forecast_highs) if forecast_highs else None
|
||||||
forecast_median = (
|
forecast_median = (
|
||||||
@@ -545,11 +541,18 @@ def analyze_weather_trend(
|
|||||||
current_forecasts["Open-Meteo"] = local_day_high
|
current_forecasts["Open-Meteo"] = local_day_high
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
forecast_highs = [h for h in current_forecasts.values() if h is not None]
|
mm_forecasts = multi_model_forecasts_for_local_date(
|
||||||
forecast_high = max(forecast_highs) if forecast_highs else None
|
weather_data.get("multi_model", {}),
|
||||||
forecast_median = (
|
local_date_str,
|
||||||
sorted(forecast_highs)[len(forecast_highs) // 2] if forecast_highs else None
|
)
|
||||||
)
|
for m_name, m_val in mm_forecasts.items():
|
||||||
|
if m_val is not None and not _is_excluded_model_name(m_name):
|
||||||
|
current_forecasts[m_name] = _sf(m_val)
|
||||||
|
forecast_highs = [h for h in current_forecasts.values() if h is not None]
|
||||||
|
forecast_high = max(forecast_highs) if forecast_highs else None
|
||||||
|
forecast_median = (
|
||||||
|
sorted(forecast_highs)[len(forecast_highs) // 2] if forecast_highs else None
|
||||||
|
)
|
||||||
|
|
||||||
# === DEB ===
|
# === DEB ===
|
||||||
deb_prediction = None
|
deb_prediction = None
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from src.data_collection.multi_model_freshness import multi_model_has_current_window
|
||||||
|
|
||||||
|
|
||||||
def _open_meteo_cache_key(
|
def _open_meteo_cache_key(
|
||||||
lat: float,
|
lat: float,
|
||||||
@@ -58,8 +60,9 @@ def _read_open_meteo_bundle_from_cache(
|
|||||||
)
|
)
|
||||||
with collector._multi_model_cache_lock:
|
with collector._multi_model_cache_lock:
|
||||||
mm_cached = collector._multi_model_cache.get(mm_key)
|
mm_cached = collector._multi_model_cache.get(mm_key)
|
||||||
if mm_cached and isinstance(mm_cached.get("data"), dict):
|
mm_data = mm_cached.get("data") if isinstance(mm_cached, dict) else None
|
||||||
results["multi_model"] = dict(mm_cached["data"])
|
if isinstance(mm_data, dict) and multi_model_has_current_window(mm_data):
|
||||||
|
results["multi_model"] = dict(mm_data)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import date, datetime, timezone
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: Any) -> Optional[date]:
|
||||||
|
text = str(value or "").strip()
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(text[:10]).date()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _numeric(value: Any) -> Optional[float]:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(value)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _date_strings_from_hourly(multi_model: Dict[str, Any]) -> set[str]:
|
||||||
|
dates = set()
|
||||||
|
for raw_time in multi_model.get("hourly_times") or []:
|
||||||
|
text = str(raw_time or "")
|
||||||
|
if len(text) >= 10:
|
||||||
|
dates.add(text[:10])
|
||||||
|
return dates
|
||||||
|
|
||||||
|
|
||||||
|
def _has_numeric_hourly_for_date(multi_model: Dict[str, Any], local_date: str) -> bool:
|
||||||
|
times = multi_model.get("hourly_times") or []
|
||||||
|
forecasts = multi_model.get("hourly_forecasts") or {}
|
||||||
|
if not isinstance(forecasts, dict):
|
||||||
|
return False
|
||||||
|
for idx, raw_time in enumerate(times):
|
||||||
|
if not str(raw_time or "").startswith(local_date):
|
||||||
|
continue
|
||||||
|
for values in forecasts.values():
|
||||||
|
if isinstance(values, list) and idx < len(values) and _numeric(values[idx]) is not None:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def multi_model_has_current_window(multi_model: Any, *, today: Optional[date] = None) -> bool:
|
||||||
|
"""Return False when cached multi-model data is definitely older than today."""
|
||||||
|
if not isinstance(multi_model, dict):
|
||||||
|
return False
|
||||||
|
today = today or datetime.now(timezone.utc).date()
|
||||||
|
raw_date_values = []
|
||||||
|
raw_date_values.extend(multi_model.get("dates") or [])
|
||||||
|
daily = multi_model.get("daily_forecasts")
|
||||||
|
if isinstance(daily, dict):
|
||||||
|
raw_date_values.extend(daily.keys())
|
||||||
|
raw_date_values.extend(_date_strings_from_hourly(multi_model))
|
||||||
|
parsed_dates = [parsed for raw in raw_date_values for parsed in [_parse_date(raw)] if parsed]
|
||||||
|
if not parsed_dates:
|
||||||
|
return True
|
||||||
|
return max(parsed_dates) >= today
|
||||||
|
|
||||||
|
|
||||||
|
def multi_model_covers_local_date(multi_model: Any, local_date: str) -> bool:
|
||||||
|
if not isinstance(multi_model, dict):
|
||||||
|
return False
|
||||||
|
wanted = str(local_date or "").strip()
|
||||||
|
if not wanted:
|
||||||
|
return bool(multi_model.get("forecasts"))
|
||||||
|
|
||||||
|
daily = multi_model.get("daily_forecasts") if isinstance(multi_model.get("daily_forecasts"), dict) else {}
|
||||||
|
day_models = daily.get(wanted) if isinstance(daily.get(wanted), dict) else {}
|
||||||
|
if any(_numeric(value) is not None for value in day_models.values()):
|
||||||
|
return True
|
||||||
|
|
||||||
|
if _has_numeric_hourly_for_date(multi_model, wanted):
|
||||||
|
return True
|
||||||
|
|
||||||
|
dates = [str(value) for value in (multi_model.get("dates") or [])]
|
||||||
|
if dates:
|
||||||
|
return wanted in dates
|
||||||
|
|
||||||
|
has_dated_payload = bool(daily) or bool(multi_model.get("hourly_times"))
|
||||||
|
return bool(multi_model.get("forecasts")) and not has_dated_payload
|
||||||
|
|
||||||
|
|
||||||
|
def multi_model_forecasts_for_local_date(multi_model: Any, local_date: str) -> Dict[str, float]:
|
||||||
|
if not isinstance(multi_model, dict) or not multi_model_covers_local_date(multi_model, local_date):
|
||||||
|
return {}
|
||||||
|
wanted = str(local_date or "").strip()
|
||||||
|
models: Dict[str, float] = {}
|
||||||
|
|
||||||
|
daily = multi_model.get("daily_forecasts") if isinstance(multi_model.get("daily_forecasts"), dict) else {}
|
||||||
|
day_models = daily.get(wanted) if isinstance(daily.get(wanted), dict) else {}
|
||||||
|
for model, value in day_models.items():
|
||||||
|
parsed = _numeric(value)
|
||||||
|
if parsed is not None:
|
||||||
|
models[str(model)] = parsed
|
||||||
|
|
||||||
|
times = multi_model.get("hourly_times") or []
|
||||||
|
hourly = multi_model.get("hourly_forecasts") if isinstance(multi_model.get("hourly_forecasts"), dict) else {}
|
||||||
|
day_indexes = [
|
||||||
|
idx
|
||||||
|
for idx, raw_time in enumerate(times)
|
||||||
|
if wanted and str(raw_time or "").startswith(wanted)
|
||||||
|
]
|
||||||
|
for model, values in hourly.items():
|
||||||
|
if not isinstance(values, list):
|
||||||
|
continue
|
||||||
|
day_values = [
|
||||||
|
parsed
|
||||||
|
for idx in day_indexes
|
||||||
|
if idx < len(values)
|
||||||
|
for parsed in [_numeric(values[idx])]
|
||||||
|
if parsed is not None
|
||||||
|
]
|
||||||
|
if day_values:
|
||||||
|
current = models.get(str(model))
|
||||||
|
models[str(model)] = max(day_values) if current is None else max(current, max(day_values))
|
||||||
|
|
||||||
|
if models:
|
||||||
|
return models
|
||||||
|
|
||||||
|
forecasts = multi_model.get("forecasts") if isinstance(multi_model.get("forecasts"), dict) else {}
|
||||||
|
for model, value in forecasts.items():
|
||||||
|
parsed = _numeric(value)
|
||||||
|
if parsed is not None:
|
||||||
|
models[str(model)] = parsed
|
||||||
|
return models
|
||||||
@@ -6,6 +6,7 @@ from typing import Any, Dict, Optional
|
|||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
|
from src.data_collection.multi_model_freshness import multi_model_has_current_window
|
||||||
from src.utils.metrics import record_source_call
|
from src.utils.metrics import record_source_call
|
||||||
|
|
||||||
|
|
||||||
@@ -868,15 +869,17 @@ class NwsOpenMeteoSourceMixin:
|
|||||||
logger.debug(f"Open-Meteo Multi-model 冷却期中,跳过请求,还需 {remaining}s")
|
logger.debug(f"Open-Meteo Multi-model 冷却期中,跳过请求,还需 {remaining}s")
|
||||||
with self._multi_model_cache_lock:
|
with self._multi_model_cache_lock:
|
||||||
stale = self._multi_model_cache.get(cache_key)
|
stale = self._multi_model_cache.get(cache_key)
|
||||||
if stale and isinstance(stale.get("data"), dict):
|
stale_data = stale.get("data") if isinstance(stale, dict) else None
|
||||||
|
if isinstance(stale_data, dict) and multi_model_has_current_window(stale_data):
|
||||||
record_source_call("open_meteo", "multi_model", "stale_cache", (time.perf_counter() - started) * 1000.0)
|
record_source_call("open_meteo", "multi_model", "stale_cache", (time.perf_counter() - started) * 1000.0)
|
||||||
return dict(stale["data"])
|
return dict(stale_data)
|
||||||
self._load_open_meteo_disk_cache()
|
self._load_open_meteo_disk_cache()
|
||||||
with self._multi_model_cache_lock:
|
with self._multi_model_cache_lock:
|
||||||
stale2 = self._multi_model_cache.get(cache_key)
|
stale2 = self._multi_model_cache.get(cache_key)
|
||||||
if stale2 and isinstance(stale2.get("data"), dict):
|
stale2_data = stale2.get("data") if isinstance(stale2, dict) else None
|
||||||
|
if isinstance(stale2_data, dict) and multi_model_has_current_window(stale2_data):
|
||||||
record_source_call("open_meteo", "multi_model", "disk_fallback", (time.perf_counter() - started) * 1000.0)
|
record_source_call("open_meteo", "multi_model", "disk_fallback", (time.perf_counter() - started) * 1000.0)
|
||||||
return dict(stale2["data"])
|
return dict(stale2_data)
|
||||||
record_source_call("open_meteo", "multi_model", "cooldown_skip", (time.perf_counter() - started) * 1000.0)
|
record_source_call("open_meteo", "multi_model", "cooldown_skip", (time.perf_counter() - started) * 1000.0)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -888,6 +891,11 @@ class NwsOpenMeteoSourceMixin:
|
|||||||
< self.open_meteo_multi_model_cache_ttl_sec
|
< self.open_meteo_multi_model_cache_ttl_sec
|
||||||
):
|
):
|
||||||
cached_data = cached.get("data")
|
cached_data = cached.get("data")
|
||||||
|
if isinstance(cached_data, dict):
|
||||||
|
if not multi_model_has_current_window(cached_data):
|
||||||
|
self._multi_model_cache.pop(cache_key, None)
|
||||||
|
record_source_call("open_meteo", "multi_model", "expired_cache_skip", (time.perf_counter() - started) * 1000.0)
|
||||||
|
cached_data = None
|
||||||
if isinstance(cached_data, dict):
|
if isinstance(cached_data, dict):
|
||||||
record_source_call("open_meteo", "multi_model", "cache_hit", (time.perf_counter() - started) * 1000.0)
|
record_source_call("open_meteo", "multi_model", "cache_hit", (time.perf_counter() - started) * 1000.0)
|
||||||
return dict(cached_data)
|
return dict(cached_data)
|
||||||
@@ -996,8 +1004,9 @@ class NwsOpenMeteoSourceMixin:
|
|||||||
logger.warning(f"Multi-model API 请求失败: {e}")
|
logger.warning(f"Multi-model API 请求失败: {e}")
|
||||||
with self._multi_model_cache_lock:
|
with self._multi_model_cache_lock:
|
||||||
stale = self._multi_model_cache.get(cache_key)
|
stale = self._multi_model_cache.get(cache_key)
|
||||||
if stale and isinstance(stale.get("data"), dict):
|
stale_data = stale.get("data") if isinstance(stale, dict) else None
|
||||||
fallback = dict(stale["data"])
|
if isinstance(stale_data, dict) and multi_model_has_current_window(stale_data):
|
||||||
|
fallback = dict(stale_data)
|
||||||
fallback["stale_cache"] = True
|
fallback["stale_cache"] = True
|
||||||
record_source_call("open_meteo", "multi_model", "stale_cache", (time.perf_counter() - started) * 1000.0)
|
record_source_call("open_meteo", "multi_model", "stale_cache", (time.perf_counter() - started) * 1000.0)
|
||||||
return fallback
|
return fallback
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
import time
|
||||||
|
|
||||||
from src.data_collection.nws_open_meteo_sources import (
|
from src.data_collection.nws_open_meteo_sources import (
|
||||||
OPEN_METEO_MULTI_MODEL_ORDER,
|
OPEN_METEO_MULTI_MODEL_ORDER,
|
||||||
_parse_open_meteo_multi_model_daily,
|
_parse_open_meteo_multi_model_daily,
|
||||||
@@ -268,6 +271,8 @@ def test_fetch_all_sources_delegates_non_hf_forecast_bundle(monkeypatch, tmp_pat
|
|||||||
def test_open_meteo_cache_only_reads_multi_model_without_forecast_cache():
|
def test_open_meteo_cache_only_reads_multi_model_without_forecast_cache():
|
||||||
from src.data_collection.forecast_source_bundle import fetch_open_meteo_forecast_bundle
|
from src.data_collection.forecast_source_bundle import fetch_open_meteo_forecast_bundle
|
||||||
|
|
||||||
|
today = datetime.now(timezone.utc).date().isoformat()
|
||||||
|
|
||||||
class DummyLock:
|
class DummyLock:
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
return None
|
return None
|
||||||
@@ -283,7 +288,7 @@ def test_open_meteo_cache_only_reads_multi_model_without_forecast_cache():
|
|||||||
_multi_model_cache = {
|
_multi_model_cache = {
|
||||||
"48.9694:2.4414:paris:c:v5": {
|
"48.9694:2.4414:paris:c:v5": {
|
||||||
"data": {
|
"data": {
|
||||||
"hourly_times": ["2026-06-16T15:00"],
|
"hourly_times": [f"{today}T15:00"],
|
||||||
"hourly_forecasts": {"ECMWF": [24.5]},
|
"hourly_forecasts": {"ECMWF": [24.5]},
|
||||||
"forecasts": {"ECMWF": 27.0},
|
"forecasts": {"ECMWF": 27.0},
|
||||||
}
|
}
|
||||||
@@ -450,6 +455,64 @@ def test_persisted_open_meteo_cooldown_skips_outbound_request(monkeypatch, tmp_p
|
|||||||
assert result is not None # cooldown returns cached data
|
assert result is not None # cooldown returns cached data
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_multi_model_ignores_cache_when_dates_are_stale(monkeypatch, tmp_path):
|
||||||
|
monkeypatch.setenv("OPEN_METEO_DISK_CACHE_PATH", str(tmp_path / "om-cache.json"))
|
||||||
|
collector = WeatherDataCollector({})
|
||||||
|
today = datetime.now(timezone.utc).date()
|
||||||
|
old_dates = [
|
||||||
|
(today - timedelta(days=11)).isoformat(),
|
||||||
|
(today - timedelta(days=10)).isoformat(),
|
||||||
|
]
|
||||||
|
fresh_dates = [today.isoformat(), (today + timedelta(days=1)).isoformat()]
|
||||||
|
cache_key = (
|
||||||
|
f"{round(float(48.9694), 4)}:{round(float(2.4414), 4)}:paris:"
|
||||||
|
f"c:{collector.multi_model_cache_version}"
|
||||||
|
)
|
||||||
|
collector._multi_model_cache[cache_key] = {
|
||||||
|
"t": time.time(),
|
||||||
|
"data": {
|
||||||
|
"dates": old_dates,
|
||||||
|
"daily_forecasts": {old_dates[0]: {"ECMWF": 24.0}},
|
||||||
|
"forecasts": {"ECMWF": 24.0},
|
||||||
|
"hourly_times": [f"{old_dates[0]}T15:00"],
|
||||||
|
"hourly_forecasts": {"ECMWF": [23.0]},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
class FakeResponse:
|
||||||
|
def raise_for_status(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return {
|
||||||
|
"daily": {
|
||||||
|
"time": fresh_dates,
|
||||||
|
"temperature_2m_max_ecmwf_ifs025": [39.2, 33.0],
|
||||||
|
"temperature_2m_max_gfs_seamless": [38.6, 32.5],
|
||||||
|
},
|
||||||
|
"hourly": {
|
||||||
|
"time": [f"{fresh_dates[0]}T15:00", f"{fresh_dates[0]}T16:00"],
|
||||||
|
"temperature_2m_ecmwf_ifs025": [38.8, 39.2],
|
||||||
|
"temperature_2m_gfs_seamless": [38.0, 38.6],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "_wait_open_meteo_slot", lambda *_args, **_kwargs: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
collector,
|
||||||
|
"_http_get",
|
||||||
|
lambda *args, **kwargs: calls.append((args, kwargs)) or FakeResponse(),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = collector.fetch_multi_model(48.9694, 2.4414, city="paris")
|
||||||
|
|
||||||
|
assert len(calls) == 1
|
||||||
|
assert result["dates"][:2] == fresh_dates
|
||||||
|
assert result["forecasts"]["ECMWF"] == 39.2
|
||||||
|
assert result["forecasts"]["GFS"] == 38.6
|
||||||
|
|
||||||
|
|
||||||
def test_multi_model_hourly_parser():
|
def test_multi_model_hourly_parser():
|
||||||
from src.data_collection.nws_open_meteo_sources import _parse_open_meteo_multi_model_hourly
|
from src.data_collection.nws_open_meteo_sources import _parse_open_meteo_multi_model_hourly
|
||||||
|
|
||||||
|
|||||||
@@ -1401,6 +1401,8 @@ def test_chart_data_cache_hit_starts_full_stale_refresh(monkeypatch):
|
|||||||
def test_chart_data_cache_hit_overlays_cached_multi_model_hourly(monkeypatch):
|
def test_chart_data_cache_hit_overlays_cached_multi_model_hourly(monkeypatch):
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
|
local_date = datetime.now(timezone.utc).date().isoformat()
|
||||||
|
|
||||||
class FakeCache:
|
class FakeCache:
|
||||||
def get_city_cache(self, kind, city):
|
def get_city_cache(self, kind, city):
|
||||||
assert kind == "full"
|
assert kind == "full"
|
||||||
@@ -1408,7 +1410,7 @@ def test_chart_data_cache_hit_overlays_cached_multi_model_hourly(monkeypatch):
|
|||||||
"payload": {
|
"payload": {
|
||||||
"name": city,
|
"name": city,
|
||||||
"display_name": city.title(),
|
"display_name": city.title(),
|
||||||
"local_date": "2026-06-16",
|
"local_date": local_date,
|
||||||
"local_time": "15:20",
|
"local_time": "15:20",
|
||||||
"temp_symbol": "°C",
|
"temp_symbol": "°C",
|
||||||
"current": {"temp": 20.0},
|
"current": {"temp": 20.0},
|
||||||
@@ -1433,7 +1435,7 @@ def test_chart_data_cache_hit_overlays_cached_multi_model_hourly(monkeypatch):
|
|||||||
monkeypatch.setattr(collector, "_multi_model_cache", {
|
monkeypatch.setattr(collector, "_multi_model_cache", {
|
||||||
"48.9694:2.4414:paris:c:v5": {
|
"48.9694:2.4414:paris:c:v5": {
|
||||||
"data": {
|
"data": {
|
||||||
"hourly_times": ["2026-06-16T15:00"],
|
"hourly_times": [f"{local_date}T15:00"],
|
||||||
"hourly_forecasts": {"ECMWF": [24.5]},
|
"hourly_forecasts": {"ECMWF": [24.5]},
|
||||||
"forecasts": {"ECMWF": 27.0},
|
"forecasts": {"ECMWF": 27.0},
|
||||||
}
|
}
|
||||||
@@ -1458,6 +1460,9 @@ def test_chart_data_cache_hit_overlays_cached_multi_model_hourly(monkeypatch):
|
|||||||
def test_chart_data_cache_hit_replaces_stale_multi_model_hourly(monkeypatch):
|
def test_chart_data_cache_hit_replaces_stale_multi_model_hourly(monkeypatch):
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
|
local_date = datetime.now(timezone.utc).date().isoformat()
|
||||||
|
stale_date = (datetime.now(timezone.utc).date() - timedelta(days=3)).isoformat()
|
||||||
|
|
||||||
class FakeCache:
|
class FakeCache:
|
||||||
def get_city_cache(self, kind, city):
|
def get_city_cache(self, kind, city):
|
||||||
assert kind == "full"
|
assert kind == "full"
|
||||||
@@ -1465,13 +1470,13 @@ def test_chart_data_cache_hit_replaces_stale_multi_model_hourly(monkeypatch):
|
|||||||
"payload": {
|
"payload": {
|
||||||
"name": city,
|
"name": city,
|
||||||
"display_name": city.title(),
|
"display_name": city.title(),
|
||||||
"local_date": "2026-06-17",
|
"local_date": local_date,
|
||||||
"local_time": "15:20",
|
"local_time": "15:20",
|
||||||
"temp_symbol": "°C",
|
"temp_symbol": "°C",
|
||||||
"current": {"temp": 20.0},
|
"current": {"temp": 20.0},
|
||||||
"hourly": {"times": ["15:00"], "temps": [20.0]},
|
"hourly": {"times": ["15:00"], "temps": [20.0]},
|
||||||
"multi_model": {
|
"multi_model": {
|
||||||
"hourly_times": ["2026-06-14T15:00", "2026-06-16T23:00"],
|
"hourly_times": [f"{stale_date}T15:00", f"{stale_date}T23:00"],
|
||||||
"hourly_forecasts": {"ECMWF": [21.0, 22.0]},
|
"hourly_forecasts": {"ECMWF": [21.0, 22.0]},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -1493,7 +1498,7 @@ def test_chart_data_cache_hit_replaces_stale_multi_model_hourly(monkeypatch):
|
|||||||
monkeypatch.setattr(collector, "_multi_model_cache", {
|
monkeypatch.setattr(collector, "_multi_model_cache", {
|
||||||
"48.9694:2.4414:paris:c:v5": {
|
"48.9694:2.4414:paris:c:v5": {
|
||||||
"data": {
|
"data": {
|
||||||
"hourly_times": ["2026-06-17T15:00", "2026-06-17T16:00"],
|
"hourly_times": [f"{local_date}T15:00", f"{local_date}T16:00"],
|
||||||
"hourly_forecasts": {"ECMWF": [24.5, 25.0]},
|
"hourly_forecasts": {"ECMWF": [24.5, 25.0]},
|
||||||
"forecasts": {"ECMWF": 27.0},
|
"forecasts": {"ECMWF": 27.0},
|
||||||
}
|
}
|
||||||
@@ -1512,10 +1517,29 @@ def test_chart_data_cache_hit_replaces_stale_multi_model_hourly(monkeypatch):
|
|||||||
|
|
||||||
payload = asyncio.run(city_api._get_city_chart_data("paris", force_refresh=False))
|
payload = asyncio.run(city_api._get_city_chart_data("paris", force_refresh=False))
|
||||||
|
|
||||||
assert payload["multi_model"]["hourly_times"] == ["2026-06-17T15:00", "2026-06-17T16:00"]
|
assert payload["multi_model"]["hourly_times"] == [f"{local_date}T15:00", f"{local_date}T16:00"]
|
||||||
assert payload["multi_model"]["hourly_forecasts"]["ECMWF"] == [24.5, 25.0]
|
assert payload["multi_model"]["hourly_forecasts"]["ECMWF"] == [24.5, 25.0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_multi_model_daily_models_for_date_rejects_stale_dated_forecasts():
|
||||||
|
local_date = datetime.now(timezone.utc).date().isoformat()
|
||||||
|
stale_date = (datetime.now(timezone.utc).date() - timedelta(days=7)).isoformat()
|
||||||
|
|
||||||
|
models = city_api._multi_model_daily_models_for_date(
|
||||||
|
{
|
||||||
|
"daily_forecasts": {
|
||||||
|
stale_date: {"ECMWF": 24.0},
|
||||||
|
},
|
||||||
|
"hourly_times": [f"{stale_date}T15:00"],
|
||||||
|
"hourly_forecasts": {"ECMWF": [24.0]},
|
||||||
|
"forecasts": {"ECMWF": 24.0},
|
||||||
|
},
|
||||||
|
local_date,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert models == {}
|
||||||
|
|
||||||
|
|
||||||
def test_chart_data_cache_hit_refreshes_when_multi_model_cache_is_stale(monkeypatch):
|
def test_chart_data_cache_hit_refreshes_when_multi_model_cache_is_stale(monkeypatch):
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
@@ -1592,6 +1616,8 @@ def test_chart_data_cache_hit_refreshes_when_multi_model_cache_is_stale(monkeypa
|
|||||||
|
|
||||||
def test_chart_data_floors_stale_forecast_and_deb_with_observed_high(monkeypatch):
|
def test_chart_data_floors_stale_forecast_and_deb_with_observed_high(monkeypatch):
|
||||||
import asyncio
|
import asyncio
|
||||||
|
local_date = datetime.now(timezone.utc).date().isoformat()
|
||||||
|
stale_date = (datetime.now(timezone.utc).date() - timedelta(days=7)).isoformat()
|
||||||
|
|
||||||
class FakeCache:
|
class FakeCache:
|
||||||
def get_city_cache(self, kind, city):
|
def get_city_cache(self, kind, city):
|
||||||
@@ -1600,7 +1626,7 @@ def test_chart_data_floors_stale_forecast_and_deb_with_observed_high(monkeypatch
|
|||||||
"payload": {
|
"payload": {
|
||||||
"name": city,
|
"name": city,
|
||||||
"display_name": "Lucknow",
|
"display_name": "Lucknow",
|
||||||
"local_date": "2026-06-21",
|
"local_date": local_date,
|
||||||
"local_time": "13:30",
|
"local_time": "13:30",
|
||||||
"temp_symbol": "°C",
|
"temp_symbol": "°C",
|
||||||
"current": {"temp": 38.0, "max_so_far": 38.0},
|
"current": {"temp": 38.0, "max_so_far": 38.0},
|
||||||
@@ -1608,11 +1634,18 @@ def test_chart_data_floors_stale_forecast_and_deb_with_observed_high(monkeypatch
|
|||||||
"airport_primary": {"temp": 38.0, "max_so_far": 38.0},
|
"airport_primary": {"temp": 38.0, "max_so_far": 38.0},
|
||||||
"forecast": {
|
"forecast": {
|
||||||
"today_high": 36.2,
|
"today_high": 36.2,
|
||||||
"daily": [{"date": "2026-06-14", "max_temp": 36.2}],
|
"daily": [{"date": stale_date, "max_temp": 36.2}],
|
||||||
|
},
|
||||||
|
"deb": {
|
||||||
|
"prediction": 36.0,
|
||||||
|
"raw_prediction": 36.0,
|
||||||
|
"hourly_path": {
|
||||||
|
"times": ["13:00", "14:00"],
|
||||||
|
"temps": [36.0, 37.0],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"deb": {"prediction": 36.0, "raw_prediction": 36.0},
|
|
||||||
"multi_model_daily": {
|
"multi_model_daily": {
|
||||||
"2026-06-14": {"models": {"Open-Meteo": 36.2}},
|
stale_date: {"models": {"Open-Meteo": 36.2}},
|
||||||
},
|
},
|
||||||
"multi_model": {},
|
"multi_model": {},
|
||||||
"hourly": {"times": ["13:00"], "temps": [36.0]},
|
"hourly": {"times": ["13:00"], "temps": [36.0]},
|
||||||
@@ -1649,16 +1682,16 @@ def test_chart_data_floors_stale_forecast_and_deb_with_observed_high(monkeypatch
|
|||||||
cache_key: {
|
cache_key: {
|
||||||
"data": {
|
"data": {
|
||||||
"hourly_times": [
|
"hourly_times": [
|
||||||
"2026-06-21T13:00",
|
f"{local_date}T13:00",
|
||||||
"2026-06-21T14:00",
|
f"{local_date}T14:00",
|
||||||
"2026-06-21T15:00",
|
f"{local_date}T15:00",
|
||||||
],
|
],
|
||||||
"hourly_forecasts": {
|
"hourly_forecasts": {
|
||||||
"ECMWF": [38.7, 39.0, 38.0],
|
"ECMWF": [38.7, 39.0, 38.0],
|
||||||
"GFS": [42.0, 44.1, 43.0],
|
"GFS": [42.0, 44.1, 43.0],
|
||||||
},
|
},
|
||||||
"daily_forecasts": {
|
"daily_forecasts": {
|
||||||
"2026-06-21": {"ECMWF": 39.0, "GFS": 44.1},
|
local_date: {"ECMWF": 39.0, "GFS": 44.1},
|
||||||
},
|
},
|
||||||
"forecasts": {"ECMWF": 39.0, "GFS": 44.1},
|
"forecasts": {"ECMWF": 39.0, "GFS": 44.1},
|
||||||
}
|
}
|
||||||
@@ -1681,10 +1714,12 @@ def test_chart_data_floors_stale_forecast_and_deb_with_observed_high(monkeypatch
|
|||||||
|
|
||||||
assert payload["forecast"]["today_high"] >= 38.0
|
assert payload["forecast"]["today_high"] >= 38.0
|
||||||
assert payload["deb"]["prediction"] >= 38.0
|
assert payload["deb"]["prediction"] >= 38.0
|
||||||
assert payload["multi_model_daily"]["2026-06-21"]["models"]["GFS"] == 44.1
|
assert max(payload["deb"]["hourly_path"]["temps"]) >= 38.0
|
||||||
|
assert payload["multi_model_daily"][local_date]["models"]["GFS"] == 44.1
|
||||||
assert detail["forecast"]["today_high"] >= 38.0
|
assert detail["forecast"]["today_high"] >= 38.0
|
||||||
assert detail["overview"]["deb_prediction"] >= 38.0
|
assert detail["overview"]["deb_prediction"] >= 38.0
|
||||||
assert detail["multi_model_daily"]["2026-06-21"]["models"]["GFS"] == 44.1
|
assert max(detail["deb"]["hourly_path"]["temps"]) >= 38.0
|
||||||
|
assert detail["multi_model_daily"][local_date]["models"]["GFS"] == 44.1
|
||||||
|
|
||||||
|
|
||||||
def test_chart_data_cache_hit_overlays_latest_amsc_raw(monkeypatch):
|
def test_chart_data_cache_hit_overlays_latest_amsc_raw(monkeypatch):
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from src.data_collection.country_networks import build_country_network_snapshot
|
|||||||
from src.data_collection.city_registry import ALIASES, CITY_REGISTRY
|
from src.data_collection.city_registry import ALIASES, CITY_REGISTRY
|
||||||
from src.data_collection.city_time import get_city_utc_offset_seconds
|
from src.data_collection.city_time import get_city_utc_offset_seconds
|
||||||
from src.data_collection.forecast_source_bundle import ensure_multi_model_hourly_payload
|
from src.data_collection.forecast_source_bundle import ensure_multi_model_hourly_payload
|
||||||
|
from src.data_collection.multi_model_freshness import multi_model_forecasts_for_local_date
|
||||||
from src.database.runtime_state import IntradayPathSnapshotRepository
|
from src.database.runtime_state import IntradayPathSnapshotRepository
|
||||||
from web.services.city_payloads import (
|
from web.services.city_payloads import (
|
||||||
build_city_chart_detail_payload as _city_chart_payload_detail,
|
build_city_chart_detail_payload as _city_chart_payload_detail,
|
||||||
@@ -894,7 +895,7 @@ def _analyze(
|
|||||||
current_forecasts: Dict[str, float] = {}
|
current_forecasts: Dict[str, float] = {}
|
||||||
if om_today is not None:
|
if om_today is not None:
|
||||||
current_forecasts["Open-Meteo"] = om_today
|
current_forecasts["Open-Meteo"] = om_today
|
||||||
for m, v in mm.get("forecasts", {}).items():
|
for m, v in multi_model_forecasts_for_local_date(mm, local_date_str).items():
|
||||||
if v is not None and not _is_excluded_model_name(m):
|
if v is not None and not _is_excluded_model_name(m):
|
||||||
temp_val = _sf(v)
|
temp_val = _sf(v)
|
||||||
if temp_val is not None:
|
if temp_val is not None:
|
||||||
@@ -1924,7 +1925,7 @@ def _analyze_summary(city: str, force_refresh: bool = False) -> Dict[str, Any]:
|
|||||||
current_forecasts: Dict[str, float] = {}
|
current_forecasts: Dict[str, float] = {}
|
||||||
if om_today is not None:
|
if om_today is not None:
|
||||||
current_forecasts["Open-Meteo"] = om_today
|
current_forecasts["Open-Meteo"] = om_today
|
||||||
for m, v in mm.get("forecasts", {}).items():
|
for m, v in multi_model_forecasts_for_local_date(mm, local_date_str).items():
|
||||||
if v is not None and not _is_excluded_model_name(m):
|
if v is not None and not _is_excluded_model_name(m):
|
||||||
temp_val = _sf(v)
|
temp_val = _sf(v)
|
||||||
if temp_val is not None:
|
if temp_val is not None:
|
||||||
|
|||||||
+37
-25
@@ -18,6 +18,7 @@ from src.data_collection.forecast_source_bundle import (
|
|||||||
_multi_model_cache_key,
|
_multi_model_cache_key,
|
||||||
fetch_open_meteo_forecast_bundle,
|
fetch_open_meteo_forecast_bundle,
|
||||||
)
|
)
|
||||||
|
from src.data_collection.multi_model_freshness import multi_model_forecasts_for_local_date
|
||||||
import web.routes as legacy_routes
|
import web.routes as legacy_routes
|
||||||
from web.analysis_service import _runway_history_temp_for_city
|
from web.analysis_service import _runway_history_temp_for_city
|
||||||
from web.services.canonical_temperature import build_city_weather_from_canonical
|
from web.services.canonical_temperature import build_city_weather_from_canonical
|
||||||
@@ -868,6 +869,7 @@ def _floor_chart_forecast_with_observed_high(payload: Dict[str, Any]) -> Dict[st
|
|||||||
rounded_floor = round(float(observed_floor), 1)
|
rounded_floor = round(float(observed_floor), 1)
|
||||||
next_payload = _floor_forecast_today_high(next_payload, local_date, rounded_floor)
|
next_payload = _floor_forecast_today_high(next_payload, local_date, rounded_floor)
|
||||||
next_payload = _floor_deb_prediction(next_payload, rounded_floor)
|
next_payload = _floor_deb_prediction(next_payload, rounded_floor)
|
||||||
|
next_payload = _floor_deb_hourly_path(next_payload, rounded_floor)
|
||||||
next_payload = _floor_multi_model_daily_deb(next_payload, local_date, rounded_floor)
|
next_payload = _floor_multi_model_daily_deb(next_payload, local_date, rounded_floor)
|
||||||
next_payload = _floor_probability_mu(next_payload, rounded_floor)
|
next_payload = _floor_probability_mu(next_payload, rounded_floor)
|
||||||
return next_payload
|
return next_payload
|
||||||
@@ -923,31 +925,13 @@ def _first_float(block: Dict[str, Any], keys: Tuple[str, ...]) -> Optional[float
|
|||||||
|
|
||||||
|
|
||||||
def _multi_model_daily_models_for_date(multi_model: Any, local_date: str) -> Dict[str, float]:
|
def _multi_model_daily_models_for_date(multi_model: Any, local_date: str) -> Dict[str, float]:
|
||||||
if not isinstance(multi_model, dict) or not local_date:
|
return {
|
||||||
return {}
|
model: round(value, 1)
|
||||||
|
for model, value in multi_model_forecasts_for_local_date(
|
||||||
models: Dict[str, float] = {}
|
multi_model,
|
||||||
daily = multi_model.get("daily_forecasts") if isinstance(multi_model.get("daily_forecasts"), dict) else {}
|
local_date,
|
||||||
day_models = daily.get(local_date) if isinstance(daily, dict) else {}
|
).items()
|
||||||
if isinstance(day_models, dict):
|
}
|
||||||
for model, value in day_models.items():
|
|
||||||
parsed = _float_or_none(value)
|
|
||||||
if parsed is not None:
|
|
||||||
models[str(model)] = round(parsed, 1)
|
|
||||||
|
|
||||||
hourly_models = _multi_model_daily_models_from_hourly(multi_model, local_date)
|
|
||||||
for model, value in hourly_models.items():
|
|
||||||
current = models.get(model)
|
|
||||||
models[model] = value if current is None else round(max(current, value), 1)
|
|
||||||
|
|
||||||
if not models:
|
|
||||||
forecasts = multi_model.get("forecasts") if isinstance(multi_model.get("forecasts"), dict) else {}
|
|
||||||
for model, value in forecasts.items():
|
|
||||||
parsed = _float_or_none(value)
|
|
||||||
if parsed is not None:
|
|
||||||
models[str(model)] = round(parsed, 1)
|
|
||||||
|
|
||||||
return models
|
|
||||||
|
|
||||||
|
|
||||||
def _multi_model_daily_models_from_hourly(multi_model: Dict[str, Any], local_date: str) -> Dict[str, float]:
|
def _multi_model_daily_models_from_hourly(multi_model: Dict[str, Any], local_date: str) -> Dict[str, float]:
|
||||||
@@ -1062,6 +1046,34 @@ def _floor_deb_prediction(payload: Dict[str, Any], observed_floor: float) -> Dic
|
|||||||
return next_payload
|
return next_payload
|
||||||
|
|
||||||
|
|
||||||
|
def _floor_deb_hourly_path(payload: Dict[str, Any], observed_floor: float) -> Dict[str, Any]:
|
||||||
|
deb = payload.get("deb") if isinstance(payload.get("deb"), dict) else {}
|
||||||
|
path = deb.get("hourly_path") if isinstance(deb.get("hourly_path"), dict) else {}
|
||||||
|
temps = path.get("temps") if isinstance(path.get("temps"), list) else []
|
||||||
|
parsed_temps = [_float_or_none(value) for value in temps]
|
||||||
|
valid_temps = [value for value in parsed_temps if value is not None]
|
||||||
|
if not valid_temps:
|
||||||
|
return payload
|
||||||
|
path_max = max(valid_temps)
|
||||||
|
if path_max >= observed_floor:
|
||||||
|
return payload
|
||||||
|
|
||||||
|
offset = observed_floor - path_max
|
||||||
|
next_payload = deepcopy(payload)
|
||||||
|
next_deb = dict(next_payload.get("deb") or {})
|
||||||
|
next_path = dict(next_deb.get("hourly_path") or {})
|
||||||
|
next_path["temps"] = [
|
||||||
|
round(value + offset, 1) if value is not None else raw_value
|
||||||
|
for raw_value, value in zip(temps, parsed_temps)
|
||||||
|
]
|
||||||
|
next_path["observed_floor_applied"] = True
|
||||||
|
next_path["observed_floor_offset"] = round(offset, 1)
|
||||||
|
next_deb["hourly_path"] = next_path
|
||||||
|
next_deb["observed_floor_applied"] = True
|
||||||
|
next_payload["deb"] = next_deb
|
||||||
|
return next_payload
|
||||||
|
|
||||||
|
|
||||||
def _floor_multi_model_daily_deb(
|
def _floor_multi_model_daily_deb(
|
||||||
payload: Dict[str, Any],
|
payload: Dict[str, Any],
|
||||||
local_date: str,
|
local_date: str,
|
||||||
|
|||||||
Reference in New Issue
Block a user