Add WeatherNext2 worker and prevent empty scan cache
This commit is contained in:
2569718930@qq.com
@@ -110,6 +110,7 @@ def _deb_model_priority(model_name: str) -> int:
|
||||
"hko": 45,
|
||||
"lgbm": 50,
|
||||
"openmeteo": 15,
|
||||
"weathernext2": 30,
|
||||
}.get(normalized, 10)
|
||||
|
||||
|
||||
|
||||
@@ -50,6 +50,49 @@ def _sf(v):
|
||||
return None
|
||||
|
||||
|
||||
def _weathernext2_probability_payload(
|
||||
weather_data: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
source = weather_data.get("weathernext2")
|
||||
if not isinstance(source, dict):
|
||||
return None
|
||||
raw_buckets = source.get("buckets")
|
||||
if not isinstance(raw_buckets, list) or not raw_buckets:
|
||||
return None
|
||||
|
||||
buckets = []
|
||||
for bucket in raw_buckets:
|
||||
if not isinstance(bucket, dict):
|
||||
continue
|
||||
probability = _sf(bucket.get("probability"))
|
||||
if probability is None or probability <= 0:
|
||||
continue
|
||||
copied = dict(bucket)
|
||||
copied["probability"] = round(probability, 3)
|
||||
buckets.append(copied)
|
||||
if not buckets:
|
||||
return None
|
||||
|
||||
summary = source.get("summary") if isinstance(source.get("summary"), dict) else {}
|
||||
mu = _sf(summary.get("median"))
|
||||
if mu is None:
|
||||
mu = _sf(summary.get("mean"))
|
||||
if mu is None:
|
||||
top_bucket = max(buckets, key=lambda item: _sf(item.get("probability")) or 0)
|
||||
mu = _sf(top_bucket.get("value"))
|
||||
|
||||
return {
|
||||
"engine": "weathernext2",
|
||||
"mu": mu,
|
||||
"probabilities": sorted(
|
||||
buckets,
|
||||
key=lambda item: _sf(item.get("probability")) or 0,
|
||||
reverse=True,
|
||||
)[:4],
|
||||
"probabilities_all": buckets,
|
||||
}
|
||||
|
||||
|
||||
def _median(values: List[float]) -> Optional[float]:
|
||||
if not values:
|
||||
return None
|
||||
@@ -548,6 +591,18 @@ def analyze_weather_trend(
|
||||
for m_name, m_val in mm_forecasts.items():
|
||||
if m_val is not None and not _is_excluded_model_name(m_name):
|
||||
current_forecasts[m_name] = _sf(m_val)
|
||||
weathernext2 = weather_data.get("weathernext2")
|
||||
if isinstance(weathernext2, dict):
|
||||
weathernext2_summary = (
|
||||
weathernext2.get("summary")
|
||||
if isinstance(weathernext2.get("summary"), dict)
|
||||
else {}
|
||||
)
|
||||
weathernext2_median = _sf(weathernext2_summary.get("median"))
|
||||
if weathernext2_median is None:
|
||||
weathernext2_median = _sf(weathernext2_summary.get("mean"))
|
||||
if weathernext2_median is not None:
|
||||
current_forecasts["WeatherNext 2"] = weathernext2_median
|
||||
forecast_highs = [h for h in current_forecasts.values() if h is not None]
|
||||
forecast_high = max(forecast_highs) if forecast_highs else None
|
||||
forecast_median = (
|
||||
@@ -865,9 +920,12 @@ def analyze_weather_trend(
|
||||
# === Probability Engine ===
|
||||
probabilities: List[Dict[str, Any]] = []
|
||||
probabilities_all: List[Dict[str, Any]] = []
|
||||
probability_engine = "legacy"
|
||||
forecast_miss_deg = 0.0
|
||||
weathernext2_probs = _weathernext2_probability_payload(weather_data)
|
||||
|
||||
if is_dead_market:
|
||||
probability_engine = "dead_market"
|
||||
settled_wu = apply_city_settlement(city_name, max_so_far) if max_so_far is not None else 0
|
||||
dead_msg = (
|
||||
f"🎲 <b>结算预测</b>：已锁定 {settled_wu}{temp_symbol} "
|
||||
@@ -881,6 +939,24 @@ def analyze_weather_trend(
|
||||
{"value": settled_wu, "range": f"[{settled_wu-0.5}~{settled_wu+0.5})", "probability": 1.0}
|
||||
]
|
||||
probabilities_all = probabilities
|
||||
elif weathernext2_probs:
|
||||
if max_so_far is not None and forecast_median is not None:
|
||||
forecast_miss_deg = round(forecast_median - max_so_far, 1)
|
||||
probability_engine = "weathernext2"
|
||||
mu = weathernext2_probs.get("mu") or mu
|
||||
probabilities = weathernext2_probs.get("probabilities", [])
|
||||
probabilities_all = weathernext2_probs.get("probabilities_all", probabilities)
|
||||
prob_parts = []
|
||||
for bucket in probabilities[:4]:
|
||||
label = str(bucket.get("label") or bucket.get("range") or bucket.get("value") or "").strip()
|
||||
probability = _sf(bucket.get("probability"))
|
||||
if label and probability is not None:
|
||||
prob_parts.append(f"{label} {probability * 100:.0f}%")
|
||||
if prob_parts:
|
||||
mu_label = f"μ={mu:.1f}" if mu is not None else "μ=--"
|
||||
prob_str = " | ".join(prob_parts)
|
||||
insights.append(f"🎲 <b>WeatherNext 2 概率</b> ({mu_label})：{prob_str}")
|
||||
ai_features.append(f"🎲 WeatherNext 2 概率分布：{prob_str}")
|
||||
elif (ens_p10 is not None and ens_p90 is not None) or fallback_sigma:
|
||||
# Forecast miss magnitude
|
||||
if max_so_far is not None and forecast_median is not None:
|
||||
@@ -1117,7 +1193,7 @@ def analyze_weather_trend(
|
||||
"mu": mu,
|
||||
"probabilities": probabilities,
|
||||
"probabilities_all": probabilities_all or probabilities,
|
||||
"probability_engine": "legacy",
|
||||
"probability_engine": probability_engine,
|
||||
"trend_info": {
|
||||
"direction": trend_direction if 'trend_direction' in dir() else "unknown",
|
||||
"recent": recent_list,
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, Optional
|
||||
|
||||
from src.data_collection.weathernext2_sources import build_weathernext2_city_probability
|
||||
|
||||
|
||||
QUANTILES = {"q10": 0.10, "q50": 0.50, "q90": 0.90}
|
||||
FEATURE_NAMES = [
|
||||
"city_code",
|
||||
"wn2_mean",
|
||||
"wn2_median",
|
||||
"wn2_p10",
|
||||
"wn2_p25",
|
||||
"wn2_p75",
|
||||
"wn2_p90",
|
||||
"wn2_spread",
|
||||
"deb_prediction_c",
|
||||
"model_median_c",
|
||||
"model_spread",
|
||||
"current_max_so_far_c",
|
||||
"local_hour",
|
||||
"month",
|
||||
"day_of_year",
|
||||
"observation_progress",
|
||||
]
|
||||
|
||||
|
||||
def _sf(value: Any) -> Optional[float]:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if math.isfinite(parsed) else None
|
||||
|
||||
|
||||
def _date_parts(value: Any) -> tuple[float, float]:
|
||||
text = str(value or "").strip()
|
||||
try:
|
||||
parsed = time.strptime(text[:10], "%Y-%m-%d")
|
||||
return float(parsed.tm_mon), float(parsed.tm_yday)
|
||||
except Exception:
|
||||
return 0.0, 0.0
|
||||
|
||||
|
||||
def _summary_from_record(record: Dict[str, Any]) -> Dict[str, Any]:
|
||||
wn2 = record.get("weathernext2") if isinstance(record.get("weathernext2"), dict) else {}
|
||||
summary = wn2.get("summary") if isinstance(wn2.get("summary"), dict) else {}
|
||||
return summary if isinstance(summary, dict) else {}
|
||||
|
||||
|
||||
def _city_key(value: Any) -> str:
|
||||
return str(value or "").strip().lower()
|
||||
|
||||
|
||||
def _build_city_index(records: Iterable[Dict[str, Any]]) -> Dict[str, int]:
|
||||
cities = sorted({_city_key(record.get("city")) for record in records if _city_key(record.get("city"))})
|
||||
return {city: idx for idx, city in enumerate(cities)}
|
||||
|
||||
|
||||
def _feature_row(record: Dict[str, Any], city_index: Dict[str, int]) -> Optional[list[float]]:
|
||||
summary = _summary_from_record(record)
|
||||
median = _sf(summary.get("median"))
|
||||
if median is None:
|
||||
return None
|
||||
target_date = record.get("target_date") or record.get("date")
|
||||
month, day_of_year = _date_parts(target_date)
|
||||
city = _city_key(record.get("city"))
|
||||
current = _sf(record.get("current_max_so_far_c"))
|
||||
progress = _sf(record.get("observation_progress"))
|
||||
if progress is None:
|
||||
local_hour = _sf(record.get("local_hour"))
|
||||
progress = min(max((local_hour or 0.0) / 24.0, 0.0), 1.0)
|
||||
|
||||
def fallback(name: str, default: float) -> float:
|
||||
parsed = _sf(summary.get(name))
|
||||
return default if parsed is None else parsed
|
||||
|
||||
return [
|
||||
float(city_index.get(city, -1)),
|
||||
fallback("mean", median),
|
||||
median,
|
||||
fallback("p10", median),
|
||||
fallback("p25", median),
|
||||
fallback("p75", median),
|
||||
fallback("p90", median),
|
||||
fallback("spread", 0.0),
|
||||
_sf(record.get("deb_prediction_c")) or median,
|
||||
_sf(record.get("model_median_c")) or median,
|
||||
_sf(record.get("model_spread")) or 0.0,
|
||||
current if current is not None else median,
|
||||
_sf(record.get("local_hour")) or 0.0,
|
||||
month,
|
||||
day_of_year,
|
||||
progress,
|
||||
]
|
||||
|
||||
|
||||
def _training_xy(
|
||||
records: Iterable[Dict[str, Any]],
|
||||
city_index: Dict[str, int],
|
||||
) -> tuple[list[list[float]], list[float], list[Dict[str, Any]]]:
|
||||
features: list[list[float]] = []
|
||||
residuals: list[float] = []
|
||||
kept: list[Dict[str, Any]] = []
|
||||
for record in records:
|
||||
summary = _summary_from_record(record)
|
||||
median = _sf(summary.get("median"))
|
||||
actual = _sf(record.get("actual_high_c", record.get("actual_high")))
|
||||
row = _feature_row(record, city_index)
|
||||
if median is None or actual is None or row is None:
|
||||
continue
|
||||
features.append(row)
|
||||
residuals.append(actual - median)
|
||||
kept.append(record)
|
||||
return features, residuals, kept
|
||||
|
||||
|
||||
def train_lightgbm_quantile_calibrator(
|
||||
records: Iterable[Dict[str, Any]],
|
||||
*,
|
||||
model_dir: os.PathLike[str] | str,
|
||||
min_global_samples: int = 150,
|
||||
min_city_samples: int = 5,
|
||||
) -> Dict[str, Any]:
|
||||
rows = [record for record in records if isinstance(record, dict)]
|
||||
city_index = _build_city_index(rows)
|
||||
features, residuals, kept = _training_xy(rows, city_index)
|
||||
if len(features) < int(min_global_samples):
|
||||
return {
|
||||
"trained": False,
|
||||
"reason": "insufficient_global_samples",
|
||||
"samples": len(features),
|
||||
}
|
||||
|
||||
city_counts: Dict[str, int] = {}
|
||||
for record in kept:
|
||||
city_counts[_city_key(record.get("city"))] = city_counts.get(_city_key(record.get("city")), 0) + 1
|
||||
if not any(count >= int(min_city_samples) for count in city_counts.values()):
|
||||
return {
|
||||
"trained": False,
|
||||
"reason": "insufficient_city_samples",
|
||||
"samples": len(features),
|
||||
}
|
||||
|
||||
try:
|
||||
import joblib # type: ignore
|
||||
from lightgbm import LGBMRegressor # type: ignore
|
||||
except Exception as exc:
|
||||
return {
|
||||
"trained": False,
|
||||
"reason": "missing_lightgbm",
|
||||
"samples": len(features),
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
target_dir = Path(model_dir)
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
models = {}
|
||||
for key, alpha in QUANTILES.items():
|
||||
model = LGBMRegressor(
|
||||
objective="quantile",
|
||||
alpha=alpha,
|
||||
n_estimators=45,
|
||||
learning_rate=0.08,
|
||||
num_leaves=15,
|
||||
min_child_samples=5,
|
||||
random_state=42,
|
||||
n_jobs=2,
|
||||
verbosity=-1,
|
||||
)
|
||||
model.fit(features, residuals)
|
||||
models[key] = model
|
||||
joblib.dump(model, target_dir / f"{key}.pkl")
|
||||
|
||||
metadata = {
|
||||
"model_version": f"weathernext2_lightgbm_quantile_{int(time.time())}",
|
||||
"engine": "lightgbm_quantile",
|
||||
"samples": len(features),
|
||||
"feature_names": FEATURE_NAMES,
|
||||
"city_index": city_index,
|
||||
"city_counts": city_counts,
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
}
|
||||
(target_dir / "metadata.json").write_text(
|
||||
json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
ordered = True
|
||||
for feature in features[: min(len(features), 50)]:
|
||||
preds = sorted(float(models[key].predict([feature])[0]) for key in ("q10", "q50", "q90"))
|
||||
if preds[0] > preds[1] or preds[1] > preds[2]:
|
||||
ordered = False
|
||||
break
|
||||
return {
|
||||
"trained": True,
|
||||
"samples": len(features),
|
||||
"model_dir": str(target_dir),
|
||||
"model_version": metadata["model_version"],
|
||||
"validation": {"ordered_quantiles": ordered},
|
||||
}
|
||||
|
||||
|
||||
def _load_model_bundle(model_dir: os.PathLike[str] | str) -> Optional[Dict[str, Any]]:
|
||||
target_dir = Path(model_dir)
|
||||
metadata_path = target_dir / "metadata.json"
|
||||
if not metadata_path.is_file():
|
||||
return None
|
||||
try:
|
||||
import joblib # type: ignore
|
||||
|
||||
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||||
return {
|
||||
"metadata": metadata,
|
||||
"models": {
|
||||
key: joblib.load(target_dir / f"{key}.pkl")
|
||||
for key in QUANTILES
|
||||
},
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _predict_residual_quantiles(record: Dict[str, Any], bundle: Dict[str, Any]) -> Optional[Dict[str, float]]:
|
||||
metadata = bundle.get("metadata") or {}
|
||||
city_index = metadata.get("city_index") if isinstance(metadata.get("city_index"), dict) else {}
|
||||
feature = _feature_row(record, city_index)
|
||||
if feature is None:
|
||||
return None
|
||||
raw = {
|
||||
key: float(model.predict([feature])[0])
|
||||
for key, model in (bundle.get("models") or {}).items()
|
||||
}
|
||||
values = sorted([raw.get("q10", 0.0), raw.get("q50", 0.0), raw.get("q90", 0.0)])
|
||||
return {"q10": values[0], "q50": values[1], "q90": values[2]}
|
||||
|
||||
|
||||
def _calibrate_members(
|
||||
member_highs: Dict[str, Any],
|
||||
raw_summary: Dict[str, Any],
|
||||
residuals: Dict[str, float],
|
||||
) -> Dict[str, float]:
|
||||
raw_median = _sf(raw_summary.get("median"))
|
||||
raw_p10 = _sf(raw_summary.get("p10"))
|
||||
raw_p90 = _sf(raw_summary.get("p90"))
|
||||
if raw_median is None:
|
||||
return {}
|
||||
target_p10 = (raw_p10 if raw_p10 is not None else raw_median) + residuals["q10"]
|
||||
target_median = raw_median + residuals["q50"]
|
||||
target_p90 = (raw_p90 if raw_p90 is not None else raw_median) + residuals["q90"]
|
||||
|
||||
calibrated = {}
|
||||
for member_id, value in (member_highs or {}).items():
|
||||
parsed = _sf(value)
|
||||
if parsed is None:
|
||||
continue
|
||||
if parsed <= raw_median:
|
||||
raw_span = max(raw_median - (raw_p10 if raw_p10 is not None else raw_median), 0.1)
|
||||
target_span = max(target_median - target_p10, 0.1)
|
||||
adjusted = target_median - (raw_median - parsed) / raw_span * target_span
|
||||
else:
|
||||
raw_span = max((raw_p90 if raw_p90 is not None else raw_median) - raw_median, 0.1)
|
||||
target_span = max(target_p90 - target_median, 0.1)
|
||||
adjusted = target_median + (parsed - raw_median) / raw_span * target_span
|
||||
calibrated[str(member_id)] = round(adjusted, 1)
|
||||
return calibrated
|
||||
|
||||
|
||||
def apply_quantile_calibration_to_payload(
|
||||
payload: Dict[str, Any],
|
||||
*,
|
||||
model_dir: os.PathLike[str] | str,
|
||||
) -> Dict[str, Any]:
|
||||
bundle = _load_model_bundle(model_dir)
|
||||
if not bundle:
|
||||
return dict(payload)
|
||||
raw_summary = payload.get("summary") if isinstance(payload.get("summary"), dict) else {}
|
||||
member_highs = payload.get("member_highs") if isinstance(payload.get("member_highs"), dict) else {}
|
||||
if not raw_summary or not member_highs:
|
||||
return dict(payload)
|
||||
|
||||
record = {
|
||||
"city": payload.get("city"),
|
||||
"target_date": payload.get("target_date"),
|
||||
"weathernext2": {"summary": raw_summary},
|
||||
}
|
||||
residuals = _predict_residual_quantiles(record, bundle)
|
||||
if residuals is None:
|
||||
return dict(payload)
|
||||
|
||||
calibrated_members = _calibrate_members(member_highs, raw_summary, residuals)
|
||||
if not calibrated_members:
|
||||
return dict(payload)
|
||||
|
||||
calibrated = build_weathernext2_city_probability(
|
||||
city=str(payload.get("city") or ""),
|
||||
member_highs=calibrated_members,
|
||||
temp_symbol=str(payload.get("temp_symbol") or "°C"),
|
||||
target_date=payload.get("target_date"),
|
||||
source_run=payload.get("source_run"),
|
||||
generated_at=payload.get("generated_at"),
|
||||
)
|
||||
metadata = bundle.get("metadata") or {}
|
||||
calibrated["calibration"] = {
|
||||
"engine": "lightgbm_quantile",
|
||||
"model_version": metadata.get("model_version"),
|
||||
"samples": metadata.get("samples"),
|
||||
"residual_quantiles": {key: round(value, 3) for key, value in residuals.items()},
|
||||
"raw_summary": raw_summary,
|
||||
"calibrated_summary": calibrated.get("summary"),
|
||||
}
|
||||
calibrated["raw_weathernext2"] = {
|
||||
"summary": raw_summary,
|
||||
"buckets": payload.get("buckets") or [],
|
||||
"top_bucket": payload.get("top_bucket"),
|
||||
}
|
||||
return calibrated
|
||||
@@ -18,6 +18,7 @@ from src.data_collection.metar_sources import MetarSourceMixin
|
||||
from src.data_collection.mgm_sources import MgmSourceMixin
|
||||
from src.data_collection.jma_amedas_sources import JmaAmedasSourceMixin
|
||||
from src.data_collection.nws_open_meteo_sources import NwsOpenMeteoSourceMixin
|
||||
from src.data_collection.weathernext2_sources import WeatherNext2SourceMixin
|
||||
from src.data_collection.amos_station_sources import AmosStationSourceMixin
|
||||
from src.data_collection.amsc_awos_sources import AmscAwosSourceMixin
|
||||
from src.data_collection.fmi_sources import FmiSourceMixin
|
||||
@@ -35,7 +36,7 @@ from src.data_collection.forecast_source_bundle import fetch_open_meteo_forecast
|
||||
from src.database.db_manager import DBManager
|
||||
|
||||
|
||||
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, CowinSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin, ImsSourceMixin, NcmSourceMixin, AerowebSourceMixin, WundergroundHistoricalMixin):
|
||||
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, NwsOpenMeteoSourceMixin, WeatherNext2SourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, CowinSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin, ImsSourceMixin, NcmSourceMixin, AerowebSourceMixin, WundergroundHistoricalMixin):
|
||||
"""
|
||||
Multi-source weather data collector
|
||||
|
||||
@@ -189,9 +190,11 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
self._open_meteo_cache: Dict[str, Dict] = {}
|
||||
self._ensemble_cache: Dict[str, Dict] = {}
|
||||
self._multi_model_cache: Dict[str, Dict] = {}
|
||||
self._weathernext2_cache: Dict[str, Dict] = {}
|
||||
self._open_meteo_cache_lock = threading.Lock()
|
||||
self._ensemble_cache_lock = threading.Lock()
|
||||
self._multi_model_cache_lock = threading.Lock()
|
||||
self._weathernext2_cache_lock = threading.Lock()
|
||||
# Open-Meteo 共享 429 冷却计时器：触发限流后所有 OM 端点暂停请求
|
||||
self._open_meteo_rate_limit_until: float = 0.0
|
||||
self._open_meteo_rl_cooldown: int = int(
|
||||
@@ -1740,6 +1743,26 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
if multi_model_data:
|
||||
results["multi_model"] = multi_model_data
|
||||
|
||||
def _attach_weathernext2_model(
|
||||
self,
|
||||
results: Dict,
|
||||
city: str,
|
||||
lat: float,
|
||||
lon: float,
|
||||
use_fahrenheit: bool,
|
||||
*,
|
||||
timezone_offset_seconds: Optional[int] = None,
|
||||
) -> None:
|
||||
payload = self.fetch_weathernext2_probability(
|
||||
city,
|
||||
lat,
|
||||
lon,
|
||||
use_fahrenheit=use_fahrenheit,
|
||||
timezone_offset_seconds=timezone_offset_seconds,
|
||||
)
|
||||
if payload:
|
||||
results["weathernext2"] = payload
|
||||
|
||||
def fetch_all_sources(
|
||||
self,
|
||||
city: str,
|
||||
@@ -1794,6 +1817,14 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
results["open-meteo"] = open_meteo
|
||||
# 获取时区偏移以过滤 METAR
|
||||
utc_offset = open_meteo.get("utc_offset", 0)
|
||||
self._attach_weathernext2_model(
|
||||
results,
|
||||
city_lower,
|
||||
lat,
|
||||
lon,
|
||||
use_fahrenheit,
|
||||
timezone_offset_seconds=utc_offset,
|
||||
)
|
||||
if supports_aviationweather:
|
||||
metar_data = self.fetch_metar(
|
||||
city, use_fahrenheit=use_fahrenheit, utc_offset=utc_offset
|
||||
@@ -1844,6 +1875,14 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
fallback_utc_offset = int(
|
||||
self.CITY_REGISTRY.get(city_lower, {}).get("tz_offset", 0)
|
||||
)
|
||||
self._attach_weathernext2_model(
|
||||
results,
|
||||
city_lower,
|
||||
lat,
|
||||
lon,
|
||||
use_fahrenheit,
|
||||
timezone_offset_seconds=fallback_utc_offset,
|
||||
)
|
||||
if supports_aviationweather:
|
||||
metar_data = self.fetch_metar(
|
||||
city,
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Mapping, Optional, Sequence
|
||||
|
||||
|
||||
TEMPERATURE_VARIABLE_CANDIDATES = (
|
||||
"temperature_2m",
|
||||
"2m_temperature",
|
||||
"t2m",
|
||||
"air_temperature_2m",
|
||||
"temperature",
|
||||
)
|
||||
|
||||
|
||||
def _mapping_keys(dataset: Any) -> list[str]:
|
||||
if isinstance(dataset, Mapping):
|
||||
return [str(key) for key in dataset.keys()]
|
||||
data_vars = getattr(dataset, "data_vars", None)
|
||||
if data_vars is not None:
|
||||
try:
|
||||
return [str(key) for key in data_vars.keys()]
|
||||
except Exception:
|
||||
return []
|
||||
return []
|
||||
|
||||
|
||||
def select_temperature_variable(dataset: Any, preferred: Optional[str] = None) -> str:
|
||||
if preferred:
|
||||
keys = set(_mapping_keys(dataset))
|
||||
if preferred in keys:
|
||||
return preferred
|
||||
raise KeyError(f"WeatherNext2 temperature variable not found: {preferred}")
|
||||
|
||||
keys = _mapping_keys(dataset)
|
||||
lowered = {key.lower(): key for key in keys}
|
||||
for candidate in TEMPERATURE_VARIABLE_CANDIDATES:
|
||||
if candidate.lower() in lowered:
|
||||
return lowered[candidate.lower()]
|
||||
|
||||
for key in keys:
|
||||
normalized = key.lower().replace("-", "_")
|
||||
if "temperature" in normalized and ("2m" in normalized or "two_meter" in normalized):
|
||||
return key
|
||||
raise KeyError("WeatherNext2 2m temperature variable was not found")
|
||||
|
||||
|
||||
def normalize_temperature_value(
|
||||
value: Any,
|
||||
units: str = "",
|
||||
*,
|
||||
use_fahrenheit: bool = False,
|
||||
) -> Optional[float]:
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not math.isfinite(parsed):
|
||||
return None
|
||||
|
||||
unit = str(units or "").strip().lower()
|
||||
if unit in {"k", "kelvin"} or parsed > 150:
|
||||
celsius = parsed - 273.15
|
||||
elif unit in {"f", "fahrenheit", "degf", "degree_fahrenheit"}:
|
||||
celsius = (parsed - 32.0) * 5.0 / 9.0
|
||||
else:
|
||||
celsius = parsed
|
||||
|
||||
if use_fahrenheit:
|
||||
return round(celsius * 9.0 / 5.0 + 32.0, 1)
|
||||
return round(celsius, 1)
|
||||
|
||||
|
||||
def _get_dataset_value(dataset: Any, key: str) -> Any:
|
||||
if isinstance(dataset, Mapping):
|
||||
return dataset[key]
|
||||
return dataset[key]
|
||||
|
||||
|
||||
def _as_list(values: Any) -> list[Any]:
|
||||
if hasattr(values, "values"):
|
||||
try:
|
||||
values = values.values
|
||||
except Exception:
|
||||
pass
|
||||
if hasattr(values, "tolist"):
|
||||
try:
|
||||
return values.tolist()
|
||||
except Exception:
|
||||
pass
|
||||
return list(values or [])
|
||||
|
||||
|
||||
def _nearest_index(values: Sequence[Any], target: float) -> int:
|
||||
numeric_values = [float(value) for value in values]
|
||||
return min(range(len(numeric_values)), key=lambda idx: abs(numeric_values[idx] - target))
|
||||
|
||||
|
||||
def _variable_units(dataset: Any, temp_var: str) -> str:
|
||||
if isinstance(dataset, Mapping):
|
||||
units = dataset.get("units")
|
||||
if isinstance(units, Mapping):
|
||||
return str(units.get(temp_var) or "")
|
||||
return ""
|
||||
variable = dataset[temp_var]
|
||||
attrs = getattr(variable, "attrs", {}) or {}
|
||||
return str(attrs.get("units") or attrs.get("unit") or "")
|
||||
|
||||
|
||||
def _variable_values(dataset: Any, temp_var: str) -> Any:
|
||||
variable = _get_dataset_value(dataset, temp_var)
|
||||
if hasattr(variable, "values"):
|
||||
return variable.values
|
||||
return variable
|
||||
|
||||
|
||||
def _index_nested(values: Any, indexes: Sequence[int]) -> Any:
|
||||
current = values
|
||||
for idx in indexes:
|
||||
current = current[idx]
|
||||
return current
|
||||
|
||||
|
||||
def _dataset_dimension_values(dataset: Any, candidates: Sequence[str]) -> tuple[str, list[Any]]:
|
||||
keys = _mapping_keys(dataset)
|
||||
lowered = {key.lower(): key for key in keys}
|
||||
for candidate in candidates:
|
||||
key = lowered.get(candidate.lower())
|
||||
if key is not None:
|
||||
return key, _as_list(_get_dataset_value(dataset, key))
|
||||
|
||||
coords = getattr(dataset, "coords", None)
|
||||
if coords is not None:
|
||||
for candidate in candidates:
|
||||
if candidate in coords:
|
||||
return candidate, _as_list(coords[candidate])
|
||||
raise KeyError(f"WeatherNext2 dimension not found: {candidates}")
|
||||
|
||||
|
||||
def _parse_time(value: Any) -> str:
|
||||
if isinstance(value, datetime):
|
||||
dt = value
|
||||
else:
|
||||
text = str(value or "").strip()
|
||||
if text.endswith("Z"):
|
||||
text = f"{text[:-1]}+00:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return str(value)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def extract_member_hourly_from_grid_dataset(
|
||||
dataset: Any,
|
||||
*,
|
||||
lat: float,
|
||||
lon: float,
|
||||
temp_var: Optional[str] = None,
|
||||
use_fahrenheit: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Extract all member hourly temperatures for the nearest grid point."""
|
||||
selected_var = select_temperature_variable(dataset, temp_var or os.getenv("WEATHERNEXT2_TEMP_VAR") or None)
|
||||
lat_name, lat_values = _dataset_dimension_values(dataset, ("lat", "latitude"))
|
||||
lon_name, lon_values = _dataset_dimension_values(dataset, ("lon", "longitude"))
|
||||
member_name, member_values = _dataset_dimension_values(dataset, ("member", "realization", "ensemble_member"))
|
||||
time_name, time_values = _dataset_dimension_values(dataset, ("time", "valid_time"))
|
||||
|
||||
lat_idx = _nearest_index(lat_values, lat)
|
||||
lon_idx = _nearest_index(lon_values, lon)
|
||||
units = _variable_units(dataset, selected_var)
|
||||
|
||||
variable = _get_dataset_value(dataset, selected_var)
|
||||
dims = list(getattr(variable, "dims", []) or [member_name, time_name, lat_name, lon_name])
|
||||
values = _variable_values(dataset, selected_var)
|
||||
dim_indexes = {
|
||||
member_name: None,
|
||||
time_name: None,
|
||||
lat_name: lat_idx,
|
||||
lon_name: lon_idx,
|
||||
}
|
||||
|
||||
member_hourly: Dict[str, list[Optional[float]]] = {}
|
||||
for member_idx, member in enumerate(member_values):
|
||||
hourly = []
|
||||
for time_idx, _time_value in enumerate(time_values):
|
||||
dim_indexes[member_name] = member_idx
|
||||
dim_indexes[time_name] = time_idx
|
||||
indexes = [int(dim_indexes[dim]) for dim in dims]
|
||||
hourly.append(
|
||||
normalize_temperature_value(
|
||||
_index_nested(values, indexes),
|
||||
units,
|
||||
use_fahrenheit=use_fahrenheit,
|
||||
)
|
||||
)
|
||||
try:
|
||||
member_label = f"member_{int(member):02d}"
|
||||
except Exception:
|
||||
member_label = f"member_{member_idx:02d}"
|
||||
member_hourly[member_label] = hourly
|
||||
|
||||
return {
|
||||
"temp_var": selected_var,
|
||||
"units": units,
|
||||
"lat_dim": lat_name,
|
||||
"lon_dim": lon_name,
|
||||
"member_dim": member_name,
|
||||
"time_dim": time_name,
|
||||
"nearest_lat": float(lat_values[lat_idx]),
|
||||
"nearest_lon": float(lon_values[lon_idx]),
|
||||
"utc_times": [_parse_time(value) for value in time_values],
|
||||
"member_hourly": member_hourly,
|
||||
}
|
||||
|
||||
|
||||
def open_weathernext2_zarr_dataset(uri: str):
|
||||
"""Open a WeatherNext 2 Zarr dataset lazily with optional runtime dependencies."""
|
||||
try:
|
||||
import xarray as xr # type: ignore
|
||||
except Exception as exc: # pragma: no cover - depends on deployment extras
|
||||
raise RuntimeError("xarray is required for WeatherNext2 GCS/Zarr access") from exc
|
||||
|
||||
storage_options = {}
|
||||
credentials = str(os.getenv("GOOGLE_APPLICATION_CREDENTIALS", "") or "").strip()
|
||||
if credentials:
|
||||
storage_options["token"] = "google_default"
|
||||
try:
|
||||
return xr.open_zarr(
|
||||
uri,
|
||||
storage_options=storage_options or None,
|
||||
consolidated=True,
|
||||
)
|
||||
except Exception:
|
||||
return xr.open_zarr(
|
||||
uri,
|
||||
storage_options=storage_options or None,
|
||||
consolidated=False,
|
||||
)
|
||||
@@ -0,0 +1,581 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Any, Dict, Iterable, Mapping, Optional, Sequence, Union
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
WEATHERNEXT2_SOURCE = "weathernext2"
|
||||
WEATHERNEXT2_PROVIDER = "google_deepmind"
|
||||
WEATHERNEXT2_GCS_ZARR_URI = "gs://weathernext/weathernext_2_0_0/zarr"
|
||||
WEATHERNEXT2_MEAN_GCS_ZARR_URI = "gs://weathernext/weathernext_2_0_0_mean/zarr"
|
||||
|
||||
Number = Union[int, float]
|
||||
|
||||
|
||||
def _numeric(value: Any) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if math.isfinite(parsed) else None
|
||||
|
||||
|
||||
def _round1(value: Number) -> float:
|
||||
return round(float(value), 1)
|
||||
|
||||
|
||||
def _round3(value: Number) -> float:
|
||||
return round(float(value), 3)
|
||||
|
||||
|
||||
def _settle_integer(value: Number) -> int:
|
||||
parsed = float(value)
|
||||
if parsed >= 0:
|
||||
return int(math.floor(parsed + 0.5))
|
||||
return int(math.ceil(parsed - 0.5))
|
||||
|
||||
|
||||
def _is_fahrenheit(temp_symbol: str) -> bool:
|
||||
return "F" in str(temp_symbol or "").upper()
|
||||
|
||||
|
||||
def _unit(temp_symbol: str) -> str:
|
||||
return "°F" if _is_fahrenheit(temp_symbol) else "°C"
|
||||
|
||||
|
||||
def market_bucket_for_temperature(value: Number, temp_symbol: str = "°C") -> Dict[str, Any]:
|
||||
"""Return the tradable market option bucket for a single member temperature."""
|
||||
unit = _unit(temp_symbol)
|
||||
settled = _settle_integer(value)
|
||||
if _is_fahrenheit(unit):
|
||||
lower_value = settled if settled % 2 == 0 else settled - 1
|
||||
upper_value = lower_value + 1
|
||||
return {
|
||||
"key": f"{lower_value}-{upper_value}{unit}",
|
||||
"label": f"{lower_value}-{upper_value}{unit}",
|
||||
"lower": float(lower_value) - 0.5,
|
||||
"upper": float(upper_value) + 0.5,
|
||||
"sort_value": float(lower_value),
|
||||
}
|
||||
return {
|
||||
"key": f"{settled}{unit}",
|
||||
"label": f"{settled}{unit}",
|
||||
"lower": float(settled) - 0.5,
|
||||
"upper": float(settled) + 0.5,
|
||||
"sort_value": float(settled),
|
||||
}
|
||||
|
||||
|
||||
def _member_items(member_highs: Union[Mapping[str, Any], Sequence[Any]]) -> Iterable[tuple[str, float]]:
|
||||
if isinstance(member_highs, Mapping):
|
||||
iterable = member_highs.items()
|
||||
else:
|
||||
iterable = ((f"member_{idx:02d}", value) for idx, value in enumerate(member_highs))
|
||||
for member_id, value in iterable:
|
||||
parsed = _numeric(value)
|
||||
if parsed is not None:
|
||||
yield str(member_id), parsed
|
||||
|
||||
|
||||
def summarize_member_highs(member_highs: Union[Mapping[str, Any], Sequence[Any]]) -> Dict[str, Any]:
|
||||
values = sorted(value for _member_id, value in _member_items(member_highs))
|
||||
if not values:
|
||||
return {
|
||||
"members": 0,
|
||||
"mean": None,
|
||||
"median": None,
|
||||
"p10": None,
|
||||
"p25": None,
|
||||
"p75": None,
|
||||
"p90": None,
|
||||
"min": None,
|
||||
"max": None,
|
||||
"spread": None,
|
||||
}
|
||||
|
||||
def percentile(q: float) -> float:
|
||||
if len(values) == 1:
|
||||
return values[0]
|
||||
position = (len(values) - 1) * q
|
||||
lower_idx = int(math.floor(position))
|
||||
upper_idx = int(math.ceil(position))
|
||||
if lower_idx == upper_idx:
|
||||
return values[lower_idx]
|
||||
weight = position - lower_idx
|
||||
return values[lower_idx] * (1 - weight) + values[upper_idx] * weight
|
||||
|
||||
return {
|
||||
"members": len(values),
|
||||
"mean": _round1(sum(values) / len(values)),
|
||||
"median": _round1(percentile(0.5)),
|
||||
"p10": _round1(percentile(0.1)),
|
||||
"p25": _round1(percentile(0.25)),
|
||||
"p75": _round1(percentile(0.75)),
|
||||
"p90": _round1(percentile(0.9)),
|
||||
"min": _round1(values[0]),
|
||||
"max": _round1(values[-1]),
|
||||
"spread": _round1(values[-1] - values[0]),
|
||||
}
|
||||
|
||||
|
||||
def build_market_bucket_probabilities(
|
||||
member_highs: Union[Mapping[str, Any], Sequence[Any]],
|
||||
temp_symbol: str = "°C",
|
||||
) -> list[Dict[str, Any]]:
|
||||
grouped: Dict[str, Dict[str, Any]] = {}
|
||||
total_members = 0
|
||||
|
||||
for _member_id, value in _member_items(member_highs):
|
||||
total_members += 1
|
||||
option = market_bucket_for_temperature(value, temp_symbol)
|
||||
bucket = grouped.setdefault(
|
||||
option["key"],
|
||||
{
|
||||
"key": option["key"],
|
||||
"label": option["label"],
|
||||
"lower": option["lower"],
|
||||
"upper": option["upper"],
|
||||
"sort_value": option["sort_value"],
|
||||
"weighted_sum": 0.0,
|
||||
"member_count": 0,
|
||||
},
|
||||
)
|
||||
bucket["weighted_sum"] += value
|
||||
bucket["member_count"] += 1
|
||||
|
||||
if total_members <= 0:
|
||||
return []
|
||||
|
||||
buckets = []
|
||||
for bucket in grouped.values():
|
||||
member_count = int(bucket["member_count"])
|
||||
buckets.append(
|
||||
{
|
||||
"key": bucket["key"],
|
||||
"label": bucket["label"],
|
||||
"lower": bucket["lower"],
|
||||
"upper": bucket["upper"],
|
||||
"value": _round1(bucket["weighted_sum"] / member_count),
|
||||
"probability": _round3(member_count / total_members),
|
||||
"member_count": member_count,
|
||||
"total_members": total_members,
|
||||
}
|
||||
)
|
||||
|
||||
return sorted(buckets, key=lambda item: (float(item["lower"]), float(item["upper"])))
|
||||
|
||||
|
||||
def _parse_utc_time(value: Any) -> Optional[datetime]:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
else:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
if text.endswith("Z"):
|
||||
text = f"{text[:-1]}+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _parse_date(value: Any) -> Optional[date]:
|
||||
if isinstance(value, date) and not isinstance(value, datetime):
|
||||
return value
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(text[:10]).date()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def build_city_local_daily_highs_from_hourly(
|
||||
member_hourly: Mapping[str, Sequence[Any]],
|
||||
utc_times: Sequence[Any],
|
||||
timezone_offset_seconds: int,
|
||||
target_local_date: Any,
|
||||
) -> Dict[str, float]:
|
||||
"""Reduce hourly ensemble member temperatures to local-date daily highs."""
|
||||
target_date = _parse_date(target_local_date)
|
||||
if target_date is None:
|
||||
return {}
|
||||
|
||||
parsed_times = [_parse_utc_time(value) for value in utc_times]
|
||||
highs: Dict[str, float] = {}
|
||||
offset = timedelta(seconds=int(timezone_offset_seconds or 0))
|
||||
|
||||
for member_id, values in member_hourly.items():
|
||||
member_values = []
|
||||
for idx, utc_time in enumerate(parsed_times):
|
||||
if utc_time is None or idx >= len(values):
|
||||
continue
|
||||
local_date = (utc_time + offset).date()
|
||||
if local_date != target_date:
|
||||
continue
|
||||
parsed = _numeric(values[idx])
|
||||
if parsed is not None:
|
||||
member_values.append(parsed)
|
||||
if member_values:
|
||||
highs[str(member_id)] = _round1(max(member_values))
|
||||
|
||||
return highs
|
||||
|
||||
|
||||
def build_weathernext2_city_probability(
|
||||
*,
|
||||
city: str,
|
||||
member_highs: Union[Mapping[str, Any], Sequence[Any]],
|
||||
temp_symbol: str = "°C",
|
||||
target_date: Optional[str] = None,
|
||||
source_run: Optional[str] = None,
|
||||
generated_at: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build a WeatherNext 2 probability payload aligned with tradable market options."""
|
||||
buckets = build_market_bucket_probabilities(member_highs, temp_symbol=temp_symbol)
|
||||
summary = summarize_member_highs(member_highs)
|
||||
normalized_member_highs = {
|
||||
member_id: _round1(value)
|
||||
for member_id, value in _member_items(member_highs)
|
||||
}
|
||||
top_bucket = (
|
||||
max(buckets, key=lambda item: (float(item["probability"]), float(item["lower"])))
|
||||
if buckets
|
||||
else None
|
||||
)
|
||||
|
||||
return {
|
||||
"source": WEATHERNEXT2_SOURCE,
|
||||
"provider": WEATHERNEXT2_PROVIDER,
|
||||
"city": str(city or "").strip(),
|
||||
"target_date": target_date,
|
||||
"source_run": source_run,
|
||||
"generated_at": generated_at or datetime.now(timezone.utc).isoformat(),
|
||||
"temp_symbol": _unit(temp_symbol),
|
||||
"members": int(summary["members"] or 0),
|
||||
"member_highs": normalized_member_highs,
|
||||
"summary": summary,
|
||||
"buckets": buckets,
|
||||
"top_bucket": top_bucket,
|
||||
"bucket_policy": "celsius_single_fahrenheit_two_degree_market_options",
|
||||
"gcs_zarr_uri": os.getenv("WEATHERNEXT2_GCS_ZARR_URI", WEATHERNEXT2_GCS_ZARR_URI),
|
||||
"mean_gcs_zarr_uri": os.getenv("WEATHERNEXT2_MEAN_GCS_ZARR_URI", WEATHERNEXT2_MEAN_GCS_ZARR_URI),
|
||||
}
|
||||
|
||||
|
||||
def _env_enabled(name: str, default: str = "0") -> bool:
|
||||
return str(os.getenv(name, default) or "").strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
|
||||
def _city_key(city: str) -> str:
|
||||
return str(city or "").strip().lower()
|
||||
|
||||
|
||||
def _load_fixture_payload(path: str, city: str) -> Optional[Dict[str, Any]]:
|
||||
if not path:
|
||||
return None
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
payload = json.load(fh)
|
||||
except Exception as exc:
|
||||
logger.warning("WeatherNext2 fixture load failed path={}: {}", path, exc)
|
||||
return None
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
|
||||
key = _city_key(city)
|
||||
if isinstance(payload.get(key), dict):
|
||||
return dict(payload[key])
|
||||
for candidate, value in payload.items():
|
||||
if _city_key(candidate) == key and isinstance(value, dict):
|
||||
return dict(value)
|
||||
if "member_highs" in payload or "member_hourly" in payload:
|
||||
return dict(payload)
|
||||
return None
|
||||
|
||||
|
||||
def _weathernext2_data_root() -> str:
|
||||
return str(os.getenv("WEATHERNEXT2_DATA_ROOT", "/app/data/weathernext2") or "").strip()
|
||||
|
||||
|
||||
def _weathernext2_artifact_path() -> str:
|
||||
configured = str(os.getenv("WEATHERNEXT2_CITY_HIGHS_PATH", "") or "").strip()
|
||||
if configured:
|
||||
return configured
|
||||
return os.path.join(_weathernext2_data_root(), "weathernext2_city_highs.json")
|
||||
|
||||
|
||||
def _weathernext2_model_dir() -> str:
|
||||
return str(
|
||||
os.getenv(
|
||||
"WEATHERNEXT2_MODEL_DIR",
|
||||
"/app/data/models/weathernext2_calibrator",
|
||||
)
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
|
||||
def _load_artifact_payload(path: str, city: str) -> Optional[Dict[str, Any]]:
|
||||
if not path or not os.path.isfile(path):
|
||||
return None
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
payload = json.load(fh)
|
||||
except Exception as exc:
|
||||
logger.warning("WeatherNext2 artifact load failed path={}: {}", path, exc)
|
||||
return None
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
|
||||
key = _city_key(city)
|
||||
city_payloads: Any = payload.get("cities")
|
||||
if isinstance(city_payloads, dict):
|
||||
if isinstance(city_payloads.get(key), dict):
|
||||
return dict(city_payloads[key])
|
||||
for candidate, value in city_payloads.items():
|
||||
if _city_key(candidate) == key and isinstance(value, dict):
|
||||
return dict(value)
|
||||
if isinstance(city_payloads, list):
|
||||
for value in city_payloads:
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
if _city_key(value.get("city")) == key:
|
||||
return dict(value)
|
||||
|
||||
if isinstance(payload.get(key), dict):
|
||||
return dict(payload[key])
|
||||
for candidate, value in payload.items():
|
||||
if _city_key(candidate) == key and isinstance(value, dict):
|
||||
return dict(value)
|
||||
return None
|
||||
|
||||
|
||||
class WeatherNext2SourceMixin:
|
||||
"""Optional WeatherNext 2 probability source.
|
||||
|
||||
The live Google datasets require project access and optional heavy client
|
||||
dependencies. This mixin keeps production safe by supporting a fixture or
|
||||
prepared payload first, while reserving backend configuration for the real
|
||||
GCS/BigQuery fetcher.
|
||||
"""
|
||||
|
||||
def _weathernext2_enabled(self) -> bool:
|
||||
return _env_enabled("WEATHERNEXT2_ENABLED")
|
||||
|
||||
def _weathernext2_cache_key(
|
||||
self,
|
||||
city: str,
|
||||
lat: float,
|
||||
lon: float,
|
||||
use_fahrenheit: bool,
|
||||
target_date: Optional[str],
|
||||
) -> str:
|
||||
return (
|
||||
f"{_city_key(city)}:{round(float(lat), 4)}:{round(float(lon), 4)}:"
|
||||
f"{'f' if use_fahrenheit else 'c'}:{target_date or ''}"
|
||||
)
|
||||
|
||||
def _weathernext2_cache_state(self) -> tuple[Dict[str, Dict[str, Any]], threading.Lock, int]:
|
||||
if not hasattr(self, "_weathernext2_cache"):
|
||||
self._weathernext2_cache = {}
|
||||
if not hasattr(self, "_weathernext2_cache_lock"):
|
||||
self._weathernext2_cache_lock = threading.Lock()
|
||||
ttl_sec = int(os.getenv("WEATHERNEXT2_CACHE_TTL_SEC", "21600"))
|
||||
return self._weathernext2_cache, self._weathernext2_cache_lock, ttl_sec
|
||||
|
||||
def _weathernext2_target_date(
|
||||
self,
|
||||
timezone_offset_seconds: Optional[int],
|
||||
target_date: Optional[str],
|
||||
) -> str:
|
||||
if target_date:
|
||||
return str(target_date)
|
||||
offset = timedelta(seconds=int(timezone_offset_seconds or 0))
|
||||
return (datetime.now(timezone.utc) + offset).date().isoformat()
|
||||
|
||||
def _weathernext2_from_fixture(
|
||||
self,
|
||||
*,
|
||||
city: str,
|
||||
use_fahrenheit: bool,
|
||||
target_date: str,
|
||||
timezone_offset_seconds: Optional[int],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
fixture_path = str(os.getenv("WEATHERNEXT2_FIXTURE_PATH", "") or "").strip()
|
||||
fixture = _load_fixture_payload(fixture_path, city)
|
||||
if not fixture:
|
||||
return None
|
||||
|
||||
temp_symbol = "°F" if use_fahrenheit else "°C"
|
||||
fixture_target_date = str(fixture.get("target_date") or target_date)
|
||||
member_highs = fixture.get("member_highs")
|
||||
if member_highs is None and isinstance(fixture.get("member_hourly"), dict):
|
||||
member_highs = build_city_local_daily_highs_from_hourly(
|
||||
fixture["member_hourly"],
|
||||
fixture.get("utc_times") or fixture.get("times") or [],
|
||||
int(timezone_offset_seconds or fixture.get("timezone_offset_seconds") or 0),
|
||||
fixture_target_date,
|
||||
)
|
||||
if member_highs is None:
|
||||
return None
|
||||
|
||||
return build_weathernext2_city_probability(
|
||||
city=city,
|
||||
member_highs=member_highs,
|
||||
temp_symbol=temp_symbol,
|
||||
target_date=fixture_target_date,
|
||||
source_run=fixture.get("source_run"),
|
||||
generated_at=fixture.get("generated_at"),
|
||||
)
|
||||
|
||||
def _weathernext2_normalize_prepared_payload(
|
||||
self,
|
||||
*,
|
||||
payload: Mapping[str, Any],
|
||||
city: str,
|
||||
use_fahrenheit: bool,
|
||||
target_date: str,
|
||||
timezone_offset_seconds: Optional[int],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
temp_symbol = "°F" if use_fahrenheit else "°C"
|
||||
payload_target_date = str(payload.get("target_date") or target_date)
|
||||
member_highs = payload.get("member_highs")
|
||||
if member_highs is None and isinstance(payload.get("member_hourly"), dict):
|
||||
member_highs = build_city_local_daily_highs_from_hourly(
|
||||
payload["member_hourly"],
|
||||
payload.get("utc_times") or payload.get("times") or [],
|
||||
int(timezone_offset_seconds or payload.get("timezone_offset_seconds") or 0),
|
||||
payload_target_date,
|
||||
)
|
||||
if member_highs is None and isinstance(payload.get("buckets"), list):
|
||||
normalized = dict(payload)
|
||||
normalized.setdefault("source", WEATHERNEXT2_SOURCE)
|
||||
normalized.setdefault("provider", WEATHERNEXT2_PROVIDER)
|
||||
normalized.setdefault("city", city)
|
||||
normalized.setdefault("target_date", payload_target_date)
|
||||
normalized.setdefault("temp_symbol", temp_symbol)
|
||||
normalized.setdefault(
|
||||
"gcs_zarr_uri",
|
||||
os.getenv("WEATHERNEXT2_GCS_ZARR_URI", WEATHERNEXT2_GCS_ZARR_URI),
|
||||
)
|
||||
return normalized
|
||||
if member_highs is None:
|
||||
return None
|
||||
return build_weathernext2_city_probability(
|
||||
city=city,
|
||||
member_highs=member_highs,
|
||||
temp_symbol=temp_symbol,
|
||||
target_date=payload_target_date,
|
||||
source_run=payload.get("source_run"),
|
||||
generated_at=payload.get("generated_at"),
|
||||
)
|
||||
|
||||
def _weathernext2_from_artifact(
|
||||
self,
|
||||
*,
|
||||
city: str,
|
||||
use_fahrenheit: bool,
|
||||
target_date: str,
|
||||
timezone_offset_seconds: Optional[int],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
artifact = _load_artifact_payload(_weathernext2_artifact_path(), city)
|
||||
if not artifact:
|
||||
return None
|
||||
payload = self._weathernext2_normalize_prepared_payload(
|
||||
payload=artifact,
|
||||
city=city,
|
||||
use_fahrenheit=use_fahrenheit,
|
||||
target_date=target_date,
|
||||
timezone_offset_seconds=timezone_offset_seconds,
|
||||
)
|
||||
if payload is None:
|
||||
return None
|
||||
try:
|
||||
from src.analysis.weathernext2_calibration import apply_quantile_calibration_to_payload
|
||||
|
||||
return apply_quantile_calibration_to_payload(
|
||||
payload,
|
||||
model_dir=_weathernext2_model_dir(),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("WeatherNext2 calibration apply failed city={}: {}", city, exc)
|
||||
return payload
|
||||
|
||||
def fetch_weathernext2_probability(
|
||||
self,
|
||||
city: str,
|
||||
lat: float,
|
||||
lon: float,
|
||||
*,
|
||||
use_fahrenheit: bool = False,
|
||||
target_date: Optional[str] = None,
|
||||
timezone_offset_seconds: Optional[int] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not self._weathernext2_enabled():
|
||||
return None
|
||||
if lat is None or lon is None:
|
||||
return None
|
||||
|
||||
resolved_target_date = self._weathernext2_target_date(
|
||||
timezone_offset_seconds,
|
||||
target_date,
|
||||
)
|
||||
cache_key = self._weathernext2_cache_key(
|
||||
city,
|
||||
lat,
|
||||
lon,
|
||||
use_fahrenheit,
|
||||
resolved_target_date,
|
||||
)
|
||||
cache, lock, ttl_sec = self._weathernext2_cache_state()
|
||||
now_ts = time.time()
|
||||
with lock:
|
||||
cached = cache.get(cache_key)
|
||||
cached_data = cached.get("data") if isinstance(cached, dict) else None
|
||||
if isinstance(cached_data, dict) and now_ts - float(cached.get("t", 0)) < ttl_sec:
|
||||
return dict(cached_data)
|
||||
|
||||
payload = self._weathernext2_from_artifact(
|
||||
city=city,
|
||||
use_fahrenheit=use_fahrenheit,
|
||||
target_date=resolved_target_date,
|
||||
timezone_offset_seconds=timezone_offset_seconds,
|
||||
)
|
||||
if payload is None:
|
||||
payload = self._weathernext2_from_fixture(
|
||||
city=city,
|
||||
use_fahrenheit=use_fahrenheit,
|
||||
target_date=resolved_target_date,
|
||||
timezone_offset_seconds=timezone_offset_seconds,
|
||||
)
|
||||
if payload is None:
|
||||
backend = str(os.getenv("WEATHERNEXT2_BACKEND", "") or "").strip().lower()
|
||||
if backend:
|
||||
logger.warning(
|
||||
"WeatherNext2 backend={} configured but live fetcher is not installed; use fixture/prepared payload first",
|
||||
backend,
|
||||
)
|
||||
return None
|
||||
|
||||
with lock:
|
||||
cache[cache_key] = {"t": now_ts, "data": dict(payload)}
|
||||
return payload
|
||||
Reference in New Issue
Block a user