gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add LightGBM daily high forecasting pipeline

Browse Source
This commit is contained in:
2569718930@qq.com
2026-03-29 23:29:17 +08:00
parent 4a2db4727b
commit 0cc4f4f0d3
25 changed files with 3038 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/analysis/probability_snapshot_archive.py
+12
View File
@@ -186,6 +186,11 @@ def append_probability_snapshot(
ens_data: Optional[Dict[str, Any]],
current_forecasts: Optional[Dict[str, Any]],
max_so_far: Optional[float],
current_temp: Optional[float] = None,
humidity: Optional[float] = None,
wind_speed_kt: Optional[float] = None,
visibility_mi: Optional[float] = None,
local_hour: Optional[float] = None,
peak_status: Optional[str],
probabilities: Optional[List[Dict[str, Any]]],
shadow_probabilities: Optional[List[Dict[str, Any]]],
@@ -227,6 +232,13 @@ def append_probability_snapshot(
if _sf(value) is not None
},
"max_so_far": _sf(max_so_far),
"observation": {
"current_temp": _sf(current_temp),
"humidity": _sf(humidity),
"wind_speed_kt": _sf(wind_speed_kt),
"visibility_mi": _sf(visibility_mi),
"local_hour": _sf(local_hour),
},
"peak_status": peak_status,
"prob_snapshot": _compact_snapshot(probabilities),
"shadow_prob_snapshot": _compact_snapshot(shadow_probabilities),
src/analysis/trend_engine.py
+38
View File
@@ -23,6 +23,7 @@ from src.analysis.probability_snapshot_archive import append_probability_snapsho
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
from src.data_collection.city_registry import CITY_REGISTRY
from src.data_collection.city_risk_profiles import get_city_risk_profile
from src.models.lgbm_daily_high import predict_lgbm_daily_high
SETTLEMENT_SOURCE_LABELS = {
"metar": "METAR",
@@ -418,6 +419,38 @@ def analyze_weather_trend(
else:
peak_status = "before"
if city_name and current_forecasts and deb_prediction is not None:
lgbm_prediction, _ = predict_lgbm_daily_high(
city_name=city_name,
current_forecasts=current_forecasts,
deb_prediction=deb_prediction,
current_temp=cur_temp,
max_so_far=max_so_far,
humidity=_sf(primary_current.get("humidity")),
wind_speed_kt=_sf(primary_current.get("wind_speed_kt")),
visibility_mi=_sf(primary_current.get("visibility_mi")),
local_hour=local_hour,
local_date=local_date_str,
peak_status=peak_status,
)
if lgbm_prediction is not None:
current_forecasts["LGBM"] = lgbm_prediction
blended_high, weight_info = calculate_dynamic_weights(
city_name, current_forecasts
)
if blended_high is not None:
deb_prediction = blended_high
deb_weights = weight_info
_deb_to_save = blended_high
if insights and "DEB 融合预测" in insights[0]:
insights[0] = (
f"🧬 <b>DEB 融合预测</b>：<b>{blended_high}{temp_symbol}</b> ({weight_info})"
)
if ai_features and "DEB系统已通过历史偏差矫正算出期待点是" in ai_features[0]:
ai_features[0] = (
f"🧬 DEB系统已通过历史偏差矫正算出期待点是: {blended_high}{temp_symbol}。"
)
if trend_direction == "stagnant":
if peak_status == "before":
trend_desc = (
@@ -871,6 +904,11 @@ def analyze_weather_trend(
ens_data=ens_data,
current_forecasts=current_forecasts,
max_so_far=max_so_far,
current_temp=cur_temp,
humidity=_sf(primary_current.get("humidity")),
wind_speed_kt=_sf(primary_current.get("wind_speed_kt")),
visibility_mi=_sf(primary_current.get("visibility_mi")),
local_hour=local_hour_frac,
peak_status=peak_status,
probabilities=_prob_list,
shadow_probabilities=_shadow_prob_list,
src/data_collection/city_registry.py
+97 -43
View File
@@ -93,44 +93,6 @@ CITY_REGISTRY = {
"distance_km": 2.0,
"warning": "海风与地形共同作用，午后对流触发后温度回落可能偏快。",
},
"shek kong": {
"name": "Shek Kong",
"lat": 22.4366,
"lon": 114.0800,
"icao": "VHSK",
"settlement_source": "hko",
"settlement_station_code": "SEK",
"settlement_station_label": "Shek Kong",
"settlement_station_candidates": ["Shek Kong"],
"disable_aviationweather": True,
"tz_offset": 28800,
"use_fahrenheit": False,
"is_major": False,
"risk_level": "medium",
"risk_emoji": "🟡",
"airport_name": "石岗机场",
"distance_km": 0.8,
"warning": "HKO 结算取石岗站分钟级观测，机场报文与 HKO 站点最高温不可直接混用。",
},
"lau fau shan": {
"name": "Lau Fau Shan",
"lat": 22.4674,
"lon": 113.9841,
"icao": "LFS",
"settlement_source": "hko",
"settlement_station_code": "LFS",
"settlement_station_label": "Lau Fau Shan",
"settlement_station_candidates": ["Lau Fau Shan"],
"disable_aviationweather": True,
"tz_offset": 28800,
"use_fahrenheit": False,
"is_major": False,
"risk_level": "medium",
"risk_emoji": "🟡",
"airport_name": "流浮山站",
"distance_km": 0.6,
"warning": "HKO 结算取流浮山站分钟级观测；该站不是机场 METAR，不能与 VHHH/VHSK 报文混用。",
},
"taipei": {
"name": "Taipei",
"lat": 25.0777,
@@ -260,6 +222,90 @@ CITY_REGISTRY = {
"distance_km": 14.5,
"warning": "东河水汽可能在春季产生温差。",
},
"los angeles": {
"name": "Los Angeles",
"lat": 33.9416,
"lon": -118.4085,
"icao": "KLAX",
"tz_offset": -28800,
"use_fahrenheit": True,
"is_major": True,
"risk_level": "medium",
"risk_emoji": "🟡",
"airport_name": "Los Angeles International Airport",
"distance_km": 29.0,
"warning": "海风与沿海层云对午后升温影响明显，LAX 口径常明显低于内陆城区。",
},
"san francisco": {
"name": "San Francisco",
"lat": 37.6213,
"lon": -122.3790,
"icao": "KSFO",
"tz_offset": -28800,
"use_fahrenheit": True,
"is_major": True,
"risk_level": "medium",
"risk_emoji": "🟡",
"airport_name": "San Francisco International Airport",
"distance_km": 20.0,
"warning": "海湾冷空气和平流雾常压制机场升温，SFO 与市区体感差异可快速放大。",
},
"aurora": {
"name": "Aurora",
"lat": 39.7017,
"lon": -104.7518,
"icao": "KBKF",
"tz_offset": -25200,
"use_fahrenheit": True,
"is_major": False,
"risk_level": "medium",
"risk_emoji": "🟡",
"airport_name": "Buckley Space Force Base",
"distance_km": 3.5,
"warning": "丹佛高原地形下日照与下沉增温切换快，Buckley 与 Denver 核心区午后峰值常有明显错位。",
},
"austin": {
"name": "Austin",
"lat": 30.1945,
"lon": -97.6699,
"icao": "KAUS",
"tz_offset": -21600,
"use_fahrenheit": True,
"is_major": True,
"risk_level": "medium",
"risk_emoji": "🟡",
"airport_name": "Austin-Bergstrom International Airport",
"distance_km": 12.0,
"warning": "德州中部午后混合层增强快，干热与局地对流会让峰值窗口前后偏差放大。",
},
"houston": {
"name": "Houston",
"lat": 29.6454,
"lon": -95.2789,
"icao": "KHOU",
"tz_offset": -21600,
"use_fahrenheit": True,
"is_major": True,
"risk_level": "medium",
"risk_emoji": "🟡",
"airport_name": "William P. Hobby Airport",
"distance_km": 12.0,
"warning": "墨西哥湾水汽与海风回流显著，湿热条件下机场白天升温节奏容易偏慢。",
},
"mexico city": {
"name": "Mexico City",
"lat": 19.4363,
"lon": -99.0721,
"icao": "MMMX",
"tz_offset": -21600,
"use_fahrenheit": False,
"is_major": True,
"risk_level": "high",
"risk_emoji": "🔴",
"airport_name": "Benito Juárez International Airport",
"distance_km": 6.5,
"warning": "高海拔盆地城市，辐射增温与午后对流并存，峰值前后 1-2°C 偏差较常见。",
},
"chicago": {
"name": "Chicago",
"lat": 41.9742,
@@ -497,11 +543,15 @@ ALIASES = {
# English shortcuts
"ank": "ankara", "ist": "istanbul", "ltfm": "istanbul", "lon": "london", "par": "paris",
"nyc": "new york", "ny": "new york", "chi": "chicago",
"la": "los angeles", "lax": "los angeles", "losangeles": "los angeles",
"sf": "san francisco", "sfo": "san francisco", "sanfrancisco": "san francisco",
"aur": "aurora", "denver": "aurora", "buckley": "aurora", "kbkf": "aurora",
"aus": "austin", "kaus": "austin",
"hou": "houston", "khou": "houston", "hobby": "houston",
"cdmx": "mexico city", "mexicocity": "mexico city", "mmmx": "mexico city",
"dal": "dallas", "mia": "miami", "atl": "atlanta",
"sea": "seattle", "tor": "toronto", "sel": "seoul",
"seo": "seoul", "hkg": "hong kong", "hk": "hong kong",
"vhsk": "shek kong", "shekkong": "shek kong",
"lfs": "lau fau shan", "laufaushan": "lau fau shan",
"tpe": "taipei", "tp": "taipei", "taipei": "taipei",
"sha": "shanghai", "sh": "shanghai", "sin": "singapore",
"sg": "singapore", "tok": "tokyo", "tyo": "tokyo",
@@ -517,6 +567,13 @@ ALIASES = {
"伦敦": "london",
"巴黎": "paris",
"纽约": "new york",
"洛杉矶": "los angeles",
"旧金山": "san francisco",
"奥罗拉": "aurora",
"丹佛": "aurora",
"奥斯汀": "austin",
"休斯顿": "houston",
"墨西哥城": "mexico city",
"芝加哥": "chicago",
"达拉斯": "dallas",
"迈阿密": "miami",
@@ -525,9 +582,6 @@ ALIASES = {
"多伦多": "toronto",
"首尔": "seoul",
"香港": "hong kong",
"石岗": "shek kong",
"石崗": "shek kong",
"流浮山": "lau fau shan",
"台北": "taipei",
"臺北": "taipei",
"上海": "shanghai",
src/data_collection/weather_sources.py
+7 -2
View File
@@ -35,11 +35,15 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
"istanbul": ["LTFM", "LTBA", "LTFJ"],
"london": ["EGLL", "EGLC", "EGKK", "EGSS", "EGGW"],
"new york": ["KLGA", "KJFK", "KEWR", "KTEB", "KHPN"],
"los angeles": ["KLAX", "KBUR", "KLGB", "KSNA", "KVNY"],
"san francisco": ["KSFO", "KOAK", "KSJC", "KHAF"],
"aurora": ["KBKF", "KDEN", "KAPA", "KBJC"],
"austin": ["KAUS", "KEDC", "KSAT"],
"houston": ["KHOU", "KIAH", "KSGR", "KCXO"],
"mexico city": ["MMMX", "MMSM", "MMTO"],
"paris": ["LFPG", "LFPO", "LFPB"],
"seoul": ["RKSI", "RKSS"],
"hong kong": ["VHHH", "VMMC", "ZGSZ"],
"shek kong": ["VHSK", "VHHH", "VMMC", "ZGSZ"],
"lau fau shan": ["VHHH", "VMMC", "ZGSZ"],
"taipei": ["RCSS", "RCTP"],
"chengdu": ["ZUUU", "ZUTF"],
"chongqing": ["ZUCK", "ZUPS"],
@@ -81,6 +85,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
"new york's central park",
"portland",
"denver",
"aurora",
"austin",
"san diego",
"detroit",
src/models/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Model adapters and feature builders."""
src/models/lgbm_daily_high.py
+171
View File
@@ -0,0 +1,171 @@
from __future__ import annotations
import json
import os
from typing import Any, Dict, List, Optional, Tuple
from loguru import logger
from src.models.lgbm_features import (
FEATURE_NAMES,
build_runtime_feature_map,
)
_MODEL_CACHE: Dict[str, Any] = {"path": None, "mtime": None, "booster": None}
_SCHEMA_CACHE: Dict[str, Any] = {"path": None, "mtime": None, "schema": None}
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _truthy_env(name: str, default: str = "false") -> bool:
return str(os.getenv(name, default)).strip().lower() in {"1", "true", "yes", "on"}
def lgbm_model_path() -> str:
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
return str(
os.getenv(
"POLYWEATHER_LGBM_MODEL_PATH",
os.path.join(root, "artifacts", "models", "lgbm_daily_high.txt"),
)
).strip()
def lgbm_schema_path() -> str:
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
return str(
os.getenv(
"POLYWEATHER_LGBM_SCHEMA_PATH",
os.path.join(root, "artifacts", "models", "lgbm_daily_high_schema.json"),
)
).strip()
def lgbm_min_history_points() -> int:
try:
return max(1, int(os.getenv("POLYWEATHER_LGBM_MIN_HISTORY_POINTS", "3")))
except Exception:
return 3
def is_lgbm_enabled() -> bool:
return _truthy_env("POLYWEATHER_LGBM_ENABLED", "false")
def _load_schema(schema_path: str) -> Optional[Dict[str, Any]]:
if not schema_path or not os.path.exists(schema_path):
return None
mtime = os.path.getmtime(schema_path)
if (
_SCHEMA_CACHE["schema"] is not None
and _SCHEMA_CACHE["path"] == schema_path
and _SCHEMA_CACHE["mtime"] == mtime
):
return _SCHEMA_CACHE["schema"]
with open(schema_path, "r", encoding="utf-8") as fh:
data = json.load(fh)
if not isinstance(data, dict):
return None
_SCHEMA_CACHE.update({"path": schema_path, "mtime": mtime, "schema": data})
return data
def _load_booster(model_path: str):
if not model_path or not os.path.exists(model_path):
return None
mtime = os.path.getmtime(model_path)
if (
_MODEL_CACHE["booster"] is not None
and _MODEL_CACHE["path"] == model_path
and _MODEL_CACHE["mtime"] == mtime
):
return _MODEL_CACHE["booster"]
try:
import lightgbm as lgb
except Exception as exc:
logger.warning(f"LGBM runtime dependency missing: {exc}")
return None
booster = lgb.Booster(model_file=model_path)
_MODEL_CACHE.update({"path": model_path, "mtime": mtime, "booster": booster})
return booster
def _vector_from_features(
feature_map: Dict[str, Optional[float]],
schema: Optional[Dict[str, Any]],
) -> List[float]:
feature_names = schema.get("feature_names") if isinstance(schema, dict) else None
ordered_names = feature_names if isinstance(feature_names, list) and feature_names else FEATURE_NAMES
vector: List[float] = []
for name in ordered_names:
value = feature_map.get(str(name))
vector.append(float("nan") if value is None else float(value))
return vector
def predict_lgbm_daily_high(
*,
city_name: str,
current_forecasts: Dict[str, Any],
deb_prediction: Optional[float],
current_temp: Optional[float],
max_so_far: Optional[float],
humidity: Optional[float],
wind_speed_kt: Optional[float],
visibility_mi: Optional[float],
local_hour: int,
local_date: str,
peak_status: str,
history_data: Optional[Dict[str, Any]] = None,
) -> Tuple[Optional[float], Dict[str, Any]]:
if not is_lgbm_enabled():
return None, {"reason": "disabled"}
schema = _load_schema(lgbm_schema_path())
booster = _load_booster(lgbm_model_path())
if schema is None or booster is None:
return None, {"reason": "artifact_missing"}
feature_map, meta = build_runtime_feature_map(
city_name=city_name,
current_forecasts=current_forecasts,
deb_prediction=deb_prediction,
current_temp=current_temp,
max_so_far=max_so_far,
humidity=humidity,
wind_speed_kt=wind_speed_kt,
visibility_mi=visibility_mi,
local_hour=local_hour,
local_date=local_date,
peak_status=peak_status,
history_data=history_data,
)
if not feature_map:
return None, meta
if int(meta.get("history_count") or 0) < lgbm_min_history_points():
return None, {
"reason": "insufficient_history",
"history_count": int(meta.get("history_count") or 0),
}
try:
vector = _vector_from_features(feature_map, schema)
prediction = booster.predict([vector], num_iteration=booster.best_iteration)
value = _sf(prediction[0] if prediction is not None else None)
if value is None:
return None, {"reason": "empty_prediction"}
return round(float(value), 1), {
"reason": "ok",
"history_count": int(meta.get("history_count") or 0),
}
except Exception as exc:
logger.warning(f"LGBM prediction failed for {city_name}: {exc}")
return None, {"reason": "predict_failed", "error": str(exc)}
src/models/lgbm_features.py
+356
View File
@@ -0,0 +1,356 @@
from __future__ import annotations
import json
import os
from datetime import datetime
from statistics import mean
from typing import Any, Dict, List, Optional, Tuple
from src.analysis.deb_algorithm import load_history
from src.data_collection.city_registry import ALIASES
BASE_MODEL_COLUMNS: List[Tuple[str, str]] = [
("Open-Meteo", "open_meteo"),
("ECMWF", "ecmwf"),
("GFS", "gfs"),
("GEM", "gem"),
("JMA", "jma"),
("ICON", "icon"),
("MGM", "mgm"),
("NWS", "nws"),
]
FEATURE_NAMES: List[str] = [
"actual_high_lag_1",
"actual_high_lag_2",
"actual_high_lag_3",
"actual_high_lag_7",
"actual_high_mean_7",
"actual_high_mean_14",
"actual_high_trend_3",
*[column for _, column in BASE_MODEL_COLUMNS],
"deb_prediction",
"model_median",
"model_spread",
"current_temp",
"max_so_far",
"humidity",
"wind_speed_kt",
"visibility_mi",
"local_hour",
"month",
"weekday",
"peak_status_code",
]
PEAK_STATUS_CODES = {
"before": 0.0,
"in_window": 1.0,
"past": 2.0,
}
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _parse_date(value: Any) -> Optional[datetime]:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.strptime(text, "%Y-%m-%d")
except Exception:
return None
def _parse_timestamp(value: Any) -> Optional[datetime]:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.fromisoformat(text.replace("Z", "+00:00"))
except Exception:
return None
def _history_file_path() -> str:
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
return os.path.join(root, "data", "daily_records.json")
def _snapshot_archive_path() -> str:
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
return os.path.join(root, "data", "probability_training_snapshots.jsonl")
def _normalized_city_key(city_name: str) -> str:
city_key = str(city_name or "").strip().lower()
return ALIASES.get(city_key, city_key)
def _safe_mean(values: List[Optional[float]]) -> Optional[float]:
valid = [float(v) for v in values if v is not None]
if not valid:
return None
return float(mean(valid))
def _peak_status_code(value: Any) -> Optional[float]:
status = str(value or "").strip().lower()
if not status:
return None
return PEAK_STATUS_CODES.get(status, -1.0)
def _compute_model_summary(features: Dict[str, Optional[float]]) -> Tuple[Optional[float], Optional[float]]:
values = [
features.get(column)
for _, column in BASE_MODEL_COLUMNS
if features.get(column) is not None
]
values = [float(v) for v in values if v is not None]
if not values:
return None, None
ordered = sorted(values)
median_value = ordered[len(ordered) // 2]
spread_value = ordered[-1] - ordered[0] if len(ordered) >= 2 else 0.0
return float(median_value), float(spread_value)
def load_snapshot_index(archive_path: Optional[str] = None) -> Dict[Tuple[str, str], Dict[str, Any]]:
path = archive_path or _snapshot_archive_path()
if not os.path.exists(path):
return {}
latest_rows: Dict[Tuple[str, str], Dict[str, Any]] = {}
with open(path, "r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except Exception:
continue
if not isinstance(row, dict):
continue
city = _normalized_city_key(str(row.get("city") or ""))
date_str = str(row.get("date") or "").strip()
if not city or not date_str:
continue
key = (city, date_str)
current_best = latest_rows.get(key)
if current_best is None or str(row.get("timestamp") or "") >= str(
current_best.get("timestamp") or ""
):
latest_rows[key] = row
return latest_rows
def _extract_history_rows(
history_data: Dict[str, Any],
city_name: str,
exclude_date: Optional[str] = None,
) -> List[Tuple[str, float]]:
city_key = _normalized_city_key(city_name)
city_rows = history_data.get(city_key) if isinstance(history_data, dict) else None
if not isinstance(city_rows, dict):
return []
rows: List[Tuple[str, float]] = []
for date_str, record in city_rows.items():
if exclude_date and str(date_str) >= str(exclude_date):
continue
if not isinstance(record, dict):
continue
actual = _sf(record.get("actual_high"))
if actual is None:
continue
rows.append((str(date_str), float(actual)))
rows.sort(key=lambda item: item[0])
return rows
def _lag(values: List[float], distance: int) -> Optional[float]:
if len(values) < distance:
return None
return float(values[-distance])
def build_runtime_feature_map(
*,
city_name: str,
current_forecasts: Dict[str, Any],
deb_prediction: Optional[float],
current_temp: Optional[float],
max_so_far: Optional[float],
humidity: Optional[float],
wind_speed_kt: Optional[float],
visibility_mi: Optional[float],
local_hour: int,
local_date: str,
peak_status: str,
history_data: Optional[Dict[str, Any]] = None,
) -> Tuple[Optional[Dict[str, Optional[float]]], Dict[str, Any]]:
data = history_data if isinstance(history_data, dict) else load_history(_history_file_path())
history_rows = _extract_history_rows(data, city_name, exclude_date=local_date)
history_values = [value for _, value in history_rows]
if not history_values:
return None, {"reason": "no_history", "history_count": 0}
date_obj = _parse_date(local_date)
if date_obj is None:
return None, {"reason": "invalid_date", "history_count": len(history_values)}
features: Dict[str, Optional[float]] = {
"actual_high_lag_1": _lag(history_values, 1),
"actual_high_lag_2": _lag(history_values, 2),
"actual_high_lag_3": _lag(history_values, 3),
"actual_high_lag_7": _lag(history_values, 7),
"actual_high_mean_7": _safe_mean(history_values[-7:]),
"actual_high_mean_14": _safe_mean(history_values[-14:]),
"actual_high_trend_3": (
history_values[-1] - history_values[-3] if len(history_values) >= 3 else None
),
"deb_prediction": _sf(deb_prediction),
"current_temp": _sf(current_temp),
"max_so_far": _sf(max_so_far),
"humidity": _sf(humidity),
"wind_speed_kt": _sf(wind_speed_kt),
"visibility_mi": _sf(visibility_mi),
"local_hour": float(local_hour),
"month": float(date_obj.month),
"weekday": float(date_obj.weekday()),
"peak_status_code": _peak_status_code(peak_status),
}
for model_name, column in BASE_MODEL_COLUMNS:
features[column] = _sf(current_forecasts.get(model_name))
model_median, model_spread = _compute_model_summary(features)
features["model_median"] = model_median
features["model_spread"] = model_spread
return features, {
"reason": "ok",
"history_count": len(history_values),
}
def _features_to_vector(features: Dict[str, Optional[float]], feature_names: Optional[List[str]] = None) -> List[float]:
ordered_names = feature_names or FEATURE_NAMES
vector: List[float] = []
for name in ordered_names:
value = features.get(name)
vector.append(float("nan") if value is None else float(value))
return vector
def build_training_samples(
history_data: Optional[Dict[str, Any]] = None,
snapshot_index: Optional[Dict[Tuple[str, str], Dict[str, Any]]] = None,
) -> List[Dict[str, Any]]:
data = history_data if isinstance(history_data, dict) else load_history(_history_file_path())
snapshots = snapshot_index if isinstance(snapshot_index, dict) else load_snapshot_index()
samples: List[Dict[str, Any]] = []
for city_name, city_records in (data or {}).items():
if not isinstance(city_records, dict):
continue
ordered_dates = sorted(city_records.keys())
for date_str in ordered_dates:
record = city_records.get(date_str)
if not isinstance(record, dict):
continue
target = _sf(record.get("actual_high"))
forecasts = record.get("forecasts") if isinstance(record.get("forecasts"), dict) else {}
if target is None or not forecasts:
continue
feature_map, meta = build_runtime_feature_map(
city_name=city_name,
current_forecasts=forecasts,
deb_prediction=_sf(record.get("deb_prediction")),
current_temp=None,
max_so_far=None,
humidity=None,
wind_speed_kt=None,
visibility_mi=None,
local_hour=12,
local_date=str(date_str),
peak_status="before",
history_data=data,
)
if not feature_map:
continue
snapshot = snapshots.get((_normalized_city_key(city_name), str(date_str))) or {}
observation = snapshot.get("observation") if isinstance(snapshot.get("observation"), dict) else {}
feature_map["max_so_far"] = _sf(snapshot.get("max_so_far"))
feature_map["current_temp"] = _sf(observation.get("current_temp"))
feature_map["humidity"] = _sf(observation.get("humidity"))
feature_map["wind_speed_kt"] = _sf(observation.get("wind_speed_kt"))
feature_map["visibility_mi"] = _sf(observation.get("visibility_mi"))
local_hour = _sf(observation.get("local_hour"))
if local_hour is not None:
feature_map["local_hour"] = local_hour
else:
timestamp = _parse_timestamp(snapshot.get("timestamp"))
if timestamp is not None:
feature_map["local_hour"] = float(timestamp.hour)
peak_code = _peak_status_code(snapshot.get("peak_status"))
if peak_code is not None:
feature_map["peak_status_code"] = peak_code
model_median, model_spread = _compute_model_summary(feature_map)
feature_map["model_median"] = model_median
feature_map["model_spread"] = model_spread
samples.append(
{
"city": _normalized_city_key(city_name),
"date": str(date_str),
"target": float(target),
"features": feature_map,
"vector": _features_to_vector(feature_map),
"history_count": int(meta.get("history_count") or 0),
"deb_prediction": _sf(record.get("deb_prediction")),
"forecasts": {
key: _sf(value)
for key, value in forecasts.items()
if _sf(value) is not None
},
}
)
samples.sort(key=lambda row: (row["date"], row["city"]))
return samples
def schema_payload(
*,
model_path: str,
sample_count: int,
train_count: int,
validation_count: int,
metrics: Dict[str, Any],
) -> Dict[str, Any]:
return {
"model_type": "LightGBMRegressor",
"target": "actual_high",
"horizon": "D0",
"feature_names": FEATURE_NAMES,
"base_model_columns": [column for _, column in BASE_MODEL_COLUMNS],
"model_path": model_path,
"sample_count": sample_count,
"train_count": train_count,
"validation_count": validation_count,
"metrics": metrics,
"generated_at": datetime.utcnow().isoformat() + "Z",
}