Add HKO-backed Shek Kong settlement and history support

This commit is contained in:
2569718930@qq.com
2026-03-27 20:11:08 +08:00
parent 1ab7db065c
commit 2007ded86d
7 changed files with 312 additions and 98 deletions
@@ -58,6 +58,23 @@ const CITY_SPECIFIC_SOURCES: Record<string, OfficialSourceLink[]> = {
kind: "metar",
},
],
"shek kong": [
{
label: "香港天文台",
href: "https://www.hko.gov.hk/en/index.html",
kind: "agency",
},
{
label: "HKO 区域天气数据",
href: "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather/latest_1min_temperature.csv",
kind: "agency",
},
{
label: "VHSK Timeseries",
href: "https://www.weather.gov/wrh/timeseries?site=VHSK",
kind: "metar",
},
],
taipei: [
{
label: "NOAA RCTP Timeseries",
+160 -29
View File
@@ -150,18 +150,44 @@ def _parse_metar_row_time(row):
return None
def reconcile_recent_actual_highs(city_name: str, lookback_days: int = 7):
def _get_history_file_path():
project_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
return os.path.join(project_root, "data", "daily_records.json")
def _resolve_city_history_context(city_name: str):
from src.data_collection.city_registry import CITY_REGISTRY, ALIASES
city_key = str(city_name or "").strip().lower()
city_key = ALIASES.get(city_key, city_key)
city_meta = CITY_REGISTRY.get(city_key)
if not isinstance(city_meta, dict):
return None, None
return city_key, city_meta
def _parse_hko_ryes_max_temp(payload):
if not isinstance(payload, dict):
return None
for key, value in payload.items():
if not str(key or "").endswith("MaxTemp"):
continue
parsed = _sf(value)
if parsed is not None:
return parsed
return None
def _reconcile_recent_metar_actual_highs(city_name: str, lookback_days: int = 7):
"""
Reconcile recent `actual_high` values using historical METAR data from
aviationweather.gov to fix stale/wrong daily records.
"""
try:
from src.data_collection.city_registry import CITY_REGISTRY, ALIASES
city_key = str(city_name or "").strip().lower()
city_key = ALIASES.get(city_key, city_key)
city_meta = CITY_REGISTRY.get(city_key)
if not isinstance(city_meta, dict):
city_key, city_meta = _resolve_city_history_context(city_name)
if not city_key or not isinstance(city_meta, dict):
return {"ok": False, "reason": "unknown_city", "updated": 0}
icao = str(city_meta.get("icao") or "").strip().upper()
@@ -171,10 +197,7 @@ def reconcile_recent_actual_highs(city_name: str, lookback_days: int = 7):
tz_offset = int(city_meta.get("tz_offset") or 0)
use_fahrenheit = bool(city_meta.get("use_fahrenheit"))
project_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
history_file = os.path.join(project_root, "data", "daily_records.json")
history_file = _get_history_file_path()
data = load_history(history_file)
city_data = data.get(city_key) or {}
if not isinstance(city_data, dict) or not city_data:
@@ -257,42 +280,145 @@ def reconcile_recent_actual_highs(city_name: str, lookback_days: int = 7):
"scanned_dates": len(target_dates),
"metar_rows": len(rows),
"icao": icao,
"source": "metar",
}
except Exception as e:
return {"ok": False, "reason": str(e), "updated": 0}
def bootstrap_recent_daily_history_if_missing(city_name: str, lookback_days: int = 14):
def _reconcile_recent_hko_actual_highs(city_name: str, lookback_days: int = 14):
"""
For METAR-settled cities added after launch, ensure recent daily_records rows
exist so the history page can show recent actual highs without requiring a
one-off backfill script.
Reconcile recent `actual_high` values using HKO's RYES daily summary endpoint.
This covers HKO-settled cities such as Hong Kong and Shek Kong.
"""
try:
from src.data_collection.city_registry import CITY_REGISTRY, ALIASES
city_key, city_meta = _resolve_city_history_context(city_name)
if not city_key or not isinstance(city_meta, dict):
return {"ok": False, "reason": "unknown_city", "updated": 0}
city_key = str(city_name or "").strip().lower()
city_key = ALIASES.get(city_key, city_key)
city_meta = CITY_REGISTRY.get(city_key)
if not isinstance(city_meta, dict):
station_code = str(city_meta.get("settlement_station_code") or "").strip().upper()
if not station_code:
return {"ok": False, "reason": "missing_station_code", "updated": 0}
tz_offset = int(city_meta.get("tz_offset") or 0)
use_fahrenheit = bool(city_meta.get("use_fahrenheit"))
history_file = _get_history_file_path()
data = load_history(history_file)
city_data = data.get(city_key) or {}
if not isinstance(city_data, dict) or not city_data:
return {"ok": True, "reason": "no_city_history", "updated": 0}
local_now = datetime.utcnow() + timedelta(seconds=tz_offset)
local_today = local_now.strftime("%Y-%m-%d")
cutoff = (local_now - timedelta(days=max(lookback_days, 1) + 1)).strftime(
"%Y-%m-%d"
)
target_dates = sorted(
d for d in city_data.keys() if isinstance(d, str) and cutoff <= d < local_today
)
if not target_dates:
return {"ok": True, "reason": "no_target_dates", "updated": 0}
updated = 0
scanned_dates = 0
base_url = "https://data.weather.gov.hk/weatherAPI/opendata/opendata.php"
for date_str in target_dates:
date_token = date_str.replace("-", "")
try:
resp = requests.get(
base_url,
params={
"dataType": "RYES",
"date": date_token,
"lang": "en",
"station": station_code,
},
timeout=12,
)
resp.raise_for_status()
payload = resp.json() if resp.content else {}
except Exception:
continue
scanned_dates += 1
max_temp_c = _parse_hko_ryes_max_temp(payload)
if max_temp_c is None:
continue
corrected = (
round(max_temp_c * 9 / 5 + 32, 1)
if use_fahrenheit
else round(max_temp_c, 1)
)
rec = city_data.get(date_str) or {}
old = rec.get("actual_high")
try:
old_val = float(old) if old is not None else None
except Exception:
old_val = None
if old_val is None or abs(old_val - corrected) >= 0.1:
rec["actual_high"] = corrected
city_data[date_str] = rec
updated += 1
if updated > 0:
data[city_key] = city_data
save_history(history_file, data)
return {
"ok": True,
"updated": updated,
"scanned_dates": scanned_dates,
"station_code": station_code,
"source": "hko",
}
except Exception as e:
return {"ok": False, "reason": str(e), "updated": 0}
def reconcile_recent_actual_highs(city_name: str, lookback_days: int = 7):
"""
Reconcile recent `actual_high` values using the city's official settlement source.
"""
city_key, city_meta = _resolve_city_history_context(city_name)
if not city_key or not isinstance(city_meta, dict):
return {"ok": False, "reason": "unknown_city", "updated": 0}
settlement_source = str(city_meta.get("settlement_source") or "metar").strip().lower()
if settlement_source == "hko":
return _reconcile_recent_hko_actual_highs(city_key, lookback_days=lookback_days)
return _reconcile_recent_metar_actual_highs(city_key, lookback_days=lookback_days)
def bootstrap_recent_daily_history_if_missing(city_name: str, lookback_days: int = 14):
"""
For supported settlement cities added after launch, ensure recent daily_records
rows exist so the history page can show recent actual highs without requiring
a one-off backfill script.
"""
try:
city_key, city_meta = _resolve_city_history_context(city_name)
if not city_key or not isinstance(city_meta, dict):
return {"ok": False, "reason": "unknown_city", "seeded": 0, "updated": 0}
settlement_source = str(city_meta.get("settlement_source") or "metar").strip().lower()
if settlement_source != "metar":
return {"ok": True, "reason": "non_metar_city", "seeded": 0, "updated": 0}
if settlement_source not in {"metar", "hko"}:
return {"ok": True, "reason": "unsupported_settlement_source", "seeded": 0, "updated": 0}
icao = str(city_meta.get("icao") or "").strip().upper()
if not icao:
station_code = str(city_meta.get("settlement_station_code") or "").strip().upper()
if settlement_source == "metar" and not icao:
return {"ok": False, "reason": "missing_icao", "seeded": 0, "updated": 0}
if settlement_source == "hko" and not station_code:
return {"ok": False, "reason": "missing_station_code", "seeded": 0, "updated": 0}
tz_offset = int(city_meta.get("tz_offset") or 0)
local_now = datetime.utcnow() + timedelta(seconds=tz_offset)
local_today = local_now.date()
project_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
history_file = os.path.join(project_root, "data", "daily_records.json")
history_file = _get_history_file_path()
data = load_history(history_file)
city_rows = data.get(city_key)
if not isinstance(city_rows, dict):
@@ -310,13 +436,18 @@ def bootstrap_recent_daily_history_if_missing(city_name: str, lookback_days: int
save_history(history_file, data)
reconcile_result = reconcile_recent_actual_highs(city_key, lookback_days=lookback_days)
return {
result = {
"ok": True,
"reason": "bootstrapped" if seeded > 0 else "already_seeded",
"seeded": seeded,
"updated": int(reconcile_result.get("updated") or 0),
"icao": icao,
"settlement_source": settlement_source,
}
if icao:
result["icao"] = icao
if station_code:
result["station_code"] = station_code
return result
except Exception as e:
return {"ok": False, "reason": str(e), "seeded": 0, "updated": 0}
+10 -3
View File
@@ -23,14 +23,21 @@ def is_exact_settlement_city(city: str) -> bool:
"""是否为不四舍五入的精确结算城市"""
if not city:
return False
c = str(city).lower().strip()
return c in ["hong kong", "hk", "香港"]
normalized = str(city).lower().strip()
try:
from src.data_collection.city_registry import ALIASES, CITY_REGISTRY
canonical = ALIASES.get(normalized, normalized)
city_meta = CITY_REGISTRY.get(canonical) or {}
return str(city_meta.get("settlement_source") or "").strip().lower() == "hko"
except Exception:
return normalized in ["hong kong", "hk", "香港"]
def apply_city_settlement(city: str, value: Optional[Number]) -> Optional[int]:
"""
根据城市返回最终的结算值:
- 香港: 向下取整 (e.g. 28.9 -> 28)
- HKO 系城市: 向下取整 (e.g. 28.9 -> 28)
- 其他: WU 规则四舍五入
"""
if value is None:
+1 -1
View File
@@ -88,7 +88,7 @@ class DebAnalysisService:
):
lines.extend(
[
f"🔁 已用 METAR 历史回填修正 {int(reconcile_info.get('updated'))} 天实测最高温",
f"🔁 已用官方历史回填修正 {int(reconcile_info.get('updated'))} 天实测最高温",
"",
]
)
+25
View File
@@ -64,6 +64,9 @@ CITY_REGISTRY = {
"lon": 114.1742,
"icao": "VHHH",
"settlement_source": "hko",
"settlement_station_code": "HKO",
"settlement_station_label": "HK Observatory",
"settlement_station_candidates": ["HK Observatory", "Hong Kong Observatory"],
"tz_offset": 28800,
"use_fahrenheit": False,
"is_major": True,
@@ -73,6 +76,25 @@ CITY_REGISTRY = {
"distance_km": 2.0,
"warning": "海风与地形共同作用,午后对流触发后温度回落可能偏快。",
},
"shek kong": {
"name": "Shek Kong",
"lat": 22.4366,
"lon": 114.0800,
"icao": "VHSK",
"settlement_source": "hko",
"settlement_station_code": "SEK",
"settlement_station_label": "Shek Kong",
"settlement_station_candidates": ["Shek Kong"],
"disable_aviationweather": True,
"tz_offset": 28800,
"use_fahrenheit": False,
"is_major": False,
"risk_level": "medium",
"risk_emoji": "🟡",
"airport_name": "石岗机场",
"distance_km": 0.8,
"warning": "HKO 结算取石岗站分钟级观测,机场报文与 HKO 站点最高温不可直接混用。",
},
"taipei": {
"name": "Taipei",
"lat": 25.0777,
@@ -440,6 +462,7 @@ ALIASES = {
"dal": "dallas", "mia": "miami", "atl": "atlanta",
"sea": "seattle", "tor": "toronto", "sel": "seoul",
"seo": "seoul", "hkg": "hong kong", "hk": "hong kong",
"vhsk": "shek kong", "shekkong": "shek kong",
"tpe": "taipei", "tp": "taipei", "taipei": "taipei",
"sha": "shanghai", "sh": "shanghai", "sin": "singapore",
"sg": "singapore", "tok": "tokyo", "tyo": "tokyo",
@@ -462,6 +485,8 @@ ALIASES = {
"多伦多": "toronto",
"首尔": "seoul",
"香港": "hong kong",
"石岗": "shek kong",
"石崗": "shek kong",
"台北": "taipei",
"臺北": "taipei",
"上海": "shanghai",
+59 -45
View File
@@ -147,41 +147,17 @@ class SettlementSourceMixin:
def _update_hko_today_obs(
self,
*,
station_code: str,
obs_iso: Optional[str],
current_temp: Optional[float],
) -> List[Dict[str, Any]]:
if not obs_iso or current_temp is None:
return []
try:
obs_dt = datetime.fromisoformat(str(obs_iso).replace("Z", "+00:00"))
except Exception:
return []
if obs_dt.tzinfo is None:
obs_dt = obs_dt.replace(tzinfo=timezone(timedelta(hours=8)))
local_dt = obs_dt.astimezone(timezone(timedelta(hours=8)))
date_str = local_dt.strftime("%Y-%m-%d")
time_str = local_dt.strftime("%H:%M")
mode = get_state_storage_mode()
if mode not in {STATE_STORAGE_DUAL, STATE_STORAGE_SQLITE}:
return [{"time": time_str, "temp": round(float(current_temp), 1)}]
lock = self._get_settlement_series_lock()
with lock:
_official_intraday_repo.upsert_point(
source_code="hko",
station_code="HKO",
target_date=date_str,
observation_time=time_str,
value=round(float(current_temp), 1),
payload={"time": time_str, "temp": round(float(current_temp), 1)},
)
points = _official_intraday_repo.load_points(
source_code="hko",
station_code="HKO",
target_date=date_str,
)
return self._sort_temp_points(points)
return self._update_official_today_obs(
source_code="hko",
station_code=station_code,
obs_iso=obs_iso,
current_temp=current_temp,
utc_offset_seconds=28800,
)
def _update_official_today_obs(
self,
@@ -226,8 +202,24 @@ class SettlementSourceMixin:
)
return self._sort_temp_points(points)
def fetch_hko_settlement_current(self) -> Optional[Dict[str, Any]]:
cache_key = "hko:hong_kong"
def fetch_hko_settlement_current(
self,
*,
station_code: str = "HKO",
station_name: str = "HK Observatory",
station_candidates: Optional[List[str]] = None,
) -> Optional[Dict[str, Any]]:
normalized_station_code = str(station_code or "HKO").strip() or "HKO"
normalized_station_name = str(station_name or "HK Observatory").strip() or "HK Observatory"
candidate_names = [
str(item).strip()
for item in (station_candidates or [normalized_station_name])
if str(item).strip()
]
if not candidate_names:
candidate_names = [normalized_station_name]
cache_key = f"hko:{normalized_station_code.lower()}"
cached = self._get_settlement_cache(cache_key)
if cached:
return cached
@@ -248,11 +240,10 @@ class SettlementSourceMixin:
humidity_rows = self._csv_rows(humidity_csv.text)
wind_rows = self._csv_rows(wind_csv.text)
station_candidates = ["HK Observatory", "Hong Kong Observatory"]
temp_row = self._pick_station_row(temp_rows, station_candidates)
maxmin_row = self._pick_station_row(maxmin_rows, station_candidates)
humidity_row = self._pick_station_row(humidity_rows, station_candidates)
wind_row = self._pick_station_row(wind_rows, station_candidates)
temp_row = self._pick_station_row(temp_rows, candidate_names)
maxmin_row = self._pick_station_row(maxmin_rows, candidate_names)
humidity_row = self._pick_station_row(humidity_rows, candidate_names)
wind_row = self._pick_station_row(wind_rows, candidate_names)
if not temp_row or not maxmin_row:
return None
@@ -268,6 +259,7 @@ class SettlementSourceMixin:
wind_row.get("10-Minute Mean Wind Direction(Compass points)") if wind_row else None
)
today_obs = self._update_hko_today_obs(
station_code=normalized_station_code,
obs_iso=obs_iso,
current_temp=current_temp,
)
@@ -281,8 +273,8 @@ class SettlementSourceMixin:
payload: Dict[str, Any] = {
"source": "hko",
"source_label": "HKO",
"station_code": "HKO",
"station_name": "HK Observatory",
"station_code": normalized_station_code,
"station_name": normalized_station_name,
"observation_time": obs_iso,
"current": {
"temp": round(current_temp, 1) if current_temp is not None else None,
@@ -299,7 +291,7 @@ class SettlementSourceMixin:
self._set_settlement_cache(cache_key, payload)
return payload
except Exception as exc:
logger.warning(f"HKO settlement fetch failed: {exc}")
logger.warning(f"HKO settlement fetch failed station={normalized_station_code}: {exc}")
return None
def fetch_cwa_taipei_settlement_current(self) -> Optional[Dict[str, Any]]:
@@ -594,10 +586,32 @@ class SettlementSourceMixin:
station_label=str(city_meta.get("settlement_station_label") or "").strip() or None,
icao=str(city_meta.get("icao") or "").strip() or None,
)
if settlement_source == "hko":
raw_candidates = city_meta.get("settlement_station_candidates") or []
if isinstance(raw_candidates, str):
station_candidates = [raw_candidates]
else:
station_candidates = [
str(item).strip()
for item in raw_candidates
if str(item).strip()
]
station_name = (
str(city_meta.get("settlement_station_label") or "").strip()
or (station_candidates[0] if station_candidates else "HK Observatory")
)
station_code = (
str(city_meta.get("settlement_station_code") or "").strip()
or str(city_meta.get("icao") or "").strip()
or "HKO"
)
return self.fetch_hko_settlement_current(
station_code=station_code,
station_name=station_name,
station_candidates=station_candidates,
)
except Exception as exc:
logger.warning(f"Wunderground settlement dispatch failed city={city}: {exc}")
if normalized == "hong kong":
return self.fetch_hko_settlement_current()
logger.warning(f"Settlement source dispatch failed city={city}: {exc}")
if normalized == "taipei":
return self.fetch_noaa_rctp_settlement_current()
return None
+40 -20
View File
@@ -37,6 +37,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
"paris": ["LFPG", "LFPO", "LFPB"],
"seoul": ["RKSI", "RKSS"],
"hong kong": ["VHHH", "VMMC", "ZGSZ"],
"shek kong": ["VHSK", "VHHH", "VMMC", "ZGSZ"],
"taipei": ["RCSS", "RCTP"],
"chengdu": ["ZUUU", "ZUTF"],
"chongqing": ["ZUCK", "ZUPS"],
@@ -429,6 +430,10 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
"hong kong": "Hong Kong",
"hong kong international airport": "Hong Kong",
"香港": "Hong Kong",
"shek kong": "Shek Kong",
"vhsk": "Shek Kong",
"石岗": "Shek Kong",
"石崗": "Shek Kong",
"taipei": "Taipei",
"台北": "Taipei",
"臺北": "Taipei",
@@ -542,14 +547,25 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
self._metar_cache.pop(key, None)
normalized = str(city or "").strip().lower()
with self._settlement_cache_lock:
if normalized == "hong kong":
self._settlement_cache.pop("hko:hong_kong", None)
city_meta = self.CITY_REGISTRY.get(normalized) or {}
settlement_source = str(city_meta.get("settlement_source") or "").strip().lower()
if settlement_source == "hko":
station_code = (
str(city_meta.get("settlement_station_code") or "").strip()
or str(city_meta.get("icao") or "").strip()
or normalized
)
self._settlement_cache.pop(f"hko:{station_code.lower()}", None)
elif normalized == "taipei":
self._settlement_cache.pop("noaa:rctp", None)
def _uses_fahrenheit(self, city_lower: str) -> bool:
return city_lower in self.US_CITIES
def _supports_aviationweather(self, city_lower: str) -> bool:
city_meta = self.CITY_REGISTRY.get(str(city_lower or "").strip().lower(), {}) or {}
return not bool(city_meta.get("disable_aviationweather"))
def _log_temperature_unit(self, city: str, use_fahrenheit: bool) -> None:
unit = "华氏度 (°F)" if use_fahrenheit else "摄氏度 (°C)"
logger.info(f"🌡️ {city} 使用{unit}")
@@ -651,6 +667,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
results = {}
city_lower = city.lower().strip()
use_fahrenheit = self._uses_fahrenheit(city_lower)
supports_aviationweather = self._supports_aviationweather(city_lower)
if force_refresh:
self._evict_city_caches(
@@ -670,12 +687,13 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
results["open-meteo"] = open_meteo
# 获取时区偏移以过滤 METAR
utc_offset = open_meteo.get("utc_offset", 0)
metar_data = self.fetch_metar(
city, use_fahrenheit=use_fahrenheit, utc_offset=utc_offset
)
if metar_data:
results["metar"] = metar_data
if city_lower != "hong kong":
if supports_aviationweather:
metar_data = self.fetch_metar(
city, use_fahrenheit=use_fahrenheit, utc_offset=utc_offset
)
if metar_data:
results["metar"] = metar_data
if supports_aviationweather and city_lower != "hong kong":
taf_data = self.fetch_taf(city, utc_offset=utc_offset)
if taf_data:
results["taf"] = taf_data
@@ -693,14 +711,15 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
fallback_utc_offset = int(
self.CITY_REGISTRY.get(city_lower, {}).get("tz_offset", 0)
)
metar_data = self.fetch_metar(
city,
use_fahrenheit=use_fahrenheit,
utc_offset=fallback_utc_offset,
)
if metar_data:
results["metar"] = metar_data
if city_lower != "hong kong":
if supports_aviationweather:
metar_data = self.fetch_metar(
city,
use_fahrenheit=use_fahrenheit,
utc_offset=fallback_utc_offset,
)
if metar_data:
results["metar"] = metar_data
if supports_aviationweather and city_lower != "hong kong":
taf_data = self.fetch_taf(city, utc_offset=fallback_utc_offset)
if taf_data:
results["taf"] = taf_data
@@ -715,10 +734,11 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
results, city, lat, lon, use_fahrenheit
)
else:
metar_data = self.fetch_metar(city, use_fahrenheit=use_fahrenheit)
if metar_data:
results["metar"] = metar_data
if city_lower != "hong kong":
if supports_aviationweather:
metar_data = self.fetch_metar(city, use_fahrenheit=use_fahrenheit)
if metar_data:
results["metar"] = metar_data
if supports_aviationweather and city_lower != "hong kong":
taf_data = self.fetch_taf(city)
if taf_data:
results["taf"] = taf_data