Files
PolyWeather/src/data_collection/amos_station_sources.py
T
2026-06-05 14:38:58 +08:00

633 lines
24 KiB
Python

"""AMOS (Aerodrome Meteorological Observation System) real-time data source.
Fetches runway-level observations from global.amo.go.kr for Korean airports.
Provides per-runway wind, temperature, pressure, visibility, RVR, cloud data.
"""
from __future__ import annotations
import re
import os
import time
from html import unescape
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from loguru import logger
from src.data_collection.observation_source_gate import run_observation_source
from src.utils.metrics import record_source_call
AMOS_BASE_URL = os.getenv("AMOS_BASE_URL", "").strip() or "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do"
AMOS_AIRPORT_QUERY_KEYS = (
"stnCd",
"icao",
"airport",
"airportCd",
"airPort",
)
AMOS_AIRPORT_CODES: Dict[str, Dict[str, str]] = {
"seoul": {
"icao": "RKSI",
"stn_id": "113",
"label_ko": "인천공항",
"label_en": "Incheon Intl",
},
"busan": {
"icao": "RKPK",
"stn_id": "153",
"label_ko": "김해공항",
"label_en": "Gimhae Intl",
},
}
def _amos_safe_float(value: str | None) -> Optional[float]:
if value is None:
return None
text = str(value).strip()
if not text or text in ("-", "null", ""):
return None
try:
return float(text)
except (ValueError, TypeError):
return None
def _amos_extract_metar_temperature(metar_line: str) -> tuple[Optional[float], Optional[float]]:
"""Extract temperature and dew point from a METAR string like 'RKSI ... 17/08 ...'."""
match = re.search(r"\b(\d{2})/(\d{2})\b", metar_line)
if match:
t = _amos_safe_float(match.group(1))
d = _amos_safe_float(match.group(2))
if t is not None and t > 50:
t = None # unlikely air temp
return t, d
return None, None
def _amos_extract_metar_qnh(metar_line: str) -> Optional[float]:
"""Extract QNH from METAR like 'Q1015'."""
match = re.search(r"\bQ(\d{4})\b", metar_line)
if match:
return _amos_safe_float(match.group(1))
return None
def _amos_extract_observation_time(text: str, icao: str) -> tuple[Optional[str], Optional[str]]:
"""Return AMOS page timestamp as (UTC ISO, KST local display)."""
icao_pattern = re.escape(str(icao or "").strip().upper())
patterns = []
if icao_pattern:
patterns.append(
rf"\({icao_pattern}\)\s*(\d{{4}})년\s*(\d{{1,2}})월\s*(\d{{1,2}})일\s*(\d{{1,2}}):(\d{{2}})\s*KST"
)
patterns.append(
r"(\d{4})년\s*(\d{1,2})월\s*(\d{1,2})일\s*(\d{1,2}):(\d{2})\s*KST"
)
for pattern in patterns:
match = re.search(pattern, text or "", re.I)
if not match:
continue
try:
year, month, day, hour, minute = [int(part) for part in match.groups()[:5]]
local_dt = datetime(
year,
month,
day,
hour,
minute,
tzinfo=timezone(timedelta(hours=9)),
)
utc_dt = local_dt.astimezone(timezone.utc).replace(microsecond=0)
return (
utc_dt.isoformat().replace("+00:00", "Z"),
local_dt.strftime("%Y-%m-%d %H:%M:%S"),
)
except (TypeError, ValueError):
continue
return None, None
def _amos_extract_metar_wind(metar_line: str) -> Optional[float]:
"""Extract wind speed in knots from METAR like '22014KT'."""
match = re.search(r"\b(\d{3})(\d{2,3})KT\b", metar_line)
if match:
return _amos_safe_float(match.group(2))
return None
def _amos_to_lines(text: str) -> list[str]:
"""Convert the AMOS HTML/plain text page to parseable text lines."""
normalized = unescape(str(text or ""))
# The public AMOS page is table-heavy. Preserve cell boundaries as
# whitespace/newlines so regexes work both on raw HTML and crawler text.
normalized = re.sub(r"(?i)<\s*br\s*/?\s*>", "\n", normalized)
normalized = re.sub(r"(?i)</\s*(?:tr|div|p|li|h\d|table)\s*>", "\n", normalized)
normalized = re.sub(r"<[^>]+>", " ", normalized)
normalized = normalized.replace("\xa0", " ")
return [re.sub(r"\s+", " ", line).strip() for line in normalized.splitlines() if line.strip()]
def _amos_is_runway_token(value: str) -> bool:
text = str(value or "").strip()
return bool(
re.match(r"^\d{2}[LRC]?$", text, re.I)
or re.match(r"^[NS]\s+[LR]$", text, re.I)
)
def _amos_normalize_runway_label(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "").strip().upper())
def _amos_runway_pair_label(pair: Any, index: int) -> str:
if isinstance(pair, (list, tuple)) and len(pair) >= 2:
left = _amos_normalize_runway_label(pair[0])
right = _amos_normalize_runway_label(pair[1])
if left and right:
return f"{left}/{right}"
return f"RWY {index + 1}"
def _amos_build_point_temperatures(
runway_pairs: list[tuple[str, str]],
temperatures: list[tuple[Any, Any]],
) -> list[dict[str, Any]]:
points: list[dict[str, Any]] = []
for index, pair in enumerate(runway_pairs):
if index >= len(temperatures):
continue
temp = _amos_safe_float(temperatures[index][0])
if temp is None:
continue
points.append(
{
"runway": _amos_runway_pair_label(pair, index),
"temp": temp,
"target_runway_max": temp,
}
)
return points
def _amos_parse_cell_table(lines: list[str]) -> Optional[dict[str, Any]]:
"""Parse the actual AMOS HTML table after it has been flattened to cells."""
runway_rows: list[dict[str, Any]] = []
i = 0
while i < len(lines):
token = lines[i].strip()
if (
_amos_is_runway_token(token)
and i + 3 < len(lines)
and lines[i + 1].upper() == "AVG"
and lines[i + 2].upper() == "MIN"
and lines[i + 3].upper() == "MAX"
):
row: dict[str, Any] = {"runway": token.upper()}
i += 4
while i < len(lines):
label = lines[i].strip()
upper = label.upper()
if (
_amos_is_runway_token(label)
and i + 3 < len(lines)
and lines[i + 1].upper() == "AVG"
and lines[i + 2].upper() == "MIN"
and lines[i + 3].upper() == "MAX"
):
break
if upper == "WD" and i + 3 < len(lines):
wd = [_amos_safe_float(lines[i + j]) for j in range(1, 4)]
if all(v is not None for v in wd):
row["wind_direction"] = (int(wd[0]), int(wd[1]), int(wd[2]))
i += 4
continue
if upper == "WS" and i + 3 < len(lines):
ws = [_amos_safe_float(lines[i + j]) for j in range(1, 4)]
if all(v is not None for v in ws):
row["wind_speed"] = (float(ws[0]), float(ws[1]), float(ws[2]))
i += 4
continue
if upper == "MOR" and i + 1 < len(lines):
mor = _amos_safe_float(lines[i + 1])
if mor is not None and "visibility_mor" not in row:
row["visibility_mor"] = int(mor)
i += 2
continue
if upper == "RVR" and i + 1 < len(lines):
rvr = _amos_safe_float(str(lines[i + 1]).lstrip("P"))
if rvr is not None and "rvr" not in row:
row["rvr"] = int(rvr)
i += 2
continue
if upper.startswith("TEMP") and i + 1 < len(lines):
row["temp"] = _amos_safe_float(lines[i + 1])
i += 2
continue
if upper.startswith("DEW") and i + 1 < len(lines):
row["dew"] = _amos_safe_float(lines[i + 1])
i += 2
continue
if upper == "QNH (HPA)" and i + 1 < len(lines):
row["pressure_hpa"] = _amos_safe_float(lines[i + 1])
i += 2
continue
i += 1
runway_rows.append(row)
continue
i += 1
if len(runway_rows) < 2:
return None
runway_pairs: list[tuple[str, str]] = []
temperatures: list[tuple[Optional[float], Optional[float]]] = []
pressures_hpa: list[Optional[float]] = []
wind_directions: list[Optional[tuple[int, int, int]]] = []
wind_speeds: list[Optional[tuple[float, float, float]]] = []
visibility_mor: list[Optional[int]] = []
rvr_values: list[Optional[int]] = []
for idx in range(0, len(runway_rows) - 1, 2):
first = runway_rows[idx]
second = runway_rows[idx + 1]
runway_pairs.append((str(first["runway"]), str(second["runway"])))
temp = first.get("temp")
dew = first.get("dew")
if temp is None and second.get("temp") is not None:
temp = second.get("temp")
dew = second.get("dew")
temperatures.append((temp, dew))
pressures_hpa.append(first.get("pressure_hpa") or second.get("pressure_hpa"))
wind_directions.append(first.get("wind_direction") or second.get("wind_direction"))
wind_speeds.append(first.get("wind_speed") or second.get("wind_speed"))
visibility_mor.append(first.get("visibility_mor") or second.get("visibility_mor"))
rvr_values.append(first.get("rvr") or second.get("rvr"))
return {
"runway_pairs": runway_pairs,
"temperatures": temperatures,
"point_temperatures": _amos_build_point_temperatures(runway_pairs, temperatures),
"pressures_hpa": pressures_hpa,
"wind_directions": wind_directions,
"wind_speeds": wind_speeds,
"visibility_mor": visibility_mor,
"rvr": rvr_values,
}
def _amos_parse_runway_table(text: str) -> dict[str, Any]:
"""Parse the runway-level data from AMOS page HTML text.
The page shows data organized by runway direction pairs.
We match patterns like:
WD 230 (220-250)
WS 14.2 (10.9-18.7)
CROSS R14
HEADTAIL +8
MOR 10000 RVR P2000
TEMP/DEW 16.5/9.2
PRECIP 0 QNH 1015.8
"""
lines = _amos_to_lines(text)
cell_table = _amos_parse_cell_table(lines)
if cell_table and cell_table.get("runway_pairs"):
return cell_table
normalized_text = "\n".join(lines)
runway_pairs: list[tuple[str, str]] = []
temperatures: list[tuple[float, float]] = []
pressures_hpa: list[float] = []
wind_directions: list[tuple[int, int, int]] = []
wind_speeds: list[tuple[float, float, float]] = []
visibility_mor: list[int] = []
rvr: list[int] = []
pending_temp: float | None = None
# Current public page format is line/table-cell based:
# 15R AVG MIN MAX
# WD 240 220 250
# WS 4.7 2.9 6.8
# TEMP(℃) 13.7
# DEW (℃) 9.8
# QNH (hPa) 1021.0
# 33L AVG MIN MAX
# Older crawler output may use "15R/33L" and "TEMP/DEW 13.7/9.8".
for line in lines:
runway_header = re.match(r"^(\d{2}[LR]?)\s+AVG\s+MIN\s+MAX\b", line, re.I)
if runway_header:
continue
pair_match = re.match(r"^(\d{2}[LR]?)\s*/\s*(\d{2}[LR]?)$", line)
if not pair_match:
pair_match = re.match(r"^(\d{2}[LR]?)\s+(\d{2}[LR]?)$", line)
if pair_match:
pair = (pair_match.group(1), pair_match.group(2))
# Ignore bare duplicated orientation rows such as "15 33" when
# richer L/R pair labels are present nearby, but keep them as a
# fallback for airports without side designators.
if pair not in runway_pairs:
runway_pairs.append(pair)
continue
wd = re.match(r"^WD\s+(\d+)\s+(\d+)\s+(\d+)\b", line, re.I)
if wd:
wind_directions.append((int(wd.group(1)), int(wd.group(2)), int(wd.group(3))))
continue
ws = re.match(r"^WS\s+(\d+(?:\.\d+)?)\s+(\d+(?:\.\d+)?)\s+(\d+(?:\.\d+)?)\b", line, re.I)
if ws:
wind_speeds.append((float(ws.group(1)), float(ws.group(2)), float(ws.group(3))))
continue
temp_dew = re.search(
r"TEMP\s*/\s*DEW\s*(\d+(?:\.\d+)?)\s*/\s*(\d+(?:\.\d+)?)",
line,
re.I,
)
if temp_dew:
temperatures.append((float(temp_dew.group(1)), float(temp_dew.group(2))))
continue
temp_match = re.search(r"TEMP\s*\([^)]*\)\s*(\d+(?:\.\d+)?)", line, re.I)
if temp_match:
pending_temp = float(temp_match.group(1))
continue
dew_match = re.search(r"DEW\s*\([^)]*\)\s*(\d+(?:\.\d+)?)", line, re.I)
if dew_match and pending_temp is not None:
temperatures.append((pending_temp, float(dew_match.group(1))))
pending_temp = None
continue
qnh = re.search(r"QNH\s*(?:\(\s*hPa\s*\))?\s*(\d+(?:\.\d+)?)", line, re.I)
if qnh:
pressures_hpa.append(float(qnh.group(1)))
continue
mor = re.match(r"^MOR\s+(\d+)", line, re.I)
if mor:
visibility_mor.append(int(mor.group(1)))
continue
rvr_match = re.match(r"^RVR\s+P?(\d+)", line, re.I)
if rvr_match:
rvr.append(int(rvr_match.group(1)))
# Prefer concrete runway-side pairs (15L/33R) over repeated orientation
# rows (15/33). If no paired label exists, pair runway headers in order.
side_pairs = [p for p in runway_pairs if any(ch in "".join(p) for ch in ("L", "R", "C"))]
if side_pairs:
runway_pairs = side_pairs
elif not runway_pairs:
headers = re.findall(r"^(\d{2}[LRC]?)\s+AVG\s+MIN\s+MAX\b", normalized_text, re.I | re.M)
runway_pairs = [
(headers[i], headers[i + 1])
for i in range(0, len(headers) - 1, 2)
]
return {
"runway_pairs": runway_pairs,
"temperatures": temperatures,
"point_temperatures": _amos_build_point_temperatures(runway_pairs, temperatures),
"pressures_hpa": pressures_hpa,
"wind_directions": wind_directions,
"wind_speeds": wind_speeds,
"visibility_mor": visibility_mor,
"rvr": rvr,
}
class AmosStationSourceMixin:
"""Mixin that adds AMOS runway-level data fetching to WeatherDataCollector."""
def _amos_get_page(self, icao: str) -> Optional[str]:
"""Fetch the AMOS page.
The AMOS site loads Incheon (RKSI) by default. Keep the default URL
for RKSI and try common airport-code query keys for other airports;
only accept a response when the requested ICAO is present, so ignored
parameters cannot accidentally attach RKSI data to Busan/RKPK.
"""
started = time.perf_counter()
icao = str(icao or "").strip().upper()
stn_id = next(
(
meta.get("stn_id")
for meta in AMOS_AIRPORT_CODES.values()
if meta.get("icao") == icao
),
None,
)
urls = [(AMOS_BASE_URL, None)]
if stn_id:
urls = [(AMOS_BASE_URL, {"stnId": stn_id})]
if icao != "RKSI" and not stn_id:
urls = [f"{AMOS_BASE_URL}?{key}={icao}" for key in AMOS_AIRPORT_QUERY_KEYS]
try:
for url_item in urls:
getter = getattr(self, "_http_get_text", None)
post_data = None
if isinstance(url_item, tuple):
url, post_data = url_item
else:
url = url_item
if post_data is None and callable(getter):
text = str(getter(url))
elif hasattr(self, "session"):
if post_data is not None:
resp = self.session.post(
url,
data=post_data,
timeout=float(getattr(self, "timeout", 4.0)),
)
else:
resp = self.session.get(url, timeout=float(getattr(self, "timeout", 4.0)))
resp.raise_for_status()
text = resp.text
else:
return None
if text and re.search(rf"\({icao}\)|\b(?:METAR|TAF)\s+{icao}\b", text, re.I):
logger.info("AMOS page matched icao={} length={}", icao, len(text))
record_source_call("amos", "page", "success", (time.perf_counter() - started) * 1000.0)
return text
logger.warning("AMOS page did not expose requested airport {} (tried {} urls)", icao, len(urls))
return None
except Exception as exc:
logger.warning("AMOS page fetch failed icao={}: {}", icao, exc)
record_source_call("amos", "page", "error", (time.perf_counter() - started) * 1000.0)
return None
def fetch_amos_official_current(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
city_key = str(city or "").strip().lower()
interval_sec = max(
30,
int(os.getenv("POLYWEATHER_OBSERVATION_COLLECTOR_AMOS_SEC", "60") or "60"),
)
return run_observation_source(
"amos",
city_key,
interval_sec,
lambda: self._fetch_amos_official_current_uncached(
city_key,
use_fahrenheit=use_fahrenheit,
),
)
def _fetch_amos_official_current_uncached(
self,
city: str,
use_fahrenheit: bool = False,
) -> Optional[Dict[str, Any]]:
"""Fetch AMOS runway-level observations for Seoul or Busan.
Temperature priority:
1. METAR temperature (official aerodrome sensor, authoritative)
2. Median of runway sensor temperatures (fallback; individual runway
sensors may differ by 0.5-1.0°C due to location/altitude on the airfield)
Returns a dict with: temp, temp_c, dew, dew_c, pressure_hpa, wind_kt,
temp_source ("metar" or "runway_median"), runway_temps (list of per-runway
(temp, dew) tuples), raw_metar, raw_taf, runway_data, source.
"""
started = time.perf_counter()
city_key = str(city or "").strip().lower()
airport_meta = AMOS_AIRPORT_CODES.get(city_key)
if not airport_meta:
return None
icao = airport_meta["icao"]
try:
html = self._amos_get_page(icao)
if not html:
logger.warning("AMOS fetch_amos_official_current: no HTML for {}", icao)
return None
logger.info("AMOS fetch_amos_official_current: got HTML for {} ({} chars), parsing METAR/runway", icao, len(html))
# Parse METAR line
icao_pattern = re.escape(icao)
metar_match = re.search(rf"METAR\s+{icao_pattern}\s.*?=", html, re.DOTALL)
metar_line = metar_match.group(0) if metar_match else ""
metar_line = re.sub(r"\s+", " ", metar_line).strip()
# Parse TAF line
taf_match = re.search(rf"TAF\s+{icao_pattern}\s.*?=", html, re.DOTALL)
taf_line = taf_match.group(0) if taf_match else ""
taf_line = re.sub(r"\s+", " ", taf_line).strip()
# METAR is the authoritative aerodrome observation
metar_temp_c, metar_dew_c = _amos_extract_metar_temperature(metar_line)
pressure_hpa = _amos_extract_metar_qnh(metar_line)
wind_kt = _amos_extract_metar_wind(metar_line)
# Runway-level temperatures from individual sensor pairs
runway_data = _amos_parse_runway_table(html)
runway_temps = runway_data.get("temperatures") or []
runway_pressures = [
float(p)
for p in (runway_data.get("pressures_hpa") or [])
if p is not None
]
# Primary: METAR (official aerodrome sensor)
# Fallback: median of runway sensors (if METAR unavailable)
# Runway sensors may differ by 0.5-1.0°C from METAR due to
# different locations/altitudes on the airfield
temp_c: Optional[float] = metar_temp_c
dew_c: Optional[float] = metar_dew_c
temp_source = "metar"
if temp_c is None and runway_temps:
runway_temps_only = [t[0] for t in runway_temps if t[0] is not None and -50 < float(t[0]) < 60]
if runway_temps_only:
sorted_t = sorted(runway_temps_only)
mid = len(sorted_t) // 2
temp_c = float(sorted_t[mid]) if len(sorted_t) % 2 else float((sorted_t[mid-1] + sorted_t[mid]) / 2)
temp_source = "runway_median"
if dew_c is None and runway_temps:
runway_dews = [t[1] for t in runway_temps if t[1] is not None and -50 < float(t[1]) < 60]
if runway_dews:
sorted_d = sorted(runway_dews)
mid = len(sorted_d) // 2
dew_c = float(sorted_d[mid]) if len(sorted_d) % 2 else float((sorted_d[mid-1] + sorted_d[mid]) / 2)
if pressure_hpa is None and runway_pressures:
sorted_p = sorted(runway_pressures)
mid = len(sorted_p) // 2
pressure_hpa = float(sorted_p[mid]) if len(sorted_p) % 2 else float((sorted_p[mid-1] + sorted_p[mid]) / 2)
observation_time, observation_time_local = _amos_extract_observation_time(html, icao)
if not observation_time:
now_utc = datetime.now(timezone.utc).replace(microsecond=0)
observation_time = now_utc.isoformat().replace("+00:00", "Z")
observation_time_local = now_utc.astimezone(
timezone(timedelta(hours=9))
).strftime("%Y-%m-%d %H:%M:%S")
temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit and temp_c is not None else temp_c
dew = round(dew_c * 9 / 5 + 32, 1) if use_fahrenheit and dew_c is not None else dew_c
result: Dict[str, Any] = {
"temp": temp,
"temp_c": temp_c,
"dew": dew,
"dew_c": dew_c,
"pressure_hpa": pressure_hpa,
"wind_kt": wind_kt,
"temp_source": temp_source,
"runway_temps": runway_temps,
"runway_temp_range": None,
"source": "amos",
"source_label": f"AMOS {airport_meta['label_en']} ({icao})",
"source_code": "amos",
"icao": icao,
"station_label": airport_meta["label_ko"],
"station_label_en": airport_meta["label_en"],
"is_official": True,
"is_airport_station": True,
"is_settlement_anchor": False,
"network_type": "amos",
"raw_metar": metar_line or None,
"raw_taf": taf_line or None,
"runway_obs": runway_data if runway_data.get("temperatures") else None,
"observation_source": "AMOS runway sensors",
"observation_source_zh": "AMOS 跑道传感器",
"observation_time": observation_time,
"observation_time_local": observation_time_local,
}
# Compute runway temp range for compact display ("14.6~15.2")
valid = [t[0] for t in runway_temps if t[0] is not None and -50 < float(t[0]) < 60]
if len(valid) >= 2:
result["runway_temp_range"] = (round(min(valid), 1), round(max(valid), 1))
elif len(valid) == 1:
result["runway_temp_range"] = (round(valid[0], 1), round(valid[0], 1))
record_source_call(
"amos", "current", "success",
(time.perf_counter() - started) * 1000.0,
)
return result
except Exception as exc:
logger.warning("AMOS fetch failed city={}: {}", city_key, exc)
record_source_call(
"amos", "current", "error",
(time.perf_counter() - started) * 1000.0,
)
return None