新增 AEROWEB (Météo-France) 和 NCM (沙特) 实时气象数据源
AEROWEB: 接入 aviation.meteo.fr,LFPB METAR 2 分钟内可获取, 实测温度替代 AROME 模式预报作为 Paris 电报推送数据源。 登录流程: PHPSESSID + MD5 密码 → ajax/login_valid.php, Session 20 分钟自动续期,XML 解析提取 tempe/td/dd/ff/qnh。 NCM: 沙特气象局 Meteomatics API 代码框架,待凭证激活。 Jeddah settlement_source 从废弃的 wunderground 迁移至 ncm。 同时清理: 日志文件 trading_system.log → polyweather.log, 删除 2026-02-07 的 1.2MB 废弃交易引擎日志。 Constraint: AEROWEB 需 AEROWEB_USERNAME/AEROWEB_PASSWORD 环境变量 Constraint: NCM 需 NCM_API_USERNAME/NCM_API_PASSWORD 环境变量 Confidence: high Tested: AEROWEB 登录→取数→解析端到端验证通过
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from src.utils.metrics import record_source_call
|
||||
|
||||
|
||||
class AerowebSourceMixin:
|
||||
"""Fetch realtime METAR from Météo-France AEROWEB.
|
||||
|
||||
AEROWEB (https://aviation.meteo.fr) delivers METAR/TAF directly from
|
||||
Météo-France's aviation network. Reports typically appear within
|
||||
2 minutes of issue time — faster than NOAA's aviationweather.gov.
|
||||
|
||||
Auth flow:
|
||||
1. GET login.php → receive PHPSESSID cookie
|
||||
2. POST ajax/login_valid.php (login=<user>&password=<md5(pass)>)
|
||||
3. GET ajax/ajax_trajet_get_stations.php (with PHPSESSID)
|
||||
|
||||
Env vars:
|
||||
AEROWEB_USERNAME — login name
|
||||
AEROWEB_PASSWORD — plain-text password (MD5 hashed before sending)
|
||||
"""
|
||||
|
||||
AEROWEB_BASE = "https://aviation.meteo.fr"
|
||||
# Tight bounding box around LFPB (Paris Le Bourget) in WGS84 degrees.
|
||||
# The WFS endpoint uses SRS=EPSG:900913 but accepts lat/lon in practice.
|
||||
LFPB_BBOX = "2.42,48.95,2.46,48.99"
|
||||
|
||||
def _aeroweb_creds(self) -> Optional[tuple]:
|
||||
user = os.getenv("AEROWEB_USERNAME", "").strip()
|
||||
pwd = os.getenv("AEROWEB_PASSWORD", "").strip()
|
||||
if not user or not pwd:
|
||||
return None
|
||||
return (user, pwd)
|
||||
|
||||
def _aeroweb_session(self) -> Optional[str]:
|
||||
"""Return a valid PHPSESSID, logging in if necessary."""
|
||||
creds = self._aeroweb_creds()
|
||||
if not creds:
|
||||
logger.warning("AEROWEB credentials not configured (AEROWEB_USERNAME / AEROWEB_PASSWORD)")
|
||||
return None
|
||||
|
||||
user, pwd = creds
|
||||
now = time.time()
|
||||
|
||||
# Check cached session
|
||||
cached = getattr(self, "_aeroweb_sid", None)
|
||||
cached_ts = getattr(self, "_aeroweb_sid_ts", 0)
|
||||
# Re-login every 20 minutes to keep the session alive
|
||||
if cached and (now - cached_ts) < 1200:
|
||||
return cached
|
||||
|
||||
try:
|
||||
# Step 1: get a fresh PHPSESSID
|
||||
resp = self._http_get(f"{self.AEROWEB_BASE}/login.php", timeout=self.timeout)
|
||||
# httpx follows redirects; grab PHPSESSID from the cookie jar
|
||||
sid = None
|
||||
for cookie in resp.cookies.jar:
|
||||
if cookie.name == "PHPSESSID":
|
||||
sid = cookie.value
|
||||
break
|
||||
if not sid:
|
||||
# Try extracting from Set-Cookie header
|
||||
set_cookie = resp.headers.get("set-cookie", "")
|
||||
for part in set_cookie.replace(",", ";").split(";"):
|
||||
part = part.strip()
|
||||
if part.startswith("PHPSESSID="):
|
||||
sid = part.split("=", 1)[1]
|
||||
break
|
||||
if not sid:
|
||||
logger.error("AEROWEB: could not obtain PHPSESSID from login page")
|
||||
return None
|
||||
|
||||
# Step 2: authenticate (httpx session POST directly)
|
||||
pwd_hash = hashlib.md5(pwd.encode()).hexdigest()
|
||||
login_url = f"{self.AEROWEB_BASE}/ajax/login_valid.php"
|
||||
login_resp = self.session.post(
|
||||
login_url,
|
||||
data={"login": user, "password": pwd_hash},
|
||||
cookies={"PHPSESSID": sid},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
if login_resp.text.strip() != "ok":
|
||||
logger.error("AEROWEB login failed: {}", login_resp.text.strip()[:100])
|
||||
return None
|
||||
|
||||
logger.info("AEROWEB login OK, sid={}...", sid[:8])
|
||||
self._aeroweb_sid = sid
|
||||
self._aeroweb_sid_ts = now
|
||||
return sid
|
||||
|
||||
except Exception:
|
||||
logger.exception("AEROWEB login error")
|
||||
return None
|
||||
|
||||
def fetch_from_aeroweb(self) -> Optional[Dict]:
|
||||
"""Fetch latest METAR for LFPB from AEROWEB."""
|
||||
started = datetime.now()
|
||||
|
||||
def _elapsed_ms() -> float:
|
||||
return (datetime.now() - started).total_seconds() * 1000.0
|
||||
|
||||
sid = self._aeroweb_session()
|
||||
if not sid:
|
||||
record_source_call("aeroweb", "station", "noauth", _elapsed_ms())
|
||||
return None
|
||||
|
||||
try:
|
||||
url = (
|
||||
f"{self.AEROWEB_BASE}/ajax/ajax_trajet_get_stations.php"
|
||||
f"?reseau=&type=oaci&SERVICE=WFS&VERSION=1.0.0"
|
||||
f"&REQUEST=GetFeature&SRS=EPSG:900913"
|
||||
f"&BBOX={self.LFPB_BBOX}&LEVEL=8"
|
||||
)
|
||||
resp = self._http_get(
|
||||
url,
|
||||
cookies={"PHPSESSID": sid},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
# If redirected to login page, session expired — clear and retry once
|
||||
if "login.php" in str(resp.url) or resp.status_code == 302:
|
||||
logger.info("AEROWEB session expired, re-logging in")
|
||||
self._aeroweb_sid = None
|
||||
sid = self._aeroweb_session()
|
||||
if not sid:
|
||||
record_source_call("aeroweb", "station", "auth_error", _elapsed_ms())
|
||||
return None
|
||||
resp = self._http_get(
|
||||
url,
|
||||
cookies={"PHPSESSID": sid},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
if resp.status_code != 200:
|
||||
logger.warning("AEROWEB API returned HTTP {}", resp.status_code)
|
||||
record_source_call("aeroweb", "station", "error", _elapsed_ms())
|
||||
return None
|
||||
|
||||
root = ET.fromstring(resp.text)
|
||||
# Find LFPB metar element
|
||||
metar_el = root.find(".//opmet[@id='LFPB']/messages/metar")
|
||||
if metar_el is None:
|
||||
record_source_call("aeroweb", "station", "empty", _elapsed_ms())
|
||||
return None
|
||||
|
||||
def _attr_int(name: str) -> Optional[int]:
|
||||
v = metar_el.get(name)
|
||||
return int(v) if v and v.strip() else None
|
||||
|
||||
def _attr_float(name: str) -> Optional[float]:
|
||||
v = metar_el.get(name)
|
||||
return float(v) if v and v.strip() else None
|
||||
|
||||
temp = _attr_float("tempe")
|
||||
td = _attr_float("td")
|
||||
wind_dir = _attr_float("dd")
|
||||
wind_kt = _attr_float("ff")
|
||||
qnh = metar_el.get("qnh", "").strip().replace(" HPA", "")
|
||||
pressure = float(qnh) if qnh else None
|
||||
visibility = _attr_int("visi")
|
||||
raw = (metar_el.text or "").strip()
|
||||
is_auto = metar_el.get("auto") == "1"
|
||||
|
||||
# Build observation time from day/hour/minute attributes
|
||||
day = _attr_int("day")
|
||||
hour = _attr_int("hour")
|
||||
minute = _attr_int("minute")
|
||||
obs_time = None
|
||||
if day is not None and hour is not None and minute is not None:
|
||||
now = datetime.utcnow()
|
||||
obs_time = datetime(now.year, now.month, day, hour, minute).isoformat()
|
||||
|
||||
result: Dict = {
|
||||
"current": {
|
||||
"temp": temp,
|
||||
"dew_point": td,
|
||||
"wind_dir": wind_dir,
|
||||
"wind_speed_kt": wind_kt,
|
||||
"pressure": pressure,
|
||||
"visibility_m": visibility,
|
||||
"raw_metar": raw,
|
||||
"is_auto": is_auto,
|
||||
},
|
||||
"obs_time": obs_time,
|
||||
"station_id": "LFPB",
|
||||
"station_label": "Paris Le Bourget (AEROWEB)",
|
||||
"lat": 48.9695,
|
||||
"lon": 2.4415,
|
||||
}
|
||||
record_source_call("aeroweb", "station", "success", _elapsed_ms())
|
||||
logger.info(
|
||||
"AEROWEB LFPB {}:{}Z: temp={}°C dew={}°C wind={}kt@{:03.0f}° QNH={}hPa",
|
||||
day,
|
||||
f"{hour:02d}{minute:02d}",
|
||||
temp,
|
||||
td,
|
||||
wind_kt,
|
||||
wind_dir or 0,
|
||||
pressure,
|
||||
)
|
||||
return result
|
||||
|
||||
except ET.ParseError:
|
||||
logger.exception("AEROWEB XML parse error")
|
||||
record_source_call("aeroweb", "station", "parse_error", _elapsed_ms())
|
||||
return None
|
||||
except Exception:
|
||||
logger.exception("AEROWEB fetch failed")
|
||||
record_source_call("aeroweb", "station", "error", _elapsed_ms())
|
||||
return None
|
||||
@@ -71,9 +71,9 @@ CITY_REGISTRY = {
|
||||
"lat": 48.9694,
|
||||
"lon": 2.4414,
|
||||
"icao": "LFPB",
|
||||
"settlement_source": "metar",
|
||||
"settlement_source": "aeroweb",
|
||||
"settlement_station_code": "LFPB",
|
||||
"settlement_station_label": "Paris-Le Bourget Airport METAR",
|
||||
"settlement_station_label": "Paris Le Bourget (AEROWEB)",
|
||||
"settlement_url": "https://www.wunderground.com/history/daily/fr/bonneuil-en-france/LFPB",
|
||||
"tz_offset": 3600,
|
||||
"use_fahrenheit": False,
|
||||
@@ -788,9 +788,9 @@ CITY_REGISTRY = {
|
||||
"lat": 21.6702,
|
||||
"lon": 39.1525,
|
||||
"icao": "OEJN",
|
||||
"settlement_source": "wunderground",
|
||||
"settlement_source": "ncm",
|
||||
"settlement_station_code": "OEJN",
|
||||
"settlement_station_label": "Jeddah/King Abdulaziz International Airport Station",
|
||||
"settlement_station_label": "Jeddah King Abdulaziz Intl (NCM)",
|
||||
"settlement_url": "https://www.wunderground.com/history/daily/sa/jeddah/OEJN",
|
||||
"tz_offset": 10800,
|
||||
"use_fahrenheit": False,
|
||||
|
||||
@@ -188,6 +188,10 @@ def _provider_code_for_city(city: str) -> str:
|
||||
return "korea_kma"
|
||||
if normalized == "tel aviv":
|
||||
return "israel_ims"
|
||||
if normalized == "jeddah":
|
||||
return "saudi_ncm"
|
||||
if normalized == "paris":
|
||||
return "fr_aeroweb"
|
||||
if normalized == "moscow":
|
||||
return "russia_metar_cluster"
|
||||
if settlement_source == "hko":
|
||||
@@ -422,6 +426,39 @@ def _airport_primary_from_raw(city: str, raw: Dict[str, Any]) -> Dict[str, Any]:
|
||||
},
|
||||
)
|
||||
|
||||
aeroweb = raw.get("aeroweb") or {}
|
||||
aw_current = aeroweb.get("current") or {}
|
||||
if aw_current.get("temp") is not None:
|
||||
return _normalize_station_row(
|
||||
station_code=meta.get("icao") or "LFPB",
|
||||
station_label=meta.get("airport_name") or aeroweb.get("station_label") or "Paris Le Bourget (AEROWEB)",
|
||||
temp=_safe_float(aw_current["temp"]),
|
||||
obs_time=aeroweb.get("obs_time") or metar.get("observation_time"),
|
||||
source_code="aeroweb",
|
||||
source_label="AEROWEB",
|
||||
is_official=True,
|
||||
is_airport_station=True,
|
||||
is_settlement_anchor=False,
|
||||
extra={
|
||||
"max_so_far": _safe_float(current.get("max_temp_so_far")),
|
||||
"max_temp_time": current.get("max_temp_time"),
|
||||
"obs_age_min": None,
|
||||
"report_time": metar.get("report_time"),
|
||||
"receipt_time": metar.get("receipt_time"),
|
||||
"obs_time_epoch": metar.get("obs_time_epoch"),
|
||||
"obs_time_utc_offset_seconds": 0,
|
||||
"wind_speed_kt": _safe_float(aw_current.get("wind_speed_kt"))
|
||||
or _safe_float(current.get("wind_speed_kt")),
|
||||
"wind_dir": _safe_float(aw_current.get("wind_dir"))
|
||||
or _safe_float(current.get("wind_dir")),
|
||||
"humidity": _safe_float(current.get("humidity")),
|
||||
"pressure": _safe_float(aw_current.get("pressure")),
|
||||
"visibility_mi": _safe_float(current.get("visibility_mi")),
|
||||
"wx_desc": current.get("wx_desc"),
|
||||
"raw_metar": aw_current.get("raw_metar") or current.get("raw_metar"),
|
||||
},
|
||||
)
|
||||
|
||||
ims = raw.get("ims") or {}
|
||||
ims_current = ims.get("current") or {}
|
||||
if ims_current.get("temp") is not None:
|
||||
@@ -455,6 +492,39 @@ def _airport_primary_from_raw(city: str, raw: Dict[str, Any]) -> Dict[str, Any]:
|
||||
},
|
||||
)
|
||||
|
||||
ncm = raw.get("ncm") or {}
|
||||
ncm_current = ncm.get("current") or {}
|
||||
if ncm_current.get("temp") is not None:
|
||||
return _normalize_station_row(
|
||||
station_code=meta.get("icao") or "OEJN",
|
||||
station_label=meta.get("airport_name") or ncm.get("station_label") or "King Abdulaziz Intl (NCM)",
|
||||
temp=_safe_float(ncm_current["temp"]),
|
||||
obs_time=ncm.get("obs_time") or metar.get("observation_time"),
|
||||
source_code="ncm",
|
||||
source_label="Saudi NCM",
|
||||
is_official=True,
|
||||
is_airport_station=True,
|
||||
is_settlement_anchor=False,
|
||||
extra={
|
||||
"max_so_far": _safe_float(current.get("max_temp_so_far")),
|
||||
"max_temp_time": current.get("max_temp_time"),
|
||||
"obs_age_min": None,
|
||||
"report_time": metar.get("report_time"),
|
||||
"receipt_time": metar.get("receipt_time"),
|
||||
"obs_time_epoch": metar.get("obs_time_epoch"),
|
||||
"obs_time_utc_offset_seconds": 0,
|
||||
"wind_speed_kt": _safe_float(ncm_current.get("wind_speed_kt"))
|
||||
or _safe_float(current.get("wind_speed_kt")),
|
||||
"wind_dir": _safe_float(current.get("wind_dir")),
|
||||
"humidity": _safe_float(ncm_current.get("humidity"))
|
||||
or _safe_float(current.get("humidity")),
|
||||
"pressure": _safe_float(ncm_current.get("pressure")),
|
||||
"visibility_mi": _safe_float(current.get("visibility_mi")),
|
||||
"wx_desc": current.get("wx_desc"),
|
||||
"raw_metar": current.get("raw_metar"),
|
||||
},
|
||||
)
|
||||
|
||||
sg_mss = raw.get("singapore_mss_current") or {}
|
||||
if sg_mss.get("temp_c") is not None:
|
||||
return _normalize_station_row(
|
||||
@@ -949,6 +1019,96 @@ class IsraelImsNetworkProvider(CountryNetworkProvider):
|
||||
}
|
||||
|
||||
|
||||
class AerowebNetworkProvider(CountryNetworkProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("fr_aeroweb", "AEROWEB")
|
||||
|
||||
def official_nearby_current(self, city: str, raw: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
aw = raw.get("aeroweb") or {}
|
||||
aw_current = aw.get("current") or {}
|
||||
if aw_current.get("temp") is not None:
|
||||
row = _normalize_station_row(
|
||||
station_code="LFPB",
|
||||
station_label=aw.get("station_label") or "Paris Le Bourget (AEROWEB)",
|
||||
temp=aw_current.get("temp"),
|
||||
lat=aw.get("lat"),
|
||||
lon=aw.get("lon"),
|
||||
obs_time=aw.get("obs_time"),
|
||||
source_code="aeroweb",
|
||||
source_label="AEROWEB",
|
||||
is_official=True,
|
||||
is_airport_station=True,
|
||||
is_settlement_anchor=False,
|
||||
)
|
||||
return [row]
|
||||
return _metar_cluster_rows(raw)
|
||||
|
||||
def official_network_status(self, city: str, raw: Dict[str, Any]) -> Dict[str, Any]:
|
||||
aw = raw.get("aeroweb") or {}
|
||||
has_aw = (aw.get("current") or {}).get("temp") is not None
|
||||
if has_aw:
|
||||
return {
|
||||
"provider_code": self.provider_code,
|
||||
"provider_label": self.provider_label,
|
||||
"available": True,
|
||||
"mode": "official_active",
|
||||
"row_count": 1,
|
||||
}
|
||||
rows = _metar_cluster_rows(raw)
|
||||
return {
|
||||
"provider_code": self.provider_code,
|
||||
"provider_label": self.provider_label,
|
||||
"available": bool(rows),
|
||||
"mode": "fallback_metar_cluster" if rows else "no_official_network",
|
||||
"row_count": len(rows),
|
||||
}
|
||||
|
||||
|
||||
class SaudiNcmNetworkProvider(CountryNetworkProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("saudi_ncm", "Saudi NCM")
|
||||
|
||||
def official_nearby_current(self, city: str, raw: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
ncm = raw.get("ncm") or {}
|
||||
ncm_current = ncm.get("current") or {}
|
||||
if ncm_current.get("temp") is not None:
|
||||
row = _normalize_station_row(
|
||||
station_code="OEJN",
|
||||
station_label=ncm.get("station_label") or "Jeddah OEJN (NCM)",
|
||||
temp=ncm_current.get("temp"),
|
||||
lat=ncm.get("lat"),
|
||||
lon=ncm.get("lon"),
|
||||
obs_time=ncm.get("obs_time"),
|
||||
source_code="ncm",
|
||||
source_label="Saudi NCM",
|
||||
is_official=True,
|
||||
is_airport_station=True,
|
||||
is_settlement_anchor=False,
|
||||
)
|
||||
return [row]
|
||||
return _metar_cluster_rows(raw)
|
||||
|
||||
def official_network_status(self, city: str, raw: Dict[str, Any]) -> Dict[str, Any]:
|
||||
ncm = raw.get("ncm") or {}
|
||||
has_ncm = (ncm.get("current") or {}).get("temp") is not None
|
||||
if has_ncm:
|
||||
return {
|
||||
"provider_code": self.provider_code,
|
||||
"provider_label": self.provider_label,
|
||||
"available": True,
|
||||
"mode": "official_active",
|
||||
"row_count": 1,
|
||||
}
|
||||
rows = _metar_cluster_rows(raw)
|
||||
return {
|
||||
"provider_code": self.provider_code,
|
||||
"provider_label": self.provider_label,
|
||||
"available": bool(rows),
|
||||
"mode": "fallback_metar_cluster" if rows else "no_official_network",
|
||||
"row_count": len(rows),
|
||||
}
|
||||
|
||||
|
||||
class HongKongHkoNetworkProvider(CountryNetworkProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("hongkong_hko", "HKO")
|
||||
@@ -981,6 +1141,10 @@ def get_country_network_provider(city: str) -> CountryNetworkProvider:
|
||||
return ChinaCmaNetworkProvider()
|
||||
if provider_code == "israel_ims":
|
||||
return IsraelImsNetworkProvider()
|
||||
if provider_code == "saudi_ncm":
|
||||
return SaudiNcmNetworkProvider()
|
||||
if provider_code == "fr_aeroweb":
|
||||
return AerowebNetworkProvider()
|
||||
if provider_code == "hongkong_hko":
|
||||
return HongKongHkoNetworkProvider()
|
||||
if provider_code == "taiwan_cwa":
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from src.utils.metrics import record_source_call
|
||||
|
||||
|
||||
class NcmSourceMixin:
|
||||
"""Fetch realtime observations from Saudi NCM via the Meteomatics API.
|
||||
|
||||
API base: https://api-mm.ncm.gov.sa/
|
||||
Docs: https://api-doc.ncm.gov.sa/en/api/getting-started
|
||||
|
||||
Auth via HTTP Basic Auth: username + password (set in .env as
|
||||
NCM_API_USERNAME / NCM_API_PASSWORD).
|
||||
|
||||
Station for Jeddah: "Mataar Municipality" (ID 36) near OEJN.
|
||||
Uses source=mix-obs for real-time station observations.
|
||||
|
||||
Parameters:
|
||||
t_2m:C — temperature at 2 m (°C)
|
||||
wind_speed_10m:ms — wind speed at 10 m (m/s)
|
||||
sfc_pressure:hPa — surface pressure (hPa)
|
||||
dew_point_2m:C — dew point at 2 m (°C)
|
||||
relative_humidity_2m:p — relative humidity (%)
|
||||
"""
|
||||
|
||||
NCM_API_BASE = "https://api-mm.ncm.gov.sa"
|
||||
# Station 36 = Mataar Municipality (21.60 N, 39.25 E), closest to OEJN.
|
||||
# Use lat/lon for automatic nearest-station routing.
|
||||
JEDDAH_LAT = "21.6702"
|
||||
JEDDAH_LON = "39.1525"
|
||||
JEDDAH_STATION_ID = "36"
|
||||
PARAMS = "t_2m:C,wind_speed_10m:ms,sfc_pressure:hPa,dew_point_2m:C,relative_humidity_2m:p"
|
||||
|
||||
def _ncm_auth(self) -> Optional[tuple]:
|
||||
username = os.getenv("NCM_API_USERNAME", "").strip()
|
||||
password = os.getenv("NCM_API_PASSWORD", "").strip()
|
||||
if not username or not password:
|
||||
return None
|
||||
return (username, password)
|
||||
|
||||
def fetch_from_ncm(self) -> Optional[Dict]:
|
||||
"""Fetch latest observation for Jeddah from NCM."""
|
||||
started = datetime.now()
|
||||
auth = self._ncm_auth()
|
||||
if not auth:
|
||||
logger.warning("NCM credentials not configured (NCM_API_USERNAME / NCM_API_PASSWORD)")
|
||||
record_source_call("ncm", "station", "noauth", 0)
|
||||
return None
|
||||
|
||||
def _elapsed_ms() -> float:
|
||||
return (datetime.now() - started).total_seconds() * 1000.0
|
||||
|
||||
now_utc = datetime.utcnow()
|
||||
valid_time = now_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
location = f"{self.JEDDAH_LAT},{self.JEDDAH_LON}"
|
||||
url = f"{self.NCM_API_BASE}/{valid_time}/{self.PARAMS}/{location}/json?source=mix-obs"
|
||||
|
||||
try:
|
||||
resp = self._http_get(url, auth=auth, timeout=self.timeout)
|
||||
if resp.status_code == 401:
|
||||
logger.error("NCM API authentication failed — check NCM_API_USERNAME / NCM_API_PASSWORD")
|
||||
record_source_call("ncm", "station", "auth_error", _elapsed_ms())
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
logger.warning("NCM API returned HTTP {}", resp.status_code)
|
||||
record_source_call("ncm", "station", "error", _elapsed_ms())
|
||||
return None
|
||||
|
||||
body = resp.json()
|
||||
if not body:
|
||||
record_source_call("ncm", "station", "empty", _elapsed_ms())
|
||||
return None
|
||||
|
||||
def _first_value(key: str) -> Optional[float]:
|
||||
"""NCM returns per-parameter arrays; extract the first value."""
|
||||
arr = body.get(key) if isinstance(body, dict) else None
|
||||
if arr and isinstance(arr, list) and len(arr) > 0:
|
||||
try:
|
||||
return float(arr[0])
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
temp = _first_value("t_2m:C")
|
||||
wind_ms = _first_value("wind_speed_10m:ms")
|
||||
pressure = _first_value("sfc_pressure:hPa")
|
||||
dew_point = _first_value("dew_point_2m:C")
|
||||
humidity = _first_value("relative_humidity_2m:p")
|
||||
|
||||
wind_kmh = round(wind_ms * 3.6, 1) if wind_ms is not None else None
|
||||
wind_kt = round(wind_ms * 1.94384, 1) if wind_ms is not None else None
|
||||
|
||||
result: Dict = {
|
||||
"current": {
|
||||
"temp": temp,
|
||||
"humidity": humidity,
|
||||
"wind_speed_kmh": wind_kmh,
|
||||
"wind_speed_kt": wind_kt,
|
||||
"pressure": pressure,
|
||||
"dew_point": dew_point,
|
||||
},
|
||||
"obs_time": now_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"station_label": "Jeddah OEJN (NCM)",
|
||||
"lat": float(self.JEDDAH_LAT),
|
||||
"lon": float(self.JEDDAH_LON),
|
||||
}
|
||||
record_source_call("ncm", "station", "success", _elapsed_ms())
|
||||
logger.info(
|
||||
"NCM Jeddah {}: temp={}°C RH={}% wind={}km/h p={}hPa",
|
||||
valid_time,
|
||||
temp,
|
||||
humidity,
|
||||
wind_kmh,
|
||||
pressure,
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception:
|
||||
logger.exception("NCM fetch failed for Jeddah")
|
||||
record_source_call("ncm", "station", "error", _elapsed_ms())
|
||||
return None
|
||||
@@ -20,10 +20,12 @@ from src.data_collection.hko_obs_sources import HkoObsSourceMixin
|
||||
from src.data_collection.madis_sources import MadisSourceMixin
|
||||
from src.data_collection.singapore_mss_sources import SingaporeMssSourceMixin
|
||||
from src.data_collection.ims_sources import ImsSourceMixin
|
||||
from src.data_collection.ncm_sources import NcmSourceMixin
|
||||
from src.data_collection.aeroweb_sources import AerowebSourceMixin
|
||||
from src.database.db_manager import DBManager
|
||||
|
||||
|
||||
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin, ImsSourceMixin):
|
||||
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, JmaAmedasSourceMixin, NwsOpenMeteoSourceMixin, AmosStationSourceMixin, AmscAwosSourceMixin, FmiSourceMixin, KnmiSourceMixin, HkoObsSourceMixin, MadisSourceMixin, SingaporeMssSourceMixin, ImsSourceMixin, NcmSourceMixin, AerowebSourceMixin):
|
||||
"""
|
||||
Multi-source weather data collector
|
||||
|
||||
@@ -859,7 +861,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
if (
|
||||
city_lower not in self.CITY_METAR_CLUSTERS
|
||||
or "mgm_nearby" in results
|
||||
or settlement_source in {"hko", "cwa", "ims"}
|
||||
or settlement_source in {"hko", "cwa", "ims", "ncm", "aeroweb"}
|
||||
):
|
||||
return
|
||||
cluster_icaos = self.CITY_METAR_CLUSTERS[city_lower]
|
||||
@@ -905,6 +907,42 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
except Exception:
|
||||
logger.exception("airport_obs_log append failed for ims city={}", city_lower)
|
||||
|
||||
def _attach_saudi_ncm_data(self, results: Dict, city_lower: str) -> None:
|
||||
if city_lower != "jeddah":
|
||||
return
|
||||
ncm_data = self.fetch_from_ncm()
|
||||
if ncm_data:
|
||||
results["ncm"] = ncm_data
|
||||
try:
|
||||
ncm_current = ncm_data.get("current") or {}
|
||||
DBManager().append_airport_obs(
|
||||
icao="OEJN",
|
||||
city=city_lower,
|
||||
temp_c=ncm_current.get("temp"),
|
||||
wind_kt=ncm_current.get("wind_speed_kt"),
|
||||
obs_time=ncm_data.get("obs_time") or datetime.now().isoformat(),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("airport_obs_log append failed for ncm city={}", city_lower)
|
||||
|
||||
def _attach_paris_aeroweb_data(self, results: Dict, city_lower: str) -> None:
|
||||
if city_lower != "paris":
|
||||
return
|
||||
aw_data = self.fetch_from_aeroweb()
|
||||
if aw_data:
|
||||
results["aeroweb"] = aw_data
|
||||
try:
|
||||
aw_current = aw_data.get("current") or {}
|
||||
DBManager().append_airport_obs(
|
||||
icao="LFPB",
|
||||
city=city_lower,
|
||||
temp_c=aw_current.get("temp"),
|
||||
wind_kt=aw_current.get("wind_speed_kt"),
|
||||
obs_time=aw_data.get("obs_time") or datetime.now().isoformat(),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("airport_obs_log append failed for aeroweb city={}", city_lower)
|
||||
|
||||
def _attach_china_official_nearby(
|
||||
self, results: Dict, city_lower: str, use_fahrenheit: bool
|
||||
) -> None:
|
||||
@@ -1384,6 +1422,8 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
self._attach_madis_hfmetar_data(results, city_lower)
|
||||
self._attach_singapore_mss_data(results, city_lower)
|
||||
self._attach_israel_ims_data(results, city_lower)
|
||||
self._attach_saudi_ncm_data(results, city_lower)
|
||||
self._attach_paris_aeroweb_data(results, city_lower)
|
||||
if include_nearby:
|
||||
self._attach_china_official_nearby(results, city_lower, use_fahrenheit)
|
||||
self._attach_japan_official_nearby(results, city_lower, use_fahrenheit)
|
||||
@@ -1433,6 +1473,8 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
|
||||
self._attach_madis_hfmetar_data(results, city_lower)
|
||||
self._attach_singapore_mss_data(results, city_lower)
|
||||
self._attach_israel_ims_data(results, city_lower)
|
||||
self._attach_saudi_ncm_data(results, city_lower)
|
||||
self._attach_paris_aeroweb_data(results, city_lower)
|
||||
if include_nearby:
|
||||
self._attach_china_official_nearby(results, city_lower, use_fahrenheit)
|
||||
self._attach_japan_official_nearby(results, city_lower, use_fahrenheit)
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ def setup_logger(level="DEBUG"):
|
||||
|
||||
# 文件输出
|
||||
logger.add(
|
||||
"data/logs/trading_system.log",
|
||||
"data/logs/polyweather.log",
|
||||
rotation="10 MB",
|
||||
retention="10 days",
|
||||
level=level,
|
||||
|
||||
@@ -1236,12 +1236,15 @@ def _process_airport_city(
|
||||
if not current_obs_time:
|
||||
current_obs_time = str(airport_primary.get("obs_time") or "")
|
||||
if city == "paris":
|
||||
arome_temp = _fetch_arome_temp()
|
||||
if arome_temp is not None:
|
||||
current_temp = arome_temp
|
||||
city_weather.setdefault("current", {})["temp"] = arome_temp
|
||||
if not current_obs_time:
|
||||
current_obs_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
# AEROWEB provides real observations; prefer over AROME model nowcast
|
||||
airport_primary = city_weather.get("airport_primary") or {}
|
||||
if airport_primary.get("source_code") != "aeroweb":
|
||||
arome_temp = _fetch_arome_temp()
|
||||
if arome_temp is not None:
|
||||
current_temp = arome_temp
|
||||
city_weather.setdefault("current", {})["temp"] = arome_temp
|
||||
if not current_obs_time:
|
||||
current_obs_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
if current_temp is None or deb_pred is None:
|
||||
return None
|
||||
|
||||
|
||||
@@ -611,7 +611,7 @@ def get_ops_logs(
|
||||
|
||||
# Fallback to local log file if docker logs returns empty
|
||||
if not log_text.strip():
|
||||
log_file = "data/logs/trading_system.log"
|
||||
log_file = "data/logs/polyweather.log"
|
||||
if os.path.exists(log_file):
|
||||
try:
|
||||
with open(log_file, "r", encoding="utf-8", errors="ignore") as f:
|
||||
|
||||
Reference in New Issue
Block a user