feat: add multiple weather data source scrapers and integrate dashboard temperature chart logic components

frontend/components/dashboard/scan-terminal/TemperatureTooltipContent.tsx

@@ -3,6 +3,11 @@
 import { validNumber, type EvidenceSeries } from "@/components/dashboard/scan-terminal/temperature-chart-logic";
 
 type TooltipSeries = Pick<EvidenceSeries, "key" | "label" | "color">;
+type TooltipRow = TooltipSeries & { value: number };
+
+function isRunwayTooltipSeries(seriesKey: string) {
+  return seriesKey.startsWith("runway_");
+}
 
 function nearestSeriesValue(
   data: Array<Record<string, any>>,
@@ -42,14 +47,7 @@ export function TemperatureTooltipContent({
 }) {
   if (!active || !payload?.length || !series.length) return null;
   const activePoint = payload[0]?.payload || {};
-  const activeIndex = data.findIndex((point) => point.ts === activePoint.ts);
+  const rows = buildTooltipRows(activePoint, data, series);
-  const rows = series
-    .map((item) => {
-      const directValue = validNumber(activePoint[item.key]);
-      const value = directValue ?? nearestSeriesValue(data, item.key, activeIndex);
-      return value === null ? null : { ...item, value };
-    })
-    .filter((item): item is TooltipSeries & { value: number } => item !== null);
   if (!rows.length) return null;
 
   return (
@@ -69,3 +67,22 @@ export function TemperatureTooltipContent({
     </div>
   );
 }
+
+function buildTooltipRows(
+  activePoint: Record<string, any>,
+  data: Array<Record<string, any>>,
+  series: TooltipSeries[],
+): TooltipRow[] {
+  const activeIndex = data.findIndex((point) => point.ts === activePoint.ts);
+  return series
+    .map((item) => {
+      const directValue = validNumber(activePoint[item.key]);
+      const value = directValue ?? (
+        isRunwayTooltipSeries(item.key) ? null : nearestSeriesValue(data, item.key, activeIndex)
+      );
+      return value === null ? null : { ...item, value };
+    })
+    .filter((item): item is TooltipRow => item !== null);
+}
+
+export const __buildTemperatureTooltipRowsForTest = buildTooltipRows;

frontend/components/dashboard/scan-terminal/__tests__/temperatureDefaultVisibilityPolicy.test.ts

@@ -601,6 +601,38 @@ export function runTests() {
     "AMOS runway_obs patch should use the runway temperature, not ignore the SR/SL point",
   );
 
+  const busanSnapshotWithLocalAndUtc = __buildTemperatureChartDataForTest(
+    {
+      city: "busan",
+      local_date: "2026-05-27",
+      local_time: "09:58",
+      tz_offset_seconds: 9 * 60 * 60,
+      temp_symbol: "°C",
+    } as any,
+    {
+      localTime: "09:58",
+      times: ["00:00", "12:00", "18:00", "23:00"],
+      temps: [19.6, 21.1, 20.0, 19.0],
+      amos: {
+        source: "amos",
+        observation_time: "2026-05-27T00:58:00Z",
+        observation_time_local: "2026-05-27 09:58:00",
+        runway_obs: {
+          runway_pairs: [["S R", "S L"]],
+          temperatures: [[21.5, 12.4]],
+        },
+      },
+    } as any,
+    "1D",
+  );
+  const busanSnapshotLabels = busanSnapshotWithLocalAndUtc.data
+    .filter((point: any) => point[runwayKey("SR/SL")] === 21.5)
+    .map((point: any) => point.label);
+  assert(
+    busanSnapshotLabels.includes("09:58:00"),
+    "AMOS snapshot fallback should prefer UTC observation_time over naive observation_time_local for chart positioning",
+  );
+
   const busanCurrentOnly = __buildTemperatureChartDataForTest(
     {
       city: "busan",

frontend/components/dashboard/scan-terminal/__tests__/temperatureTooltipContent.test.ts

@@ -0,0 +1,51 @@
+import { __buildTemperatureTooltipRowsForTest } from "@/components/dashboard/scan-terminal/TemperatureTooltipContent";
+
+function assert(condition: unknown, message: string) {
+  if (!condition) throw new Error(message);
+}
+
+export function runTests() {
+  const data = [
+    {
+      ts: Date.UTC(2026, 4, 27, 9, 10),
+      label: "09:10:00",
+      runway_35R_17L: 26,
+      gfs: 24.2,
+      deb: null,
+    },
+    {
+      ts: Date.UTC(2026, 4, 27, 14, 0),
+      label: "14:00:00",
+      runway_35R_17L: null,
+      gfs: null,
+      deb: 26.7,
+    },
+    {
+      ts: Date.UTC(2026, 4, 27, 19, 0),
+      label: "19:00:00",
+      runway_35R_17L: null,
+      gfs: 24.8,
+      deb: 23.5,
+    },
+  ];
+  const series = [
+    { key: "runway_35R_17L", label: "35R/17L 结算跑道", color: "#009688" },
+    { key: "gfs", label: "GFS", color: "#10b981" },
+    { key: "deb", label: "DEB Forecast", color: "#f97316" },
+  ];
+
+  const rows = __buildTemperatureTooltipRowsForTest(data[1], data, series);
+
+  assert(
+    !rows.some((row) => row.key === "runway_35R_17L"),
+    "runway tooltip rows should not use nearest-value fallback when the active x slot has no runway value",
+  );
+  assert(
+    rows.some((row) => row.key === "deb" && row.value === 26.7),
+    "tooltip should still show direct values at the active x slot",
+  );
+  assert(
+    rows.some((row) => row.key === "gfs" && row.value === 24.2),
+    "non-runway sparse series should keep nearest-value fallback",
+  );
+}

frontend/components/dashboard/scan-terminal/temperature-chart-logic.ts

@@ -1069,7 +1069,7 @@ function buildRunwayHistorySeries(
   const pointTemps = runwayObs?.point_temperatures || [];
   const isAmosTempDewTuple = String(amos?.source || "").toLowerCase() === "amos";
   const anchor =
-    getCityLocalUtcTimestamp(amos?.observation_time_local || amos?.observation_time || hourly?.localTime || row?.local_time, tzOffset, localDateStr) ??
+    getCityLocalUtcTimestamp(amos?.observation_time || amos?.observation_time_local || hourly?.localTime || row?.local_time, tzOffset, localDateStr) ??
     getCityLocalUtcTimestamp(row?.local_time, tzOffset, localDateStr);
 
   if (!anchor || !Array.isArray(runwayTemps)) return [];

src/data_collection/aeroweb_sources.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import hashlib
 import os
 import time
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import Dict, Optional
 from xml.etree import ElementTree as ET
 
@@ -12,6 +12,37 @@ from loguru import logger
 from src.utils.metrics import record_source_call
 
 
+def _aeroweb_obs_time_from_parts(
+    day: Optional[int],
+    hour: Optional[int],
+    minute: Optional[int],
+    *,
+    now_utc: Optional[datetime] = None,
+) -> Optional[str]:
+    if day is None or hour is None or minute is None:
+        return None
+
+    base = now_utc or datetime.now(timezone.utc)
+    if base.tzinfo is None:
+        base = base.replace(tzinfo=timezone.utc)
+    else:
+        base = base.astimezone(timezone.utc)
+
+    def _candidate(year: int, month: int) -> Optional[datetime]:
+        try:
+            return datetime(year, month, int(day), int(hour), int(minute), tzinfo=timezone.utc)
+        except (TypeError, ValueError):
+            return None
+
+    obs_dt = _candidate(base.year, base.month)
+    if obs_dt and obs_dt - base > timedelta(hours=18):
+        previous_month = (base.replace(day=1) - timedelta(days=1)).date()
+        obs_dt = _candidate(previous_month.year, previous_month.month) or obs_dt
+    if not obs_dt:
+        return None
+    return obs_dt.isoformat().replace("+00:00", "Z")
+
+
 class AerowebSourceMixin:
     """Fetch realtime METAR from Météo-France AEROWEB.
 
@@ -172,10 +203,7 @@ class AerowebSourceMixin:
             day = _attr_int("day")
             hour = _attr_int("hour")
             minute = _attr_int("minute")
-            obs_time = None
+            obs_time = _aeroweb_obs_time_from_parts(day, hour, minute)
-            if day is not None and hour is not None and minute is not None:
-                now = datetime.utcnow()
-                obs_time = datetime(now.year, now.month, day, hour, minute).isoformat()
 
             result: Dict = {
                 "current": {

src/data_collection/cowin_sources.py

@@ -23,6 +23,21 @@ COWIN_STATION_ID = int(os.getenv("COWIN_HK_STATION_ID", "6087"))
 COWIN_STATION_LABEL = os.getenv("COWIN_HK_STATION_LABEL", "").strip() or "保良局陳守仁小學 1min (CoWIN)"
 
 
+def _cowin_obs_time_to_iso(value: Any) -> Optional[str]:
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    try:
+        dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except (ValueError, TypeError):
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    else:
+        dt = dt.astimezone(timezone.utc)
+    return dt.isoformat().replace("+00:00", "Z")
+
+
 class CowinSourceMixin:
 
     def _cowin_http_get(self, url: str) -> requests.Response:
@@ -95,7 +110,7 @@ class CowinSourceMixin:
                                (time.perf_counter() - started) * 1000.0)
             return None
 
-        obs_time = str(latest.get("obstime") or "").strip()
+        obs_time = _cowin_obs_time_to_iso(latest.get("obstime")) or str(latest.get("obstime") or "").strip()
 
         temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1)
 

src/data_collection/hko_obs_sources.py

@@ -11,7 +11,7 @@ import csv
 import os
 import io
 import time
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, Optional
 
 from loguru import logger
@@ -34,6 +34,19 @@ HKO_STATIONS = {
 }
 
 
+def _hko_obs_time_to_iso(raw_yyyymmddhhmm: Any) -> Optional[str]:
+    raw = str(raw_yyyymmddhhmm or "").strip()
+    if len(raw) != 12 or not raw.isdigit():
+        return None
+    try:
+        dt = datetime.strptime(raw, "%Y%m%d%H%M").replace(
+            tzinfo=timezone(timedelta(hours=8))
+        )
+        return dt.isoformat()
+    except Exception:
+        return None
+
+
 class HkoObsSourceMixin:
     session: Any
     timeout: Any
@@ -92,13 +105,7 @@ class HkoObsSourceMixin:
                 return None
 
             temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1)
-            obs_iso = None
+            obs_iso = _hko_obs_time_to_iso(obs_time) or obs_time
-            if obs_time and len(obs_time) == 12:
-                try:
-                    dt = datetime.strptime(obs_time, "%Y%m%d%H%M")
-                    obs_iso = dt.isoformat()
-                except Exception:
-                    obs_iso = obs_time
 
             result = {
                 "source": "hko_obs",

src/data_collection/ims_sources.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import Dict, Optional
 
 from loguru import logger
@@ -8,6 +8,48 @@ from loguru import logger
 from src.utils.metrics import record_source_call
 
 
+def _last_sunday(year: int, month: int) -> datetime:
+    if month == 12:
+        last_day = datetime(year + 1, 1, 1) - timedelta(days=1)
+    else:
+        last_day = datetime(year, month + 1, 1) - timedelta(days=1)
+    while last_day.weekday() != 6:
+        last_day -= timedelta(days=1)
+    return last_day
+
+
+def _is_israel_dst(local_dt: datetime) -> bool:
+    start = (_last_sunday(local_dt.year, 3) - timedelta(days=2)).replace(
+        hour=2,
+        minute=0,
+        second=0,
+        microsecond=0,
+    )
+    end = _last_sunday(local_dt.year, 10).replace(
+        hour=2,
+        minute=0,
+        second=0,
+        microsecond=0,
+    )
+    return start <= local_dt < end
+
+
+def _ims_obs_time_to_iso(value: str) -> Optional[str]:
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    try:
+        local_dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except (ValueError, TypeError):
+        return None
+    if local_dt.tzinfo is not None:
+        return local_dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+    offset_hours = 3 if _is_israel_dst(local_dt) else 2
+    return local_dt.replace(
+        tzinfo=timezone(timedelta(hours=offset_hours))
+    ).isoformat()
+
+
 class ImsSourceMixin:
     """Fetch realtime observations from Israel Meteorological Service (IMS).
 
@@ -40,8 +82,8 @@ class ImsSourceMixin:
                 record_source_call("ims", "station", "empty", _elapsed_ms())
                 return None
 
-            latest_time = max(obs_map.keys())
+            latest_time_raw = max(obs_map.keys())
-            latest = obs_map[latest_time].get(station_id) or {}
+            latest = obs_map[latest_time_raw].get(station_id) or {}
             if not latest:
                 record_source_call("ims", "station", "empty", _elapsed_ms())
                 return None
@@ -65,7 +107,7 @@ class ImsSourceMixin:
             wind_dir = _f("WD")
 
             # Compute max / min so far today from all 10-min slots
-            today_prefix = latest_time[:10]
+            today_prefix = latest_time_raw[:10]
             td_vals = []
             for t, stations in obs_map.items():
                 if t.startswith(today_prefix):
@@ -74,6 +116,7 @@ class ImsSourceMixin:
                         td_vals.append(td)
             max_so_far = round(max(td_vals), 1) if td_vals else None
             min_so_far = round(min(td_vals), 1) if td_vals else None
+            obs_time = _ims_obs_time_to_iso(latest_time_raw) or latest_time_raw
 
             result: Dict = {
                 "current": {
@@ -85,7 +128,7 @@ class ImsSourceMixin:
                     "max_temp_so_far": max_so_far,
                     "min_temp_so_far": min_so_far,
                 },
-                "obs_time": latest_time,
+                "obs_time": obs_time,
                 "station_id": station_id,
                 "station_label": "Lod Airport",
                 "lat": 32.002943,
@@ -96,7 +139,7 @@ class ImsSourceMixin:
             logger.info(
                 "IMS Lod Airport (s{}) {}: temp={}°C RH={}% wind={}km/h max={} min={}",
                 station_id,
-                latest_time,
+                obs_time,
                 temp,
                 rh,
                 wind_kmh,

src/data_collection/jma_amedas_sources.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import os
 import time
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional
 
 from loguru import logger
@@ -22,6 +22,19 @@ JMA_AMEDAS_STATIONS: Dict[str, Dict[str, Any]] = {
 }
 
 
+def _jma_obs_time_from_key(value: Any) -> Optional[str]:
+    raw = str(value or "").strip()
+    if len(raw) != 14 or not raw.isdigit():
+        return None
+    try:
+        dt = datetime.strptime(raw, "%Y%m%d%H%M%S").replace(
+            tzinfo=timezone(timedelta(hours=9))
+        )
+        return dt.isoformat()
+    except Exception:
+        return None
+
+
 class JmaAmedasSourceMixin:
     def _jma_http_get_text(self, url: str) -> str:
         getter = getattr(self, "_http_get", None)
@@ -92,15 +105,11 @@ class JmaAmedasSourceMixin:
                 return None
 
             temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1)
-            obs_time = None
+            obs_time = _jma_obs_time_from_key(latest_key) or str(latest_key)
-            try:
-                obs_time = datetime.strptime(str(latest_key), "%Y%m%d%H%M%S").isoformat()
-            except Exception:
-                obs_time = str(latest_key)
 
             result = {
                 "source": "jma_amedas",
-                "timestamp": datetime.utcnow().isoformat(),
+                "timestamp": datetime.now(timezone.utc).isoformat(),
                 "station_code": station_code,
                 "station_name": meta.get("station_label") or "羽田 10分实况 (JMA)",
                 "obs_time": obs_time,

src/data_collection/knmi_sources.py

@@ -8,7 +8,7 @@ Uses netCDF4; requires libhdf5-dev on Linux.
 from __future__ import annotations
 
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Dict, Optional
 
 from loguru import logger
@@ -27,6 +27,21 @@ KNMI_STATION = {
 }
 
 
+def _knmi_obs_time_from_filename(filename: Any) -> Optional[str]:
+    import re
+
+    time_match = re.search(r"(\d{12})", str(filename or ""))
+    if not time_match:
+        return None
+    try:
+        obs_dt = datetime.strptime(time_match.group(1), "%Y%m%d%H%M").replace(
+            tzinfo=timezone.utc
+        )
+        return obs_dt.isoformat().replace("+00:00", "Z")
+    except Exception:
+        return None
+
+
 class KnmiSourceMixin:
     def _knmi_api_key(self) -> str:
         import os
@@ -185,21 +200,15 @@ class KnmiSourceMixin:
                 temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else temp_c
 
                 # Extract obs time from filename: KMDS__OPER_P___10M_OBS_L2_202605121150.nc
-                import re
+                obs_time = _knmi_obs_time_from_filename(fname)
-                obs_time = None
-                time_match = re.search(r"(\d{12})", fname)
-                if time_match:
-                    ts = time_match.group(1)
-                    obs_dt = datetime.strptime(ts, "%Y%m%d%H%M")
-                    obs_time = obs_dt.isoformat()
 
                 result = {
                     "source": "knmi",
-                    "timestamp": datetime.utcnow().isoformat(),
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
                     "station_code": stn,
                     "station_name": meta["label"],
                     "icao": meta["icao"],
-                    "obs_time": obs_time or datetime.utcnow().isoformat(),
+                    "obs_time": obs_time or datetime.now(timezone.utc).isoformat(),
                     "current": {
                         "temp": temp,
                     },

src/data_collection/mgm_sources.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import os
 import unicodedata
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import Dict, Optional
 
 from loguru import logger
@@ -10,6 +10,20 @@ from loguru import logger
 from src.utils.metrics import record_source_call
 
 
+def _mgm_obs_time_to_iso(value: object) -> Optional[str]:
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    try:
+        dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except (TypeError, ValueError):
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone(timedelta(hours=3)))
+        return dt.isoformat()
+    return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
 class MgmSourceMixin:
     @staticmethod
     def _normalize_mgm_text(value: str) -> str:
@@ -49,6 +63,8 @@ class MgmSourceMixin:
                     def _valid(v):
                         return v is not None and v > -9000
 
+                    obs_time = _mgm_obs_time_to_iso(latest.get("veriZamani"))
+                    results["obs_time"] = obs_time or latest.get("veriZamani")
                     results["current"] = {
                         "temp": latest.get("sicaklik")
                         if _valid(latest.get("sicaklik"))
@@ -78,7 +94,7 @@ class MgmSourceMixin:
                         "mgm_max_temp": latest.get("maxSicaklik")
                         if _valid(latest.get("maxSicaklik"))
                         else None,
-                        "time": latest.get("veriZamani"),
+                        "time": obs_time or latest.get("veriZamani"),
                         "station_name": latest.get("istasyonAd")
                         or latest.get("adi")
                         or latest.get("merkezAd")
@@ -305,7 +321,7 @@ class MgmSourceMixin:
                             temp = obs.get("sicaklik")
                             wind_speed = obs.get("ruzgarHiz")
                             wind_dir = obs.get("ruzgarYon")
-                            obs_time = obs.get("veriZamani")
+                            obs_time = _mgm_obs_time_to_iso(obs.get("veriZamani")) or obs.get("veriZamani")
                             if temp is not None and temp > -9000:
                                 return ist_no, {
                                     "temp": temp,

tests/test_observation_time_sources.py

@@ -0,0 +1,51 @@
+from datetime import datetime, timezone
+
+from src.data_collection.aeroweb_sources import _aeroweb_obs_time_from_parts
+from src.data_collection.cowin_sources import _cowin_obs_time_to_iso
+from src.data_collection.hko_obs_sources import _hko_obs_time_to_iso
+from src.data_collection.ims_sources import _ims_obs_time_to_iso
+from src.data_collection.jma_amedas_sources import _jma_obs_time_from_key
+from src.data_collection.knmi_sources import _knmi_obs_time_from_filename
+from src.data_collection.mgm_sources import _mgm_obs_time_to_iso
+
+
+def test_aeroweb_obs_time_is_utc_aware():
+    assert (
+        _aeroweb_obs_time_from_parts(
+            27,
+            1,
+            0,
+            now_utc=datetime(2026, 5, 27, 1, 5, tzinfo=timezone.utc),
+        )
+        == "2026-05-27T01:00:00Z"
+    )
+
+
+def test_cowin_obs_time_uses_utc_window_when_timezone_is_missing():
+    assert _cowin_obs_time_to_iso("2026-05-27T01:15:00") == "2026-05-27T01:15:00Z"
+    assert _cowin_obs_time_to_iso("2026-05-27T09:15:00+08:00") == "2026-05-27T01:15:00Z"
+
+
+def test_hko_one_minute_obs_time_keeps_hong_kong_timezone():
+    assert _hko_obs_time_to_iso("202605270858") == "2026-05-27T08:58:00+08:00"
+
+
+def test_ims_obs_time_keeps_israel_timezone():
+    assert _ims_obs_time_to_iso("2026-05-27 03:50:00") == "2026-05-27T03:50:00+03:00"
+    assert _ims_obs_time_to_iso("2026-01-27 03:50:00") == "2026-01-27T03:50:00+02:00"
+
+
+def test_jma_amedas_obs_time_keeps_japan_timezone():
+    assert _jma_obs_time_from_key("20260527095800") == "2026-05-27T09:58:00+09:00"
+
+
+def test_knmi_filename_obs_time_is_utc_aware():
+    assert (
+        _knmi_obs_time_from_filename("KMDS__OPER_P___10M_OBS_L2_202605121150.nc")
+        == "2026-05-12T11:50:00Z"
+    )
+
+
+def test_mgm_obs_time_is_exposed_as_utc_aware():
+    assert _mgm_obs_time_to_iso("2026-05-27T01:00:00.000Z") == "2026-05-27T01:00:00Z"
+    assert _mgm_obs_time_to_iso("2026-05-27T04:00:00") == "2026-05-27T04:00:00+03:00"