feat: add multiple weather data source scrapers and integrate dashboard temperature chart logic components

src/data_collection/aeroweb_sources.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import hashlib
 import os
 import time
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import Dict, Optional
 from xml.etree import ElementTree as ET
 
@@ -12,6 +12,37 @@ from loguru import logger
 from src.utils.metrics import record_source_call
 
 
+def _aeroweb_obs_time_from_parts(
+    day: Optional[int],
+    hour: Optional[int],
+    minute: Optional[int],
+    *,
+    now_utc: Optional[datetime] = None,
+) -> Optional[str]:
+    if day is None or hour is None or minute is None:
+        return None
+
+    base = now_utc or datetime.now(timezone.utc)
+    if base.tzinfo is None:
+        base = base.replace(tzinfo=timezone.utc)
+    else:
+        base = base.astimezone(timezone.utc)
+
+    def _candidate(year: int, month: int) -> Optional[datetime]:
+        try:
+            return datetime(year, month, int(day), int(hour), int(minute), tzinfo=timezone.utc)
+        except (TypeError, ValueError):
+            return None
+
+    obs_dt = _candidate(base.year, base.month)
+    if obs_dt and obs_dt - base > timedelta(hours=18):
+        previous_month = (base.replace(day=1) - timedelta(days=1)).date()
+        obs_dt = _candidate(previous_month.year, previous_month.month) or obs_dt
+    if not obs_dt:
+        return None
+    return obs_dt.isoformat().replace("+00:00", "Z")
+
+
 class AerowebSourceMixin:
     """Fetch realtime METAR from Météo-France AEROWEB.
 
@@ -172,10 +203,7 @@ class AerowebSourceMixin:
             day = _attr_int("day")
             hour = _attr_int("hour")
             minute = _attr_int("minute")
-            obs_time = None
-            if day is not None and hour is not None and minute is not None:
-                now = datetime.utcnow()
-                obs_time = datetime(now.year, now.month, day, hour, minute).isoformat()
+            obs_time = _aeroweb_obs_time_from_parts(day, hour, minute)
 
             result: Dict = {
                 "current": {

src/data_collection/cowin_sources.py

@@ -23,6 +23,21 @@ COWIN_STATION_ID = int(os.getenv("COWIN_HK_STATION_ID", "6087"))
 COWIN_STATION_LABEL = os.getenv("COWIN_HK_STATION_LABEL", "").strip() or "保良局陳守仁小學 1min (CoWIN)"
 
 
+def _cowin_obs_time_to_iso(value: Any) -> Optional[str]:
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    try:
+        dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except (ValueError, TypeError):
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    else:
+        dt = dt.astimezone(timezone.utc)
+    return dt.isoformat().replace("+00:00", "Z")
+
+
 class CowinSourceMixin:
 
     def _cowin_http_get(self, url: str) -> requests.Response:
@@ -95,7 +110,7 @@ class CowinSourceMixin:
                                (time.perf_counter() - started) * 1000.0)
             return None
 
-        obs_time = str(latest.get("obstime") or "").strip()
+        obs_time = _cowin_obs_time_to_iso(latest.get("obstime")) or str(latest.get("obstime") or "").strip()
 
         temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1)
 

src/data_collection/hko_obs_sources.py

@@ -11,7 +11,7 @@ import csv
 import os
 import io
 import time
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, Optional
 
 from loguru import logger
@@ -34,6 +34,19 @@ HKO_STATIONS = {
 }
 
 
+def _hko_obs_time_to_iso(raw_yyyymmddhhmm: Any) -> Optional[str]:
+    raw = str(raw_yyyymmddhhmm or "").strip()
+    if len(raw) != 12 or not raw.isdigit():
+        return None
+    try:
+        dt = datetime.strptime(raw, "%Y%m%d%H%M").replace(
+            tzinfo=timezone(timedelta(hours=8))
+        )
+        return dt.isoformat()
+    except Exception:
+        return None
+
+
 class HkoObsSourceMixin:
     session: Any
     timeout: Any
@@ -92,13 +105,7 @@ class HkoObsSourceMixin:
                 return None
 
             temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1)
-            obs_iso = None
-            if obs_time and len(obs_time) == 12:
-                try:
-                    dt = datetime.strptime(obs_time, "%Y%m%d%H%M")
-                    obs_iso = dt.isoformat()
-                except Exception:
-                    obs_iso = obs_time
+            obs_iso = _hko_obs_time_to_iso(obs_time) or obs_time
 
             result = {
                 "source": "hko_obs",

src/data_collection/ims_sources.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import Dict, Optional
 
 from loguru import logger
@@ -8,6 +8,48 @@ from loguru import logger
 from src.utils.metrics import record_source_call
 
 
+def _last_sunday(year: int, month: int) -> datetime:
+    if month == 12:
+        last_day = datetime(year + 1, 1, 1) - timedelta(days=1)
+    else:
+        last_day = datetime(year, month + 1, 1) - timedelta(days=1)
+    while last_day.weekday() != 6:
+        last_day -= timedelta(days=1)
+    return last_day
+
+
+def _is_israel_dst(local_dt: datetime) -> bool:
+    start = (_last_sunday(local_dt.year, 3) - timedelta(days=2)).replace(
+        hour=2,
+        minute=0,
+        second=0,
+        microsecond=0,
+    )
+    end = _last_sunday(local_dt.year, 10).replace(
+        hour=2,
+        minute=0,
+        second=0,
+        microsecond=0,
+    )
+    return start <= local_dt < end
+
+
+def _ims_obs_time_to_iso(value: str) -> Optional[str]:
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    try:
+        local_dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except (ValueError, TypeError):
+        return None
+    if local_dt.tzinfo is not None:
+        return local_dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+    offset_hours = 3 if _is_israel_dst(local_dt) else 2
+    return local_dt.replace(
+        tzinfo=timezone(timedelta(hours=offset_hours))
+    ).isoformat()
+
+
 class ImsSourceMixin:
     """Fetch realtime observations from Israel Meteorological Service (IMS).
 
@@ -40,8 +82,8 @@ class ImsSourceMixin:
                 record_source_call("ims", "station", "empty", _elapsed_ms())
                 return None
 
-            latest_time = max(obs_map.keys())
-            latest = obs_map[latest_time].get(station_id) or {}
+            latest_time_raw = max(obs_map.keys())
+            latest = obs_map[latest_time_raw].get(station_id) or {}
             if not latest:
                 record_source_call("ims", "station", "empty", _elapsed_ms())
                 return None
@@ -65,7 +107,7 @@ class ImsSourceMixin:
             wind_dir = _f("WD")
 
             # Compute max / min so far today from all 10-min slots
-            today_prefix = latest_time[:10]
+            today_prefix = latest_time_raw[:10]
             td_vals = []
             for t, stations in obs_map.items():
                 if t.startswith(today_prefix):
@@ -74,6 +116,7 @@ class ImsSourceMixin:
                         td_vals.append(td)
             max_so_far = round(max(td_vals), 1) if td_vals else None
             min_so_far = round(min(td_vals), 1) if td_vals else None
+            obs_time = _ims_obs_time_to_iso(latest_time_raw) or latest_time_raw
 
             result: Dict = {
                 "current": {
@@ -85,7 +128,7 @@ class ImsSourceMixin:
                     "max_temp_so_far": max_so_far,
                     "min_temp_so_far": min_so_far,
                 },
-                "obs_time": latest_time,
+                "obs_time": obs_time,
                 "station_id": station_id,
                 "station_label": "Lod Airport",
                 "lat": 32.002943,
@@ -96,7 +139,7 @@ class ImsSourceMixin:
             logger.info(
                 "IMS Lod Airport (s{}) {}: temp={}°C RH={}% wind={}km/h max={} min={}",
                 station_id,
-                latest_time,
+                obs_time,
                 temp,
                 rh,
                 wind_kmh,

src/data_collection/jma_amedas_sources.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import os
 import time
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional
 
 from loguru import logger
@@ -22,6 +22,19 @@ JMA_AMEDAS_STATIONS: Dict[str, Dict[str, Any]] = {
 }
 
 
+def _jma_obs_time_from_key(value: Any) -> Optional[str]:
+    raw = str(value or "").strip()
+    if len(raw) != 14 or not raw.isdigit():
+        return None
+    try:
+        dt = datetime.strptime(raw, "%Y%m%d%H%M%S").replace(
+            tzinfo=timezone(timedelta(hours=9))
+        )
+        return dt.isoformat()
+    except Exception:
+        return None
+
+
 class JmaAmedasSourceMixin:
     def _jma_http_get_text(self, url: str) -> str:
         getter = getattr(self, "_http_get", None)
@@ -92,15 +105,11 @@ class JmaAmedasSourceMixin:
                 return None
 
             temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else round(temp_c, 1)
-            obs_time = None
-            try:
-                obs_time = datetime.strptime(str(latest_key), "%Y%m%d%H%M%S").isoformat()
-            except Exception:
-                obs_time = str(latest_key)
+            obs_time = _jma_obs_time_from_key(latest_key) or str(latest_key)
 
             result = {
                 "source": "jma_amedas",
-                "timestamp": datetime.utcnow().isoformat(),
+                "timestamp": datetime.now(timezone.utc).isoformat(),
                 "station_code": station_code,
                 "station_name": meta.get("station_label") or "羽田 10分实况 (JMA)",
                 "obs_time": obs_time,

src/data_collection/knmi_sources.py

@@ -8,7 +8,7 @@ Uses netCDF4; requires libhdf5-dev on Linux.
 from __future__ import annotations
 
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Dict, Optional
 
 from loguru import logger
@@ -27,6 +27,21 @@ KNMI_STATION = {
 }
 
 
+def _knmi_obs_time_from_filename(filename: Any) -> Optional[str]:
+    import re
+
+    time_match = re.search(r"(\d{12})", str(filename or ""))
+    if not time_match:
+        return None
+    try:
+        obs_dt = datetime.strptime(time_match.group(1), "%Y%m%d%H%M").replace(
+            tzinfo=timezone.utc
+        )
+        return obs_dt.isoformat().replace("+00:00", "Z")
+    except Exception:
+        return None
+
+
 class KnmiSourceMixin:
     def _knmi_api_key(self) -> str:
         import os
@@ -185,21 +200,15 @@ class KnmiSourceMixin:
                 temp = round(temp_c * 9 / 5 + 32, 1) if use_fahrenheit else temp_c
 
                 # Extract obs time from filename: KMDS__OPER_P___10M_OBS_L2_202605121150.nc
-                import re
-                obs_time = None
-                time_match = re.search(r"(\d{12})", fname)
-                if time_match:
-                    ts = time_match.group(1)
-                    obs_dt = datetime.strptime(ts, "%Y%m%d%H%M")
-                    obs_time = obs_dt.isoformat()
+                obs_time = _knmi_obs_time_from_filename(fname)
 
                 result = {
                     "source": "knmi",
-                    "timestamp": datetime.utcnow().isoformat(),
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
                     "station_code": stn,
                     "station_name": meta["label"],
                     "icao": meta["icao"],
-                    "obs_time": obs_time or datetime.utcnow().isoformat(),
+                    "obs_time": obs_time or datetime.now(timezone.utc).isoformat(),
                     "current": {
                         "temp": temp,
                     },

src/data_collection/mgm_sources.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import os
 import unicodedata
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import Dict, Optional
 
 from loguru import logger
@@ -10,6 +10,20 @@ from loguru import logger
 from src.utils.metrics import record_source_call
 
 
+def _mgm_obs_time_to_iso(value: object) -> Optional[str]:
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    try:
+        dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except (TypeError, ValueError):
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone(timedelta(hours=3)))
+        return dt.isoformat()
+    return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
 class MgmSourceMixin:
     @staticmethod
     def _normalize_mgm_text(value: str) -> str:
@@ -49,6 +63,8 @@ class MgmSourceMixin:
                     def _valid(v):
                         return v is not None and v > -9000
 
+                    obs_time = _mgm_obs_time_to_iso(latest.get("veriZamani"))
+                    results["obs_time"] = obs_time or latest.get("veriZamani")
                     results["current"] = {
                         "temp": latest.get("sicaklik")
                         if _valid(latest.get("sicaklik"))
@@ -78,7 +94,7 @@ class MgmSourceMixin:
                         "mgm_max_temp": latest.get("maxSicaklik")
                         if _valid(latest.get("maxSicaklik"))
                         else None,
-                        "time": latest.get("veriZamani"),
+                        "time": obs_time or latest.get("veriZamani"),
                         "station_name": latest.get("istasyonAd")
                         or latest.get("adi")
                         or latest.get("merkezAd")
@@ -305,7 +321,7 @@ class MgmSourceMixin:
                             temp = obs.get("sicaklik")
                             wind_speed = obs.get("ruzgarHiz")
                             wind_dir = obs.get("ruzgarYon")
-                            obs_time = obs.get("veriZamani")
+                            obs_time = _mgm_obs_time_to_iso(obs.get("veriZamani")) or obs.get("veriZamani")
                             if temp is not None and temp > -9000:
                                 return ist_no, {
                                     "temp": temp,