Add METAR backfill script and remove settlement history fallback

scripts/backfill_recent_daily_actuals_from_metar.py

@@ -0,0 +1,105 @@
+import argparse
+import os
+import sys
+from datetime import datetime, timedelta
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+from src.analysis.deb_algorithm import load_history, reconcile_recent_actual_highs, save_history  # noqa: E402
+from src.data_collection.city_registry import CITY_REGISTRY  # noqa: E402
+
+
+def _target_dates(city_info: dict, lookback_days: int) -> list[str]:
+    tz_offset = int(city_info.get("tz_offset") or 0)
+    local_now = datetime.utcnow() + timedelta(seconds=tz_offset)
+    local_today = local_now.date()
+    dates = []
+    for offset in range(max(lookback_days, 1), 0, -1):
+        day = local_today - timedelta(days=offset)
+        dates.append(day.strftime("%Y-%m-%d"))
+    return dates
+
+
+def _is_metar_city(city_info: dict) -> bool:
+    source = str(city_info.get("settlement_source") or "metar").strip().lower()
+    return source == "metar"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Seed recent daily_records rows and backfill actual_high from aviationweather METAR history."
+    )
+    parser.add_argument(
+        "--cities",
+        nargs="*",
+        default=[],
+        help="Optional subset of city registry keys.",
+    )
+    parser.add_argument(
+        "--lookback-days",
+        type=int,
+        default=14,
+        help="How many recent local days to seed/backfill (excluding today).",
+    )
+    parser.add_argument(
+        "--only-missing-cities",
+        action="store_true",
+        help="Only process cities that do not exist in daily_records yet.",
+    )
+    args = parser.parse_args()
+
+    history_file = os.path.join(PROJECT_ROOT, "data", "daily_records.json")
+    data = load_history(history_file)
+
+    selected = {str(item).strip().lower() for item in args.cities if str(item).strip()}
+    candidates: list[str] = []
+    for city_name, city_info in sorted(CITY_REGISTRY.items()):
+        if selected and city_name not in selected:
+            continue
+        if not isinstance(city_info, dict) or not _is_metar_city(city_info):
+            continue
+        if not str(city_info.get("icao") or "").strip():
+            continue
+        if args.only_missing_cities and city_name in data:
+            continue
+        candidates.append(city_name)
+
+    seeded_rows = 0
+    seeded_cities = 0
+    for city_name in candidates:
+        city_info = CITY_REGISTRY[city_name]
+        city_rows = data.get(city_name)
+        if not isinstance(city_rows, dict):
+            city_rows = {}
+            data[city_name] = city_rows
+
+        before = len(city_rows)
+        for date_str in _target_dates(city_info, args.lookback_days):
+            city_rows.setdefault(date_str, {})
+        if len(city_rows) > before:
+            seeded_cities += 1
+            seeded_rows += len(city_rows) - before
+
+    if seeded_rows > 0:
+        save_history(history_file, data)
+
+    results = []
+    for city_name in candidates:
+        result = reconcile_recent_actual_highs(city_name, lookback_days=args.lookback_days)
+        results.append((city_name, result))
+
+    print(
+        {
+            "lookback_days": args.lookback_days,
+            "candidate_count": len(candidates),
+            "seeded_cities": seeded_cities,
+            "seeded_rows": seeded_rows,
+            "results": results,
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()

web/routes.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import json
 import os
 from datetime import datetime, timedelta
 from typing import Optional
@@ -49,29 +48,6 @@ from web.core import (
 
 router = APIRouter()
 
-_SETTLEMENT_HISTORY_CACHE: Optional[dict] = None
-
-
-def _load_settlement_history() -> dict:
-    global _SETTLEMENT_HISTORY_CACHE
-    if _SETTLEMENT_HISTORY_CACHE is not None:
-        return _SETTLEMENT_HISTORY_CACHE
-
-    project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    history_path = os.path.join(
-        project_root,
-        "artifacts",
-        "probability_calibration",
-        "settlement_history.json",
-    )
-    try:
-        with open(history_path, "r", encoding="utf-8") as handle:
-            payload = json.load(handle)
-        _SETTLEMENT_HISTORY_CACHE = payload if isinstance(payload, dict) else {}
-    except Exception:
-        _SETTLEMENT_HISTORY_CACHE = {}
-    return _SETTLEMENT_HISTORY_CACHE
-
 
 def _parse_snapshot_dt(value: object) -> Optional[datetime]:
     raw = str(value or "").strip()
@@ -222,11 +198,9 @@ async def city_history(request: Request, name: str):
     project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     history_file = os.path.join(project_root, "data", "daily_records.json")
     data = load_history(history_file)
-    settlement_history = _load_settlement_history()
-    settlement_city_data = settlement_history.get(city, {})
 
     city_data = data.get(city, {}) if isinstance(data.get(city, {}), dict) else {}
-    if not city_data and not isinstance(settlement_city_data, dict):
+    if not city_data:
         source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
         return {
             "history": [],
@@ -234,20 +208,12 @@ async def city_history(request: Request, name: str):
             "settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
         }
 
-    settlement_city_data = settlement_city_data if isinstance(settlement_city_data, dict) else {}
     out = []
-    all_days = sorted(set(city_data.keys()) | set(settlement_city_data.keys()))
-    for day in all_days:
-        rec = city_data.get(day, {})
+    for day, rec in sorted(city_data.items()):
         if not isinstance(rec, dict):
             rec = {}
-        settlement_rec = settlement_city_data.get(day, {})
-        if not isinstance(settlement_rec, dict):
-            settlement_rec = {}
 
         act = rec.get("actual_high")
-        if act is None:
-            act = settlement_rec.get("max_temp")
         deb = rec.get("deb_prediction")
         mu = rec.get("mu")
         snapshots = load_snapshot_rows_for_day(city, day)