Add ops dashboards for training data and model coverage

scripts/backfill_recent_daily_actuals_from_metar.py

@@ -25,12 +25,12 @@ def _target_dates(city_info: dict, lookback_days: int) -> list[str]:
 
 def _is_metar_city(city_info: dict) -> bool:
     source = str(city_info.get("settlement_source") or "metar").strip().lower()
-    return source == "metar"
+    return source in {"metar", "hko", "noaa", "wunderground"}
 
 
 def main() -> None:
     parser = argparse.ArgumentParser(
-        description="Seed recent daily_records rows and backfill actual_high from aviationweather METAR history."
+        description="Seed recent runtime daily_records rows and backfill actual_high from the city's settlement source."
     )
     parser.add_argument(
         "--cities",

scripts/fit_probability_calibration.py

@@ -15,9 +15,12 @@ from src.analysis.probability_calibration import (  # noqa: E402
 )
 from src.analysis.deb_algorithm import load_history  # noqa: E402
 from src.database.runtime_state import (  # noqa: E402
+    DailyRecordRepository,
     ProbabilitySnapshotRepository,
     STATE_STORAGE_FILE,
     STATE_STORAGE_SQLITE,
+    TrainingFeatureRecordRepository,
+    TruthRecordRepository,
     get_state_storage_mode,
 )
 
@@ -56,12 +59,34 @@ def _default_snapshot_arg():
 
 
 def _load_history_with_fallback(path):
+    if not path:
+        if get_state_storage_mode() == STATE_STORAGE_SQLITE:
+            return DailyRecordRepository().load_all()
+        return {}
     data = load_history(path)
     if data:
         return data
     return _load_json_if_exists(path)
 
 
+def _load_truth_history():
+    if get_state_storage_mode() != STATE_STORAGE_SQLITE:
+        return {}
+    try:
+        return TruthRecordRepository().load_all()
+    except Exception:
+        return {}
+
+
+def _load_training_feature_history():
+    if get_state_storage_mode() != STATE_STORAGE_SQLITE:
+        return {}
+    try:
+        return TrainingFeatureRecordRepository().load_all()
+    except Exception:
+        return {}
+
+
 def _load_snapshot_rows(path):
     if get_state_storage_mode() == STATE_STORAGE_SQLITE:
         return ProbabilitySnapshotRepository().load_all_rows()
@@ -82,18 +107,28 @@ def _load_snapshot_rows(path):
     return rows
 
 
-def _actual_high_for(history, settlement_history, city, date_str):
+def _actual_high_for(history, truth_history, settlement_history, city, date_str):
     city_rows = (history or {}).get(city) or {}
     record = city_rows.get(date_str) or {}
     actual_high = _sf(record.get("actual_high")) if isinstance(record, dict) else None
+    truth_record = ((truth_history.get(city) or {}).get(date_str) or {})
+    if actual_high is None and isinstance(truth_record, dict):
+        actual_high = _sf(truth_record.get("actual_high"))
     filled = False
     if actual_high is None:
         actual_high = _sf(((settlement_history.get(city) or {}).get(date_str) or {}).get("max_temp"))
         filled = actual_high is not None
-    return actual_high, filled
+    metadata = {
+        "settlement_source": truth_record.get("settlement_source"),
+        "settlement_station_code": truth_record.get("settlement_station_code"),
+        "truth_version": truth_record.get("truth_version"),
+        "truth_updated_by": truth_record.get("updated_by"),
+        "truth_updated_at": truth_record.get("truth_updated_at"),
+    }
+    return actual_high, filled, metadata
 
 
-def _extract_snapshot_samples(history, snapshot_rows, settlement_history=None):
+def _extract_snapshot_samples(history, truth_history=None, snapshot_rows=None, settlement_history=None):
     samples = []
     filled_actual_from_history = 0
     today = datetime.utcnow().strftime("%Y-%m-%d")
@@ -105,7 +140,13 @@ def _extract_snapshot_samples(history, snapshot_rows, settlement_history=None):
         if not city or not date_str or date_str == today:
             continue
 
-        actual_high, filled = _actual_high_for(history, settlement_history, city, date_str)
+        actual_high, filled, truth_meta = _actual_high_for(
+            history,
+            truth_history or {},
+            settlement_history,
+            city,
+            date_str,
+        )
         if actual_high is None:
             continue
         if filled:
@@ -167,13 +208,20 @@ def _extract_snapshot_samples(history, snapshot_rows, settlement_history=None):
                 "max_so_far_gap": max_so_far_gap,
                 "peak_flag": peak_flag,
                 "sample_source": "snapshot",
+                **truth_meta,
             }
         )
 
     return samples, filled_actual_from_history
 
 
-def _extract_daily_record_samples(history, settlement_history=None, excluded_keys=None):
+def _extract_daily_record_samples(
+    history,
+    training_feature_history=None,
+    truth_history=None,
+    settlement_history=None,
+    excluded_keys=None,
+):
     samples = []
     filled_actual_from_history = 0
     today = datetime.utcnow().strftime("%Y-%m-%d")
@@ -189,13 +237,18 @@ def _extract_daily_record_samples(history, settlement_history=None, excluded_key
             if (city, date_str) in excluded_keys:
                 continue
             actual_high = _sf(record.get("actual_high"))
+            truth_meta = ((truth_history or {}).get(city) or {}).get(date_str) or {}
+            if actual_high is None:
+                actual_high = _sf(truth_meta.get("actual_high"))
             if actual_high is None:
                 actual_high = _sf((city_settlement.get(date_str) or {}).get("max_temp"))
                 if actual_high is not None:
                     filled_actual_from_history += 1
-            deb_prediction = _sf(record.get("deb_prediction"))
-            raw_mu = _sf(record.get("mu")) or deb_prediction
-            forecasts = record.get("forecasts") or {}
+            feature_record = ((training_feature_history or {}).get(city) or {}).get(date_str) or {}
+            source_record = feature_record if isinstance(feature_record, dict) and feature_record else record
+            deb_prediction = _sf(source_record.get("deb_prediction"))
+            raw_mu = _sf(source_record.get("mu")) or deb_prediction
+            forecasts = source_record.get("forecasts") or {}
             if not isinstance(forecasts, dict):
                 forecasts = {}
             forecast_values = [val for val in (_sf(v) for v in forecasts.values()) if val is not None]
@@ -203,7 +256,7 @@ def _extract_daily_record_samples(history, settlement_history=None, excluded_key
             forecast_median = (
                 forecast_values[len(forecast_values) // 2] if forecast_values else None
             )
-            feature_snapshot = record.get("probability_features") or {}
+            feature_snapshot = source_record.get("probability_features") or {}
             if not isinstance(feature_snapshot, dict):
                 feature_snapshot = {}
 
@@ -243,15 +296,21 @@ def _extract_daily_record_samples(history, settlement_history=None, excluded_key
                     "max_so_far_gap": max_so_far_gap,
                     "peak_flag": peak_flag,
                     "sample_source": "daily_record",
+                    "settlement_source": truth_meta.get("settlement_source"),
+                    "settlement_station_code": truth_meta.get("settlement_station_code"),
+                    "truth_version": truth_meta.get("truth_version"),
+                    "truth_updated_by": truth_meta.get("updated_by"),
+                    "truth_updated_at": truth_meta.get("truth_updated_at"),
                 }
             )
     return samples, filled_actual_from_history
 
 
-def _extract_samples(history, settlement_history=None, snapshot_rows=None):
+def _extract_samples(history, training_feature_history=None, truth_history=None, settlement_history=None, snapshot_rows=None):
     snapshot_samples, snapshot_filled = _extract_snapshot_samples(
         history,
-        snapshot_rows or [],
+        truth_history=truth_history,
+        snapshot_rows=snapshot_rows or [],
         settlement_history=settlement_history,
     )
     excluded_keys = {
@@ -260,6 +319,8 @@ def _extract_samples(history, settlement_history=None, snapshot_rows=None):
     }
     daily_samples, daily_filled = _extract_daily_record_samples(
         history,
+        training_feature_history=training_feature_history,
+        truth_history=truth_history,
         settlement_history=settlement_history,
         excluded_keys=excluded_keys,
     )
@@ -301,10 +362,14 @@ def main():
     args = parser.parse_args()
 
     history = _load_history_with_fallback(args.history_file)
+    training_feature_history = _load_training_feature_history()
+    truth_history = _load_truth_history()
     settlement_history = _load_json_if_exists(args.settlement_history)
     snapshot_rows = _load_snapshot_rows(args.snapshot_file)
     samples, filled_actual_from_history = _extract_samples(
         history,
+        training_feature_history=training_feature_history,
+        truth_history=truth_history,
         settlement_history=settlement_history,
         snapshot_rows=snapshot_rows,
     )

scripts/restore_training_feature_history.py

@@ -0,0 +1,114 @@
+import argparse
+import json
+import os
+import sys
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+from src.database.runtime_state import (  # noqa: E402
+    ProbabilitySnapshotRepository,
+    TrainingFeatureRecordRepository,
+    get_state_storage_mode,
+)
+
+
+def _load_legacy_snapshot_rows(path: str):
+    rows = []
+    if not path or not os.path.exists(path):
+        return rows
+    with open(path, "r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except Exception:
+                continue
+            if isinstance(row, dict):
+                rows.append(row)
+    return rows
+
+
+def _spread_from_ensemble(ensemble: dict):
+    if not isinstance(ensemble, dict):
+        return None
+    try:
+        p10 = float(ensemble.get("p10"))
+        p90 = float(ensemble.get("p90"))
+    except Exception:
+        return None
+    if p90 < p10:
+        return None
+    return max(0.1, round((p90 - p10) / 2.56, 3))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Restore permanent training feature history from snapshot archives."
+    )
+    parser.add_argument(
+        "--snapshot-file",
+        default=os.path.join(PROJECT_ROOT, "data", "probability_training_snapshots.jsonl"),
+    )
+    args = parser.parse_args()
+
+    rows = []
+    if get_state_storage_mode() == "sqlite":
+        rows.extend(ProbabilitySnapshotRepository().load_all_rows())
+    rows.extend(_load_legacy_snapshot_rows(args.snapshot_file))
+
+    latest = {}
+    for row in rows:
+        city = str(row.get("city") or "").strip().lower()
+        date_str = str(row.get("date") or "").strip()
+        ts = str(row.get("timestamp") or "")
+        if not city or not date_str:
+            continue
+        key = (city, date_str)
+        current = latest.get(key)
+        if current is None or ts >= str(current.get("timestamp") or ""):
+            latest[key] = row
+
+    repo = TrainingFeatureRecordRepository()
+    restored = 0
+    for (city, date_str), row in latest.items():
+        repo.upsert_record(
+            city,
+            date_str,
+            {
+                "forecasts": row.get("multi_model") or {},
+                "deb_prediction": row.get("deb_prediction"),
+                "mu": row.get("raw_mu"),
+                "probability_features": {
+                    "raw_mu": row.get("raw_mu"),
+                    "raw_sigma": row.get("raw_sigma"),
+                    "deb_prediction": row.get("deb_prediction"),
+                    "ens_median": ((row.get("ensemble") or {}).get("median")),
+                    "ensemble_spread": _spread_from_ensemble(row.get("ensemble") or {}),
+                    "max_so_far": row.get("max_so_far"),
+                    "peak_status": row.get("peak_status"),
+                },
+                "prob_snapshot": row.get("prob_snapshot") or [],
+                "shadow_prob_snapshot": row.get("shadow_prob_snapshot") or [],
+                "probability_calibration": {
+                    "engine": row.get("probability_engine"),
+                    "mode": row.get("probability_mode"),
+                    "calibration_version": row.get("calibration_version"),
+                    "calibration_source": row.get("calibration_source"),
+                    "calibrated_mu": row.get("calibrated_mu"),
+                    "calibrated_sigma": row.get("calibrated_sigma"),
+                },
+                "observation": row.get("observation") or {},
+                "snapshot_timestamp": row.get("timestamp"),
+            },
+        )
+        restored += 1
+
+    print(json.dumps({"restored_feature_records": restored}, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

scripts/restore_training_truth_history.py

@@ -0,0 +1,140 @@
+import argparse
+import json
+import os
+import sys
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+from src.data_collection.city_registry import CITY_REGISTRY  # noqa: E402
+from src.database.runtime_state import TruthRecordRepository  # noqa: E402
+from scripts.fit_probability_calibration import (  # noqa: E402
+    _default_history_arg,
+    _load_history_with_fallback,
+    _load_json_if_exists,
+)
+
+
+def _sf(value):
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except Exception:
+        return None
+
+
+def _truth_meta(city: str) -> dict:
+    city_meta = CITY_REGISTRY.get(city) or {}
+    return {
+        "settlement_source": str(city_meta.get("settlement_source") or "metar").strip().lower(),
+        "settlement_station_code": str(
+            city_meta.get("settlement_station_code") or city_meta.get("icao") or ""
+        ).strip().upper()
+        or None,
+        "settlement_station_label": str(
+            city_meta.get("settlement_station_label")
+            or city_meta.get("airport_name")
+            or city_meta.get("name")
+            or ""
+        ).strip()
+        or None,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Restore permanent training truth history from settlement history and recent runtime cache."
+    )
+    parser.add_argument(
+        "--history-file",
+        default=_default_history_arg(),
+    )
+    parser.add_argument(
+        "--settlement-history",
+        default=os.path.join(
+            PROJECT_ROOT,
+            "artifacts",
+            "probability_calibration",
+            "settlement_history.json",
+        ),
+    )
+    parser.add_argument(
+        "--truth-version",
+        default="v1",
+    )
+    args = parser.parse_args()
+
+    repo = TruthRecordRepository()
+    settlement_history = _load_json_if_exists(args.settlement_history)
+    runtime_history = _load_history_with_fallback(args.history_file)
+    restored = 0
+
+    for city, city_rows in (settlement_history or {}).items():
+        if not isinstance(city_rows, dict):
+            continue
+        meta = _truth_meta(city)
+        for date_str, payload in city_rows.items():
+            if not isinstance(payload, dict):
+                continue
+            actual_high = _sf(payload.get("max_temp"))
+            if actual_high is None:
+                continue
+            repo.upsert_truth(
+                city=city,
+                target_date=str(date_str),
+                actual_high=actual_high,
+                settlement_source=meta["settlement_source"],
+                settlement_station_code=meta["settlement_station_code"],
+                settlement_station_label=meta["settlement_station_label"],
+                truth_version=args.truth_version,
+                updated_by="restore:settlement_history",
+                source_payload=payload,
+                is_final=True,
+                reason="restore_training_truth_history",
+            )
+            restored += 1
+
+    merged_recent = 0
+    for city, city_rows in (runtime_history or {}).items():
+        if not isinstance(city_rows, dict):
+            continue
+        meta = _truth_meta(city)
+        for date_str, payload in city_rows.items():
+            if not isinstance(payload, dict):
+                continue
+            actual_high = _sf(payload.get("actual_high"))
+            if actual_high is None:
+                continue
+            repo.upsert_truth(
+                city=city,
+                target_date=str(date_str),
+                actual_high=actual_high,
+                settlement_source=meta["settlement_source"],
+                settlement_station_code=meta["settlement_station_code"],
+                settlement_station_label=meta["settlement_station_label"],
+                truth_version=args.truth_version,
+                updated_by="restore:runtime_daily_records",
+                source_payload={
+                    "actual_high": actual_high,
+                    "payload_json": payload,
+                },
+                is_final=True,
+                reason="restore_training_truth_history",
+            )
+            merged_recent += 1
+
+    print(
+        json.dumps(
+            {
+                "restored_from_settlement_history": restored,
+                "merged_recent_runtime_records": merged_recent,
+            },
+            ensure_ascii=False,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()