diff --git a/scripts/backfill_recent_daily_actuals_from_metar.py b/scripts/backfill_recent_daily_actuals_from_metar.py new file mode 100644 index 00000000..2489de8e --- /dev/null +++ b/scripts/backfill_recent_daily_actuals_from_metar.py @@ -0,0 +1,105 @@ +import argparse +import os +import sys +from datetime import datetime, timedelta + +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + +from src.analysis.deb_algorithm import load_history, reconcile_recent_actual_highs, save_history # noqa: E402 +from src.data_collection.city_registry import CITY_REGISTRY # noqa: E402 + + +def _target_dates(city_info: dict, lookback_days: int) -> list[str]: + tz_offset = int(city_info.get("tz_offset") or 0) + local_now = datetime.utcnow() + timedelta(seconds=tz_offset) + local_today = local_now.date() + dates = [] + for offset in range(max(lookback_days, 1), 0, -1): + day = local_today - timedelta(days=offset) + dates.append(day.strftime("%Y-%m-%d")) + return dates + + +def _is_metar_city(city_info: dict) -> bool: + source = str(city_info.get("settlement_source") or "metar").strip().lower() + return source == "metar" + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Seed recent daily_records rows and backfill actual_high from aviationweather METAR history." + ) + parser.add_argument( + "--cities", + nargs="*", + default=[], + help="Optional subset of city registry keys.", + ) + parser.add_argument( + "--lookback-days", + type=int, + default=14, + help="How many recent local days to seed/backfill (excluding today).", + ) + parser.add_argument( + "--only-missing-cities", + action="store_true", + help="Only process cities that do not exist in daily_records yet.", + ) + args = parser.parse_args() + + history_file = os.path.join(PROJECT_ROOT, "data", "daily_records.json") + data = load_history(history_file) + + selected = {str(item).strip().lower() for item in args.cities if str(item).strip()} + candidates: list[str] = [] + for city_name, city_info in sorted(CITY_REGISTRY.items()): + if selected and city_name not in selected: + continue + if not isinstance(city_info, dict) or not _is_metar_city(city_info): + continue + if not str(city_info.get("icao") or "").strip(): + continue + if args.only_missing_cities and city_name in data: + continue + candidates.append(city_name) + + seeded_rows = 0 + seeded_cities = 0 + for city_name in candidates: + city_info = CITY_REGISTRY[city_name] + city_rows = data.get(city_name) + if not isinstance(city_rows, dict): + city_rows = {} + data[city_name] = city_rows + + before = len(city_rows) + for date_str in _target_dates(city_info, args.lookback_days): + city_rows.setdefault(date_str, {}) + if len(city_rows) > before: + seeded_cities += 1 + seeded_rows += len(city_rows) - before + + if seeded_rows > 0: + save_history(history_file, data) + + results = [] + for city_name in candidates: + result = reconcile_recent_actual_highs(city_name, lookback_days=args.lookback_days) + results.append((city_name, result)) + + print( + { + "lookback_days": args.lookback_days, + "candidate_count": len(candidates), + "seeded_cities": seeded_cities, + "seeded_rows": seeded_rows, + "results": results, + } + ) + + +if __name__ == "__main__": + main() diff --git a/web/routes.py b/web/routes.py index bbccc781..135c36c9 100644 --- a/web/routes.py +++ b/web/routes.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import os from datetime import datetime, timedelta from typing import Optional @@ -49,29 +48,6 @@ from web.core import ( router = APIRouter() -_SETTLEMENT_HISTORY_CACHE: Optional[dict] = None - - -def _load_settlement_history() -> dict: - global _SETTLEMENT_HISTORY_CACHE - if _SETTLEMENT_HISTORY_CACHE is not None: - return _SETTLEMENT_HISTORY_CACHE - - project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - history_path = os.path.join( - project_root, - "artifacts", - "probability_calibration", - "settlement_history.json", - ) - try: - with open(history_path, "r", encoding="utf-8") as handle: - payload = json.load(handle) - _SETTLEMENT_HISTORY_CACHE = payload if isinstance(payload, dict) else {} - except Exception: - _SETTLEMENT_HISTORY_CACHE = {} - return _SETTLEMENT_HISTORY_CACHE - def _parse_snapshot_dt(value: object) -> Optional[datetime]: raw = str(value or "").strip() @@ -222,11 +198,9 @@ async def city_history(request: Request, name: str): project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) history_file = os.path.join(project_root, "data", "daily_records.json") data = load_history(history_file) - settlement_history = _load_settlement_history() - settlement_city_data = settlement_history.get(city, {}) city_data = data.get(city, {}) if isinstance(data.get(city, {}), dict) else {} - if not city_data and not isinstance(settlement_city_data, dict): + if not city_data: source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower() return { "history": [], @@ -234,20 +208,12 @@ async def city_history(request: Request, name: str): "settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()), } - settlement_city_data = settlement_city_data if isinstance(settlement_city_data, dict) else {} out = [] - all_days = sorted(set(city_data.keys()) | set(settlement_city_data.keys())) - for day in all_days: - rec = city_data.get(day, {}) + for day, rec in sorted(city_data.items()): if not isinstance(rec, dict): rec = {} - settlement_rec = settlement_city_data.get(day, {}) - if not isinstance(settlement_rec, dict): - settlement_rec = {} act = rec.get("actual_high") - if act is None: - act = settlement_rec.get("max_temp") deb = rec.get("deb_prediction") mu = rec.get("mu") snapshots = load_snapshot_rows_for_day(city, day)