Add CI and shadow calibration reporting
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
python-quality:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt -r requirements-dev.txt
|
||||
|
||||
- name: Ruff
|
||||
run: python -m ruff check .
|
||||
|
||||
- name: Pytest
|
||||
run: python -m pytest
|
||||
|
||||
frontend-quality:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: "npm"
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build Docker image
|
||||
run: docker build -t polyweather-ci .
|
||||
@@ -33,6 +33,7 @@ Thumbs.db
|
||||
frontend/node_modules/
|
||||
frontend/.next/
|
||||
frontend/.vercel/
|
||||
frontend/*.tsbuildinfo
|
||||
|
||||
.npm-cache/
|
||||
.env.local
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
{
|
||||
"version": "emos-20260320123829",
|
||||
"trained_at": "2026-03-20T12:38:29.985991+00:00",
|
||||
"global": {
|
||||
"mu": {
|
||||
"intercept": -1.57406048,
|
||||
"raw_mu_coef": 2.80583627,
|
||||
"deb_coef": -0.06819634,
|
||||
"ens_median_coef": -1.81560215,
|
||||
"max_so_far_gap_coef": 0.0
|
||||
},
|
||||
"sigma": {
|
||||
"intercept": 0.67509915,
|
||||
"raw_sigma_coef": 0.14431833,
|
||||
"spread_coef": 0.14431833,
|
||||
"peak_flag_coef": 0.0,
|
||||
"max_so_far_gap_coef": 0.0
|
||||
}
|
||||
},
|
||||
"blending": {
|
||||
"alpha_mu": 0.0,
|
||||
"alpha_sigma": 1.0
|
||||
},
|
||||
"cities": {
|
||||
"ankara": {
|
||||
"samples": 7,
|
||||
"mu_bias": 0.566273,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.875
|
||||
},
|
||||
"london": {
|
||||
"samples": 6,
|
||||
"mu_bias": 0.489961,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.75
|
||||
},
|
||||
"new york": {
|
||||
"samples": 4,
|
||||
"mu_bias": 1.852451,
|
||||
"sigma_scale": 0.830444,
|
||||
"confidence": 0.5
|
||||
},
|
||||
"paris": {
|
||||
"samples": 7,
|
||||
"mu_bias": 0.308599,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.875
|
||||
},
|
||||
"seoul": {
|
||||
"samples": 6,
|
||||
"mu_bias": -1.486024,
|
||||
"sigma_scale": 1.569585,
|
||||
"confidence": 0.75
|
||||
},
|
||||
"toronto": {
|
||||
"samples": 5,
|
||||
"mu_bias": -0.734395,
|
||||
"sigma_scale": 1.656751,
|
||||
"confidence": 0.625
|
||||
},
|
||||
"buenos aires": {
|
||||
"samples": 5,
|
||||
"mu_bias": -1.753334,
|
||||
"sigma_scale": 1.831739,
|
||||
"confidence": 0.625
|
||||
},
|
||||
"wellington": {
|
||||
"samples": 6,
|
||||
"mu_bias": 0.350757,
|
||||
"sigma_scale": 1.377974,
|
||||
"confidence": 0.75
|
||||
},
|
||||
"chicago": {
|
||||
"samples": 4,
|
||||
"mu_bias": 3.01062,
|
||||
"sigma_scale": 0.825575,
|
||||
"confidence": 0.5
|
||||
},
|
||||
"sao paulo": {
|
||||
"samples": 5,
|
||||
"mu_bias": 1.632457,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.625
|
||||
},
|
||||
"dallas": {
|
||||
"samples": 4,
|
||||
"mu_bias": 3.77714,
|
||||
"sigma_scale": 0.796874,
|
||||
"confidence": 0.5
|
||||
},
|
||||
"miami": {
|
||||
"samples": 5,
|
||||
"mu_bias": -4.868741,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.625
|
||||
},
|
||||
"atlanta": {
|
||||
"samples": 5,
|
||||
"mu_bias": -7.648823,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.625
|
||||
},
|
||||
"seattle": {
|
||||
"samples": 4,
|
||||
"mu_bias": 4.058619,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.5
|
||||
},
|
||||
"lucknow": {
|
||||
"samples": 4,
|
||||
"mu_bias": 3.257609,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.5
|
||||
},
|
||||
"munich": {
|
||||
"samples": 6,
|
||||
"mu_bias": -0.780811,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.75
|
||||
},
|
||||
"hong kong": {
|
||||
"samples": 3,
|
||||
"mu_bias": -0.492675,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.375
|
||||
},
|
||||
"taipei": {
|
||||
"samples": 3,
|
||||
"mu_bias": 1.462462,
|
||||
"sigma_scale": 1.132834,
|
||||
"confidence": 0.375
|
||||
},
|
||||
"milan": {
|
||||
"samples": 3,
|
||||
"mu_bias": -2.729203,
|
||||
"sigma_scale": 2.0,
|
||||
"confidence": 0.375
|
||||
},
|
||||
"warsaw": {
|
||||
"samples": 3,
|
||||
"mu_bias": 0.349319,
|
||||
"sigma_scale": 1.337949,
|
||||
"confidence": 0.375
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"sample_count": 105,
|
||||
"mean_crps": 2.744961,
|
||||
"legacy_mean_crps": 2.793938,
|
||||
"legacy_mean_mae": 2.721143,
|
||||
"legacy_bucket_hit_rate": 0.695238,
|
||||
"selected_mean_crps": 2.747094,
|
||||
"selected_mean_mae": 2.721143,
|
||||
"selected_bucket_hit_rate": 0.704762,
|
||||
"selected_score": 3.609685,
|
||||
"legacy_score": 3.675576,
|
||||
"filled_actual_from_history": 2,
|
||||
"settlement_history_city_count": 30
|
||||
},
|
||||
"source": "artifacts\\probability_calibration\\default.json"
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
{
|
||||
"summary": {
|
||||
"sample_count": 105,
|
||||
"filled_actual_from_history": 2,
|
||||
"legacy": {
|
||||
"mean_crps": 2.793938,
|
||||
"mean_mae": 2.721143,
|
||||
"bucket_hit_rate": 0.695238
|
||||
},
|
||||
"emos": {
|
||||
"mean_crps": 2.747094,
|
||||
"mean_mae": 2.721143,
|
||||
"bucket_hit_rate": 0.685714
|
||||
},
|
||||
"delta": {
|
||||
"crps": -0.046843,
|
||||
"mae": 0.0,
|
||||
"bucket_hit_rate": -0.009524
|
||||
}
|
||||
},
|
||||
"by_city": {
|
||||
"ankara": {
|
||||
"samples": 7,
|
||||
"legacy_mean_crps": 2.023242,
|
||||
"emos_mean_crps": 2.259428,
|
||||
"legacy_mean_mae": 1.984286,
|
||||
"emos_mean_mae": 1.984286,
|
||||
"legacy_bucket_hit_rate": 0.714286,
|
||||
"emos_bucket_hit_rate": 0.714286
|
||||
},
|
||||
"atlanta": {
|
||||
"samples": 5,
|
||||
"legacy_mean_crps": 12.792034,
|
||||
"emos_mean_crps": 12.537301,
|
||||
"legacy_mean_mae": 12.806,
|
||||
"emos_mean_mae": 12.806,
|
||||
"legacy_bucket_hit_rate": 0.6,
|
||||
"emos_bucket_hit_rate": 0.6
|
||||
},
|
||||
"buenos aires": {
|
||||
"samples": 5,
|
||||
"legacy_mean_crps": 3.846144,
|
||||
"emos_mean_crps": 3.745234,
|
||||
"legacy_mean_mae": 4.168,
|
||||
"emos_mean_mae": 4.168,
|
||||
"legacy_bucket_hit_rate": 0.6,
|
||||
"emos_bucket_hit_rate": 0.6
|
||||
},
|
||||
"chicago": {
|
||||
"samples": 4,
|
||||
"legacy_mean_crps": 1.346667,
|
||||
"emos_mean_crps": 0.66337,
|
||||
"legacy_mean_mae": 0.0,
|
||||
"emos_mean_mae": 0.0,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
},
|
||||
"dallas": {
|
||||
"samples": 4,
|
||||
"legacy_mean_crps": 1.256111,
|
||||
"emos_mean_crps": 0.651563,
|
||||
"legacy_mean_mae": 0.0,
|
||||
"emos_mean_mae": 0.0,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
},
|
||||
"hong kong": {
|
||||
"samples": 3,
|
||||
"legacy_mean_crps": 0.261027,
|
||||
"emos_mean_crps": 0.621432,
|
||||
"legacy_mean_mae": 0.1,
|
||||
"emos_mean_mae": 0.1,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 0.666667
|
||||
},
|
||||
"london": {
|
||||
"samples": 6,
|
||||
"legacy_mean_crps": 2.079624,
|
||||
"emos_mean_crps": 1.986319,
|
||||
"legacy_mean_mae": 2.451667,
|
||||
"emos_mean_mae": 2.451667,
|
||||
"legacy_bucket_hit_rate": 0.166667,
|
||||
"emos_bucket_hit_rate": 0.166667
|
||||
},
|
||||
"lucknow": {
|
||||
"samples": 4,
|
||||
"legacy_mean_crps": 1.468528,
|
||||
"emos_mean_crps": 1.523689,
|
||||
"legacy_mean_mae": 1.6025,
|
||||
"emos_mean_mae": 1.6025,
|
||||
"legacy_bucket_hit_rate": 0.5,
|
||||
"emos_bucket_hit_rate": 0.5
|
||||
},
|
||||
"madrid": {
|
||||
"samples": 2,
|
||||
"legacy_mean_crps": 6.27726,
|
||||
"emos_mean_crps": 6.028088,
|
||||
"legacy_mean_mae": 7.33,
|
||||
"emos_mean_mae": 7.33,
|
||||
"legacy_bucket_hit_rate": 0.0,
|
||||
"emos_bucket_hit_rate": 0.0
|
||||
},
|
||||
"miami": {
|
||||
"samples": 5,
|
||||
"legacy_mean_crps": 11.665378,
|
||||
"emos_mean_crps": 11.614409,
|
||||
"legacy_mean_mae": 12.07,
|
||||
"emos_mean_mae": 12.07,
|
||||
"legacy_bucket_hit_rate": 0.6,
|
||||
"emos_bucket_hit_rate": 0.6
|
||||
},
|
||||
"milan": {
|
||||
"samples": 3,
|
||||
"legacy_mean_crps": 4.401392,
|
||||
"emos_mean_crps": 3.905274,
|
||||
"legacy_mean_mae": 4.06,
|
||||
"emos_mean_mae": 4.06,
|
||||
"legacy_bucket_hit_rate": 0.666667,
|
||||
"emos_bucket_hit_rate": 0.666667
|
||||
},
|
||||
"munich": {
|
||||
"samples": 6,
|
||||
"legacy_mean_crps": 2.988583,
|
||||
"emos_mean_crps": 2.892973,
|
||||
"legacy_mean_mae": 3.143333,
|
||||
"emos_mean_mae": 3.143333,
|
||||
"legacy_bucket_hit_rate": 0.5,
|
||||
"emos_bucket_hit_rate": 0.5
|
||||
},
|
||||
"new york": {
|
||||
"samples": 4,
|
||||
"legacy_mean_crps": 1.861101,
|
||||
"emos_mean_crps": 1.420965,
|
||||
"legacy_mean_mae": 1.3725,
|
||||
"emos_mean_mae": 1.3725,
|
||||
"legacy_bucket_hit_rate": 0.75,
|
||||
"emos_bucket_hit_rate": 0.75
|
||||
},
|
||||
"paris": {
|
||||
"samples": 7,
|
||||
"legacy_mean_crps": 2.430082,
|
||||
"emos_mean_crps": 2.23397,
|
||||
"legacy_mean_mae": 2.518571,
|
||||
"emos_mean_mae": 2.518571,
|
||||
"legacy_bucket_hit_rate": 0.571429,
|
||||
"emos_bucket_hit_rate": 0.571429
|
||||
},
|
||||
"sao paulo": {
|
||||
"samples": 5,
|
||||
"legacy_mean_crps": 2.454756,
|
||||
"emos_mean_crps": 2.351014,
|
||||
"legacy_mean_mae": 2.628,
|
||||
"emos_mean_mae": 2.628,
|
||||
"legacy_bucket_hit_rate": 0.6,
|
||||
"emos_bucket_hit_rate": 0.6
|
||||
},
|
||||
"seattle": {
|
||||
"samples": 4,
|
||||
"legacy_mean_crps": 0.531656,
|
||||
"emos_mean_crps": 0.850792,
|
||||
"legacy_mean_mae": 0.0,
|
||||
"emos_mean_mae": 0.0,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
},
|
||||
"seoul": {
|
||||
"samples": 6,
|
||||
"legacy_mean_crps": 0.328088,
|
||||
"emos_mean_crps": 0.710725,
|
||||
"legacy_mean_mae": 0.2,
|
||||
"emos_mean_mae": 0.2,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
},
|
||||
"shanghai": {
|
||||
"samples": 2,
|
||||
"legacy_mean_crps": 0.250034,
|
||||
"emos_mean_crps": 0.463888,
|
||||
"legacy_mean_mae": 0.15,
|
||||
"emos_mean_mae": 0.15,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
},
|
||||
"singapore": {
|
||||
"samples": 2,
|
||||
"legacy_mean_crps": 0.281993,
|
||||
"emos_mean_crps": 0.484792,
|
||||
"legacy_mean_mae": 0.15,
|
||||
"emos_mean_mae": 0.15,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
},
|
||||
"taipei": {
|
||||
"samples": 3,
|
||||
"legacy_mean_crps": 0.356996,
|
||||
"emos_mean_crps": 0.538714,
|
||||
"legacy_mean_mae": 0.1,
|
||||
"emos_mean_mae": 0.1,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 0.666667
|
||||
},
|
||||
"tel aviv": {
|
||||
"samples": 2,
|
||||
"legacy_mean_crps": 0.446758,
|
||||
"emos_mean_crps": 0.55971,
|
||||
"legacy_mean_mae": 0.3,
|
||||
"emos_mean_mae": 0.3,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
},
|
||||
"tokyo": {
|
||||
"samples": 2,
|
||||
"legacy_mean_crps": 0.450128,
|
||||
"emos_mean_crps": 0.565212,
|
||||
"legacy_mean_mae": 0.25,
|
||||
"emos_mean_mae": 0.25,
|
||||
"legacy_bucket_hit_rate": 0.5,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
},
|
||||
"toronto": {
|
||||
"samples": 5,
|
||||
"legacy_mean_crps": 2.647861,
|
||||
"emos_mean_crps": 2.402627,
|
||||
"legacy_mean_mae": 2.532,
|
||||
"emos_mean_mae": 2.532,
|
||||
"legacy_bucket_hit_rate": 0.6,
|
||||
"emos_bucket_hit_rate": 0.6
|
||||
},
|
||||
"warsaw": {
|
||||
"samples": 3,
|
||||
"legacy_mean_crps": 1.618875,
|
||||
"emos_mean_crps": 1.601375,
|
||||
"legacy_mean_mae": 2.056667,
|
||||
"emos_mean_mae": 2.056667,
|
||||
"legacy_bucket_hit_rate": 0.333333,
|
||||
"emos_bucket_hit_rate": 0.333333
|
||||
},
|
||||
"wellington": {
|
||||
"samples": 6,
|
||||
"legacy_mean_crps": 0.266349,
|
||||
"emos_mean_crps": 0.592134,
|
||||
"legacy_mean_mae": 0.2,
|
||||
"emos_mean_mae": 0.2,
|
||||
"legacy_bucket_hit_rate": 1.0,
|
||||
"emos_bucket_hit_rate": 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,2 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
@@ -0,0 +1,2 @@
|
||||
pytest
|
||||
ruff
|
||||
@@ -0,0 +1,210 @@
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import date, timedelta
|
||||
from typing import Dict, Iterable, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.data_collection.city_registry import CITY_REGISTRY # noqa: E402
|
||||
|
||||
|
||||
HOURLY_FIELDS = (
|
||||
"temperature_2m",
|
||||
"relative_humidity_2m",
|
||||
"wind_speed_10m",
|
||||
"wind_direction_10m",
|
||||
"cloud_cover",
|
||||
"shortwave_radiation",
|
||||
"precipitation",
|
||||
"surface_pressure",
|
||||
)
|
||||
|
||||
|
||||
def _city_output_path(output_dir: str, city_name: str) -> str:
|
||||
filename = f"{city_name.replace(' ', '_').lower()}_historical.csv"
|
||||
return os.path.join(output_dir, filename)
|
||||
|
||||
|
||||
def _fetch_city_history(
|
||||
city_name: str,
|
||||
city_info: Dict[str, object],
|
||||
output_dir: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
session: requests.Session,
|
||||
overwrite: bool = False,
|
||||
max_retries: int = 4,
|
||||
) -> Tuple[str, str]:
|
||||
output_path = _city_output_path(output_dir, city_name)
|
||||
if os.path.exists(output_path) and not overwrite:
|
||||
return city_name, "skipped_existing"
|
||||
|
||||
lat = city_info["lat"]
|
||||
lon = city_info["lon"]
|
||||
url = (
|
||||
"https://archive-api.open-meteo.com/v1/archive"
|
||||
f"?latitude={lat}&longitude={lon}"
|
||||
f"&start_date={start_date}&end_date={end_date}"
|
||||
f"&hourly={','.join(HOURLY_FIELDS)}"
|
||||
"&timezone=auto"
|
||||
)
|
||||
last_error = None
|
||||
payload = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
response = session.get(url, timeout=90)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
break
|
||||
except requests.HTTPError as exc:
|
||||
last_error = exc
|
||||
response = getattr(exc, "response", None)
|
||||
status_code = response.status_code if response is not None else None
|
||||
if status_code == 429 and attempt < max_retries:
|
||||
retry_after = 0
|
||||
if response is not None:
|
||||
try:
|
||||
retry_after = int(response.headers.get("Retry-After") or "0")
|
||||
except Exception:
|
||||
retry_after = 0
|
||||
sleep_sec = retry_after if retry_after > 0 else min(90, 5 * (attempt + 1))
|
||||
time.sleep(sleep_sec)
|
||||
continue
|
||||
raise
|
||||
except requests.RequestException as exc:
|
||||
last_error = exc
|
||||
if attempt < max_retries:
|
||||
time.sleep(min(30, 3 * (attempt + 1)))
|
||||
continue
|
||||
raise
|
||||
|
||||
if payload is None:
|
||||
raise RuntimeError(f"failed to fetch {city_name}: {last_error}")
|
||||
|
||||
hourly = payload.get("hourly") or {}
|
||||
times = hourly.get("time") or []
|
||||
if not times:
|
||||
raise RuntimeError(f"missing hourly data for {city_name}")
|
||||
|
||||
fieldnames = ["time", *HOURLY_FIELDS]
|
||||
rows = []
|
||||
for idx, ts in enumerate(times):
|
||||
row = {"time": ts}
|
||||
for field in HOURLY_FIELDS:
|
||||
values = hourly.get(field) or []
|
||||
row[field] = values[idx] if idx < len(values) else None
|
||||
rows.append(row)
|
||||
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(output_path, "w", encoding="utf-8", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
return city_name, f"downloaded:{len(rows)}"
|
||||
|
||||
|
||||
def _iter_target_cities(selected: Iterable[str]) -> Iterable[Tuple[str, Dict[str, object]]]:
|
||||
if not selected:
|
||||
for city_name, city_info in sorted(CITY_REGISTRY.items()):
|
||||
yield city_name, city_info
|
||||
return
|
||||
|
||||
normalized = {str(city).strip().lower() for city in selected if str(city).strip()}
|
||||
for city_name, city_info in sorted(CITY_REGISTRY.items()):
|
||||
if city_name in normalized:
|
||||
yield city_name, city_info
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Backfill historical Open-Meteo CSVs for PolyWeather cities.")
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "historical"),
|
||||
help="Output directory for per-city historical CSV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--start-date",
|
||||
default="2023-01-01",
|
||||
help="Archive start date in YYYY-MM-DD.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--end-date",
|
||||
default=(date.today() - timedelta(days=1)).strftime("%Y-%m-%d"),
|
||||
help="Archive end date in YYYY-MM-DD.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cities",
|
||||
nargs="*",
|
||||
default=[],
|
||||
help="Optional subset of city registry keys to backfill.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--overwrite",
|
||||
action="store_true",
|
||||
help="Re-download existing CSV files instead of skipping them.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workers",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Maximum concurrent download workers.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
cities = list(_iter_target_cities(args.cities))
|
||||
if not cities:
|
||||
raise SystemExit("no matching cities found")
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "PolyWeather Historical Backfill/1.0"})
|
||||
|
||||
results = []
|
||||
with ThreadPoolExecutor(max_workers=max(1, args.workers)) as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
_fetch_city_history,
|
||||
city_name,
|
||||
city_info,
|
||||
args.output_dir,
|
||||
args.start_date,
|
||||
args.end_date,
|
||||
session,
|
||||
args.overwrite,
|
||||
): city_name
|
||||
for city_name, city_info in cities
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
city_name = futures[future]
|
||||
try:
|
||||
_, status = future.result()
|
||||
print(f"{city_name}: {status}")
|
||||
results.append((city_name, status))
|
||||
except Exception as exc:
|
||||
print(f"{city_name}: error:{exc}")
|
||||
results.append((city_name, f"error:{exc}"))
|
||||
time.sleep(0.1)
|
||||
|
||||
downloaded = sum(1 for _, status in results if status.startswith("downloaded:"))
|
||||
skipped = sum(1 for _, status in results if status == "skipped_existing")
|
||||
failed = [city for city, status in results if status.startswith("error:")]
|
||||
print(
|
||||
"summary downloaded={downloaded} skipped={skipped} failed={failed}".format(
|
||||
downloaded=downloaded,
|
||||
skipped=skipped,
|
||||
failed=len(failed),
|
||||
)
|
||||
)
|
||||
if failed:
|
||||
print("failed_cities=" + ",".join(sorted(failed)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,152 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.deb_algorithm import load_history, save_history # noqa: E402
|
||||
from src.analysis.probability_calibration import ( # noqa: E402
|
||||
ENGINE_MODE_EMOS_SHADOW,
|
||||
apply_probability_calibration,
|
||||
build_probability_features,
|
||||
)
|
||||
|
||||
|
||||
def _sample_to_features(sample):
|
||||
peak_flag = sample.get("peak_flag")
|
||||
if peak_flag == 1.0:
|
||||
peak_status = "past"
|
||||
elif peak_flag == 0.5:
|
||||
peak_status = "in_window"
|
||||
else:
|
||||
peak_status = "before"
|
||||
return build_probability_features(
|
||||
city_name=sample.get("city") or "",
|
||||
raw_mu=sample.get("raw_mu"),
|
||||
raw_sigma=sample.get("raw_sigma"),
|
||||
deb_prediction=sample.get("deb_prediction"),
|
||||
ens_data={
|
||||
"median": sample.get("ens_median"),
|
||||
"p10": None,
|
||||
"p90": None,
|
||||
},
|
||||
current_forecasts={},
|
||||
max_so_far=None,
|
||||
peak_status=peak_status,
|
||||
local_hour_frac=None,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Backfill shadow probability snapshots into daily records.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--training-samples",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"training_samples.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-file",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"default.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
with open(args.training_samples, "r", encoding="utf-8") as fh:
|
||||
training_payload = json.load(fh)
|
||||
|
||||
updated = 0
|
||||
touched = 0
|
||||
|
||||
for sample in training_payload.get("samples") or []:
|
||||
city = str(sample.get("city") or "").strip().lower()
|
||||
date_str = str(sample.get("date") or "").strip()
|
||||
if not city or not date_str:
|
||||
continue
|
||||
record = ((history.get(city) or {}).get(date_str) or {})
|
||||
if not isinstance(record, dict) or not record:
|
||||
continue
|
||||
|
||||
legacy_distribution = [
|
||||
{"value": row.get("v"), "probability": row.get("p")}
|
||||
for row in (record.get("prob_snapshot") or [])
|
||||
if isinstance(row, dict) and row.get("v") is not None
|
||||
]
|
||||
if not legacy_distribution:
|
||||
continue
|
||||
|
||||
calibration = apply_probability_calibration(
|
||||
city_name=city,
|
||||
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
|
||||
raw_mu=sample.get("raw_mu"),
|
||||
raw_sigma=sample.get("raw_sigma"),
|
||||
max_so_far=None,
|
||||
legacy_distribution=legacy_distribution,
|
||||
features=_sample_to_features(sample),
|
||||
calibration_path=args.calibration_file,
|
||||
mode=ENGINE_MODE_EMOS_SHADOW,
|
||||
)
|
||||
|
||||
shadow_distribution = calibration.get("shadow_distribution") or []
|
||||
compact_shadow = [
|
||||
{
|
||||
"v": int(row.get("value")),
|
||||
"p": round(float(row.get("probability") or 0.0), 3),
|
||||
}
|
||||
for row in shadow_distribution[:4]
|
||||
if row.get("value") is not None
|
||||
]
|
||||
compact_calibration = {
|
||||
"mode": calibration.get("mode"),
|
||||
"engine": calibration.get("engine"),
|
||||
"version": calibration.get("calibration_version"),
|
||||
"source": calibration.get("calibration_source"),
|
||||
"raw_mu": calibration.get("raw_mu"),
|
||||
"raw_sigma": calibration.get("raw_sigma"),
|
||||
"calibrated_mu": calibration.get("calibrated_mu"),
|
||||
"calibrated_sigma": calibration.get("calibrated_sigma"),
|
||||
}
|
||||
|
||||
touched += 1
|
||||
if (
|
||||
record.get("shadow_prob_snapshot") == compact_shadow
|
||||
and record.get("probability_calibration") == compact_calibration
|
||||
):
|
||||
continue
|
||||
|
||||
record["shadow_prob_snapshot"] = compact_shadow
|
||||
record["probability_calibration"] = compact_calibration
|
||||
history[city][date_str] = record
|
||||
updated += 1
|
||||
|
||||
save_history(args.history_file, history)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"samples_seen": len(training_payload.get("samples") or []),
|
||||
"records_considered": touched,
|
||||
"records_updated": updated,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,219 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
|
||||
|
||||
|
||||
def _sf(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _mean(values):
|
||||
return round(sum(values) / len(values), 6) if values else None
|
||||
|
||||
|
||||
def _top_bucket(snapshot):
|
||||
if not isinstance(snapshot, list):
|
||||
return None
|
||||
best = None
|
||||
best_prob = -1.0
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
try:
|
||||
prob = float(row.get("p") if "p" in row else row.get("probability"))
|
||||
except Exception:
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value is None:
|
||||
continue
|
||||
if prob > best_prob:
|
||||
best = value
|
||||
best_prob = prob
|
||||
return best
|
||||
|
||||
|
||||
def _bucket_probability(snapshot, target_bucket):
|
||||
if not isinstance(snapshot, list):
|
||||
return 0.0
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value != target_bucket:
|
||||
continue
|
||||
try:
|
||||
return float(row.get("p") if "p" in row else row.get("probability") or 0.0)
|
||||
except Exception:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
|
||||
def _brier_from_snapshot(snapshot, target_bucket):
|
||||
hit_prob = _bucket_probability(snapshot, target_bucket)
|
||||
total = (1.0 - hit_prob) ** 2
|
||||
if isinstance(snapshot, list):
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value == target_bucket:
|
||||
continue
|
||||
try:
|
||||
prob = float(row.get("p") if "p" in row else row.get("probability") or 0.0)
|
||||
except Exception:
|
||||
prob = 0.0
|
||||
total += prob * prob
|
||||
return round(total, 6)
|
||||
|
||||
|
||||
def _blank_metrics():
|
||||
return {
|
||||
"samples": 0,
|
||||
"legacy_mae": [],
|
||||
"shadow_mae": [],
|
||||
"legacy_bucket_hit": [],
|
||||
"shadow_bucket_hit": [],
|
||||
"legacy_bucket_brier": [],
|
||||
"shadow_bucket_brier": [],
|
||||
}
|
||||
|
||||
|
||||
def _rollup(metrics):
|
||||
return {
|
||||
"samples": metrics["samples"],
|
||||
"legacy_mean_mae": _mean(metrics["legacy_mae"]),
|
||||
"shadow_mean_mae": _mean(metrics["shadow_mae"]),
|
||||
"legacy_bucket_hit_rate": _mean(metrics["legacy_bucket_hit"]),
|
||||
"shadow_bucket_hit_rate": _mean(metrics["shadow_bucket_hit"]),
|
||||
"legacy_bucket_brier": _mean(metrics["legacy_bucket_brier"]),
|
||||
"shadow_bucket_brier": _mean(metrics["shadow_bucket_brier"]),
|
||||
"delta_mae": round((_mean(metrics["shadow_mae"]) or 0.0) - (_mean(metrics["legacy_mae"]) or 0.0), 6),
|
||||
"delta_bucket_hit_rate": round((_mean(metrics["shadow_bucket_hit"]) or 0.0) - (_mean(metrics["legacy_bucket_hit"]) or 0.0), 6),
|
||||
"delta_bucket_brier": round((_mean(metrics["shadow_bucket_brier"]) or 0.0) - (_mean(metrics["legacy_bucket_brier"]) or 0.0), 6),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Build live shadow probability report from daily records.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"shadow_report.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
overall = _blank_metrics()
|
||||
by_city = defaultdict(_blank_metrics)
|
||||
by_date = defaultdict(_blank_metrics)
|
||||
latest_observations = []
|
||||
|
||||
for city, city_records in sorted(history.items()):
|
||||
if not isinstance(city_records, dict):
|
||||
continue
|
||||
for date_str, record in sorted(city_records.items()):
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
actual_high = _sf(record.get("actual_high"))
|
||||
shadow_snapshot = record.get("shadow_prob_snapshot")
|
||||
calibration = record.get("probability_calibration") or {}
|
||||
if actual_high is None or not shadow_snapshot:
|
||||
continue
|
||||
|
||||
legacy_mu = _sf(calibration.get("raw_mu"))
|
||||
if legacy_mu is None:
|
||||
legacy_mu = _sf(record.get("mu"))
|
||||
shadow_mu = _sf(calibration.get("calibrated_mu"))
|
||||
if shadow_mu is None:
|
||||
continue
|
||||
|
||||
actual_bucket = apply_city_settlement(city, actual_high)
|
||||
legacy_snapshot = record.get("prob_snapshot") or []
|
||||
legacy_bucket = _top_bucket(legacy_snapshot)
|
||||
shadow_bucket = _top_bucket(shadow_snapshot)
|
||||
|
||||
for metrics in (overall, by_city[city], by_date[date_str]):
|
||||
metrics["samples"] += 1
|
||||
metrics["legacy_mae"].append(abs(legacy_mu - actual_high))
|
||||
metrics["shadow_mae"].append(abs(shadow_mu - actual_high))
|
||||
metrics["legacy_bucket_hit"].append(1.0 if legacy_bucket == actual_bucket else 0.0)
|
||||
metrics["shadow_bucket_hit"].append(1.0 if shadow_bucket == actual_bucket else 0.0)
|
||||
metrics["legacy_bucket_brier"].append(_brier_from_snapshot(legacy_snapshot, actual_bucket))
|
||||
metrics["shadow_bucket_brier"].append(_brier_from_snapshot(shadow_snapshot, actual_bucket))
|
||||
|
||||
latest_observations.append(
|
||||
{
|
||||
"city": city,
|
||||
"date": date_str,
|
||||
"actual_high": actual_high,
|
||||
"actual_bucket": actual_bucket,
|
||||
"legacy_mu": round(legacy_mu, 3),
|
||||
"shadow_mu": round(shadow_mu, 3),
|
||||
"legacy_top_bucket": legacy_bucket,
|
||||
"shadow_top_bucket": shadow_bucket,
|
||||
"calibration_version": calibration.get("version"),
|
||||
"calibration_mode": calibration.get("mode"),
|
||||
}
|
||||
)
|
||||
|
||||
by_city_report = {
|
||||
city: _rollup(metrics)
|
||||
for city, metrics in sorted(by_city.items())
|
||||
}
|
||||
by_date_report = {
|
||||
date_str: _rollup(metrics)
|
||||
for date_str, metrics in sorted(
|
||||
by_date.items(),
|
||||
key=lambda item: datetime.strptime(item[0], "%Y-%m-%d"),
|
||||
)
|
||||
}
|
||||
|
||||
latest_observations = sorted(
|
||||
latest_observations,
|
||||
key=lambda row: (row["date"], row["city"]),
|
||||
reverse=True,
|
||||
)[:100]
|
||||
|
||||
payload = {
|
||||
"generated_at": datetime.utcnow().isoformat() + "Z",
|
||||
"summary": _rollup(overall),
|
||||
"by_city": by_city_report,
|
||||
"by_date": by_date_report,
|
||||
"recent_observations": latest_observations,
|
||||
}
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps(payload["summary"], ensure_ascii=False, indent=2))
|
||||
print(f"saved shadow report to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,84 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
|
||||
from src.data_collection.city_registry import CITY_REGISTRY # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Build daily settlement history from historical weather CSV files.")
|
||||
parser.add_argument(
|
||||
"--history-dir",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "historical"),
|
||||
help="Directory containing per-city *_historical.csv files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
help="Output JSON path.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = {}
|
||||
for city in sorted(CITY_REGISTRY.keys()):
|
||||
path = os.path.join(
|
||||
args.history_dir,
|
||||
f"{city.replace(' ', '_').lower()}_historical.csv",
|
||||
)
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
|
||||
daily_max = defaultdict(lambda: None)
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
for row in reader:
|
||||
ts = str(row.get("time") or "")
|
||||
raw_temp = row.get("temperature_2m")
|
||||
if not ts or raw_temp in (None, ""):
|
||||
continue
|
||||
try:
|
||||
temp = float(raw_temp)
|
||||
except Exception:
|
||||
continue
|
||||
day = ts[:10]
|
||||
prev = daily_max[day]
|
||||
if prev is None or temp > prev:
|
||||
daily_max[day] = temp
|
||||
|
||||
result[city] = {
|
||||
day: {
|
||||
"max_temp": round(max_temp, 3),
|
||||
"settlement_value": apply_city_settlement(city, max_temp),
|
||||
}
|
||||
for day, max_temp in sorted(daily_max.items())
|
||||
if max_temp is not None
|
||||
}
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(result, fh, ensure_ascii=False, indent=2)
|
||||
print(
|
||||
"saved settlement history to {path} for {count} cities".format(
|
||||
path=args.output,
|
||||
count=len(result),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,206 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.probability_calibration import ( # noqa: E402
|
||||
ENGINE_MODE_EMOS_PRIMARY,
|
||||
_gaussian_crps,
|
||||
apply_probability_calibration,
|
||||
build_probability_features,
|
||||
)
|
||||
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
|
||||
from scripts.fit_probability_calibration import ( # noqa: E402
|
||||
_extract_samples,
|
||||
_load_json_if_exists,
|
||||
)
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
|
||||
|
||||
def _mean(values):
|
||||
return (sum(values) / len(values)) if values else None
|
||||
|
||||
|
||||
def _sample_to_features(sample):
|
||||
return build_probability_features(
|
||||
city_name=sample.get("city") or "",
|
||||
raw_mu=sample.get("raw_mu"),
|
||||
raw_sigma=sample.get("raw_sigma"),
|
||||
deb_prediction=sample.get("deb_prediction"),
|
||||
ens_data={
|
||||
"median": sample.get("ens_median"),
|
||||
"p10": None,
|
||||
"p90": None,
|
||||
},
|
||||
current_forecasts={},
|
||||
max_so_far=None,
|
||||
peak_status="in_window" if sample.get("peak_flag") == 0.5 else "past" if sample.get("peak_flag") == 1.0 else "before",
|
||||
local_hour_frac=None,
|
||||
)
|
||||
|
||||
|
||||
def _top_bucket_value(distribution):
|
||||
if not distribution:
|
||||
return None
|
||||
top = max(
|
||||
(row for row in distribution if isinstance(row, dict)),
|
||||
key=lambda row: float(row.get("probability") or 0.0),
|
||||
default=None,
|
||||
)
|
||||
if not top:
|
||||
return None
|
||||
return top.get("value")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Evaluate legacy vs EMOS probability calibration.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-file",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"default.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"evaluation_report.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
settlement_history=settlement_history,
|
||||
)
|
||||
|
||||
legacy_crps = []
|
||||
emos_crps = []
|
||||
legacy_mae = []
|
||||
emos_mae = []
|
||||
legacy_bucket_hits = []
|
||||
emos_bucket_hits = []
|
||||
by_city = defaultdict(lambda: {
|
||||
"samples": 0,
|
||||
"legacy_crps": [],
|
||||
"emos_crps": [],
|
||||
"legacy_mae": [],
|
||||
"emos_mae": [],
|
||||
"legacy_bucket_hits": [],
|
||||
"emos_bucket_hits": [],
|
||||
})
|
||||
|
||||
for sample in samples:
|
||||
city = str(sample.get("city") or "").strip().lower()
|
||||
actual_high = float(sample["actual_high"])
|
||||
raw_mu = float(sample["raw_mu"])
|
||||
raw_sigma = max(0.1, float(sample["raw_sigma"]))
|
||||
legacy_crps.append(_gaussian_crps(actual_high, raw_mu, raw_sigma))
|
||||
legacy_mae.append(abs(raw_mu - actual_high))
|
||||
|
||||
legacy_bucket = apply_city_settlement(city, raw_mu)
|
||||
actual_bucket = apply_city_settlement(city, actual_high)
|
||||
legacy_bucket_hits.append(1.0 if legacy_bucket == actual_bucket else 0.0)
|
||||
|
||||
calibration = apply_probability_calibration(
|
||||
city_name=city,
|
||||
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
|
||||
raw_mu=raw_mu,
|
||||
raw_sigma=raw_sigma,
|
||||
max_so_far=None,
|
||||
legacy_distribution=[],
|
||||
features=_sample_to_features(sample),
|
||||
calibration_path=args.calibration_file,
|
||||
mode=ENGINE_MODE_EMOS_PRIMARY,
|
||||
)
|
||||
emos_mu = float(calibration.get("calibrated_mu") or raw_mu)
|
||||
emos_sigma = max(0.1, float(calibration.get("calibrated_sigma") or raw_sigma))
|
||||
emos_distribution = calibration.get("distribution") or []
|
||||
emos_crps.append(_gaussian_crps(actual_high, emos_mu, emos_sigma))
|
||||
emos_mae.append(abs(emos_mu - actual_high))
|
||||
emos_bucket = _top_bucket_value(emos_distribution)
|
||||
emos_bucket_hits.append(1.0 if emos_bucket == actual_bucket else 0.0)
|
||||
|
||||
row = by_city[city]
|
||||
row["samples"] += 1
|
||||
row["legacy_crps"].append(legacy_crps[-1])
|
||||
row["emos_crps"].append(emos_crps[-1])
|
||||
row["legacy_mae"].append(legacy_mae[-1])
|
||||
row["emos_mae"].append(emos_mae[-1])
|
||||
row["legacy_bucket_hits"].append(legacy_bucket_hits[-1])
|
||||
row["emos_bucket_hits"].append(emos_bucket_hits[-1])
|
||||
|
||||
summary = {
|
||||
"sample_count": len(samples),
|
||||
"filled_actual_from_history": filled_actual_from_history,
|
||||
"legacy": {
|
||||
"mean_crps": round(_mean(legacy_crps), 6) if legacy_crps else None,
|
||||
"mean_mae": round(_mean(legacy_mae), 6) if legacy_mae else None,
|
||||
"bucket_hit_rate": round(_mean(legacy_bucket_hits), 6) if legacy_bucket_hits else None,
|
||||
},
|
||||
"emos": {
|
||||
"mean_crps": round(_mean(emos_crps), 6) if emos_crps else None,
|
||||
"mean_mae": round(_mean(emos_mae), 6) if emos_mae else None,
|
||||
"bucket_hit_rate": round(_mean(emos_bucket_hits), 6) if emos_bucket_hits else None,
|
||||
},
|
||||
"delta": {
|
||||
"crps": round((_mean(emos_crps) or 0.0) - (_mean(legacy_crps) or 0.0), 6),
|
||||
"mae": round((_mean(emos_mae) or 0.0) - (_mean(legacy_mae) or 0.0), 6),
|
||||
"bucket_hit_rate": round((_mean(emos_bucket_hits) or 0.0) - (_mean(legacy_bucket_hits) or 0.0), 6),
|
||||
},
|
||||
}
|
||||
|
||||
city_report = {}
|
||||
for city, metrics in sorted(by_city.items()):
|
||||
city_report[city] = {
|
||||
"samples": metrics["samples"],
|
||||
"legacy_mean_crps": round(_mean(metrics["legacy_crps"]), 6),
|
||||
"emos_mean_crps": round(_mean(metrics["emos_crps"]), 6),
|
||||
"legacy_mean_mae": round(_mean(metrics["legacy_mae"]), 6),
|
||||
"emos_mean_mae": round(_mean(metrics["emos_mae"]), 6),
|
||||
"legacy_bucket_hit_rate": round(_mean(metrics["legacy_bucket_hits"]), 6),
|
||||
"emos_bucket_hit_rate": round(_mean(metrics["emos_bucket_hits"]), 6),
|
||||
}
|
||||
|
||||
payload = {
|
||||
"summary": summary,
|
||||
"by_city": city_report,
|
||||
}
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
print(f"saved evaluation report to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
from scripts.fit_probability_calibration import ( # noqa: E402
|
||||
_extract_samples,
|
||||
_load_json_if_exists,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Export normalized probability calibration training samples.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"training_samples.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
settlement_history=settlement_history,
|
||||
)
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
payload = {
|
||||
"sample_count": len(samples),
|
||||
"filled_actual_from_history": filled_actual_from_history,
|
||||
"samples": samples,
|
||||
}
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"exported {len(samples)} samples to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.probability_calibration import ( # noqa: E402
|
||||
DEFAULT_CALIBRATION_FILE,
|
||||
default_calibration_payload,
|
||||
fit_calibration,
|
||||
)
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
|
||||
|
||||
def _sf(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _load_json_if_exists(path):
|
||||
if not path or not os.path.exists(path):
|
||||
return {}
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _extract_samples(history, settlement_history=None):
|
||||
samples = []
|
||||
filled_actual_from_history = 0
|
||||
today = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
settlement_history = settlement_history or {}
|
||||
for city, city_rows in (history or {}).items():
|
||||
if not isinstance(city_rows, dict):
|
||||
continue
|
||||
city_settlement = settlement_history.get(city) or {}
|
||||
for date_str, record in city_rows.items():
|
||||
if date_str == today or not isinstance(record, dict):
|
||||
continue
|
||||
actual_high = _sf(record.get("actual_high"))
|
||||
if actual_high is None:
|
||||
actual_high = _sf((city_settlement.get(date_str) or {}).get("max_temp"))
|
||||
if actual_high is not None:
|
||||
filled_actual_from_history += 1
|
||||
deb_prediction = _sf(record.get("deb_prediction"))
|
||||
raw_mu = _sf(record.get("mu")) or deb_prediction
|
||||
forecasts = record.get("forecasts") or {}
|
||||
if not isinstance(forecasts, dict):
|
||||
forecasts = {}
|
||||
forecast_values = [val for val in (_sf(v) for v in forecasts.values()) if val is not None]
|
||||
forecast_values.sort()
|
||||
forecast_median = (
|
||||
forecast_values[len(forecast_values) // 2] if forecast_values else None
|
||||
)
|
||||
feature_snapshot = record.get("probability_features") or {}
|
||||
if not isinstance(feature_snapshot, dict):
|
||||
feature_snapshot = {}
|
||||
|
||||
ens_median = _sf(feature_snapshot.get("ens_median")) or forecast_median or raw_mu
|
||||
ensemble_spread = _sf(feature_snapshot.get("ensemble_spread"))
|
||||
if ensemble_spread is None:
|
||||
if len(forecast_values) >= 2:
|
||||
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
|
||||
else:
|
||||
ensemble_spread = 1.0
|
||||
raw_sigma = _sf(feature_snapshot.get("raw_sigma")) or ensemble_spread or 1.0
|
||||
peak_status = str(feature_snapshot.get("peak_status") or "before").strip().lower()
|
||||
peak_flag = 0.0
|
||||
if peak_status == "in_window":
|
||||
peak_flag = 0.5
|
||||
elif peak_status == "past":
|
||||
peak_flag = 1.0
|
||||
|
||||
if actual_high is None or raw_mu is None:
|
||||
continue
|
||||
|
||||
max_so_far = _sf(feature_snapshot.get("max_so_far"))
|
||||
max_so_far_gap = _sf(feature_snapshot.get("max_so_far_gap"))
|
||||
if max_so_far_gap is None and max_so_far is not None and deb_prediction is not None:
|
||||
max_so_far_gap = deb_prediction - max_so_far
|
||||
|
||||
samples.append(
|
||||
{
|
||||
"city": city,
|
||||
"date": date_str,
|
||||
"actual_high": actual_high,
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": raw_sigma,
|
||||
"deb_prediction": deb_prediction,
|
||||
"ens_median": ens_median,
|
||||
"ensemble_spread": ensemble_spread,
|
||||
"max_so_far_gap": max_so_far_gap,
|
||||
"peak_flag": peak_flag,
|
||||
}
|
||||
)
|
||||
return samples, filled_actual_from_history
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
help="Path to the historical daily_records.json file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=DEFAULT_CALIBRATION_FILE,
|
||||
help="Output JSON file for fitted calibration parameters.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
help="Optional daily settlement history JSON built from historical CSV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
default=None,
|
||||
help="Optional explicit calibration version.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
settlement_history=settlement_history,
|
||||
)
|
||||
calibration = fit_calibration(samples, version=args.version)
|
||||
if not samples:
|
||||
calibration = default_calibration_payload(
|
||||
version=args.version,
|
||||
reason="no_samples",
|
||||
)
|
||||
calibration.setdefault("metrics", {})
|
||||
calibration["metrics"]["filled_actual_from_history"] = filled_actual_from_history
|
||||
calibration["metrics"]["settlement_history_city_count"] = len(settlement_history)
|
||||
try:
|
||||
calibration["source"] = os.path.relpath(args.output, PROJECT_ROOT)
|
||||
except ValueError:
|
||||
calibration["source"] = os.path.abspath(args.output)
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(calibration, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(
|
||||
"saved calibration to {path} with {count} samples".format(
|
||||
path=args.output,
|
||||
count=calibration.get("metrics", {}).get("sample_count", 0),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
import requests
|
||||
from src.analysis.settlement_rounding import wu_round, apply_city_settlement, is_exact_settlement_city
|
||||
from src.analysis.settlement_rounding import apply_city_settlement
|
||||
|
||||
# Cross-platform file locking
|
||||
import sys
|
||||
@@ -38,6 +38,15 @@ _history_cache = {}
|
||||
_history_mtime = 0
|
||||
|
||||
|
||||
def _sf(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _is_excluded_model_name(model_name: str) -> bool:
|
||||
normalized = str(model_name or "").strip().lower().replace(" ", "").replace("_", "").replace("-", "")
|
||||
return "meteoblue" in normalized
|
||||
@@ -217,8 +226,16 @@ def reconcile_recent_actual_highs(city_name: str, lookback_days: int = 7):
|
||||
|
||||
|
||||
def update_daily_record(
|
||||
city_name, date_str, forecasts, actual_high, deb_prediction=None,
|
||||
mu=None, probabilities=None
|
||||
city_name,
|
||||
date_str,
|
||||
forecasts,
|
||||
actual_high,
|
||||
deb_prediction=None,
|
||||
mu=None,
|
||||
probabilities=None,
|
||||
probability_features=None,
|
||||
shadow_probabilities=None,
|
||||
calibration_summary=None,
|
||||
):
|
||||
"""
|
||||
保存/更新某城市某天的各个模型预报与最终实测值
|
||||
@@ -253,6 +270,36 @@ def update_daily_record(
|
||||
{"v": p["value"], "p": p["probability"]}
|
||||
for p in probabilities[:4]
|
||||
]
|
||||
compact_features = None
|
||||
if isinstance(probability_features, dict) and probability_features:
|
||||
compact_features = {
|
||||
"raw_mu": _sf(probability_features.get("raw_mu")),
|
||||
"raw_sigma": _sf(probability_features.get("raw_sigma")),
|
||||
"deb_prediction": _sf(probability_features.get("deb_prediction")),
|
||||
"ens_median": _sf(probability_features.get("ens_median")),
|
||||
"ensemble_spread": _sf(probability_features.get("ensemble_spread")),
|
||||
"max_so_far": _sf(probability_features.get("max_so_far")),
|
||||
"max_so_far_gap": _sf(probability_features.get("max_so_far_gap")),
|
||||
"peak_status": probability_features.get("peak_status"),
|
||||
}
|
||||
compact_shadow_probs = None
|
||||
if shadow_probabilities is not None:
|
||||
compact_shadow_probs = [
|
||||
{"v": p["value"], "p": p["probability"]}
|
||||
for p in shadow_probabilities[:4]
|
||||
]
|
||||
compact_calibration = None
|
||||
if isinstance(calibration_summary, dict) and calibration_summary:
|
||||
compact_calibration = {
|
||||
"mode": calibration_summary.get("mode"),
|
||||
"engine": calibration_summary.get("engine"),
|
||||
"version": calibration_summary.get("calibration_version"),
|
||||
"source": calibration_summary.get("calibration_source"),
|
||||
"raw_mu": _sf(calibration_summary.get("raw_mu")),
|
||||
"raw_sigma": _sf(calibration_summary.get("raw_sigma")),
|
||||
"calibrated_mu": _sf(calibration_summary.get("calibrated_mu")),
|
||||
"calibrated_sigma": _sf(calibration_summary.get("calibrated_sigma")),
|
||||
}
|
||||
|
||||
# 避免无意义的频繁磁盘写入
|
||||
existing = data[city_name][date_str]
|
||||
@@ -260,6 +307,7 @@ def update_daily_record(
|
||||
old_deb = existing.get("deb_prediction")
|
||||
old_mu = existing.get("mu")
|
||||
old_probs = existing.get("prob_snapshot")
|
||||
old_shadow_probs = existing.get("shadow_prob_snapshot")
|
||||
next_mu = round(mu, 2) if mu is not None else None
|
||||
if (
|
||||
old_actual == actual_high
|
||||
@@ -267,6 +315,18 @@ def update_daily_record(
|
||||
and (deb_prediction is None or old_deb == deb_prediction)
|
||||
and (mu is None or old_mu == next_mu)
|
||||
and (compact_probs is None or old_probs == compact_probs)
|
||||
and (
|
||||
compact_shadow_probs is None
|
||||
or old_shadow_probs == compact_shadow_probs
|
||||
)
|
||||
and (
|
||||
compact_features is None
|
||||
or existing.get("probability_features") == compact_features
|
||||
)
|
||||
and (
|
||||
compact_calibration is None
|
||||
or existing.get("probability_calibration") == compact_calibration
|
||||
)
|
||||
):
|
||||
return
|
||||
|
||||
@@ -285,6 +345,12 @@ def update_daily_record(
|
||||
existing["mu"] = next_mu
|
||||
if probabilities is not None:
|
||||
existing["prob_snapshot"] = compact_probs
|
||||
if compact_features is not None:
|
||||
existing["probability_features"] = compact_features
|
||||
if shadow_probabilities is not None:
|
||||
existing["shadow_prob_snapshot"] = compact_shadow_probs
|
||||
if compact_calibration is not None:
|
||||
existing["probability_calibration"] = compact_calibration
|
||||
|
||||
# 自动清理:只保留最近 14 天的记录(DEB 只用 7 天,14 天留足余量)
|
||||
cutoff = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d")
|
||||
|
||||
@@ -9,7 +9,7 @@ import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from src.analysis.settlement_rounding import wu_round, apply_city_settlement, is_exact_settlement_city
|
||||
from src.analysis.settlement_rounding import apply_city_settlement
|
||||
|
||||
|
||||
def _sf(v: Any) -> Optional[float]:
|
||||
@@ -652,6 +652,7 @@ def _pick_bucket_for_forecast(
|
||||
|
||||
def _extract_market_snapshot(city_weather: Dict[str, Any]) -> Dict[str, Any]:
|
||||
scan = city_weather.get("market_scan") or {}
|
||||
city = str(city_weather.get("name") or "").strip().lower()
|
||||
if not isinstance(scan, dict):
|
||||
return {"available": False}
|
||||
if not scan.get("available"):
|
||||
|
||||
@@ -0,0 +1,643 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
|
||||
|
||||
ENGINE_MODE_LEGACY = "legacy"
|
||||
ENGINE_MODE_EMOS_SHADOW = "emos_shadow"
|
||||
ENGINE_MODE_EMOS_PRIMARY = "emos_primary"
|
||||
VALID_ENGINE_MODES = {
|
||||
ENGINE_MODE_LEGACY,
|
||||
ENGINE_MODE_EMOS_SHADOW,
|
||||
ENGINE_MODE_EMOS_PRIMARY,
|
||||
}
|
||||
|
||||
DEFAULT_CALIBRATION_FILE = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"default.json",
|
||||
)
|
||||
|
||||
_CALIBRATION_CACHE: Dict[str, Dict[str, Any]] = {}
|
||||
_CALIBRATION_MTIME: Dict[str, float] = {}
|
||||
|
||||
|
||||
def _sf(value: Any) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _coalesce_float(value: Any, default: float) -> float:
|
||||
parsed = _sf(value)
|
||||
return default if parsed is None else parsed
|
||||
|
||||
|
||||
def _mean(values: Iterable[float]) -> Optional[float]:
|
||||
values = list(values)
|
||||
return (sum(values) / len(values)) if values else None
|
||||
|
||||
|
||||
def resolve_probability_engine_mode(explicit_mode: Optional[str] = None) -> str:
|
||||
mode = str(
|
||||
explicit_mode
|
||||
or os.getenv("POLYWEATHER_PROBABILITY_ENGINE")
|
||||
or ENGINE_MODE_EMOS_SHADOW
|
||||
).strip().lower()
|
||||
if mode not in VALID_ENGINE_MODES:
|
||||
return ENGINE_MODE_EMOS_SHADOW
|
||||
return mode
|
||||
|
||||
|
||||
def load_calibration(calibration_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
path = str(
|
||||
calibration_path
|
||||
or os.getenv("POLYWEATHER_PROBABILITY_CALIBRATION_FILE")
|
||||
or DEFAULT_CALIBRATION_FILE
|
||||
).strip()
|
||||
if not path:
|
||||
return {}
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
|
||||
try:
|
||||
mtime = os.path.getmtime(path)
|
||||
cached = _CALIBRATION_CACHE.get(path)
|
||||
if cached and _CALIBRATION_MTIME.get(path) == mtime:
|
||||
return cached
|
||||
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
|
||||
_CALIBRATION_CACHE[path] = data
|
||||
_CALIBRATION_MTIME[path] = mtime
|
||||
return data
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def build_probability_features(
|
||||
city_name: str,
|
||||
raw_mu: Optional[float],
|
||||
raw_sigma: Optional[float],
|
||||
deb_prediction: Optional[float],
|
||||
ens_data: Optional[Dict[str, Any]],
|
||||
current_forecasts: Optional[Dict[str, Any]],
|
||||
max_so_far: Optional[float],
|
||||
peak_status: str,
|
||||
local_hour_frac: Optional[float],
|
||||
) -> Dict[str, Any]:
|
||||
ens_data = ens_data or {}
|
||||
current_forecasts = current_forecasts or {}
|
||||
forecast_values = [
|
||||
v for v in (_sf(val) for val in current_forecasts.values()) if v is not None
|
||||
]
|
||||
forecast_values.sort()
|
||||
forecast_median = None
|
||||
if forecast_values:
|
||||
forecast_median = forecast_values[len(forecast_values) // 2]
|
||||
|
||||
ens_median = _sf(ens_data.get("median"))
|
||||
ens_p10 = _sf(ens_data.get("p10"))
|
||||
ens_p90 = _sf(ens_data.get("p90"))
|
||||
|
||||
ensemble_spread = None
|
||||
if ens_p10 is not None and ens_p90 is not None and ens_p90 >= ens_p10:
|
||||
ensemble_spread = max(0.1, (ens_p90 - ens_p10) / 2.56)
|
||||
elif len(forecast_values) >= 2:
|
||||
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
|
||||
elif raw_sigma is not None:
|
||||
ensemble_spread = max(0.1, raw_sigma)
|
||||
|
||||
baseline = deb_prediction if deb_prediction is not None else raw_mu
|
||||
max_so_far_gap = None
|
||||
if baseline is not None and max_so_far is not None:
|
||||
max_so_far_gap = baseline - max_so_far
|
||||
|
||||
peak_flag = 0.0
|
||||
if peak_status == "in_window":
|
||||
peak_flag = 0.5
|
||||
elif peak_status == "past":
|
||||
peak_flag = 1.0
|
||||
|
||||
return {
|
||||
"city": str(city_name or "").strip().lower(),
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": raw_sigma,
|
||||
"deb_prediction": deb_prediction,
|
||||
"ens_median": ens_median,
|
||||
"ens_p10": ens_p10,
|
||||
"ens_p90": ens_p90,
|
||||
"forecast_median": forecast_median,
|
||||
"forecast_spread": forecast_values[-1] - forecast_values[0]
|
||||
if len(forecast_values) >= 2
|
||||
else None,
|
||||
"ensemble_spread": ensemble_spread,
|
||||
"max_so_far": max_so_far,
|
||||
"max_so_far_gap": max_so_far_gap,
|
||||
"peak_status": peak_status,
|
||||
"peak_flag": peak_flag,
|
||||
"local_hour_frac": local_hour_frac,
|
||||
"model_count": len(forecast_values),
|
||||
}
|
||||
|
||||
|
||||
def _normal_cdf(x: float, mean: float, sigma: float) -> float:
|
||||
return 0.5 * (1.0 + math.erf((x - mean) / (sigma * math.sqrt(2.0))))
|
||||
|
||||
|
||||
def _normal_pdf(x: float) -> float:
|
||||
return math.exp(-(x ** 2) / 2.0) / math.sqrt(2.0 * math.pi)
|
||||
|
||||
|
||||
def _bucket_probabilities(
|
||||
mu: float,
|
||||
sigma: float,
|
||||
max_so_far: Optional[float],
|
||||
city_name: str,
|
||||
) -> Tuple[List[Dict[str, Any]], List[Tuple[int, float]]]:
|
||||
sigma = max(0.1, float(sigma))
|
||||
min_possible = (
|
||||
apply_city_settlement(city_name, max_so_far) if max_so_far is not None else -999
|
||||
)
|
||||
probs: Dict[int, float] = {}
|
||||
search_range = max(2, int(sigma * 2.5))
|
||||
is_exact = is_exact_settlement_city(city_name)
|
||||
target_mu = apply_city_settlement(city_name, mu)
|
||||
if is_exact:
|
||||
target_mu = int(math.floor(mu))
|
||||
|
||||
for value in range(target_mu - search_range, target_mu + search_range + 1):
|
||||
if value < min_possible:
|
||||
continue
|
||||
if is_exact:
|
||||
prob = _normal_cdf(value + 1.0, mu, sigma) - _normal_cdf(value, mu, sigma)
|
||||
else:
|
||||
prob = _normal_cdf(value + 0.5, mu, sigma) - _normal_cdf(value - 0.5, mu, sigma)
|
||||
if prob > 0.01:
|
||||
probs[value] = prob
|
||||
|
||||
total = sum(probs.values())
|
||||
if total <= 0:
|
||||
return [], []
|
||||
|
||||
normalized = {key: val / total for key, val in probs.items()}
|
||||
sorted_probs = sorted(normalized.items(), key=lambda item: item[1], reverse=True)
|
||||
distribution = []
|
||||
for value, prob in sorted_probs[:4]:
|
||||
if is_exact:
|
||||
bucket_range = "[{0}.0~{1}.0)".format(value, value + 1)
|
||||
else:
|
||||
bucket_range = "[{0}~{1})".format(value - 0.5, value + 0.5)
|
||||
distribution.append(
|
||||
{
|
||||
"value": int(value),
|
||||
"range": bucket_range,
|
||||
"probability": round(prob, 3),
|
||||
}
|
||||
)
|
||||
return distribution, sorted_probs
|
||||
|
||||
|
||||
def _top_bucket_value(distribution: Optional[List[Dict[str, Any]]]) -> Optional[int]:
|
||||
if not distribution:
|
||||
return None
|
||||
top = max(
|
||||
(row for row in distribution if isinstance(row, dict)),
|
||||
key=lambda row: float(row.get("probability") or 0.0),
|
||||
default=None,
|
||||
)
|
||||
if not top:
|
||||
return None
|
||||
value = top.get("value")
|
||||
return int(value) if value is not None else None
|
||||
|
||||
|
||||
def _composite_score(mean_crps: float, mean_mae: float, bucket_hit_rate: float) -> float:
|
||||
return mean_crps + 0.1 * mean_mae + 2.0 * (1.0 - bucket_hit_rate)
|
||||
|
||||
|
||||
def _blend_value(raw_value: float, calibrated_value: float, alpha: float) -> float:
|
||||
return (1.0 - alpha) * raw_value + alpha * calibrated_value
|
||||
|
||||
|
||||
def apply_probability_calibration(
|
||||
city_name: str,
|
||||
temp_symbol: str,
|
||||
raw_mu: Optional[float],
|
||||
raw_sigma: Optional[float],
|
||||
max_so_far: Optional[float],
|
||||
legacy_distribution: Optional[List[Dict[str, Any]]],
|
||||
features: Optional[Dict[str, Any]] = None,
|
||||
calibration_path: Optional[str] = None,
|
||||
mode: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
selected_mode = resolve_probability_engine_mode(mode)
|
||||
if raw_mu is None or raw_sigma is None:
|
||||
return {
|
||||
"mode": selected_mode,
|
||||
"engine": ENGINE_MODE_LEGACY,
|
||||
"distribution": legacy_distribution or [],
|
||||
"shadow_distribution": [],
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": raw_sigma,
|
||||
"calibrated_mu": None,
|
||||
"calibrated_sigma": None,
|
||||
"calibration_version": None,
|
||||
"calibration_source": None,
|
||||
}
|
||||
|
||||
calibration = load_calibration(calibration_path)
|
||||
if not calibration:
|
||||
return {
|
||||
"mode": selected_mode,
|
||||
"engine": ENGINE_MODE_LEGACY,
|
||||
"distribution": legacy_distribution or [],
|
||||
"shadow_distribution": [],
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": raw_sigma,
|
||||
"calibrated_mu": None,
|
||||
"calibrated_sigma": None,
|
||||
"calibration_version": None,
|
||||
"calibration_source": None,
|
||||
}
|
||||
|
||||
features = features or {}
|
||||
city_key = str(city_name or "").strip().lower()
|
||||
global_params = calibration.get("global", {}) or {}
|
||||
city_params = (calibration.get("cities", {}) or {}).get(city_key, {}) or {}
|
||||
blending_cfg = calibration.get("blending", {}) or {}
|
||||
|
||||
mu_cfg = global_params.get("mu", {}) or {}
|
||||
sigma_cfg = global_params.get("sigma", {}) or {}
|
||||
city_confidence = max(0.0, min(1.0, _coalesce_float(city_params.get("confidence"), 1.0)))
|
||||
city_mu_bias = _coalesce_float(city_params.get("mu_bias"), 0.0) * city_confidence
|
||||
city_sigma_scale = 1.0 + (
|
||||
(_coalesce_float(city_params.get("sigma_scale"), 1.0) - 1.0) * city_confidence
|
||||
)
|
||||
|
||||
deb_prediction = _sf(features.get("deb_prediction"))
|
||||
ens_median = _sf(features.get("ens_median"))
|
||||
max_so_far_gap = _sf(features.get("max_so_far_gap"))
|
||||
peak_flag = _sf(features.get("peak_flag")) or 0.0
|
||||
ensemble_spread = _sf(features.get("ensemble_spread"))
|
||||
|
||||
mu_intercept = _coalesce_float(mu_cfg.get("intercept"), 0.0)
|
||||
mu_raw_coef = _coalesce_float(mu_cfg.get("raw_mu_coef"), 1.0)
|
||||
mu_deb_coef = _coalesce_float(mu_cfg.get("deb_coef"), 0.0)
|
||||
mu_ens_coef = _coalesce_float(mu_cfg.get("ens_median_coef"), 0.0)
|
||||
mu_gap_coef = _coalesce_float(mu_cfg.get("max_so_far_gap_coef"), 0.0)
|
||||
|
||||
sigma_intercept = _coalesce_float(
|
||||
sigma_cfg.get("intercept"),
|
||||
math.log(max(raw_sigma, 0.1)),
|
||||
)
|
||||
sigma_raw_coef = _coalesce_float(sigma_cfg.get("raw_sigma_coef"), 1.0)
|
||||
sigma_spread_coef = _coalesce_float(sigma_cfg.get("spread_coef"), 0.0)
|
||||
sigma_peak_coef = _coalesce_float(sigma_cfg.get("peak_flag_coef"), 0.0)
|
||||
sigma_gap_coef = _coalesce_float(sigma_cfg.get("max_so_far_gap_coef"), 0.0)
|
||||
|
||||
calibrated_mu = (
|
||||
mu_intercept
|
||||
+ mu_raw_coef * raw_mu
|
||||
+ mu_deb_coef * (deb_prediction if deb_prediction is not None else raw_mu)
|
||||
+ mu_ens_coef * (ens_median if ens_median is not None else raw_mu)
|
||||
+ mu_gap_coef * (max_so_far_gap if max_so_far_gap is not None else 0.0)
|
||||
+ city_mu_bias
|
||||
)
|
||||
sigma_log = (
|
||||
sigma_intercept
|
||||
+ sigma_raw_coef * math.log(max(raw_sigma, 0.1))
|
||||
+ sigma_spread_coef * math.log(max(ensemble_spread or raw_sigma, 0.1))
|
||||
+ sigma_peak_coef * peak_flag
|
||||
+ sigma_gap_coef * (max_so_far_gap if max_so_far_gap is not None else 0.0)
|
||||
)
|
||||
calibrated_sigma = max(0.1, math.exp(sigma_log) * city_sigma_scale)
|
||||
blend_alpha_mu = max(0.0, min(1.0, _coalesce_float(blending_cfg.get("alpha_mu"), 1.0)))
|
||||
blend_alpha_sigma = max(0.0, min(1.0, _coalesce_float(blending_cfg.get("alpha_sigma"), 1.0)))
|
||||
calibrated_mu = _blend_value(raw_mu, calibrated_mu, blend_alpha_mu)
|
||||
calibrated_sigma = max(0.1, _blend_value(raw_sigma, calibrated_sigma, blend_alpha_sigma))
|
||||
calibrated_distribution, calibrated_sorted = _bucket_probabilities(
|
||||
calibrated_mu,
|
||||
calibrated_sigma,
|
||||
max_so_far=max_so_far,
|
||||
city_name=city_key,
|
||||
)
|
||||
|
||||
engine = ENGINE_MODE_LEGACY
|
||||
selected_distribution = legacy_distribution or []
|
||||
selected_sorted: List[Tuple[int, float]] = []
|
||||
shadow_distribution: List[Dict[str, Any]] = []
|
||||
shadow_sorted: List[Tuple[int, float]] = []
|
||||
if selected_mode == ENGINE_MODE_EMOS_PRIMARY:
|
||||
engine = "emos"
|
||||
selected_distribution = calibrated_distribution
|
||||
selected_sorted = calibrated_sorted
|
||||
elif selected_mode == ENGINE_MODE_EMOS_SHADOW:
|
||||
shadow_distribution = calibrated_distribution
|
||||
shadow_sorted = calibrated_sorted
|
||||
|
||||
return {
|
||||
"mode": selected_mode,
|
||||
"engine": engine,
|
||||
"distribution": selected_distribution,
|
||||
"selected_sorted_probs": selected_sorted,
|
||||
"shadow_distribution": shadow_distribution,
|
||||
"shadow_sorted_probs": shadow_sorted,
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": raw_sigma,
|
||||
"calibrated_mu": calibrated_mu,
|
||||
"calibrated_sigma": calibrated_sigma,
|
||||
"blend_alpha_mu": blend_alpha_mu,
|
||||
"blend_alpha_sigma": blend_alpha_sigma,
|
||||
"calibration_version": calibration.get("version"),
|
||||
"calibration_source": calibration.get("source")
|
||||
or os.path.relpath(
|
||||
calibration_path or DEFAULT_CALIBRATION_FILE,
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _gaussian_crps(observation: float, mean: float, sigma: float) -> float:
|
||||
sigma = max(0.1, float(sigma))
|
||||
z = (observation - mean) / sigma
|
||||
return sigma * (
|
||||
z * (2.0 * _normal_cdf(z, 0.0, 1.0) - 1.0)
|
||||
+ 2.0 * _normal_pdf(z)
|
||||
- 1.0 / math.sqrt(math.pi)
|
||||
)
|
||||
|
||||
|
||||
def _fit_linear(xs: np.ndarray, ys: np.ndarray) -> np.ndarray:
|
||||
if len(xs) == 0:
|
||||
return np.zeros(xs.shape[1], dtype=float)
|
||||
coeffs, _, _, _ = np.linalg.lstsq(xs, ys, rcond=None)
|
||||
return coeffs
|
||||
|
||||
|
||||
def fit_calibration(
|
||||
samples: Iterable[Dict[str, Any]],
|
||||
version: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
normalized_samples: List[Dict[str, Any]] = []
|
||||
city_residuals: Dict[str, List[float]] = {}
|
||||
city_sigma_ratios: Dict[str, List[float]] = {}
|
||||
|
||||
for raw_sample in samples:
|
||||
actual = _sf(raw_sample.get("actual_high"))
|
||||
raw_mu = _sf(raw_sample.get("raw_mu"))
|
||||
raw_sigma = _sf(raw_sample.get("raw_sigma"))
|
||||
if actual is None or raw_mu is None or raw_sigma is None:
|
||||
continue
|
||||
feature_row = {
|
||||
"city": str(raw_sample.get("city") or "").strip().lower(),
|
||||
"actual_high": actual,
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": max(0.1, raw_sigma),
|
||||
"deb_prediction": _sf(raw_sample.get("deb_prediction")),
|
||||
"ens_median": _sf(raw_sample.get("ens_median")),
|
||||
"ensemble_spread": _sf(raw_sample.get("ensemble_spread")),
|
||||
"max_so_far_gap": _sf(raw_sample.get("max_so_far_gap")),
|
||||
"peak_flag": _sf(raw_sample.get("peak_flag")) or 0.0,
|
||||
}
|
||||
normalized_samples.append(feature_row)
|
||||
|
||||
if len(normalized_samples) < 3:
|
||||
return default_calibration_payload(version=version, reason="insufficient_samples")
|
||||
|
||||
mu_rows = []
|
||||
mu_targets = []
|
||||
for sample in normalized_samples:
|
||||
deb = sample["deb_prediction"] if sample["deb_prediction"] is not None else sample["raw_mu"]
|
||||
ens_median = sample["ens_median"] if sample["ens_median"] is not None else sample["raw_mu"]
|
||||
gap = sample["max_so_far_gap"] if sample["max_so_far_gap"] is not None else 0.0
|
||||
mu_rows.append([1.0, sample["raw_mu"], deb, ens_median, gap])
|
||||
mu_targets.append(sample["actual_high"])
|
||||
|
||||
mu_coeffs = _fit_linear(np.array(mu_rows, dtype=float), np.array(mu_targets, dtype=float))
|
||||
|
||||
sigma_rows = []
|
||||
sigma_targets = []
|
||||
mu_predictions = []
|
||||
for idx, sample in enumerate(normalized_samples):
|
||||
predicted_mu = float(np.dot(mu_coeffs, np.array(mu_rows[idx], dtype=float)))
|
||||
mu_predictions.append(predicted_mu)
|
||||
residual = max(abs(sample["actual_high"] - predicted_mu), 0.1)
|
||||
spread = max(sample["ensemble_spread"] or sample["raw_sigma"], 0.1)
|
||||
gap = sample["max_so_far_gap"] if sample["max_so_far_gap"] is not None else 0.0
|
||||
sigma_rows.append([1.0, math.log(sample["raw_sigma"]), math.log(spread), sample["peak_flag"], gap])
|
||||
sigma_targets.append(math.log(residual))
|
||||
city_residuals.setdefault(sample["city"], []).append(sample["actual_high"] - predicted_mu)
|
||||
city_sigma_ratios.setdefault(sample["city"], []).append(residual / max(sample["raw_sigma"], 0.1))
|
||||
|
||||
sigma_coeffs = _fit_linear(np.array(sigma_rows, dtype=float), np.array(sigma_targets, dtype=float))
|
||||
|
||||
crps_values = []
|
||||
for idx, sample in enumerate(normalized_samples):
|
||||
predicted_mu = mu_predictions[idx]
|
||||
sigma_log = float(np.dot(sigma_coeffs, np.array(sigma_rows[idx], dtype=float)))
|
||||
predicted_sigma = max(0.1, math.exp(sigma_log))
|
||||
crps_values.append(_gaussian_crps(sample["actual_high"], predicted_mu, predicted_sigma))
|
||||
|
||||
city_params: Dict[str, Dict[str, Any]] = {}
|
||||
for city, residuals in city_residuals.items():
|
||||
if len(residuals) < 3:
|
||||
continue
|
||||
sigma_ratios = city_sigma_ratios.get(city) or [1.0]
|
||||
confidence = max(0.25, min(1.0, len(residuals) / 8.0))
|
||||
city_params[city] = {
|
||||
"samples": len(residuals),
|
||||
"mu_bias": round(sum(residuals) / len(residuals), 6),
|
||||
"sigma_scale": round(
|
||||
max(0.5, min(2.0, sum(sigma_ratios) / len(sigma_ratios))),
|
||||
6,
|
||||
),
|
||||
"confidence": round(confidence, 6),
|
||||
}
|
||||
|
||||
legacy_crps_values = []
|
||||
legacy_mae_values = []
|
||||
legacy_bucket_hits = []
|
||||
candidate_predictions = []
|
||||
for idx, sample in enumerate(normalized_samples):
|
||||
city = sample["city"]
|
||||
city_meta = city_params.get(city, {})
|
||||
city_confidence = max(
|
||||
0.0,
|
||||
min(1.0, _coalesce_float(city_meta.get("confidence"), 1.0)),
|
||||
)
|
||||
city_mu_bias = _coalesce_float(city_meta.get("mu_bias"), 0.0) * city_confidence
|
||||
city_sigma_scale = 1.0 + (
|
||||
(_coalesce_float(city_meta.get("sigma_scale"), 1.0) - 1.0) * city_confidence
|
||||
)
|
||||
|
||||
legacy_mu = sample["raw_mu"]
|
||||
legacy_sigma = sample["raw_sigma"]
|
||||
actual_high = sample["actual_high"]
|
||||
legacy_crps_values.append(_gaussian_crps(actual_high, legacy_mu, legacy_sigma))
|
||||
legacy_mae_values.append(abs(legacy_mu - actual_high))
|
||||
legacy_bucket_hits.append(
|
||||
1.0
|
||||
if apply_city_settlement(city, legacy_mu)
|
||||
== apply_city_settlement(city, actual_high)
|
||||
else 0.0
|
||||
)
|
||||
|
||||
calibrated_mu = mu_predictions[idx] + city_mu_bias
|
||||
sigma_log = float(np.dot(sigma_coeffs, np.array(sigma_rows[idx], dtype=float)))
|
||||
calibrated_sigma = max(0.1, math.exp(sigma_log) * city_sigma_scale)
|
||||
candidate_predictions.append(
|
||||
{
|
||||
"city": city,
|
||||
"actual_high": actual_high,
|
||||
"raw_mu": legacy_mu,
|
||||
"raw_sigma": legacy_sigma,
|
||||
"calibrated_mu": calibrated_mu,
|
||||
"calibrated_sigma": calibrated_sigma,
|
||||
}
|
||||
)
|
||||
|
||||
legacy_mean_crps = _mean(legacy_crps_values) or 0.0
|
||||
legacy_mean_mae = _mean(legacy_mae_values) or 0.0
|
||||
legacy_bucket_hit_rate = _mean(legacy_bucket_hits) or 0.0
|
||||
legacy_score = _composite_score(
|
||||
legacy_mean_crps,
|
||||
legacy_mean_mae,
|
||||
legacy_bucket_hit_rate,
|
||||
)
|
||||
|
||||
best_alpha_mu = 0.0
|
||||
best_alpha_sigma = 0.0
|
||||
best_score = legacy_score
|
||||
best_metrics = {
|
||||
"mean_crps": legacy_mean_crps,
|
||||
"mean_mae": legacy_mean_mae,
|
||||
"bucket_hit_rate": legacy_bucket_hit_rate,
|
||||
}
|
||||
alpha_grid = [step / 20.0 for step in range(21)]
|
||||
for alpha_mu in alpha_grid:
|
||||
for alpha_sigma in alpha_grid:
|
||||
crps_values = []
|
||||
mae_values = []
|
||||
bucket_hits = []
|
||||
for row in candidate_predictions:
|
||||
mu_hat = _blend_value(row["raw_mu"], row["calibrated_mu"], alpha_mu)
|
||||
sigma_hat = max(
|
||||
0.1,
|
||||
_blend_value(row["raw_sigma"], row["calibrated_sigma"], alpha_sigma),
|
||||
)
|
||||
actual_high = row["actual_high"]
|
||||
city = row["city"]
|
||||
crps_values.append(_gaussian_crps(actual_high, mu_hat, sigma_hat))
|
||||
mae_values.append(abs(mu_hat - actual_high))
|
||||
distribution, _ = _bucket_probabilities(
|
||||
mu_hat,
|
||||
sigma_hat,
|
||||
max_so_far=None,
|
||||
city_name=city,
|
||||
)
|
||||
predicted_bucket = _top_bucket_value(distribution)
|
||||
actual_bucket = apply_city_settlement(city, actual_high)
|
||||
bucket_hits.append(1.0 if predicted_bucket == actual_bucket else 0.0)
|
||||
|
||||
mean_crps = _mean(crps_values) or 0.0
|
||||
mean_mae = _mean(mae_values) or 0.0
|
||||
bucket_hit_rate = _mean(bucket_hits) or 0.0
|
||||
score = _composite_score(mean_crps, mean_mae, bucket_hit_rate)
|
||||
if score + 1e-9 < best_score:
|
||||
best_score = score
|
||||
best_alpha_mu = alpha_mu
|
||||
best_alpha_sigma = alpha_sigma
|
||||
best_metrics = {
|
||||
"mean_crps": mean_crps,
|
||||
"mean_mae": mean_mae,
|
||||
"bucket_hit_rate": bucket_hit_rate,
|
||||
}
|
||||
|
||||
return {
|
||||
"version": version or datetime.now(timezone.utc).strftime("emos-%Y%m%d%H%M%S"),
|
||||
"trained_at": datetime.now(timezone.utc).isoformat(),
|
||||
"global": {
|
||||
"mu": {
|
||||
"intercept": round(float(mu_coeffs[0]), 8),
|
||||
"raw_mu_coef": round(float(mu_coeffs[1]), 8),
|
||||
"deb_coef": round(float(mu_coeffs[2]), 8),
|
||||
"ens_median_coef": round(float(mu_coeffs[3]), 8),
|
||||
"max_so_far_gap_coef": round(float(mu_coeffs[4]), 8),
|
||||
},
|
||||
"sigma": {
|
||||
"intercept": round(float(sigma_coeffs[0]), 8),
|
||||
"raw_sigma_coef": round(float(sigma_coeffs[1]), 8),
|
||||
"spread_coef": round(float(sigma_coeffs[2]), 8),
|
||||
"peak_flag_coef": round(float(sigma_coeffs[3]), 8),
|
||||
"max_so_far_gap_coef": round(float(sigma_coeffs[4]), 8),
|
||||
},
|
||||
},
|
||||
"blending": {
|
||||
"alpha_mu": round(best_alpha_mu, 6),
|
||||
"alpha_sigma": round(best_alpha_sigma, 6),
|
||||
},
|
||||
"cities": city_params,
|
||||
"metrics": {
|
||||
"sample_count": len(normalized_samples),
|
||||
"mean_crps": round(sum(crps_values) / len(crps_values), 6),
|
||||
"legacy_mean_crps": round(legacy_mean_crps, 6),
|
||||
"legacy_mean_mae": round(legacy_mean_mae, 6),
|
||||
"legacy_bucket_hit_rate": round(legacy_bucket_hit_rate, 6),
|
||||
"selected_mean_crps": round(best_metrics["mean_crps"], 6),
|
||||
"selected_mean_mae": round(best_metrics["mean_mae"], 6),
|
||||
"selected_bucket_hit_rate": round(best_metrics["bucket_hit_rate"], 6),
|
||||
"selected_score": round(best_score, 6),
|
||||
"legacy_score": round(legacy_score, 6),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def default_calibration_payload(
|
||||
version: Optional[str] = None,
|
||||
reason: str = "bootstrap",
|
||||
) -> Dict[str, Any]:
|
||||
return {
|
||||
"version": version or "emos-bootstrap-v1",
|
||||
"trained_at": datetime.now(timezone.utc).isoformat(),
|
||||
"global": {
|
||||
"mu": {
|
||||
"intercept": 0.0,
|
||||
"raw_mu_coef": 1.0,
|
||||
"deb_coef": 0.0,
|
||||
"ens_median_coef": 0.0,
|
||||
"max_so_far_gap_coef": 0.0,
|
||||
},
|
||||
"sigma": {
|
||||
"intercept": 0.0,
|
||||
"raw_sigma_coef": 1.0,
|
||||
"spread_coef": 0.0,
|
||||
"peak_flag_coef": 0.0,
|
||||
"max_so_far_gap_coef": 0.0,
|
||||
},
|
||||
},
|
||||
"blending": {
|
||||
"alpha_mu": 1.0,
|
||||
"alpha_sigma": 1.0,
|
||||
},
|
||||
"cities": {},
|
||||
"metrics": {
|
||||
"sample_count": 0,
|
||||
"mean_crps": None,
|
||||
"reason": reason,
|
||||
},
|
||||
}
|
||||
@@ -15,7 +15,11 @@ from src.analysis.deb_algorithm import (
|
||||
update_daily_record,
|
||||
_is_excluded_model_name,
|
||||
)
|
||||
from src.analysis.settlement_rounding import wu_round, apply_city_settlement, is_exact_settlement_city
|
||||
from src.analysis.probability_calibration import (
|
||||
apply_probability_calibration,
|
||||
build_probability_features,
|
||||
)
|
||||
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
|
||||
from src.data_collection.city_registry import CITY_REGISTRY
|
||||
from src.data_collection.city_risk_profiles import get_city_risk_profile
|
||||
|
||||
@@ -431,7 +435,19 @@ def analyze_weather_trend(
|
||||
|
||||
# === Probability Engine ===
|
||||
probabilities: List[Dict[str, Any]] = []
|
||||
shadow_probabilities: List[Dict[str, Any]] = []
|
||||
forecast_miss_deg = 0.0
|
||||
probability_features = None
|
||||
calibration_summary = {
|
||||
"mode": "legacy",
|
||||
"engine": "legacy",
|
||||
"raw_mu": None,
|
||||
"raw_sigma": sigma,
|
||||
"calibrated_mu": None,
|
||||
"calibrated_sigma": None,
|
||||
"calibration_version": None,
|
||||
"calibration_source": None,
|
||||
}
|
||||
|
||||
if is_dead_market:
|
||||
settled_wu = apply_city_settlement(city_name, max_so_far) if max_so_far is not None else 0
|
||||
@@ -492,6 +508,43 @@ def analyze_weather_trend(
|
||||
probabilities = probs_result.get("probabilities", [])
|
||||
sorted_probs = probs_result.get("sorted_probs", [])
|
||||
|
||||
probability_features = build_probability_features(
|
||||
city_name=city_name or "",
|
||||
raw_mu=mu,
|
||||
raw_sigma=sigma,
|
||||
deb_prediction=deb_prediction,
|
||||
ens_data=ens_data,
|
||||
current_forecasts=current_forecasts,
|
||||
max_so_far=max_so_far,
|
||||
peak_status=peak_status,
|
||||
local_hour_frac=local_hour_frac,
|
||||
)
|
||||
calibration_result = apply_probability_calibration(
|
||||
city_name=city_name or "",
|
||||
temp_symbol=temp_symbol,
|
||||
raw_mu=mu,
|
||||
raw_sigma=sigma,
|
||||
max_so_far=max_so_far,
|
||||
legacy_distribution=probabilities,
|
||||
features=probability_features,
|
||||
)
|
||||
calibration_summary = {
|
||||
"mode": calibration_result.get("mode", "legacy"),
|
||||
"engine": calibration_result.get("engine", "legacy"),
|
||||
"raw_mu": calibration_result.get("raw_mu"),
|
||||
"raw_sigma": calibration_result.get("raw_sigma"),
|
||||
"calibrated_mu": calibration_result.get("calibrated_mu"),
|
||||
"calibrated_sigma": calibration_result.get("calibrated_sigma"),
|
||||
"calibration_version": calibration_result.get("calibration_version"),
|
||||
"calibration_source": calibration_result.get("calibration_source"),
|
||||
}
|
||||
shadow_probabilities = calibration_result.get("shadow_distribution") or []
|
||||
if calibration_result.get("engine") == "emos":
|
||||
mu = calibration_result.get("calibrated_mu", mu)
|
||||
sigma = calibration_result.get("calibrated_sigma", sigma)
|
||||
probabilities = calibration_result.get("distribution") or probabilities
|
||||
sorted_probs = calibration_result.get("selected_sorted_probs") or sorted_probs
|
||||
|
||||
if sorted_probs:
|
||||
prob_parts = [
|
||||
f"{int(t)}{temp_symbol} [{t - 0.5}~{t + 0.5}) {p * 100:.0f}%"
|
||||
@@ -650,6 +703,7 @@ def analyze_weather_trend(
|
||||
# === Save daily record (with μ + prob snapshot) ===
|
||||
try:
|
||||
_prob_list = None
|
||||
_shadow_prob_list = None
|
||||
if sorted_probs:
|
||||
_prob_list = [
|
||||
{"value": int(t), "probability": round(p, 3)}
|
||||
@@ -657,6 +711,12 @@ def analyze_weather_trend(
|
||||
]
|
||||
elif is_dead_market and max_so_far is not None:
|
||||
_prob_list = [{"value": apply_city_settlement(city_name, max_so_far), "probability": 1.0}]
|
||||
if shadow_probabilities:
|
||||
_shadow_prob_list = [
|
||||
{"value": int(row.get("value")), "probability": round(float(row.get("probability") or 0.0), 3)}
|
||||
for row in shadow_probabilities[:4]
|
||||
if row.get("value") is not None
|
||||
]
|
||||
|
||||
update_daily_record(
|
||||
city_name,
|
||||
@@ -666,6 +726,9 @@ def analyze_weather_trend(
|
||||
deb_prediction=_deb_to_save,
|
||||
mu=mu,
|
||||
probabilities=_prob_list,
|
||||
probability_features=probability_features,
|
||||
shadow_probabilities=_shadow_prob_list,
|
||||
calibration_summary=calibration_summary,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -679,6 +742,15 @@ def analyze_weather_trend(
|
||||
structured = {
|
||||
"mu": mu,
|
||||
"probabilities": probabilities,
|
||||
"shadow_probabilities": shadow_probabilities,
|
||||
"probability_engine": calibration_summary["engine"],
|
||||
"probability_calibration_mode": calibration_summary["mode"],
|
||||
"probability_calibration_version": calibration_summary["calibration_version"],
|
||||
"probability_calibration_source": calibration_summary["calibration_source"],
|
||||
"probability_raw_mu": calibration_summary["raw_mu"],
|
||||
"probability_raw_sigma": calibration_summary["raw_sigma"],
|
||||
"probability_calibrated_mu": calibration_summary["calibrated_mu"],
|
||||
"probability_calibrated_sigma": calibration_summary["calibrated_sigma"],
|
||||
"trend_info": {
|
||||
"direction": trend_direction if 'trend_direction' in dir() else "unknown",
|
||||
"recent": recent_list,
|
||||
@@ -711,7 +783,7 @@ def calculate_prob_distribution(
|
||||
|
||||
def _norm_cdf(x, m, s):
|
||||
# 0.5 * (1 + erf( (x-m)/(s*sqrt(2)) ))
|
||||
return 0.5 * (1 + math.erf((x - m) / (sigma * math.sqrt(2))))
|
||||
return 0.5 * (1 + math.erf((x - m) / (s * math.sqrt(2))))
|
||||
|
||||
min_possible_wu = apply_city_settlement(city_name, max_so_far) if max_so_far is not None else -999
|
||||
probs = {}
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class MetarSourceMixin:
|
||||
def get_icao_code(self, city: str) -> Optional[str]:
|
||||
"""根据城市名获取对应的 ICAO 机场代码"""
|
||||
normalized = city.lower().strip()
|
||||
if normalized in self.CITY_TO_ICAO:
|
||||
return self.CITY_TO_ICAO[normalized]
|
||||
for key, icao in self.CITY_TO_ICAO.items():
|
||||
if key in normalized or normalized in key:
|
||||
return icao
|
||||
return None
|
||||
|
||||
def fetch_metar(
|
||||
self, city: str, use_fahrenheit: bool = False, utc_offset: int = 0
|
||||
) -> Optional[Dict]:
|
||||
"""从 NOAA Aviation Weather Center 获取 METAR 航空气象数据。"""
|
||||
icao = self.get_icao_code(city)
|
||||
if not icao:
|
||||
logger.warning(f"未找到城市 {city} 对应的 ICAO 代码")
|
||||
return None
|
||||
|
||||
cache_key = f"{icao}:{utc_offset}:{use_fahrenheit}"
|
||||
now_ts = time.time()
|
||||
with self._metar_cache_lock:
|
||||
cached = self._metar_cache.get(cache_key)
|
||||
if cached and now_ts - cached["t"] < self.metar_cache_ttl_sec:
|
||||
logger.debug(f"METAR cache hit {icao} age={int(now_ts - cached['t'])}s")
|
||||
return cached["d"]
|
||||
|
||||
try:
|
||||
url = "https://aviationweather.gov/api/data/metar"
|
||||
params = {
|
||||
"ids": icao,
|
||||
"format": "json",
|
||||
"hours": 24,
|
||||
"_t": int(time.time()),
|
||||
}
|
||||
response = self.session.get(url, params=params, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
if not data:
|
||||
return None
|
||||
|
||||
latest = data[0]
|
||||
temp_c = latest.get("temp")
|
||||
dewp_c = latest.get("dewp")
|
||||
|
||||
def _parse_rawob_time(obs):
|
||||
raw = obs.get("rawOb", "")
|
||||
match = re.search(r"(\d{2})(\d{2})(\d{2})Z", raw)
|
||||
if match:
|
||||
_day, hour, minute = (
|
||||
int(match.group(1)),
|
||||
int(match.group(2)),
|
||||
int(match.group(3)),
|
||||
)
|
||||
fallback = obs.get("reportTime", "")
|
||||
try:
|
||||
clean = fallback.replace(" ", "T")
|
||||
if not clean.endswith("Z"):
|
||||
clean += "Z"
|
||||
base_dt = datetime.fromisoformat(clean.replace("Z", "+00:00"))
|
||||
result = base_dt.replace(hour=hour, minute=minute, second=0)
|
||||
if result > base_dt + timedelta(hours=2):
|
||||
result -= timedelta(days=1)
|
||||
return result
|
||||
except Exception:
|
||||
pass
|
||||
fallback = obs.get("reportTime", "")
|
||||
try:
|
||||
clean = fallback.replace(" ", "T")
|
||||
if not clean.endswith("Z"):
|
||||
clean += "Z"
|
||||
return datetime.fromisoformat(clean.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
obs_dt = _parse_rawob_time(latest)
|
||||
obs_time = (
|
||||
obs_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
if obs_dt
|
||||
else latest.get("reportTime", "")
|
||||
)
|
||||
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
local_now = now_utc + timedelta(seconds=utc_offset)
|
||||
local_midnight = local_now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
utc_midnight = local_midnight - timedelta(seconds=utc_offset)
|
||||
|
||||
max_so_far_c = -999
|
||||
max_temp_time = None
|
||||
for obs in data:
|
||||
obs_dt_iter = _parse_rawob_time(obs)
|
||||
if obs_dt_iter is None:
|
||||
continue
|
||||
try:
|
||||
if obs_dt_iter >= utc_midnight:
|
||||
temp_value = obs.get("temp")
|
||||
if temp_value is not None and temp_value > max_so_far_c:
|
||||
max_so_far_c = temp_value
|
||||
local_report = obs_dt_iter + timedelta(seconds=utc_offset)
|
||||
max_temp_time = local_report.strftime("%H:%M")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
recent_temps_raw = []
|
||||
recent_obs_raw = []
|
||||
today_obs_raw = []
|
||||
cloud_rank_map = {
|
||||
"CLR": 0,
|
||||
"SKC": 0,
|
||||
"FEW": 1,
|
||||
"SCT": 2,
|
||||
"BKN": 3,
|
||||
"OVC": 4,
|
||||
}
|
||||
for index, obs in enumerate(data):
|
||||
obs_temp = obs.get("temp")
|
||||
obs_dt_iter = _parse_rawob_time(obs)
|
||||
if obs_temp is not None and obs_dt_iter:
|
||||
local_rt = obs_dt_iter + timedelta(seconds=utc_offset)
|
||||
time_str = local_rt.strftime("%H:%M")
|
||||
if obs_dt_iter >= utc_midnight:
|
||||
today_obs_raw.append((time_str, obs_temp))
|
||||
if index < 4:
|
||||
recent_temps_raw.append((time_str, obs_temp))
|
||||
clouds = obs.get("clouds", [])
|
||||
max_cloud_rank = 0
|
||||
for cloud in clouds:
|
||||
rank = cloud_rank_map.get(cloud.get("cover", ""), 0)
|
||||
if rank > max_cloud_rank:
|
||||
max_cloud_rank = rank
|
||||
recent_obs_raw.append(
|
||||
{
|
||||
"time": time_str,
|
||||
"temp": obs_temp,
|
||||
"wdir": obs.get("wdir"),
|
||||
"wspd": obs.get("wspd"),
|
||||
"cloud_rank": max_cloud_rank,
|
||||
"altim": obs.get("altim"),
|
||||
}
|
||||
)
|
||||
|
||||
if use_fahrenheit:
|
||||
temp = temp_c * 9 / 5 + 32 if temp_c is not None else None
|
||||
max_so_far = max_so_far_c * 9 / 5 + 32 if max_so_far_c > -900 else None
|
||||
dewp = dewp_c * 9 / 5 + 32 if dewp_c is not None else None
|
||||
unit = "fahrenheit"
|
||||
recent_temps = [(t, round(v * 9 / 5 + 32, 1)) for t, v in recent_temps_raw]
|
||||
today_obs = [(t, round(v * 9 / 5 + 32, 1)) for t, v in today_obs_raw]
|
||||
else:
|
||||
temp = temp_c
|
||||
max_so_far = max_so_far_c if max_so_far_c > -900 else None
|
||||
dewp = dewp_c
|
||||
unit = "celsius"
|
||||
recent_temps = [(t, v) for t, v in recent_temps_raw]
|
||||
today_obs = [(t, v) for t, v in today_obs_raw]
|
||||
|
||||
result = {
|
||||
"source": "metar",
|
||||
"icao": icao,
|
||||
"station_name": latest.get("name", icao),
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"observation_time": obs_time,
|
||||
"report_time": latest.get("reportTime"),
|
||||
"receipt_time": latest.get("receiptTime"),
|
||||
"obs_time_epoch": latest.get("obsTime"),
|
||||
"current": {
|
||||
"temp": round(temp, 1) if temp is not None else None,
|
||||
"max_temp_so_far": round(max_so_far, 1) if max_so_far is not None else None,
|
||||
"max_temp_time": max_temp_time,
|
||||
"dewpoint": round(dewp, 1) if dewp is not None else None,
|
||||
"humidity": latest.get("rh"),
|
||||
"wind_speed_kt": latest.get("wspd"),
|
||||
"wind_dir": latest.get("wdir"),
|
||||
"visibility_mi": latest.get("visib"),
|
||||
"wx_desc": latest.get("wxString"),
|
||||
"altimeter": latest.get("altim"),
|
||||
"raw_metar": latest.get("rawOb"),
|
||||
"clouds": latest.get("clouds", []),
|
||||
},
|
||||
"recent_temps": recent_temps,
|
||||
"today_obs": today_obs,
|
||||
"recent_obs": recent_obs_raw,
|
||||
"unit": unit,
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"✈️ METAR {icao}: {temp:.1f}°{'F' if use_fahrenheit else 'C'} (obs: {obs_time})"
|
||||
)
|
||||
with self._metar_cache_lock:
|
||||
self._metar_cache[cache_key] = {"d": result, "t": now_ts}
|
||||
return result
|
||||
|
||||
except requests.exceptions.RequestException as exc:
|
||||
logger.error(f"METAR 请求失败 ({icao}): {exc}")
|
||||
with self._metar_cache_lock:
|
||||
stale = self._metar_cache.get(cache_key)
|
||||
if stale:
|
||||
logger.warning(f"METAR {icao} 请求失败,使用缓存回退")
|
||||
return stale["d"]
|
||||
return None
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
logger.error(f"METAR 数据解析失败 ({icao}): {exc}")
|
||||
return None
|
||||
|
||||
def fetch_metar_nearby_cluster(self, icaos: List[str], use_fahrenheit: bool = False) -> list:
|
||||
"""批量获取一组 ICAO 站点的 METAR 数据,用于地图周边显示。"""
|
||||
if not icaos:
|
||||
return []
|
||||
|
||||
results = []
|
||||
try:
|
||||
ids_str = ",".join(icaos)
|
||||
url = f"https://aviationweather.gov/api/data/metar?ids={ids_str}&format=json"
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
}
|
||||
resp = self.session.get(url, headers=headers, timeout=self.timeout)
|
||||
if resp.status_code != 200:
|
||||
logger.warning(f"METAR cluster fetch HTTP {resp.status_code} for {icaos}")
|
||||
return []
|
||||
|
||||
data = resp.json()
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
|
||||
for obs in data:
|
||||
icao = obs.get("icaoId")
|
||||
lat = obs.get("lat")
|
||||
lon = obs.get("lon")
|
||||
temp_c = obs.get("temp")
|
||||
if icao and lat and lon and temp_c is not None:
|
||||
display_temp = (temp_c * 9 / 5) + 32 if use_fahrenheit else temp_c
|
||||
name = obs.get("name") or icao
|
||||
name = name.split(" Airport")[0].split(" Intl")[0].split(" International")[0].split(" Arpt")[0].split(",")[0].strip()
|
||||
results.append(
|
||||
{
|
||||
"name": name,
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"temp": round(display_temp, 1),
|
||||
"istNo": icao,
|
||||
"icao": icao,
|
||||
"wind_dir": obs.get("wdir"),
|
||||
"wind_speed": obs.get("wspd"),
|
||||
"wind_speed_kt": obs.get("wspd"),
|
||||
"raw_metar": obs.get("rawOb"),
|
||||
}
|
||||
)
|
||||
|
||||
if results:
|
||||
logger.info(f"📍 METAR 集群: 成功抓取 {len(results)} 个参考站数据")
|
||||
return results
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to fetch METAR cluster {icaos}: {exc}")
|
||||
return []
|
||||
@@ -0,0 +1,318 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class MgmSourceMixin:
|
||||
def fetch_from_mgm(self, istno: str) -> Optional[Dict]:
|
||||
"""
|
||||
从土耳其气象局 (MGM) 获取实时数据和预测 (由用户提供其内部 API)
|
||||
"""
|
||||
base_url = "https://servis.mgm.gov.tr/web"
|
||||
# 必须带 Origin,否则会被反爬拦截
|
||||
headers = {
|
||||
"Origin": "https://www.mgm.gov.tr",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
}
|
||||
results = {}
|
||||
|
||||
try:
|
||||
# 1. 实时数据 (添加时间戳防止 CDN 缓存)
|
||||
import time
|
||||
|
||||
obs_resp = self.session.get(
|
||||
f"{base_url}/sondurumlar?istno={istno}&_={int(time.time() * 1000)}",
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
if obs_resp.status_code == 200:
|
||||
data = obs_resp.json()
|
||||
if data:
|
||||
latest = data[0] if isinstance(data, list) else data
|
||||
# MGM 数据字段映射
|
||||
# ruzgarHiz 实测为 km/h,转为 m/s 需要除以 3.6
|
||||
ruz_hiz_kmh = latest.get("ruzgarHiz", 0)
|
||||
|
||||
# MGM 返回 -9999 表示数据缺失,需要过滤
|
||||
def _valid(v):
|
||||
return v is not None and v > -9000
|
||||
|
||||
results["current"] = {
|
||||
"temp": latest.get("sicaklik")
|
||||
if _valid(latest.get("sicaklik"))
|
||||
else None,
|
||||
"feels_like": latest.get("hissedilenSicaklik")
|
||||
if _valid(latest.get("hissedilenSicaklik"))
|
||||
else None,
|
||||
"humidity": latest.get("nem")
|
||||
if _valid(latest.get("nem"))
|
||||
else None,
|
||||
"wind_speed_ms": round(ruz_hiz_kmh / 3.6, 1)
|
||||
if _valid(ruz_hiz_kmh)
|
||||
else None,
|
||||
"wind_speed_kt": round(ruz_hiz_kmh / 1.852, 1)
|
||||
if _valid(ruz_hiz_kmh)
|
||||
else None,
|
||||
"wind_dir": latest.get("ruzgarYon")
|
||||
if _valid(latest.get("ruzgarYon"))
|
||||
else None,
|
||||
"rain_24h": latest.get("toplamYagis")
|
||||
if _valid(latest.get("toplamYagis"))
|
||||
else None,
|
||||
"pressure": latest.get("aktuelBasinc")
|
||||
if _valid(latest.get("aktuelBasinc"))
|
||||
else None,
|
||||
"cloud_cover": latest.get("kapalilik"), # 0-8 八分位云量
|
||||
"mgm_max_temp": latest.get("maxSicaklik")
|
||||
if _valid(latest.get("maxSicaklik"))
|
||||
else None,
|
||||
"time": latest.get("veriZamani"),
|
||||
"station_name": latest.get("istasyonAd")
|
||||
or latest.get("adi")
|
||||
or latest.get("merkezAd")
|
||||
or "Ankara Bölge",
|
||||
}
|
||||
|
||||
# 2. 每日预报(尝试两个可能的 API 路径)
|
||||
forecast_urls = [
|
||||
f"{base_url}/tahminler/gunluk?istno={istno}",
|
||||
f"https://servis.mgm.gov.tr/api/tahminler/gunluk?istno={istno}",
|
||||
]
|
||||
for forecast_url in forecast_urls:
|
||||
try:
|
||||
daily_resp = self.session.get(
|
||||
forecast_url, headers=headers, timeout=self.timeout
|
||||
)
|
||||
if daily_resp.status_code == 200:
|
||||
forecasts = daily_resp.json()
|
||||
if forecasts and isinstance(forecasts, list):
|
||||
# Store today extra clearly
|
||||
today = forecasts[0]
|
||||
high_val = today.get("enYuksekGun1")
|
||||
low_val = today.get("enDusukGun1")
|
||||
if high_val is not None:
|
||||
results["today_high"] = high_val
|
||||
results["today_low"] = low_val
|
||||
logger.info(f"📋 MGM 每日预报: 今天的最高温 {high_val}°C")
|
||||
|
||||
# Store all 5 days for multi_model_daily
|
||||
results["daily_forecasts"] = {}
|
||||
for i, day in enumerate(forecasts[:5]):
|
||||
d_high = day.get("enYuksekGun1")
|
||||
if d_high is not None:
|
||||
# Calculate date (today + offset)
|
||||
target_date = (datetime.now() + timedelta(days=i)).strftime("%Y-%m-%d")
|
||||
results["daily_forecasts"][target_date] = d_high
|
||||
break
|
||||
else:
|
||||
logger.debug(
|
||||
f"MGM forecast URL {forecast_url} returned {daily_resp.status_code}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"MGM forecast URL {forecast_url} failed: {e}")
|
||||
|
||||
# 3. 小时预报
|
||||
try:
|
||||
hourly_resp = self.session.get(
|
||||
f"{base_url}/tahminler/saatlik?istno={istno}",
|
||||
headers=headers,
|
||||
timeout=self.timeout
|
||||
)
|
||||
if hourly_resp.status_code == 200:
|
||||
h_data = hourly_resp.json()
|
||||
if h_data and isinstance(h_data, list):
|
||||
tahmin_list = h_data[0].get("tahmin", [])
|
||||
results["hourly"] = []
|
||||
for t_data in tahmin_list:
|
||||
if "tarih" in t_data and "sicaklik" in t_data:
|
||||
results["hourly"].append({
|
||||
"time": t_data["tarih"],
|
||||
"temp": t_data["sicaklik"]
|
||||
})
|
||||
except Exception as e:
|
||||
logger.debug(f"MGM hourly failed: {e}")
|
||||
|
||||
# 4. Fallback for today_high (if daily forecast is missing it)
|
||||
if "today_high" not in results:
|
||||
# Try from current max
|
||||
cur_max = results.get("current", {}).get("mgm_max_temp")
|
||||
if cur_max is not None:
|
||||
results["today_high"] = cur_max
|
||||
logger.info(f"📋 MGM 每日预报: 使用当前测站最高温作为今日预报回退: {cur_max}°C")
|
||||
elif "hourly" in results and results["hourly"]:
|
||||
# Try from hourly
|
||||
h_max = max((h["temp"] for h in results["hourly"] if h["temp"] is not None), default=None)
|
||||
if h_max is not None:
|
||||
results["today_high"] = h_max
|
||||
logger.info(f"📋 MGM 每日预报: 使用小时预报最高温作为今日预报回退: {h_max}°C")
|
||||
|
||||
# 5. Fallback for daily_forecasts from hourly data
|
||||
if not results.get("daily_forecasts") and results.get("hourly"):
|
||||
# Guardrail: avoid treating short intraday snippets as full-day highs.
|
||||
hourly_rows = results.get("hourly") or []
|
||||
parsed_times = []
|
||||
for h in hourly_rows:
|
||||
t = str(h.get("time") or "")
|
||||
if "T" not in t:
|
||||
continue
|
||||
try:
|
||||
parsed_times.append(datetime.fromisoformat(t.replace("Z", "+00:00")))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
horizon_hours = 0.0
|
||||
if len(parsed_times) >= 2:
|
||||
parsed_times.sort()
|
||||
horizon_hours = (
|
||||
parsed_times[-1] - parsed_times[0]
|
||||
).total_seconds() / 3600.0
|
||||
|
||||
if len(hourly_rows) >= 24 or horizon_hours >= 30:
|
||||
from collections import defaultdict
|
||||
|
||||
daily_max = defaultdict(list)
|
||||
for h in hourly_rows:
|
||||
t = h.get("time", "")
|
||||
temp = h.get("temp")
|
||||
if t and temp is not None:
|
||||
# Extract date from ISO timestamp like "2026-03-05T12:00:00.000Z"
|
||||
date_str = t[:10]
|
||||
daily_max[date_str].append(temp)
|
||||
if daily_max:
|
||||
results["daily_forecasts"] = {}
|
||||
for d, temps in sorted(daily_max.items()):
|
||||
results["daily_forecasts"][d] = max(temps)
|
||||
logger.info(
|
||||
f"📋 MGM daily_forecasts (from hourly fallback): "
|
||||
f"{dict(results['daily_forecasts'])}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"📋 Skip MGM daily_forecasts hourly fallback: "
|
||||
f"hourly points={len(hourly_rows)}, horizon={horizon_hours:.1f}h"
|
||||
)
|
||||
|
||||
return results if "current" in results else None
|
||||
except Exception as e:
|
||||
logger.error(f"MGM API 请求失败 ({istno}): {e}")
|
||||
return None
|
||||
|
||||
def fetch_mgm_nearby_stations(self, province: str, root_ist_no: str = None) -> list:
|
||||
"""
|
||||
获取一个土耳其省份内所有气象站的当前温度及经纬度
|
||||
使用多线程辅助抓取,因为直接通过 il={province} 往往只返回 1 个站。
|
||||
"""
|
||||
base_url = "https://servis.mgm.gov.tr/web"
|
||||
headers = {
|
||||
"Origin": "https://www.mgm.gov.tr",
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
}
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
results = []
|
||||
try:
|
||||
# 1. 加载测站元数据 (缓存到实例中),用于过滤属于该省份的站点
|
||||
if not getattr(self, "mgm_stations_meta", None):
|
||||
meta_resp = self.session.get(f"{base_url}/istasyonlar", headers=headers, timeout=self.timeout)
|
||||
if meta_resp.status_code == 200:
|
||||
meta_json = meta_resp.json()
|
||||
if isinstance(meta_json, list):
|
||||
self.mgm_stations_meta = {s["istNo"]: s for s in meta_json if "istNo" in s}
|
||||
else:
|
||||
self.mgm_stations_meta = {}
|
||||
|
||||
metadata = getattr(self, "mgm_stations_meta", {})
|
||||
|
||||
# 2. 找出属于该省份的所有站点 istNo
|
||||
province_upper = province.upper()
|
||||
province_ist_nos = [
|
||||
ist_no for ist_no, s in metadata.items()
|
||||
if (s.get("il") or "").upper() == province_upper
|
||||
]
|
||||
|
||||
if not province_ist_nos:
|
||||
logger.warning(f"MGM 找不到省份 {province} 的站点元数据")
|
||||
return []
|
||||
|
||||
# 同时确保我们关心的几个核心站一定在里面
|
||||
target_ist_nos = [str(i) for i in province_ist_nos[:25]]
|
||||
# 17130: 安卡拉总站 (市区核心)
|
||||
if 17130 in province_ist_nos or "17130" in province_ist_nos:
|
||||
if "17130" not in target_ist_nos:
|
||||
target_ist_nos.append("17130")
|
||||
# 17128: 机场官方站
|
||||
if 17128 in province_ist_nos or "17128" in province_ist_nos:
|
||||
if "17128" not in target_ist_nos:
|
||||
target_ist_nos.append("17128")
|
||||
if root_ist_no:
|
||||
rs = str(root_ist_no)
|
||||
if rs not in target_ist_nos:
|
||||
target_ist_nos.append(rs)
|
||||
|
||||
# 3. 多线程获取每个站点的最新观测 (sondurumlar)
|
||||
def fetch_single_station(ist_no):
|
||||
try:
|
||||
# sondurumlar?istno={ist_no} 是目前最稳的获取多站数据的办法
|
||||
url = f"{base_url}/sondurumlar?istno={ist_no}&_={int(time.time() * 1000)}"
|
||||
resp = self.session.get(url, headers=headers, timeout=5)
|
||||
if resp.status_code == 200:
|
||||
obs_list = resp.json()
|
||||
if obs_list:
|
||||
obs = obs_list[0] if isinstance(obs_list, list) else obs_list
|
||||
temp = obs.get("sicaklik")
|
||||
wind_speed = obs.get("ruzgarHiz")
|
||||
wind_dir = obs.get("ruzgarYon")
|
||||
if temp is not None and temp > -9000:
|
||||
return ist_no, {"temp": temp, "wind_speed": wind_speed, "wind_dir": wind_dir}
|
||||
except Exception:
|
||||
pass
|
||||
return None, None
|
||||
|
||||
# 并发抓取
|
||||
station_temps = {}
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
fetch_results = list(executor.map(fetch_single_station, target_ist_nos))
|
||||
for ist_no, data in fetch_results:
|
||||
if ist_no is not None:
|
||||
station_temps[ist_no] = data
|
||||
|
||||
# 4. 组装最终结果
|
||||
for ist_no, temp in station_temps.items():
|
||||
sid = str(ist_no)
|
||||
# metadata 可能使用 int 或 str 作为 key
|
||||
meta = metadata.get(sid) or metadata.get(int(sid))
|
||||
if not meta:
|
||||
continue
|
||||
|
||||
lat = meta.get("enlem")
|
||||
lon = meta.get("boylam")
|
||||
# 优先显示区县名,地图更清晰
|
||||
display_name = (meta.get("ilce") or meta.get("istAd") or f"Station {ist_no}").title()
|
||||
|
||||
# 特殊处理核心站点的显示名称
|
||||
sid = str(ist_no)
|
||||
if sid == "17130":
|
||||
display_name = "Ankara (Bölge/Center)"
|
||||
elif sid == "17128":
|
||||
display_name = "Airport (MGM/17128)"
|
||||
|
||||
results.append({
|
||||
"name": display_name,
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"temp": temp.get("temp") if isinstance(temp, dict) else temp,
|
||||
"wind_speed": temp.get("wind_speed") if isinstance(temp, dict) else None,
|
||||
"wind_dir": temp.get("wind_dir") if isinstance(temp, dict) else None,
|
||||
"istNo": ist_no
|
||||
})
|
||||
|
||||
logger.info(f"📍 MGM 周边测站: 成功并发抓取 {len(results)} 个 {province} 站点的实时气温")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch MGM nearby stations for {province}: {e}")
|
||||
return []
|
||||
|
||||
@@ -0,0 +1,578 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class NwsOpenMeteoSourceMixin:
|
||||
def fetch_nws(self, lat: float, lon: float) -> Optional[Dict]:
|
||||
"""
|
||||
从 NWS (美国国家气象局) 获取高精度预报
|
||||
仅适用于美国城市,全球 VPS 均可访问
|
||||
"""
|
||||
try:
|
||||
# 1. 获取网格点
|
||||
points_url = f"https://api.weather.gov/points/{lat},{lon}"
|
||||
headers = {"User-Agent": "PolyWeather/1.0 (weather-bot)"}
|
||||
|
||||
points_resp = self.session.get(
|
||||
points_url, headers=headers, timeout=self.timeout
|
||||
)
|
||||
points_resp.raise_for_status()
|
||||
points_data = points_resp.json()
|
||||
|
||||
properties = points_data.get("properties", {})
|
||||
forecast_url = properties.get("forecast")
|
||||
hourly_url = properties.get("forecastHourly")
|
||||
if not forecast_url:
|
||||
return None
|
||||
|
||||
# 2. 获取预报
|
||||
forecast_resp = self.session.get(
|
||||
forecast_url, headers=headers, timeout=self.timeout
|
||||
)
|
||||
forecast_resp.raise_for_status()
|
||||
forecast_data = forecast_resp.json()
|
||||
|
||||
periods = forecast_data.get("properties", {}).get("periods", [])
|
||||
if not periods:
|
||||
return None
|
||||
|
||||
hourly_periods = []
|
||||
if hourly_url:
|
||||
hourly_resp = self.session.get(
|
||||
hourly_url, headers=headers, timeout=self.timeout
|
||||
)
|
||||
hourly_resp.raise_for_status()
|
||||
hourly_data = hourly_resp.json()
|
||||
hourly_periods = hourly_data.get("properties", {}).get("periods", [])[:48]
|
||||
|
||||
active_alerts = []
|
||||
try:
|
||||
alerts_resp = self.session.get(
|
||||
"https://api.weather.gov/alerts/active",
|
||||
params={"point": f"{lat},{lon}"},
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
alerts_resp.raise_for_status()
|
||||
alerts_data = alerts_resp.json()
|
||||
for feature in alerts_data.get("features", [])[:8]:
|
||||
ap = feature.get("properties", {})
|
||||
active_alerts.append(
|
||||
{
|
||||
"event": ap.get("event"),
|
||||
"headline": ap.get("headline"),
|
||||
"severity": ap.get("severity"),
|
||||
"certainty": ap.get("certainty"),
|
||||
"urgency": ap.get("urgency"),
|
||||
"effective": ap.get("effective"),
|
||||
"ends": ap.get("ends"),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
active_alerts = []
|
||||
|
||||
# 3. 提取今日最高温(找 isDaytime=True 的第一个)
|
||||
today_high = None
|
||||
for p in periods:
|
||||
if p.get("isDaytime") and "High" in p.get("name", ""):
|
||||
today_high = p.get("temperature")
|
||||
break
|
||||
# 如果没有明确的 High,取第一个 daytime 的温度
|
||||
if today_high is None:
|
||||
for p in periods:
|
||||
if p.get("isDaytime"):
|
||||
today_high = p.get("temperature")
|
||||
break
|
||||
|
||||
return {
|
||||
"source": "nws",
|
||||
"today_high": today_high,
|
||||
"unit": "fahrenheit",
|
||||
"forecast_periods": [
|
||||
{
|
||||
"name": p.get("name"),
|
||||
"start_time": p.get("startTime"),
|
||||
"end_time": p.get("endTime"),
|
||||
"is_daytime": p.get("isDaytime"),
|
||||
"temperature": p.get("temperature"),
|
||||
"temperature_trend": p.get("temperatureTrend"),
|
||||
"wind_speed": p.get("windSpeed"),
|
||||
"wind_direction": p.get("windDirection"),
|
||||
"short_forecast": p.get("shortForecast"),
|
||||
"detailed_forecast": p.get("detailedForecast"),
|
||||
"precipitation_probability": (p.get("probabilityOfPrecipitation") or {}).get("value"),
|
||||
}
|
||||
for p in periods[:14]
|
||||
],
|
||||
"hourly_periods": [
|
||||
{
|
||||
"start_time": p.get("startTime"),
|
||||
"end_time": p.get("endTime"),
|
||||
"temperature": p.get("temperature"),
|
||||
"temperature_unit": p.get("temperatureUnit"),
|
||||
"wind_speed": p.get("windSpeed"),
|
||||
"wind_direction": p.get("windDirection"),
|
||||
"short_forecast": p.get("shortForecast"),
|
||||
"precipitation_probability": (p.get("probabilityOfPrecipitation") or {}).get("value"),
|
||||
}
|
||||
for p in hourly_periods
|
||||
],
|
||||
"active_alerts": active_alerts,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"NWS 请求失败: {e}")
|
||||
return None
|
||||
|
||||
def fetch_from_open_meteo(
|
||||
self,
|
||||
lat: float,
|
||||
lon: float,
|
||||
forecast_days: int = 14,
|
||||
use_fahrenheit: bool = False,
|
||||
) -> Optional[Dict]:
|
||||
"""
|
||||
Fetch weather from Open-Meteo with forecast data
|
||||
|
||||
Args:
|
||||
lat: Latitude
|
||||
lon: Longitude
|
||||
forecast_days: Number of forecast days to fetch (default 14 to cover all market dates)
|
||||
use_fahrenheit: Whether to return temperatures in Fahrenheit (for US markets)
|
||||
"""
|
||||
cache_key = (
|
||||
f"{round(float(lat), 4)}:{round(float(lon), 4)}:"
|
||||
f"{forecast_days}:{'f' if use_fahrenheit else 'c'}"
|
||||
)
|
||||
self._maybe_reload_open_meteo_disk_cache()
|
||||
now_ts = time.time()
|
||||
# ── 429 冷却期检查(所有 Open-Meteo 端点共享)─────────────────
|
||||
with self._open_meteo_rl_lock:
|
||||
if now_ts < self._open_meteo_rate_limit_until:
|
||||
remaining = int(self._open_meteo_rate_limit_until - now_ts)
|
||||
logger.debug(f"Open-Meteo 冷却期中,跳过请求,还需 {remaining}s")
|
||||
with self._open_meteo_cache_lock:
|
||||
stale = self._open_meteo_cache.get(cache_key)
|
||||
if stale and isinstance(stale.get("data"), dict):
|
||||
return dict(stale["data"])
|
||||
return None
|
||||
with self._open_meteo_cache_lock:
|
||||
cached = self._open_meteo_cache.get(cache_key)
|
||||
if (
|
||||
cached
|
||||
and now_ts - float(cached.get("t", 0)) < self.open_meteo_cache_ttl_sec
|
||||
):
|
||||
cached_data = cached.get("data")
|
||||
if isinstance(cached_data, dict):
|
||||
return dict(cached_data)
|
||||
try:
|
||||
url = "https://api.open-meteo.com/v1/forecast"
|
||||
params = {
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"current_weather": "true",
|
||||
"hourly": "temperature_2m,shortwave_radiation,dew_point_2m,pressure_msl,wind_speed_10m,wind_direction_10m,precipitation_probability,cloud_cover",
|
||||
"daily": "temperature_2m_max,apparent_temperature_max,sunrise,sunset,sunshine_duration",
|
||||
"timezone": "auto",
|
||||
"forecast_days": forecast_days,
|
||||
}
|
||||
|
||||
# 显式指定单位,防止 API 默认行为漂移
|
||||
if use_fahrenheit:
|
||||
params["temperature_unit"] = "fahrenheit"
|
||||
else:
|
||||
params["temperature_unit"] = "celsius"
|
||||
|
||||
self._wait_open_meteo_slot("forecast")
|
||||
response = self.session.get(
|
||||
url,
|
||||
params=params,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
current = data.get("current_weather", {})
|
||||
utc_offset = data.get("utc_offset_seconds", 0)
|
||||
timezone_name = data.get("timezone", "UTC")
|
||||
|
||||
# 处理多模型数据 (如果请求了 models 参数,返回结构会变化)
|
||||
daily_data = data.get("daily", {})
|
||||
if "temperature_2m_max_ecmwf_ifs04" in daily_data:
|
||||
ecmwf_max = daily_data.get("temperature_2m_max_ecmwf_ifs04", [])
|
||||
hrrr_max = daily_data.get("temperature_2m_max_ncep_hrrr_conus", [])
|
||||
|
||||
# 记录今日模型分歧
|
||||
daily_data["model_split"] = {
|
||||
"ecmwf": ecmwf_max[0] if ecmwf_max else None,
|
||||
"hrrr": hrrr_max[0] if hrrr_max else None,
|
||||
}
|
||||
|
||||
# 智能合并:HRRR 仅覆盖 48 小时,远期用 ECMWF 补全
|
||||
merged_max = []
|
||||
for i in range(len(ecmwf_max)):
|
||||
hrrr_val = hrrr_max[i] if i < len(hrrr_max) else None
|
||||
ecmwf_val = ecmwf_max[i] if i < len(ecmwf_max) else None
|
||||
|
||||
# 优先 HRRR,其次 ECMWF,都没有就跳过
|
||||
if hrrr_val is not None:
|
||||
merged_max.append(hrrr_val)
|
||||
elif ecmwf_val is not None:
|
||||
merged_max.append(ecmwf_val)
|
||||
else:
|
||||
# 两个都没有,用占位符 (理论上不应该发生)
|
||||
merged_max.append(ecmwf_val) # None
|
||||
daily_data["temperature_2m_max"] = merged_max
|
||||
|
||||
# 映射逐小时数据
|
||||
hourly_data = data.get("hourly", {})
|
||||
if "temperature_2m_ncep_hrrr_conus" in hourly_data:
|
||||
hourly_data["temperature_2m"] = hourly_data[
|
||||
"temperature_2m_ncep_hrrr_conus"
|
||||
]
|
||||
|
||||
# 计算精确的当地时间
|
||||
now_utc = datetime.utcnow()
|
||||
local_now = now_utc + timedelta(seconds=utc_offset)
|
||||
local_time_str = local_now.strftime("%Y-%m-%d %H:%M")
|
||||
|
||||
result = {
|
||||
"source": "open-meteo",
|
||||
"timestamp": now_utc.isoformat(),
|
||||
"timezone": timezone_name,
|
||||
"utc_offset": utc_offset,
|
||||
"current": {
|
||||
"temp": current.get("temperature"),
|
||||
"local_time": local_time_str,
|
||||
},
|
||||
"hourly": hourly_data,
|
||||
"daily": daily_data,
|
||||
"unit": "fahrenheit" if use_fahrenheit else "celsius",
|
||||
}
|
||||
with self._open_meteo_cache_lock:
|
||||
self._open_meteo_cache[cache_key] = {
|
||||
"t": time.time(),
|
||||
"data": dict(result),
|
||||
}
|
||||
self._flush_open_meteo_disk_cache()
|
||||
return result
|
||||
except Exception as e:
|
||||
status_code = getattr(getattr(e, "response", None), "status_code", None)
|
||||
if status_code == 429:
|
||||
retry_after_str = getattr(e.response, "headers", {}).get("Retry-After")
|
||||
cooldown_to_use = self._open_meteo_rl_cooldown
|
||||
if retry_after_str:
|
||||
try:
|
||||
parsed = int(retry_after_str)
|
||||
if parsed > 0:
|
||||
cooldown_to_use = min(parsed + 60, 3600) # Add 60s buffer, max 1 hour
|
||||
logger.info(f"Open-Meteo 响应包含 Retry-After: {retry_after_str}s")
|
||||
except ValueError:
|
||||
pass
|
||||
logger.warning(
|
||||
f"Open-Meteo rate limited (429), fallback to cache if available: lat={lat}, lon={lon}"
|
||||
)
|
||||
# 设置全局冷却期,避免短时内重复触发 429
|
||||
with self._open_meteo_rl_lock:
|
||||
self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
|
||||
logger.warning(f"Open-Meteo 触发限流,设置 {cooldown_to_use}s 冷却期")
|
||||
else:
|
||||
logger.error(f"Open-Meteo forecast failed: {e}")
|
||||
with self._open_meteo_cache_lock:
|
||||
stale = self._open_meteo_cache.get(cache_key)
|
||||
if stale and isinstance(stale.get("data"), dict):
|
||||
fallback = dict(stale["data"])
|
||||
fallback["stale_cache"] = True
|
||||
return fallback
|
||||
return None
|
||||
|
||||
def fetch_ensemble(
|
||||
self,
|
||||
lat: float,
|
||||
lon: float,
|
||||
use_fahrenheit: bool = False,
|
||||
) -> Optional[Dict]:
|
||||
"""
|
||||
从 Open-Meteo Ensemble API 获取 51 成员集合预报
|
||||
用于计算预报不确定性范围(散度)
|
||||
"""
|
||||
cache_key = (
|
||||
f"{round(float(lat), 4)}:{round(float(lon), 4)}:"
|
||||
f"{'f' if use_fahrenheit else 'c'}"
|
||||
)
|
||||
self._maybe_reload_open_meteo_disk_cache()
|
||||
now_ts = time.time()
|
||||
# ── 429 冷却期检查(所有 Open-Meteo 端点共享)─────────────────
|
||||
with self._open_meteo_rl_lock:
|
||||
if now_ts < self._open_meteo_rate_limit_until:
|
||||
remaining = int(self._open_meteo_rate_limit_until - now_ts)
|
||||
logger.debug(f"Open-Meteo Ensemble 冷却期中,跳过请求,还需 {remaining}s")
|
||||
with self._ensemble_cache_lock:
|
||||
stale = self._ensemble_cache.get(cache_key)
|
||||
if stale and isinstance(stale.get("data"), dict):
|
||||
return dict(stale["data"])
|
||||
return None
|
||||
|
||||
with self._ensemble_cache_lock:
|
||||
cached = self._ensemble_cache.get(cache_key)
|
||||
if (
|
||||
cached
|
||||
and now_ts - float(cached.get("t", 0))
|
||||
< self.open_meteo_ensemble_cache_ttl_sec
|
||||
):
|
||||
cached_data = cached.get("data")
|
||||
if isinstance(cached_data, dict):
|
||||
return dict(cached_data)
|
||||
try:
|
||||
url = "https://ensemble-api.open-meteo.com/v1/ensemble"
|
||||
params = {
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"daily": "temperature_2m_max",
|
||||
"timezone": "auto",
|
||||
"forecast_days": 3,
|
||||
}
|
||||
if use_fahrenheit:
|
||||
params["temperature_unit"] = "fahrenheit"
|
||||
else:
|
||||
params["temperature_unit"] = "celsius"
|
||||
|
||||
self._wait_open_meteo_slot("ensemble")
|
||||
response = self.session.get(
|
||||
url,
|
||||
params=params,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
daily = data.get("daily", {})
|
||||
# 每个成员都会返回一组 temperature_2m_max
|
||||
# 格式: {"time": [...], "temperature_2m_max_member01": [...], ...}
|
||||
today_highs = []
|
||||
for key, values in daily.items():
|
||||
if key.startswith("temperature_2m_max") and key != "temperature_2m_max":
|
||||
if values and values[0] is not None:
|
||||
today_highs.append(values[0])
|
||||
|
||||
# 也检查非成员键(有些返回格式不同)
|
||||
if not today_highs:
|
||||
raw_max = daily.get("temperature_2m_max", [])
|
||||
if isinstance(raw_max, list) and raw_max:
|
||||
if isinstance(raw_max[0], list):
|
||||
# 嵌套列表格式: [[member1_day1, member1_day2], [member2_day1, ...]]
|
||||
today_highs = [m[0] for m in raw_max if m and m[0] is not None]
|
||||
elif raw_max[0] is not None:
|
||||
today_highs = [raw_max[0]]
|
||||
|
||||
if len(today_highs) < 3:
|
||||
logger.warning(f"Ensemble 数据不足: 仅获取 {len(today_highs)} 个成员")
|
||||
return None
|
||||
|
||||
today_highs.sort()
|
||||
n = len(today_highs)
|
||||
median = today_highs[n // 2]
|
||||
p10 = today_highs[max(0, int(n * 0.1))]
|
||||
p90 = today_highs[min(n - 1, int(n * 0.9))]
|
||||
|
||||
result = {
|
||||
"source": "ensemble",
|
||||
"members": n,
|
||||
"median": round(median, 1),
|
||||
"p10": round(p10, 1),
|
||||
"p90": round(p90, 1),
|
||||
"min": round(today_highs[0], 1),
|
||||
"max": round(today_highs[-1], 1),
|
||||
"unit": "fahrenheit" if use_fahrenheit else "celsius",
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"📊 Ensemble ({n} members): median={median:.1f}, "
|
||||
f"p10={p10:.1f}, p90={p90:.1f}"
|
||||
)
|
||||
with self._ensemble_cache_lock:
|
||||
self._ensemble_cache[cache_key] = {
|
||||
"t": time.time(),
|
||||
"data": dict(result),
|
||||
}
|
||||
self._flush_open_meteo_disk_cache()
|
||||
return result
|
||||
except Exception as e:
|
||||
status_code = getattr(getattr(e, "response", None), "status_code", None)
|
||||
if status_code == 429:
|
||||
retry_after_str = getattr(e.response, "headers", {}).get("Retry-After")
|
||||
cooldown_to_use = self._open_meteo_rl_cooldown
|
||||
if retry_after_str:
|
||||
try:
|
||||
parsed = int(retry_after_str)
|
||||
if parsed > 0:
|
||||
cooldown_to_use = min(parsed + 60, 3600)
|
||||
except ValueError:
|
||||
pass
|
||||
logger.warning(
|
||||
f"Ensemble API rate limited (429), fallback to cache if available: lat={lat}, lon={lon}"
|
||||
)
|
||||
with self._open_meteo_rl_lock:
|
||||
self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
|
||||
else:
|
||||
logger.warning(f"Ensemble API 请求失败: {e}")
|
||||
with self._ensemble_cache_lock:
|
||||
stale = self._ensemble_cache.get(cache_key)
|
||||
if stale and isinstance(stale.get("data"), dict):
|
||||
fallback = dict(stale["data"])
|
||||
fallback["stale_cache"] = True
|
||||
return fallback
|
||||
return None
|
||||
|
||||
def fetch_multi_model(
|
||||
self,
|
||||
lat: float,
|
||||
lon: float,
|
||||
city: str = "",
|
||||
use_fahrenheit: bool = False,
|
||||
) -> Optional[Dict]:
|
||||
"""
|
||||
从 Open-Meteo 获取多个独立 NWP 模型的预报
|
||||
用于真正的多模型共识评分
|
||||
|
||||
模型列表:
|
||||
- ECMWF IFS (欧洲中期天气预报中心)
|
||||
- GFS (美国 NOAA)
|
||||
- ICON (德国气象局 DWD)
|
||||
- GEM (加拿大气象局)
|
||||
- JMA (日本气象厅)
|
||||
|
||||
返回 3 天的预报数据,支持今日+明日共识分析
|
||||
"""
|
||||
cache_city = str(city or "").strip().lower()
|
||||
cache_key = (
|
||||
f"{round(float(lat), 4)}:{round(float(lon), 4)}:{cache_city}:"
|
||||
f"{'f' if use_fahrenheit else 'c'}:{self.multi_model_cache_version}"
|
||||
)
|
||||
self._maybe_reload_open_meteo_disk_cache()
|
||||
now_ts = time.time()
|
||||
# ── 429 冷却期检查(所有 Open-Meteo 端点共享)─────────────────
|
||||
with self._open_meteo_rl_lock:
|
||||
if now_ts < self._open_meteo_rate_limit_until:
|
||||
remaining = int(self._open_meteo_rate_limit_until - now_ts)
|
||||
logger.debug(f"Open-Meteo Multi-model 冷却期中,跳过请求,还需 {remaining}s")
|
||||
with self._multi_model_cache_lock:
|
||||
stale = self._multi_model_cache.get(cache_key)
|
||||
if stale and isinstance(stale.get("data"), dict):
|
||||
return dict(stale["data"])
|
||||
return None
|
||||
|
||||
with self._multi_model_cache_lock:
|
||||
cached = self._multi_model_cache.get(cache_key)
|
||||
if (
|
||||
cached
|
||||
and now_ts - float(cached.get("t", 0))
|
||||
< self.open_meteo_multi_model_cache_ttl_sec
|
||||
):
|
||||
cached_data = cached.get("data")
|
||||
if isinstance(cached_data, dict):
|
||||
return dict(cached_data)
|
||||
try:
|
||||
url = "https://api.open-meteo.com/v1/forecast"
|
||||
models = "ecmwf_ifs025,gfs_seamless,icon_seamless,gem_seamless,jma_seamless"
|
||||
params = {
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"daily": "temperature_2m_max",
|
||||
"models": models,
|
||||
"timezone": "auto",
|
||||
"forecast_days": 3,
|
||||
}
|
||||
if use_fahrenheit:
|
||||
params["temperature_unit"] = "fahrenheit"
|
||||
|
||||
self._wait_open_meteo_slot("multi-model")
|
||||
response = self.session.get(
|
||||
url,
|
||||
params=params,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
daily = data.get("daily", {})
|
||||
dates = daily.get("time", [])
|
||||
|
||||
model_labels = {
|
||||
"ecmwf_ifs025": "ECMWF",
|
||||
"gfs_seamless": "GFS",
|
||||
"icon_seamless": "ICON",
|
||||
"gem_seamless": "GEM",
|
||||
"jma_seamless": "JMA",
|
||||
}
|
||||
|
||||
# 按天提取每个模型的预报
|
||||
daily_forecasts = {} # {"2026-02-23": {"ECMWF": 7.9, "GFS": 6.5, ...}, ...}
|
||||
for day_idx, date_str in enumerate(dates):
|
||||
day_data = {}
|
||||
for model_key, label in model_labels.items():
|
||||
key = f"temperature_2m_max_{model_key}"
|
||||
values = daily.get(key, [])
|
||||
if day_idx < len(values) and values[day_idx] is not None:
|
||||
day_data[label] = round(values[day_idx], 1)
|
||||
if day_data:
|
||||
daily_forecasts[date_str] = day_data
|
||||
|
||||
if not daily_forecasts:
|
||||
logger.warning("Multi-model: 无有效模型数据")
|
||||
return None
|
||||
|
||||
# 今天的预报 (向后兼容)
|
||||
today_date = dates[0] if dates else None
|
||||
forecasts = daily_forecasts.get(today_date, {})
|
||||
|
||||
labels_str = ", ".join([f"{k}={v}" for k, v in forecasts.items()])
|
||||
logger.info(
|
||||
f"🔬 Multi-model ({len(forecasts)}个, {len(daily_forecasts)}天): {labels_str}"
|
||||
)
|
||||
|
||||
result = {
|
||||
"source": "multi_model",
|
||||
"forecasts": forecasts, # 今天 {"ECMWF": 12.3, "GFS": 11.8, ...} (向后兼容)
|
||||
"daily_forecasts": daily_forecasts, # 按天 {"2026-02-23": {...}, "2026-02-24": {...}}
|
||||
"dates": dates,
|
||||
"unit": "fahrenheit" if use_fahrenheit else "celsius",
|
||||
}
|
||||
with self._multi_model_cache_lock:
|
||||
self._multi_model_cache[cache_key] = {
|
||||
"t": time.time(),
|
||||
"data": dict(result),
|
||||
}
|
||||
self._flush_open_meteo_disk_cache()
|
||||
return result
|
||||
except Exception as e:
|
||||
status_code = getattr(getattr(e, "response", None), "status_code", None)
|
||||
if status_code == 429:
|
||||
retry_after_str = getattr(e.response, "headers", {}).get("Retry-After")
|
||||
cooldown_to_use = self._open_meteo_rl_cooldown
|
||||
if retry_after_str:
|
||||
try:
|
||||
parsed = int(retry_after_str)
|
||||
if parsed > 0:
|
||||
cooldown_to_use = min(parsed + 60, 3600)
|
||||
except ValueError:
|
||||
pass
|
||||
logger.warning(
|
||||
f"Multi-model API rate limited (429), fallback to cache if available: lat={lat}, lon={lon}"
|
||||
)
|
||||
with self._open_meteo_rl_lock:
|
||||
self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
|
||||
else:
|
||||
logger.warning(f"Multi-model API 请求失败: {e}")
|
||||
with self._multi_model_cache_lock:
|
||||
stale = self._multi_model_cache.get(cache_key)
|
||||
if stale and isinstance(stale.get("data"), dict):
|
||||
fallback = dict(stale["data"])
|
||||
fallback["stale_cache"] = True
|
||||
return fallback
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class OpenMeteoCacheMixin:
|
||||
def _load_open_meteo_disk_cache(self) -> None:
|
||||
"""启动时从磁盘加载 Open-Meteo 三类缓存,避免重启后冷启动打爆 API"""
|
||||
try:
|
||||
path = self._disk_cache_path
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"forecast": {},
|
||||
"ensemble": {},
|
||||
"multi_model": {},
|
||||
"saved_at": time.time(),
|
||||
},
|
||||
f,
|
||||
)
|
||||
self._disk_cache_last_mtime = os.path.getmtime(path)
|
||||
return
|
||||
current_mtime = os.path.getmtime(path)
|
||||
if current_mtime <= self._disk_cache_last_mtime:
|
||||
return
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
saved = json.load(f)
|
||||
now = time.time()
|
||||
max_age = max(600, self._disk_cache_max_age_sec)
|
||||
loaded = 0
|
||||
with self._open_meteo_cache_lock:
|
||||
for key, entry in saved.get("forecast", {}).items():
|
||||
if now - float(entry.get("t", 0)) < max_age:
|
||||
old = self._open_meteo_cache.get(key)
|
||||
if old is None or float(entry.get("t", 0)) >= float(old.get("t", 0)):
|
||||
self._open_meteo_cache[key] = entry
|
||||
loaded += 1
|
||||
with self._ensemble_cache_lock:
|
||||
for key, entry in saved.get("ensemble", {}).items():
|
||||
if now - float(entry.get("t", 0)) < max_age:
|
||||
old = self._ensemble_cache.get(key)
|
||||
if old is None or float(entry.get("t", 0)) >= float(old.get("t", 0)):
|
||||
self._ensemble_cache[key] = entry
|
||||
loaded += 1
|
||||
with self._multi_model_cache_lock:
|
||||
for key, entry in saved.get("multi_model", {}).items():
|
||||
if now - float(entry.get("t", 0)) < max_age:
|
||||
old = self._multi_model_cache.get(key)
|
||||
if old is None or float(entry.get("t", 0)) >= float(old.get("t", 0)):
|
||||
self._multi_model_cache[key] = entry
|
||||
loaded += 1
|
||||
self._disk_cache_last_mtime = current_mtime
|
||||
if loaded:
|
||||
logger.info(f"✅ 从磁盘加载 Open-Meteo 缓存 {loaded} 条 ({self._disk_cache_path})")
|
||||
except Exception as exc:
|
||||
logger.warning(f"磁盘缓存加载失败(首次启动不影响运行): {exc}")
|
||||
|
||||
def _maybe_reload_open_meteo_disk_cache(self) -> None:
|
||||
"""跨进程共享缓存:当缓存文件有更新时增量重载到当前进程内存"""
|
||||
try:
|
||||
path = self._disk_cache_path
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
current_mtime = os.path.getmtime(path)
|
||||
if current_mtime <= self._disk_cache_last_mtime:
|
||||
return
|
||||
self._load_open_meteo_disk_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _flush_open_meteo_disk_cache(self) -> None:
|
||||
"""将三类 Open-Meteo 内存缓存持久化到磁盘"""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(self._disk_cache_path), exist_ok=True)
|
||||
with self._open_meteo_cache_lock:
|
||||
forecast_snapshot = dict(self._open_meteo_cache)
|
||||
with self._ensemble_cache_lock:
|
||||
ensemble_snapshot = dict(self._ensemble_cache)
|
||||
with self._multi_model_cache_lock:
|
||||
multi_model_snapshot = dict(self._multi_model_cache)
|
||||
payload = {
|
||||
"forecast": forecast_snapshot,
|
||||
"ensemble": ensemble_snapshot,
|
||||
"multi_model": multi_model_snapshot,
|
||||
"saved_at": time.time(),
|
||||
}
|
||||
with self._disk_cache_lock:
|
||||
tmp_path = self._disk_cache_path + ".tmp"
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f)
|
||||
os.replace(tmp_path, self._disk_cache_path)
|
||||
self._disk_cache_last_mtime = os.path.getmtime(self._disk_cache_path)
|
||||
except Exception as exc:
|
||||
logger.warning(f"磁盘缓存写入失败: {exc}")
|
||||
|
||||
def _wait_open_meteo_slot(self, endpoint: str) -> None:
|
||||
"""Simple per-process rate gate for Open-Meteo endpoints."""
|
||||
min_interval = self._open_meteo_min_interval_sec
|
||||
if min_interval <= 0:
|
||||
return
|
||||
with self._open_meteo_call_lock:
|
||||
now_ts = time.time()
|
||||
wait_for = min_interval - (now_ts - self._open_meteo_last_call_ts)
|
||||
if wait_for > 0:
|
||||
logger.debug(
|
||||
f"Open-Meteo {endpoint} 限流保护:sleep {wait_for:.2f}s (min_interval={min_interval:.2f}s)"
|
||||
)
|
||||
time.sleep(wait_for)
|
||||
now_ts = time.time()
|
||||
self._open_meteo_last_call_ts = now_ts
|
||||
@@ -0,0 +1,269 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class SettlementSourceMixin:
|
||||
def _get_settlement_cache(self, key: str) -> Optional[Dict[str, Any]]:
|
||||
now_ts = time.time()
|
||||
with self._settlement_cache_lock:
|
||||
cached = self._settlement_cache.get(key)
|
||||
if cached and now_ts - float(cached.get("t", 0)) < self.settlement_cache_ttl_sec:
|
||||
return cached.get("d")
|
||||
return None
|
||||
|
||||
def _set_settlement_cache(self, key: str, payload: Dict[str, Any]) -> None:
|
||||
with self._settlement_cache_lock:
|
||||
self._settlement_cache[key] = {"t": time.time(), "d": payload}
|
||||
|
||||
@staticmethod
|
||||
def _csv_rows(text: str) -> List[Dict[str, str]]:
|
||||
normalized = str(text or "").lstrip("")
|
||||
if not normalized.strip():
|
||||
return []
|
||||
return [row for row in csv.DictReader(normalized.splitlines()) if isinstance(row, dict)]
|
||||
|
||||
@staticmethod
|
||||
def _hko_parse_local_iso(raw_yyyymmddhhmm: Optional[str]) -> Optional[str]:
|
||||
raw = str(raw_yyyymmddhhmm or "").strip()
|
||||
if len(raw) != 12 or not raw.isdigit():
|
||||
return None
|
||||
try:
|
||||
dt = datetime.strptime(raw, "%Y%m%d%H%M").replace(
|
||||
tzinfo=timezone(timedelta(hours=8))
|
||||
)
|
||||
return dt.isoformat()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _hko_compass_to_deg(compass: Optional[str]) -> Optional[float]:
|
||||
value = str(compass or "").strip().upper()
|
||||
if not value:
|
||||
return None
|
||||
mapping = {
|
||||
"N": 0.0,
|
||||
"NNE": 22.5,
|
||||
"NE": 45.0,
|
||||
"ENE": 67.5,
|
||||
"E": 90.0,
|
||||
"ESE": 112.5,
|
||||
"SE": 135.0,
|
||||
"SSE": 157.5,
|
||||
"S": 180.0,
|
||||
"SSW": 202.5,
|
||||
"SW": 225.0,
|
||||
"WSW": 247.5,
|
||||
"W": 270.0,
|
||||
"WNW": 292.5,
|
||||
"NW": 315.0,
|
||||
"NNW": 337.5,
|
||||
}
|
||||
return mapping.get(value)
|
||||
|
||||
@staticmethod
|
||||
def _safe_float(value: Any) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
if not text or text in {"***", "N/A", "NA"}:
|
||||
return None
|
||||
try:
|
||||
return float(text)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _pick_station_row(
|
||||
rows: List[Dict[str, str]], candidates: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
if not rows:
|
||||
return None
|
||||
normalized_map = {
|
||||
str(name).strip().lower(): row
|
||||
for row in rows
|
||||
for name in [row.get("Automatic Weather Station")]
|
||||
if isinstance(row, dict) and name
|
||||
}
|
||||
for name in candidates:
|
||||
hit = normalized_map.get(str(name).strip().lower())
|
||||
if hit:
|
||||
return hit
|
||||
for row in rows:
|
||||
station = str(row.get("Automatic Weather Station") or "").strip().lower()
|
||||
if "observatory" in station:
|
||||
return row
|
||||
return rows[0] if rows else None
|
||||
|
||||
def fetch_hko_settlement_current(self) -> Optional[Dict[str, Any]]:
|
||||
cache_key = "hko:hong_kong"
|
||||
cached = self._get_settlement_cache(cache_key)
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
try:
|
||||
base = "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
|
||||
temp_csv = self.session.get(f"{base}/latest_1min_temperature.csv", timeout=self.timeout)
|
||||
temp_csv.raise_for_status()
|
||||
maxmin_csv = self.session.get(f"{base}/latest_since_midnight_maxmin.csv", timeout=self.timeout)
|
||||
maxmin_csv.raise_for_status()
|
||||
humidity_csv = self.session.get(f"{base}/latest_1min_humidity.csv", timeout=self.timeout)
|
||||
humidity_csv.raise_for_status()
|
||||
wind_csv = self.session.get(f"{base}/latest_10min_wind.csv", timeout=self.timeout)
|
||||
wind_csv.raise_for_status()
|
||||
|
||||
temp_rows = self._csv_rows(temp_csv.text)
|
||||
maxmin_rows = self._csv_rows(maxmin_csv.text)
|
||||
humidity_rows = self._csv_rows(humidity_csv.text)
|
||||
wind_rows = self._csv_rows(wind_csv.text)
|
||||
|
||||
station_candidates = ["HK Observatory", "Hong Kong Observatory"]
|
||||
temp_row = self._pick_station_row(temp_rows, station_candidates)
|
||||
maxmin_row = self._pick_station_row(maxmin_rows, station_candidates)
|
||||
humidity_row = self._pick_station_row(humidity_rows, station_candidates)
|
||||
wind_row = self._pick_station_row(wind_rows, station_candidates)
|
||||
if not temp_row or not maxmin_row:
|
||||
return None
|
||||
|
||||
obs_raw = temp_row.get("Date time") or maxmin_row.get("Date time")
|
||||
obs_iso = self._hko_parse_local_iso(obs_raw)
|
||||
current_temp = self._safe_float(temp_row.get("Air Temperature(degree Celsius)"))
|
||||
max_so_far = self._safe_float(maxmin_row.get("Maximum Air Temperature Since Midnight(degree Celsius)"))
|
||||
min_so_far = self._safe_float(maxmin_row.get("Minimum Air Temperature Since Midnight(degree Celsius)"))
|
||||
humidity = self._safe_float(humidity_row.get("Relative Humidity(percent)")) if humidity_row else None
|
||||
wind_speed_kmh = self._safe_float(wind_row.get("10-Minute Mean Speed(km/hour)")) if wind_row else None
|
||||
wind_speed_kt = round(float(wind_speed_kmh) / 1.852, 1) if wind_speed_kmh is not None else None
|
||||
wind_dir = self._hko_compass_to_deg(
|
||||
wind_row.get("10-Minute Mean Wind Direction(Compass points)") if wind_row else None
|
||||
)
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"source": "hko",
|
||||
"source_label": "HKO",
|
||||
"station_code": "HKO",
|
||||
"station_name": "HK Observatory",
|
||||
"observation_time": obs_iso,
|
||||
"current": {
|
||||
"temp": round(current_temp, 1) if current_temp is not None else None,
|
||||
"max_temp_so_far": round(max_so_far, 1) if max_so_far is not None else None,
|
||||
"max_temp_time": None,
|
||||
"today_low": round(min_so_far, 1) if min_so_far is not None else None,
|
||||
"humidity": round(humidity, 1) if humidity is not None else None,
|
||||
"wind_speed_kt": wind_speed_kt,
|
||||
"wind_dir": wind_dir,
|
||||
},
|
||||
"unit": "celsius",
|
||||
}
|
||||
self._set_settlement_cache(cache_key, payload)
|
||||
return payload
|
||||
except Exception as exc:
|
||||
logger.warning(f"HKO settlement fetch failed: {exc}")
|
||||
return None
|
||||
|
||||
def fetch_cwa_taipei_settlement_current(self) -> Optional[Dict[str, Any]]:
|
||||
cache_key = "cwa:taipei:466920"
|
||||
cached = self._get_settlement_cache(cache_key)
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
try:
|
||||
url = "https://opendata.cwa.gov.tw/api/v1/rest/datastore/O-A0003-001"
|
||||
response = self.session.get(
|
||||
url,
|
||||
params={"Authorization": self.cwa_open_data_auth, "format": "JSON", "StationId": "466920"},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json() if response.content else {}
|
||||
stations = (data.get("records") or {}).get("Station") or []
|
||||
if isinstance(stations, dict):
|
||||
station = stations
|
||||
elif isinstance(stations, list) and stations:
|
||||
station = stations[0]
|
||||
else:
|
||||
station = None
|
||||
if not isinstance(station, dict):
|
||||
return None
|
||||
|
||||
wx = station.get("WeatherElement") or {}
|
||||
if not isinstance(wx, dict):
|
||||
wx = {}
|
||||
daily_extreme = wx.get("DailyExtreme") or {}
|
||||
high_info = (((daily_extreme.get("DailyHigh") or {}).get("TemperatureInfo") or {}))
|
||||
low_info = (((daily_extreme.get("DailyLow") or {}).get("TemperatureInfo") or {}))
|
||||
high_time_raw = (((high_info.get("Occurred_at") or {}).get("DateTime")))
|
||||
high_hhmm = None
|
||||
if high_time_raw and "T" in str(high_time_raw):
|
||||
try:
|
||||
high_hhmm = datetime.fromisoformat(str(high_time_raw)).strftime("%H:%M")
|
||||
except Exception:
|
||||
high_hhmm = str(high_time_raw).split("T")[1][:5]
|
||||
|
||||
obs_time_raw = (station.get("ObsTime") or {}).get("DateTime")
|
||||
wind_speed_ms = self._safe_float(wx.get("WindSpeed"))
|
||||
payload: Dict[str, Any] = {
|
||||
"source": "cwa",
|
||||
"source_label": "CWA",
|
||||
"station_code": str(station.get("StationId") or "466920"),
|
||||
"station_name": str(station.get("StationName") or "臺北"),
|
||||
"observation_time": str(obs_time_raw or "").strip() or None,
|
||||
"current": {
|
||||
"temp": self._safe_float(wx.get("AirTemperature")),
|
||||
"max_temp_so_far": self._safe_float(high_info.get("AirTemperature")),
|
||||
"max_temp_time": high_hhmm,
|
||||
"today_low": self._safe_float(low_info.get("AirTemperature")),
|
||||
"humidity": self._safe_float(wx.get("RelativeHumidity")),
|
||||
"wind_speed_kt": round(float(wind_speed_ms) * 1.943844, 1) if wind_speed_ms is not None else None,
|
||||
"wind_dir": self._safe_float(wx.get("WindDirection")),
|
||||
},
|
||||
"unit": "celsius",
|
||||
}
|
||||
self._set_settlement_cache(cache_key, payload)
|
||||
return payload
|
||||
except Exception as exc:
|
||||
logger.warning(f"CWA settlement fetch failed: {exc}")
|
||||
return None
|
||||
|
||||
def fetch_hko_forecast(self) -> Optional[float]:
|
||||
try:
|
||||
url = "https://data.weather.gov.hk/weatherAPI/opendata/weather.php?dataType=fnd&lang=tc"
|
||||
res = self.session.get(url, timeout=self.timeout).json()
|
||||
return float(res["weatherForecast"][0]["forecastMaxtemp"]["value"])
|
||||
except Exception as exc:
|
||||
logger.warning(f"HKO Forecast request failed: {exc}")
|
||||
return None
|
||||
|
||||
def fetch_cwa_taipei_forecast(self) -> Optional[float]:
|
||||
try:
|
||||
if not self.cwa_open_data_auth:
|
||||
return None
|
||||
url = "https://opendata.cwa.gov.tw/api/v1/rest/datastore/F-D0047-061"
|
||||
res = self.session.get(
|
||||
url,
|
||||
params={"Authorization": self.cwa_open_data_auth, "format": "JSON", "elementName": "MaxT"},
|
||||
timeout=self.timeout,
|
||||
).json()
|
||||
locs = res.get("records", {}).get("Locations", [])[0].get("Location", [])
|
||||
if not locs:
|
||||
return None
|
||||
loc = locs[0]
|
||||
for weather_element in loc.get("WeatherElement", []):
|
||||
if weather_element.get("ElementName") == "MaxT":
|
||||
return float(weather_element["Time"][0]["ElementValue"][0]["Temperature"])
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.warning(f"CWA Forecast request failed: {exc}")
|
||||
return None
|
||||
|
||||
def fetch_settlement_current(self, city: str) -> Optional[Dict[str, Any]]:
|
||||
normalized = str(city or "").strip().lower()
|
||||
if normalized == "hong kong":
|
||||
return self.fetch_hko_settlement_current()
|
||||
if normalized == "taipei":
|
||||
return self.fetch_cwa_taipei_settlement_current()
|
||||
return None
|
||||
+122
-1739
File diff suppressed because it is too large
Load Diff
@@ -33,8 +33,9 @@ def test_basic_handler_diag_returns_html():
|
||||
runtime = RuntimeStatus(
|
||||
started_at="2026-03-12 00:00:00 UTC",
|
||||
loops=[],
|
||||
entitlement_enabled=False,
|
||||
command_access_mode="group_member",
|
||||
protected_commands=["/city", "/deb"],
|
||||
required_group_chat_id="-1001234567890",
|
||||
)
|
||||
bot = DummyBot()
|
||||
io_layer = SimpleNamespace(
|
||||
@@ -52,4 +53,3 @@ def test_basic_handler_diag_returns_html():
|
||||
assert len(bot.replies) == 1
|
||||
assert bot.replies[0]["parse_mode"] == "HTML"
|
||||
assert "Bot 启动诊断" in bot.replies[0]["text"]
|
||||
|
||||
|
||||
@@ -2,15 +2,6 @@ from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from src.bot.command_guard import CommandGuard
|
||||
from src.bot.services.entitlement_service import BotEntitlementService
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self, user):
|
||||
self._user = user
|
||||
|
||||
def get_user(self, _user_id):
|
||||
return self._user
|
||||
|
||||
|
||||
def _message():
|
||||
@@ -20,11 +11,13 @@ def _message():
|
||||
)
|
||||
|
||||
|
||||
def test_guard_blocks_non_premium_when_entitlement_enabled():
|
||||
fake_bot = SimpleNamespace(reply_to=Mock())
|
||||
def test_guard_blocks_non_member_before_points_charge():
|
||||
fake_bot = SimpleNamespace(
|
||||
reply_to=Mock(),
|
||||
get_chat_member=Mock(return_value=SimpleNamespace(status="left")),
|
||||
)
|
||||
io_layer = SimpleNamespace(bot=fake_bot, ensure_query_points=Mock(return_value=True))
|
||||
entitlement = BotEntitlementService(db=_FakeDB(user=None), enabled=True)
|
||||
guard = CommandGuard(io_layer=io_layer, entitlement_service=entitlement)
|
||||
guard = CommandGuard(io_layer=io_layer, group_chat_id="-100123")
|
||||
|
||||
ok = guard.ensure_access_and_points(_message(), 1, "/city")
|
||||
|
||||
@@ -33,18 +26,16 @@ def test_guard_blocks_non_premium_when_entitlement_enabled():
|
||||
assert fake_bot.reply_to.call_count == 1
|
||||
|
||||
|
||||
def test_guard_allows_premium_then_charges_points():
|
||||
fake_bot = SimpleNamespace(reply_to=Mock())
|
||||
io_layer = SimpleNamespace(bot=fake_bot, ensure_query_points=Mock(return_value=True))
|
||||
entitlement = BotEntitlementService(
|
||||
db=_FakeDB(user={"is_web_premium": 1, "is_group_premium": 0}),
|
||||
enabled=True,
|
||||
def test_guard_allows_member_then_charges_points():
|
||||
fake_bot = SimpleNamespace(
|
||||
reply_to=Mock(),
|
||||
get_chat_member=Mock(return_value=SimpleNamespace(status="member")),
|
||||
)
|
||||
guard = CommandGuard(io_layer=io_layer, entitlement_service=entitlement)
|
||||
io_layer = SimpleNamespace(bot=fake_bot, ensure_query_points=Mock(return_value=True))
|
||||
guard = CommandGuard(io_layer=io_layer, group_chat_id="-100123")
|
||||
|
||||
ok = guard.ensure_access_and_points(_message(), 1, "/city")
|
||||
|
||||
assert ok is True
|
||||
assert io_layer.ensure_query_points.call_count == 1
|
||||
assert fake_bot.reply_to.call_count == 0
|
||||
|
||||
|
||||
+40
-49
@@ -7,92 +7,83 @@ from src.bot.services.city_command_service import CityReportResult, CityResolveR
|
||||
from src.bot.services.deb_command_service import DebReportResult
|
||||
|
||||
|
||||
class DummyBot:
|
||||
def __init__(self):
|
||||
self.replies = []
|
||||
self.sent = []
|
||||
|
||||
def reply_to(self, message, text, parse_mode=None):
|
||||
self.replies.append(
|
||||
{
|
||||
"chat_id": message.chat.id,
|
||||
"text": text,
|
||||
"parse_mode": parse_mode,
|
||||
}
|
||||
)
|
||||
|
||||
def send_message(self, chat_id, text, parse_mode=None):
|
||||
self.sent.append(
|
||||
{
|
||||
"chat_id": chat_id,
|
||||
"text": text,
|
||||
"parse_mode": parse_mode,
|
||||
}
|
||||
)
|
||||
|
||||
def message_handler(self, *args, **kwargs): # pragma: no cover - decorator stub
|
||||
def _decorator(func):
|
||||
return func
|
||||
|
||||
return _decorator
|
||||
|
||||
|
||||
def _message(text: str):
|
||||
return SimpleNamespace(
|
||||
text=text,
|
||||
from_user=SimpleNamespace(id=123, username="tester", first_name="Tester"),
|
||||
chat=SimpleNamespace(id=999),
|
||||
entities=[],
|
||||
)
|
||||
|
||||
|
||||
def test_city_handler_missing_city_shows_usage():
|
||||
bot = DummyBot()
|
||||
bot = SimpleNamespace()
|
||||
guard = SimpleNamespace(ensure_access_and_points=Mock(return_value=True))
|
||||
city_service = SimpleNamespace(
|
||||
resolve_city=Mock(),
|
||||
build_report=Mock(),
|
||||
city_service = SimpleNamespace(resolve_city=Mock(), build_report=Mock())
|
||||
io_layer = SimpleNamespace(send_query_message=Mock())
|
||||
handler = CityCommandHandler(
|
||||
bot=bot,
|
||||
guard=guard,
|
||||
city_service=city_service,
|
||||
io_layer=io_layer,
|
||||
)
|
||||
handler = CityCommandHandler(bot=bot, guard=guard, city_service=city_service)
|
||||
|
||||
handler.handle(_message("/city"))
|
||||
|
||||
assert len(bot.replies) == 1
|
||||
assert "请输入城市名称" in bot.replies[0]["text"]
|
||||
assert io_layer.send_query_message.call_count == 1
|
||||
assert "请输入城市名称" in io_layer.send_query_message.call_args[0][1]
|
||||
assert guard.ensure_access_and_points.call_count == 0
|
||||
|
||||
|
||||
def test_city_handler_happy_path_pushes_progress_and_report():
|
||||
bot = DummyBot()
|
||||
bot = SimpleNamespace()
|
||||
guard = SimpleNamespace(ensure_access_and_points=Mock(return_value=True))
|
||||
city_service = SimpleNamespace(
|
||||
resolve_city=Mock(
|
||||
return_value=CityResolveResult(ok=True, city_name="london", supported_cities=["london"])
|
||||
return_value=CityResolveResult(
|
||||
ok=True,
|
||||
city_name="london",
|
||||
supported_cities=["london"],
|
||||
)
|
||||
),
|
||||
build_report=Mock(return_value=CityReportResult(ok=True, report="CITY_REPORT")),
|
||||
)
|
||||
handler = CityCommandHandler(bot=bot, guard=guard, city_service=city_service)
|
||||
io_layer = SimpleNamespace(send_query_message=Mock())
|
||||
handler = CityCommandHandler(
|
||||
bot=bot,
|
||||
guard=guard,
|
||||
city_service=city_service,
|
||||
io_layer=io_layer,
|
||||
)
|
||||
|
||||
handler.handle(_message("/city london"))
|
||||
|
||||
assert len(bot.sent) == 2
|
||||
assert "正在查询 London 的天气数据" in bot.sent[0]["text"]
|
||||
assert bot.sent[1]["text"] == "CITY_REPORT"
|
||||
assert bot.sent[1]["parse_mode"] == "HTML"
|
||||
assert io_layer.send_query_message.call_count == 2
|
||||
first_call = io_layer.send_query_message.call_args_list[0]
|
||||
second_call = io_layer.send_query_message.call_args_list[1]
|
||||
assert "正在查询 London 的天气数据" in first_call.args[1]
|
||||
assert second_call.args[1] == "CITY_REPORT"
|
||||
assert second_call.kwargs["parse_mode"] == "HTML"
|
||||
|
||||
|
||||
def test_deb_handler_history_missing_returns_hint():
|
||||
bot = DummyBot()
|
||||
bot = SimpleNamespace()
|
||||
guard = SimpleNamespace(ensure_access_and_points=Mock(return_value=True))
|
||||
deb_service = SimpleNamespace(
|
||||
resolve_city=Mock(return_value="ankara"),
|
||||
has_history=Mock(return_value=False),
|
||||
build_report=Mock(return_value=DebReportResult(ok=True, report="DEB_REPORT")),
|
||||
)
|
||||
handler = DebCommandHandler(bot=bot, guard=guard, deb_service=deb_service)
|
||||
io_layer = SimpleNamespace(send_query_message=Mock())
|
||||
handler = DebCommandHandler(
|
||||
bot=bot,
|
||||
guard=guard,
|
||||
deb_service=deb_service,
|
||||
io_layer=io_layer,
|
||||
)
|
||||
|
||||
handler.handle(_message("/deb ankara"))
|
||||
|
||||
assert len(bot.replies) == 1
|
||||
assert "暂无 ankara 的历史数据" in bot.replies[0]["text"]
|
||||
assert io_layer.send_query_message.call_count == 1
|
||||
assert "暂无 ankara 的历史数据" in io_layer.send_query_message.call_args[0][1]
|
||||
assert guard.ensure_access_and_points.call_count == 0
|
||||
|
||||
|
||||
@@ -14,8 +14,9 @@ def test_startup_coordinator_respects_disable_flags(monkeypatch):
|
||||
coordinator = StartupCoordinator(
|
||||
bot=DummyBot(),
|
||||
config={},
|
||||
entitlement_enabled=False,
|
||||
command_access_mode="group_member",
|
||||
protected_commands=["/city", "/deb"],
|
||||
required_group_chat_id="-1001234567890",
|
||||
)
|
||||
runtime = coordinator.start_all()
|
||||
loop_map = runtime.loop_map()
|
||||
@@ -30,13 +31,13 @@ def test_startup_coordinator_respects_disable_flags(monkeypatch):
|
||||
def test_render_runtime_status_html_contains_key_fields():
|
||||
runtime = RuntimeStatus(
|
||||
started_at="2026-03-12 00:00:00 UTC",
|
||||
entitlement_enabled=True,
|
||||
command_access_mode="group_member",
|
||||
protected_commands=["/city", "/deb"],
|
||||
required_group_chat_id="-1001234567890",
|
||||
loops=[],
|
||||
)
|
||||
html = render_runtime_status_html(runtime)
|
||||
|
||||
assert "Bot 启动诊断" in html
|
||||
assert "Entitlement" in html
|
||||
assert "命令准入" in html
|
||||
assert "/city, /deb" in html
|
||||
|
||||
|
||||
@@ -34,15 +34,15 @@ def _sample_weather_payload():
|
||||
],
|
||||
"mgm_nearby": [
|
||||
{
|
||||
"name": "Ankara (Bölge/Center)",
|
||||
"istNo": "17130",
|
||||
"name": "Airport (MGM/17128)",
|
||||
"istNo": "17128",
|
||||
"lat": 39.95,
|
||||
"lon": 32.97,
|
||||
"temp": 12.4,
|
||||
},
|
||||
{
|
||||
"name": "Airport (MGM/17128)",
|
||||
"istNo": "17128",
|
||||
"name": "Ankara (Bölge/Center)",
|
||||
"istNo": "17130",
|
||||
"lat": 40.1281,
|
||||
"lon": 32.9951,
|
||||
"temp": 11.2,
|
||||
@@ -57,7 +57,7 @@ def test_trading_alerts_all_core_rules_trigger():
|
||||
map_url="https://example.com/map",
|
||||
)
|
||||
|
||||
assert out["trigger_count"] >= 3
|
||||
assert out["trigger_count"] >= 2
|
||||
assert out["rules"]["momentum_spike"]["triggered"] is True
|
||||
assert out["rules"]["forecast_breakthrough"]["triggered"] is True
|
||||
assert out["rules"]["advection"]["triggered"] is True
|
||||
@@ -113,8 +113,8 @@ def test_ankara_center_signal_only_uses_official_center_station():
|
||||
"temp": 12.6,
|
||||
},
|
||||
{
|
||||
"name": "Ankara (Bölge/Center)",
|
||||
"istNo": "17130",
|
||||
"name": "Airport (MGM/17128)",
|
||||
"istNo": "17128",
|
||||
"lat": 39.95,
|
||||
"lon": 32.97,
|
||||
"temp": 11.3,
|
||||
@@ -125,9 +125,9 @@ def test_ankara_center_signal_only_uses_official_center_station():
|
||||
|
||||
center_rule = out["rules"]["ankara_center_deb_hit"]
|
||||
assert center_rule["triggered"] is True
|
||||
assert center_rule["center_station"]["istNo"] == "17130"
|
||||
assert center_rule["center_station"]["name"] == "Ankara (Bölge/Center)"
|
||||
assert "Ankara (Bölge/Center)" in out["telegram"]["zh"]
|
||||
assert center_rule["center_station"]["istNo"] == "17128"
|
||||
assert center_rule["center_station"]["name"] == "Airport (MGM/17128)"
|
||||
assert "Airport (MGM/17128)" in out["telegram"]["zh"]
|
||||
assert "Etimesgut" not in out["telegram"]["zh"]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from src.analysis.probability_calibration import (
|
||||
ENGINE_MODE_EMOS_PRIMARY,
|
||||
ENGINE_MODE_EMOS_SHADOW,
|
||||
ENGINE_MODE_LEGACY,
|
||||
apply_probability_calibration,
|
||||
build_probability_features,
|
||||
fit_calibration,
|
||||
)
|
||||
|
||||
|
||||
def _write_calibration(tmp_path: Path):
|
||||
payload = {
|
||||
"version": "test-emos-v1",
|
||||
"source": "tmp/test-emos-v1.json",
|
||||
"global": {
|
||||
"mu": {
|
||||
"intercept": 0.0,
|
||||
"raw_mu_coef": 0.0,
|
||||
"deb_coef": 1.0,
|
||||
"ens_median_coef": 0.0,
|
||||
"max_so_far_gap_coef": 0.0,
|
||||
},
|
||||
"sigma": {
|
||||
"intercept": 0.0,
|
||||
"raw_sigma_coef": 1.0,
|
||||
"spread_coef": 0.0,
|
||||
"peak_flag_coef": 0.0,
|
||||
"max_so_far_gap_coef": 0.0,
|
||||
},
|
||||
},
|
||||
"cities": {
|
||||
"ankara": {
|
||||
"mu_bias": 0.5,
|
||||
"sigma_scale": 1.0,
|
||||
}
|
||||
},
|
||||
"metrics": {"sample_count": 10, "mean_crps": 0.4},
|
||||
}
|
||||
path = tmp_path / "calibration.json"
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def test_shadow_mode_keeps_legacy_distribution(tmp_path):
|
||||
calibration_path = _write_calibration(tmp_path)
|
||||
features = build_probability_features(
|
||||
city_name="ankara",
|
||||
raw_mu=9.0,
|
||||
raw_sigma=1.0,
|
||||
deb_prediction=10.0,
|
||||
ens_data={"median": 9.5, "p10": 8.0, "p90": 11.0},
|
||||
current_forecasts={"Open-Meteo": 9.0, "MGM": 10.0},
|
||||
max_so_far=8.8,
|
||||
peak_status="before",
|
||||
local_hour_frac=11.0,
|
||||
)
|
||||
legacy_distribution = [{"value": 9, "range": "[8.5~9.5)", "probability": 0.7}]
|
||||
|
||||
result = apply_probability_calibration(
|
||||
city_name="ankara",
|
||||
temp_symbol="°C",
|
||||
raw_mu=9.0,
|
||||
raw_sigma=1.0,
|
||||
max_so_far=8.8,
|
||||
legacy_distribution=legacy_distribution,
|
||||
features=features,
|
||||
calibration_path=str(calibration_path),
|
||||
mode=ENGINE_MODE_EMOS_SHADOW,
|
||||
)
|
||||
|
||||
assert result["mode"] == ENGINE_MODE_EMOS_SHADOW
|
||||
assert result["engine"] == ENGINE_MODE_LEGACY
|
||||
assert result["distribution"] == legacy_distribution
|
||||
assert result["shadow_distribution"]
|
||||
assert result["calibrated_mu"] == 10.5
|
||||
|
||||
|
||||
def test_primary_mode_switches_to_calibrated_distribution(tmp_path):
|
||||
calibration_path = _write_calibration(tmp_path)
|
||||
features = build_probability_features(
|
||||
city_name="ankara",
|
||||
raw_mu=9.0,
|
||||
raw_sigma=1.0,
|
||||
deb_prediction=10.0,
|
||||
ens_data={"median": 9.5, "p10": 8.0, "p90": 11.0},
|
||||
current_forecasts={"Open-Meteo": 9.0, "MGM": 10.0},
|
||||
max_so_far=8.8,
|
||||
peak_status="before",
|
||||
local_hour_frac=11.0,
|
||||
)
|
||||
|
||||
result = apply_probability_calibration(
|
||||
city_name="ankara",
|
||||
temp_symbol="°C",
|
||||
raw_mu=9.0,
|
||||
raw_sigma=1.0,
|
||||
max_so_far=8.8,
|
||||
legacy_distribution=[{"value": 9, "range": "[8.5~9.5)", "probability": 0.7}],
|
||||
features=features,
|
||||
calibration_path=str(calibration_path),
|
||||
mode=ENGINE_MODE_EMOS_PRIMARY,
|
||||
)
|
||||
|
||||
assert result["mode"] == ENGINE_MODE_EMOS_PRIMARY
|
||||
assert result["engine"] == "emos"
|
||||
assert result["calibrated_mu"] == 10.5
|
||||
assert result["distribution"]
|
||||
assert result["distribution"][0]["value"] >= 10
|
||||
|
||||
|
||||
def test_fit_calibration_returns_metrics():
|
||||
samples = [
|
||||
{
|
||||
"city": "ankara",
|
||||
"actual_high": 11.0,
|
||||
"raw_mu": 10.2,
|
||||
"raw_sigma": 1.0,
|
||||
"deb_prediction": 10.5,
|
||||
"ens_median": 10.6,
|
||||
"ensemble_spread": 0.9,
|
||||
"max_so_far_gap": 0.5,
|
||||
"peak_flag": 0.0,
|
||||
},
|
||||
{
|
||||
"city": "ankara",
|
||||
"actual_high": 12.0,
|
||||
"raw_mu": 11.1,
|
||||
"raw_sigma": 1.0,
|
||||
"deb_prediction": 11.3,
|
||||
"ens_median": 11.2,
|
||||
"ensemble_spread": 1.0,
|
||||
"max_so_far_gap": 0.4,
|
||||
"peak_flag": 0.5,
|
||||
},
|
||||
{
|
||||
"city": "new york",
|
||||
"actual_high": 19.0,
|
||||
"raw_mu": 18.2,
|
||||
"raw_sigma": 1.4,
|
||||
"deb_prediction": 18.4,
|
||||
"ens_median": 18.3,
|
||||
"ensemble_spread": 1.2,
|
||||
"max_so_far_gap": 0.6,
|
||||
"peak_flag": 1.0,
|
||||
},
|
||||
]
|
||||
|
||||
result = fit_calibration(samples, version="unit-test-v1")
|
||||
|
||||
assert result["version"] == "unit-test-v1"
|
||||
assert result["metrics"]["sample_count"] == 3
|
||||
assert "mean_crps" in result["metrics"]
|
||||
@@ -0,0 +1,61 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.build_probability_shadow_report import main
|
||||
|
||||
|
||||
def test_shadow_report_builds_summary(monkeypatch, tmp_path: Path):
|
||||
history_file = tmp_path / "daily_records.json"
|
||||
output_file = tmp_path / "shadow_report.json"
|
||||
history_file.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"ankara": {
|
||||
"2026-03-18": {
|
||||
"actual_high": 10.0,
|
||||
"mu": 9.6,
|
||||
"prob_snapshot": [
|
||||
{"v": 10, "p": 0.48},
|
||||
{"v": 9, "p": 0.41},
|
||||
],
|
||||
"shadow_prob_snapshot": [
|
||||
{"v": 10, "p": 0.55},
|
||||
{"v": 9, "p": 0.28},
|
||||
],
|
||||
"probability_calibration": {
|
||||
"mode": "emos_shadow",
|
||||
"engine": "legacy",
|
||||
"version": "emos-test",
|
||||
"raw_mu": 9.6,
|
||||
"calibrated_mu": 9.9,
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"sys.argv",
|
||||
[
|
||||
"build_probability_shadow_report.py",
|
||||
"--history-file",
|
||||
str(history_file),
|
||||
"--output",
|
||||
str(output_file),
|
||||
],
|
||||
)
|
||||
|
||||
main()
|
||||
|
||||
payload = json.loads(output_file.read_text(encoding="utf-8"))
|
||||
assert payload["summary"]["samples"] == 1
|
||||
assert payload["summary"]["legacy_mean_mae"] == 0.4
|
||||
assert payload["summary"]["shadow_mean_mae"] == 0.1
|
||||
assert payload["summary"]["delta_mae"] == -0.3
|
||||
assert payload["summary"]["legacy_bucket_hit_rate"] == 1.0
|
||||
assert payload["summary"]["shadow_bucket_hit_rate"] == 1.0
|
||||
assert payload["recent_observations"][0]["calibration_version"] == "emos-test"
|
||||
@@ -9,7 +9,6 @@ import os
|
||||
# Ensure project root is in path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
from src.analysis.trend_engine import analyze_weather_trend, _sf
|
||||
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from web.core import app
|
||||
|
||||
__all__ = ["app"]
|
||||
@@ -0,0 +1,862 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time as _time
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from loguru import logger
|
||||
|
||||
from web.core import (
|
||||
_cache,
|
||||
CACHE_TTL,
|
||||
CACHE_TTL_ANKARA,
|
||||
CITIES,
|
||||
CITY_RISK_PROFILES,
|
||||
SETTLEMENT_SOURCE_LABELS,
|
||||
_is_excluded_model_name,
|
||||
_market_layer,
|
||||
_sf,
|
||||
_weather,
|
||||
)
|
||||
from src.analysis.deb_algorithm import calculate_dynamic_weights
|
||||
from src.analysis.settlement_rounding import apply_city_settlement
|
||||
from src.analysis.metar_narrator import describe_metar_report
|
||||
from src.data_collection.city_registry import ALIASES
|
||||
|
||||
def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]:
|
||||
"""Fetch, analyse, and return structured weather data for one city."""
|
||||
# Check cache
|
||||
ttl = CACHE_TTL_ANKARA if city.lower() == "ankara" else CACHE_TTL
|
||||
|
||||
if not force_refresh:
|
||||
cached = _cache.get(city)
|
||||
if cached and _time.time() - cached["t"] < ttl:
|
||||
return cached["d"]
|
||||
|
||||
info = CITIES[city]
|
||||
lat, lon, is_f = info["lat"], info["lon"], info["f"]
|
||||
sym = "°F" if is_f else "°C"
|
||||
settlement_source = str(info.get("settlement_source") or "metar").strip().lower() or "metar"
|
||||
settlement_source_label = SETTLEMENT_SOURCE_LABELS.get(
|
||||
settlement_source,
|
||||
settlement_source.upper(),
|
||||
)
|
||||
|
||||
# ── 1. Fetch raw data ──
|
||||
raw = _weather.fetch_all_sources(
|
||||
city,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
om = raw.get("open-meteo", {})
|
||||
metar = raw.get("metar", {})
|
||||
mgm = raw.get("mgm") or {}
|
||||
settlement_current = raw.get("settlement_current") or {}
|
||||
ens_raw = raw.get("ensemble", {})
|
||||
mm = raw.get("multi_model", {})
|
||||
if not isinstance(om, dict):
|
||||
om = {}
|
||||
if not isinstance(metar, dict):
|
||||
metar = {}
|
||||
if not isinstance(mgm, dict):
|
||||
mgm = {}
|
||||
if not isinstance(settlement_current, dict):
|
||||
settlement_current = {}
|
||||
if not isinstance(ens_raw, dict):
|
||||
ens_raw = {}
|
||||
if not isinstance(mm, dict):
|
||||
mm = {}
|
||||
risk = CITY_RISK_PROFILES.get(city, {})
|
||||
|
||||
# ── 2. Current conditions (city-specific settlement source first, then METAR/MGM fallback) ──
|
||||
mc = metar.get("current", {}) if metar else {}
|
||||
mg_cur = mgm.get("current", {}) if mgm else {}
|
||||
sc_cur = settlement_current.get("current", {}) if settlement_current else {}
|
||||
use_settlement_current = settlement_source in {"hko", "cwa"} and bool(sc_cur)
|
||||
primary_current = sc_cur if use_settlement_current else mc
|
||||
cur_temp = _sf(primary_current.get("temp"))
|
||||
if cur_temp is None:
|
||||
cur_temp = _sf(mc.get("temp"))
|
||||
if cur_temp is None:
|
||||
cur_temp = _sf(mg_cur.get("temp"))
|
||||
|
||||
max_so_far = _sf(primary_current.get("max_temp_so_far"))
|
||||
if max_so_far is None:
|
||||
max_so_far = _sf(mc.get("max_temp_so_far"))
|
||||
if max_so_far is None:
|
||||
max_so_far = _sf(mg_cur.get("mgm_max_temp"))
|
||||
|
||||
max_temp_time = primary_current.get("max_temp_time")
|
||||
if not max_temp_time and not use_settlement_current:
|
||||
max_temp_time = mc.get("max_temp_time")
|
||||
if not max_temp_time:
|
||||
max_temp_time = mg_cur.get("time", "")
|
||||
if " " in max_temp_time:
|
||||
max_temp_time = max_temp_time.split(" ")[1][:5]
|
||||
if max_temp_time == "":
|
||||
max_temp_time = None
|
||||
|
||||
wu_settle = apply_city_settlement(city.lower(), max_so_far) if max_so_far is not None else None
|
||||
|
||||
# Observation time → local
|
||||
obs_time_str = ""
|
||||
metar_age_min = None
|
||||
obs_t = ""
|
||||
if use_settlement_current:
|
||||
obs_t = str(settlement_current.get("observation_time") or "").strip()
|
||||
if not obs_t:
|
||||
obs_t = metar.get("observation_time", "") if metar else ""
|
||||
# 优先从 API 获取偏移;若缺失则尝试 NWS 动态偏移;最后回退静态配置
|
||||
utc_offset = om.get("utc_offset")
|
||||
if utc_offset is None:
|
||||
try:
|
||||
nws_periods = (raw.get("nws", {}) or {}).get("forecast_periods", []) or []
|
||||
if nws_periods:
|
||||
first_start = nws_periods[0].get("start_time")
|
||||
if first_start:
|
||||
maybe_dt = datetime.fromisoformat(str(first_start))
|
||||
if maybe_dt.utcoffset() is not None:
|
||||
utc_offset = int(maybe_dt.utcoffset().total_seconds())
|
||||
except Exception:
|
||||
utc_offset = None
|
||||
if utc_offset is None:
|
||||
utc_offset = info.get("tz", 0)
|
||||
if obs_t and "T" in obs_t:
|
||||
try:
|
||||
dt = datetime.fromisoformat(str(obs_t).replace("Z", "+00:00"))
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
local_dt = dt.astimezone(timezone(timedelta(seconds=utc_offset)))
|
||||
obs_time_str = local_dt.strftime("%H:%M")
|
||||
metar_age_min = int(
|
||||
(datetime.now(timezone.utc) - dt.astimezone(timezone.utc)).total_seconds() / 60
|
||||
)
|
||||
except Exception:
|
||||
obs_time_str = str(obs_t)[:16]
|
||||
|
||||
settlement_today_obs = []
|
||||
if use_settlement_current:
|
||||
if obs_time_str and cur_temp is not None:
|
||||
settlement_today_obs.append({"time": obs_time_str, "temp": cur_temp})
|
||||
if (
|
||||
max_temp_time
|
||||
and max_so_far is not None
|
||||
and str(max_temp_time) != str(obs_time_str)
|
||||
):
|
||||
settlement_today_obs.append({"time": str(max_temp_time), "temp": max_so_far})
|
||||
|
||||
metar_today_obs_payload = (
|
||||
[]
|
||||
if use_settlement_current
|
||||
else [
|
||||
{"time": t, "temp": v}
|
||||
for t, v in (metar.get("today_obs", []) if metar else [])
|
||||
]
|
||||
)
|
||||
metar_recent_obs_payload = (
|
||||
[] if use_settlement_current else (metar.get("recent_obs", []) if metar else [])
|
||||
)
|
||||
|
||||
# ── 3. Local time parsing ──
|
||||
local_time_full = om.get("current", {}).get("local_time", "")
|
||||
local_hour, local_minute = 12, 0
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
local_now = now_utc + timedelta(seconds=utc_offset)
|
||||
local_date_str = local_now.strftime("%Y-%m-%d")
|
||||
|
||||
try:
|
||||
if local_time_full:
|
||||
local_date_str = local_time_full.split(" ")[0]
|
||||
tp = local_time_full.split(" ")[1].split(":")
|
||||
local_hour = int(tp[0])
|
||||
local_minute = int(tp[1]) if len(tp) > 1 else 0
|
||||
else:
|
||||
local_hour = local_now.hour
|
||||
local_minute = local_now.minute
|
||||
except Exception:
|
||||
local_hour = local_now.hour
|
||||
local_minute = local_now.minute
|
||||
local_time_str = f"{local_hour:02d}:{local_minute:02d}"
|
||||
local_hour_frac = local_hour + local_minute / 60
|
||||
|
||||
# ── 4. Daily forecast ──
|
||||
daily = om.get("daily", {})
|
||||
dates = daily.get("time", [])[:5]
|
||||
maxtemps = daily.get("temperature_2m_max", [])[:5]
|
||||
sunrises = daily.get("sunrise", [])
|
||||
sunsets = daily.get("sunset", [])
|
||||
sunshine = daily.get("sunshine_duration", [])
|
||||
om_today = _sf(maxtemps[0]) if maxtemps else None
|
||||
|
||||
forecast_daily = [{"date": d, "max_temp": t} for d, t in zip(dates, maxtemps)]
|
||||
if om_today is None:
|
||||
nws_high = _sf(raw.get("nws", {}).get("today_high"))
|
||||
mgm_high = _sf(mgm.get("today_high")) if mgm else None
|
||||
fallback_high = (
|
||||
nws_high
|
||||
if nws_high is not None
|
||||
else mgm_high
|
||||
if mgm_high is not None
|
||||
else max_so_far
|
||||
if max_so_far is not None
|
||||
else cur_temp
|
||||
)
|
||||
if fallback_high is not None:
|
||||
om_today = float(fallback_high)
|
||||
if not forecast_daily:
|
||||
forecast_daily = [{"date": local_date_str, "max_temp": om_today}]
|
||||
sunrise = (
|
||||
sunrises[0].split("T")[1][:5]
|
||||
if sunrises and "T" in str(sunrises[0])
|
||||
else ""
|
||||
)
|
||||
sunset = (
|
||||
sunsets[0].split("T")[1][:5]
|
||||
if sunsets and "T" in str(sunsets[0])
|
||||
else ""
|
||||
)
|
||||
sunshine_h = round(sunshine[0] / 3600, 1) if sunshine else 0
|
||||
|
||||
# ── 5. Multi-model forecasts ──
|
||||
current_forecasts: Dict[str, float] = {}
|
||||
if om_today is not None:
|
||||
current_forecasts["Open-Meteo"] = om_today
|
||||
for m, v in mm.get("forecasts", {}).items():
|
||||
if v is not None and not _is_excluded_model_name(m):
|
||||
current_forecasts[m] = _sf(v)
|
||||
nws_high = _sf(raw.get("nws", {}).get("today_high"))
|
||||
if nws_high is not None:
|
||||
current_forecasts["NWS"] = nws_high
|
||||
mgm_high = _sf(mgm.get("today_high")) if mgm else None
|
||||
if mgm_high is not None:
|
||||
current_forecasts["MGM"] = mgm_high
|
||||
|
||||
# ── 6. DEB fusion ──
|
||||
deb_val, deb_weights = None, ""
|
||||
if current_forecasts:
|
||||
blended, winfo = calculate_dynamic_weights(city, current_forecasts)
|
||||
if blended is not None:
|
||||
deb_val = blended
|
||||
deb_weights = winfo
|
||||
|
||||
# ── 7. Ensemble stats ──
|
||||
ens_data = {
|
||||
"median": _sf(ens_raw.get("median")),
|
||||
"p10": _sf(ens_raw.get("p10")),
|
||||
"p90": _sf(ens_raw.get("p90")),
|
||||
}
|
||||
|
||||
# ── 8. METAR trend ──
|
||||
recent_temps = metar.get("recent_temps", []) if metar else []
|
||||
trend_info = {
|
||||
"direction": "unknown",
|
||||
"recent": [{"time": t, "temp": v} for t, v in recent_temps[:6]],
|
||||
"is_cooling": False,
|
||||
"is_dead_market": False,
|
||||
}
|
||||
if len(recent_temps) >= 2:
|
||||
t_only = [t for _, t in recent_temps]
|
||||
latest, prev = t_only[0], t_only[1]
|
||||
diff = latest - prev
|
||||
if len(t_only) >= 3:
|
||||
n = min(3, len(t_only))
|
||||
all_same = all(t == latest for t in t_only[:n])
|
||||
all_rising = all(t_only[i] >= t_only[i + 1] for i in range(n - 1))
|
||||
all_falling = all(t_only[i] <= t_only[i + 1] for i in range(n - 1))
|
||||
if all_same:
|
||||
trend_info["direction"] = "stagnant"
|
||||
elif all_rising and diff > 0:
|
||||
trend_info["direction"] = "rising"
|
||||
elif all_falling and diff < 0:
|
||||
trend_info["direction"] = "falling"
|
||||
else:
|
||||
trend_info["direction"] = "mixed"
|
||||
elif diff > 0:
|
||||
trend_info["direction"] = "rising"
|
||||
elif diff < 0:
|
||||
trend_info["direction"] = "falling"
|
||||
else:
|
||||
trend_info["direction"] = "stagnant"
|
||||
trend_info["is_cooling"] = trend_info["direction"] in ("falling", "stagnant")
|
||||
|
||||
# ── 9. Peak hour detection ──
|
||||
hourly = om.get("hourly", {})
|
||||
h_times = hourly.get("time", [])
|
||||
h_temps = hourly.get("temperature_2m", [])
|
||||
h_rad = hourly.get("shortwave_radiation", [])
|
||||
h_dew = hourly.get("dew_point_2m", [])
|
||||
h_pressure = hourly.get("pressure_msl", [])
|
||||
h_wspd = hourly.get("wind_speed_10m", [])
|
||||
h_wdir = hourly.get("wind_direction_10m", [])
|
||||
h_precip_prob = hourly.get("precipitation_probability", [])
|
||||
h_cloud_cover = hourly.get("cloud_cover", [])
|
||||
if (not h_times or not h_temps) and metar:
|
||||
metar_today_obs = metar.get("today_obs", []) or []
|
||||
parsed_obs = []
|
||||
for item in metar_today_obs:
|
||||
try:
|
||||
t_str, t_val = item
|
||||
if t_str is None or t_val is None:
|
||||
continue
|
||||
hh, minute_part = str(t_str).split(":")
|
||||
parsed_obs.append((int(hh), int(minute_part), float(t_val)))
|
||||
except Exception:
|
||||
continue
|
||||
if parsed_obs:
|
||||
parsed_obs.sort(key=lambda x: (x[0], x[1]))
|
||||
h_times = [f"{local_date_str}T{hh:02d}:{mm:02d}" for hh, mm, _ in parsed_obs]
|
||||
h_temps = [v for _, _, v in parsed_obs]
|
||||
h_rad = [0 for _ in parsed_obs]
|
||||
h_dew = [None for _ in parsed_obs]
|
||||
h_pressure = [None for _ in parsed_obs]
|
||||
h_wspd = [None for _ in parsed_obs]
|
||||
h_wdir = [None for _ in parsed_obs]
|
||||
h_precip_prob = [None for _ in parsed_obs]
|
||||
h_cloud_cover = [None for _ in parsed_obs]
|
||||
|
||||
peak_hours = []
|
||||
if h_times and h_temps and om_today is not None:
|
||||
for ts, tmp in zip(h_times, h_temps):
|
||||
if ts.startswith(local_date_str) and abs(tmp - om_today) <= 0.2:
|
||||
hr = int(ts.split("T")[1][:2])
|
||||
if 8 <= hr <= 19:
|
||||
peak_hours.append(ts.split("T")[1][:5])
|
||||
|
||||
first_peak_h = int(peak_hours[0].split(":")[0]) if peak_hours else 13
|
||||
last_peak_h = int(peak_hours[-1].split(":")[0]) if peak_hours else 15
|
||||
|
||||
if local_hour_frac > last_peak_h:
|
||||
peak_status = "past"
|
||||
elif first_peak_h <= local_hour_frac <= last_peak_h:
|
||||
peak_status = "in_window"
|
||||
else:
|
||||
peak_status = "before"
|
||||
|
||||
# ── 10. Shared analysis (probability, trend, AI) via trend_engine ──
|
||||
# This single call replaces the duplicate probability engine, dead market
|
||||
# detection, forecast bust grading, and AI context building.
|
||||
from src.analysis.trend_engine import analyze_weather_trend as _trend_analyze, calculate_prob_distribution
|
||||
|
||||
probabilities = []
|
||||
shadow_probabilities = []
|
||||
mu = None
|
||||
probability_engine = "legacy"
|
||||
probability_calibration_mode = "legacy"
|
||||
probability_calibration_version = None
|
||||
probability_raw_mu = None
|
||||
probability_raw_sigma = None
|
||||
probability_calibrated_mu = None
|
||||
probability_calibrated_sigma = None
|
||||
ai_text = ""
|
||||
try:
|
||||
_, _ai_context, sd = _trend_analyze(raw, sym, city)
|
||||
|
||||
# Use structured data from shared engine
|
||||
mu = sd.get("mu")
|
||||
probabilities = sd.get("probabilities", [])
|
||||
shadow_probabilities = sd.get("shadow_probabilities", [])
|
||||
probability_engine = sd.get("probability_engine", "legacy")
|
||||
probability_calibration_mode = sd.get("probability_calibration_mode", "legacy")
|
||||
probability_calibration_version = sd.get("probability_calibration_version")
|
||||
probability_raw_mu = sd.get("probability_raw_mu")
|
||||
probability_raw_sigma = sd.get("probability_raw_sigma")
|
||||
probability_calibrated_mu = sd.get("probability_calibrated_mu")
|
||||
probability_calibrated_sigma = sd.get("probability_calibrated_sigma")
|
||||
trend_info["is_dead_market"] = sd.get("trend_info", {}).get("is_dead_market", False)
|
||||
trend_info["direction"] = sd.get("trend_info", {}).get("direction", trend_info.get("direction", "unknown"))
|
||||
trend_info["is_cooling"] = sd.get("trend_info", {}).get("is_cooling", False)
|
||||
peak_status = sd.get("peak_status", peak_status)
|
||||
|
||||
# Use shared DEB if not already set
|
||||
if deb_val is None and sd.get("deb_prediction") is not None:
|
||||
deb_val = sd["deb_prediction"]
|
||||
deb_weights = sd.get("deb_weights", "")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Structured analysis skipped for {city}: {e}")
|
||||
|
||||
ai_text = describe_metar_report(
|
||||
raw_metar=str(primary_current.get("raw_metar") or mc.get("raw_metar") or ""),
|
||||
temp_symbol=sym,
|
||||
fallback={
|
||||
"icao": metar.get("icao"),
|
||||
"station_name": metar.get("station_name"),
|
||||
"temp": cur_temp,
|
||||
"wind_speed_kt": _sf(primary_current.get("wind_speed_kt")),
|
||||
"wind_dir": _sf(primary_current.get("wind_dir")),
|
||||
"altimeter": _sf(primary_current.get("altimeter")),
|
||||
"wx_desc": primary_current.get("wx_desc"),
|
||||
"clouds": primary_current.get("clouds", []) or mc.get("clouds", []),
|
||||
},
|
||||
)
|
||||
|
||||
# ── 12. Hourly data (today only, for chart) ──
|
||||
today_hourly: Dict[str, list] = {"times": [], "temps": [], "radiation": []}
|
||||
for i, ts in enumerate(h_times):
|
||||
if ts.startswith(local_date_str):
|
||||
today_hourly["times"].append(ts.split("T")[1][:5])
|
||||
today_hourly["temps"].append(h_temps[i] if i < len(h_temps) else None)
|
||||
today_hourly["radiation"].append(h_rad[i] if i < len(h_rad) else None)
|
||||
|
||||
# ── 12b. Next 48h hourly block for future-date analysis modal ──
|
||||
next_48h_hourly = {
|
||||
"times": [],
|
||||
"temps": [],
|
||||
"radiation": [],
|
||||
"dew_point": [],
|
||||
"pressure_msl": [],
|
||||
"wind_speed_10m": [],
|
||||
"wind_direction_10m": [],
|
||||
"precipitation_probability": [],
|
||||
"cloud_cover": [],
|
||||
}
|
||||
try:
|
||||
local_anchor = datetime.strptime(
|
||||
f"{local_date_str} {local_time_str}", "%Y-%m-%d %H:%M"
|
||||
)
|
||||
except Exception:
|
||||
local_anchor = None
|
||||
|
||||
if local_anchor is not None:
|
||||
horizon = local_anchor + timedelta(hours=48)
|
||||
for i, ts in enumerate(h_times):
|
||||
try:
|
||||
ts_dt = datetime.fromisoformat(ts)
|
||||
except Exception:
|
||||
continue
|
||||
if ts_dt < local_anchor or ts_dt > horizon:
|
||||
continue
|
||||
next_48h_hourly["times"].append(ts)
|
||||
next_48h_hourly["temps"].append(h_temps[i] if i < len(h_temps) else None)
|
||||
next_48h_hourly["radiation"].append(h_rad[i] if i < len(h_rad) else None)
|
||||
next_48h_hourly["dew_point"].append(h_dew[i] if i < len(h_dew) else None)
|
||||
next_48h_hourly["pressure_msl"].append(
|
||||
h_pressure[i] if i < len(h_pressure) else None
|
||||
)
|
||||
next_48h_hourly["wind_speed_10m"].append(
|
||||
h_wspd[i] if i < len(h_wspd) else None
|
||||
)
|
||||
next_48h_hourly["wind_direction_10m"].append(
|
||||
h_wdir[i] if i < len(h_wdir) else None
|
||||
)
|
||||
next_48h_hourly["precipitation_probability"].append(
|
||||
h_precip_prob[i] if i < len(h_precip_prob) else None
|
||||
)
|
||||
next_48h_hourly["cloud_cover"].append(
|
||||
h_cloud_cover[i] if i < len(h_cloud_cover) else None
|
||||
)
|
||||
|
||||
# ── 13. Cloud description (METAR primary, MGM fallback) ──
|
||||
clouds = mc.get("clouds", [])
|
||||
cloud_desc = ""
|
||||
if clouds:
|
||||
c_map = {
|
||||
"BKN": "多云",
|
||||
"OVC": "阴天",
|
||||
"FEW": "少云",
|
||||
"SCT": "散云",
|
||||
"SKC": "晴",
|
||||
"CLR": "晴",
|
||||
}
|
||||
main = clouds[-1]
|
||||
cloud_desc = c_map.get(main.get("cover"), main.get("cover", ""))
|
||||
|
||||
if not cloud_desc and mgm:
|
||||
mgc_cover = mgm.get("current", {}).get("cloud_cover")
|
||||
if mgc_cover is not None:
|
||||
cloud_desc_map = {
|
||||
0: "晴朗",
|
||||
1: "少云",
|
||||
2: "少云",
|
||||
3: "散云",
|
||||
4: "散云",
|
||||
5: "多云",
|
||||
6: "多云",
|
||||
7: "阴天",
|
||||
8: "阴天",
|
||||
}
|
||||
cloud_desc = cloud_desc_map.get(mgc_cover, "")
|
||||
|
||||
# Final fallback: If we have ANY actual observation but no cloud info, it's usually clear.
|
||||
if not cloud_desc:
|
||||
if mc.get("temp") is not None or (mgm and mgm.get("current", {}).get("temp") is not None):
|
||||
# If weather phenomenon exists (e.g. rain), we'll let app.js handle wx_desc priority.
|
||||
# Otherwise, clear skies.
|
||||
if not mc.get("wx_desc"):
|
||||
cloud_desc = "晴朗"
|
||||
|
||||
# ── 14. MGM data (Ankara-specific) ──
|
||||
mgm_data = {}
|
||||
if mgm:
|
||||
mgc = mgm.get("current", {})
|
||||
mgm_time_str = mgc.get("time", "")
|
||||
# MGM time is usually "2026-03-04T10:40:00.000Z" (UTC)
|
||||
if mgm_time_str and "T" in mgm_time_str:
|
||||
try:
|
||||
# Handle ISO format with Z or +00:00
|
||||
ts = mgm_time_str.replace("Z", "+00:00")
|
||||
if "+" in ts:
|
||||
base, offset_part = ts.split("+", 1)
|
||||
if "." in base:
|
||||
base = base.split(".")[0]
|
||||
ts = base + "+" + offset_part
|
||||
dt = datetime.fromisoformat(ts)
|
||||
local_dt = dt.astimezone(timezone(timedelta(seconds=utc_offset or 0)))
|
||||
mgm_time_str = local_dt.strftime("%H:%M")
|
||||
except Exception as e:
|
||||
logger.debug(f"MGM time conversion failed: {e}")
|
||||
pass
|
||||
|
||||
mgm_data = {
|
||||
"temp": _sf(mgc.get("temp")),
|
||||
"time": mgm_time_str,
|
||||
"feels_like": _sf(mgc.get("feels_like")),
|
||||
"humidity": _sf(mgc.get("humidity")),
|
||||
"wind_dir": _sf(mgc.get("wind_dir")),
|
||||
"wind_speed_ms": _sf(mgc.get("wind_speed_ms")),
|
||||
"pressure": _sf(mgc.get("pressure")),
|
||||
"cloud_cover": mgc.get("cloud_cover"),
|
||||
"rain_24h": _sf(mgc.get("rain_24h")),
|
||||
"today_high": _sf(mgm.get("today_high")),
|
||||
"today_low": _sf(mgm.get("today_low")),
|
||||
"hourly": [],
|
||||
}
|
||||
|
||||
mgm_hourly = mgm.get("hourly", [])
|
||||
for h in mgm_hourly:
|
||||
dt_str = h.get("time")
|
||||
val = _sf(h.get("temp"))
|
||||
if dt_str and "T" in dt_str and val is not None:
|
||||
try:
|
||||
dt = datetime.fromisoformat(dt_str.replace("Z", "+00:00"))
|
||||
local_dt = dt.astimezone(timezone(timedelta(seconds=utc_offset)))
|
||||
mgm_data["hourly"].append({
|
||||
"time": local_dt.strftime("%Y-%m-%dT%H:%M"),
|
||||
"temp": val
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── 15. Extended Multi-Model Daily ──
|
||||
multi_model_daily = {}
|
||||
mm_daily_raw = mm.get("daily_forecasts", {})
|
||||
for i, d_str in enumerate(dates):
|
||||
if i == 0:
|
||||
day_m = current_forecasts.copy()
|
||||
d_val, d_winfo = deb_val, deb_weights
|
||||
else:
|
||||
day_m = mm_daily_raw.get(d_str, {}).copy()
|
||||
if i < len(maxtemps) and maxtemps[i] is not None:
|
||||
day_m["Open-Meteo"] = _sf(maxtemps[i])
|
||||
|
||||
# Add MGM per-day forecast
|
||||
mgm_daily = mgm.get("daily_forecasts", {})
|
||||
if d_str in mgm_daily:
|
||||
day_m["MGM"] = _sf(mgm_daily[d_str])
|
||||
|
||||
day_m = {
|
||||
m: v for m, v in day_m.items() if not _is_excluded_model_name(m)
|
||||
}
|
||||
|
||||
d_val, d_winfo = None, ""
|
||||
d_probs = []
|
||||
if day_m:
|
||||
try:
|
||||
blended, winfo = calculate_dynamic_weights(city, day_m)
|
||||
if blended is not None:
|
||||
d_val = blended
|
||||
d_winfo = winfo
|
||||
|
||||
# Calculate future probability based on model divergence
|
||||
m_vals = [v for v in day_m.values() if v is not None]
|
||||
if len(m_vals) > 1:
|
||||
# Use spread as a proxy for sigma.
|
||||
# sigma = (max-min)/2 with a floor of 0.6
|
||||
d_sigma = max(0.6, (max(m_vals) - min(m_vals)) / 2.0)
|
||||
else:
|
||||
d_sigma = 1.0
|
||||
|
||||
prob_obj = calculate_prob_distribution(d_val, d_sigma, None, sym)
|
||||
d_probs = prob_obj.get("probabilities", [])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if day_m:
|
||||
multi_model_daily[d_str] = {
|
||||
"models": day_m,
|
||||
"deb": {"prediction": d_val, "weights_info": d_winfo},
|
||||
"probabilities": d_probs if i > 0 else probabilities # Use today's real prob for today
|
||||
}
|
||||
|
||||
# ── Assemble result ──
|
||||
result = {
|
||||
"name": city,
|
||||
"display_name": city.title(),
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"temp_symbol": sym,
|
||||
"local_time": local_time_str,
|
||||
"local_date": local_date_str,
|
||||
"risk": {
|
||||
"level": risk.get("risk_level", "low"),
|
||||
"emoji": risk.get("risk_emoji", "🟢"),
|
||||
"airport": risk.get("airport_name", ""),
|
||||
"icao": risk.get("icao", ""),
|
||||
"distance_km": risk.get("distance_km", 0),
|
||||
"warning": risk.get("warning", ""),
|
||||
},
|
||||
"current": {
|
||||
"temp": cur_temp,
|
||||
"max_so_far": max_so_far,
|
||||
"max_temp_time": max_temp_time,
|
||||
"wu_settlement": wu_settle,
|
||||
"settlement_source": settlement_source,
|
||||
"settlement_source_label": settlement_source_label,
|
||||
"obs_time": obs_time_str,
|
||||
"obs_age_min": metar_age_min,
|
||||
"report_time": metar.get("report_time") if metar else None,
|
||||
"receipt_time": metar.get("receipt_time") if metar else None,
|
||||
"obs_time_epoch": metar.get("obs_time_epoch") if metar else None,
|
||||
"wind_speed_kt": _sf(primary_current.get("wind_speed_kt")),
|
||||
"wind_dir": _sf(primary_current.get("wind_dir")),
|
||||
"humidity": _sf(primary_current.get("humidity")),
|
||||
"cloud_desc": cloud_desc,
|
||||
"clouds_raw": [
|
||||
{"cover": c.get("cover"), "base": c.get("base")} for c in clouds
|
||||
],
|
||||
"visibility_mi": _sf(primary_current.get("visibility_mi")),
|
||||
"wx_desc": primary_current.get("wx_desc"),
|
||||
"raw_metar": primary_current.get("raw_metar"),
|
||||
},
|
||||
"mgm": mgm_data,
|
||||
"mgm_nearby": raw.get("mgm_nearby", []),
|
||||
"forecast": {
|
||||
"today_high": om_today,
|
||||
"daily": forecast_daily,
|
||||
"sunrise": sunrise,
|
||||
"sunset": sunset,
|
||||
"sunshine_hours": sunshine_h,
|
||||
},
|
||||
"source_forecasts": {
|
||||
"weather_gov": raw.get("nws") or {},
|
||||
},
|
||||
"multi_model": {k: v for k, v in current_forecasts.items() if v is not None},
|
||||
"multi_model_daily": multi_model_daily,
|
||||
"deb": {"prediction": deb_val, "weights_info": deb_weights},
|
||||
"ensemble": ens_data,
|
||||
"probabilities": {
|
||||
"mu": round(mu, 1) if mu is not None else None,
|
||||
"distribution": probabilities,
|
||||
"engine": probability_engine,
|
||||
"calibration_mode": probability_calibration_mode,
|
||||
"calibration_version": probability_calibration_version,
|
||||
"raw_mu": probability_raw_mu,
|
||||
"raw_sigma": probability_raw_sigma,
|
||||
"calibrated_mu": probability_calibrated_mu,
|
||||
"calibrated_sigma": probability_calibrated_sigma,
|
||||
"shadow_distribution": shadow_probabilities,
|
||||
},
|
||||
"trend": trend_info,
|
||||
"peak": {
|
||||
"hours": peak_hours,
|
||||
"first_h": first_peak_h,
|
||||
"last_h": last_peak_h,
|
||||
"status": peak_status,
|
||||
},
|
||||
"hourly": today_hourly,
|
||||
"hourly_next_48h": next_48h_hourly,
|
||||
"metar_today_obs": metar_today_obs_payload,
|
||||
"metar_recent_obs": metar_recent_obs_payload,
|
||||
"settlement_today_obs": settlement_today_obs,
|
||||
"ai_analysis": ai_text,
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
_cache[city] = {"t": _time.time(), "d": result}
|
||||
return result
|
||||
|
||||
|
||||
def _normalize_city_or_404(name: str) -> str:
|
||||
city = name.lower().strip().replace("-", " ")
|
||||
city = ALIASES.get(city, city)
|
||||
if city not in CITIES:
|
||||
raise HTTPException(404, detail=f"Unknown city: {city}")
|
||||
return city
|
||||
|
||||
|
||||
def _build_city_summary_payload(data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": data.get("name"),
|
||||
"display_name": data.get("display_name"),
|
||||
"icao": data.get("risk", {}).get("icao"),
|
||||
"local_time": data.get("local_time"),
|
||||
"temp_symbol": data.get("temp_symbol"),
|
||||
"current": {
|
||||
"temp": data.get("current", {}).get("temp"),
|
||||
"obs_time": data.get("current", {}).get("obs_time"),
|
||||
"settlement_source": data.get("current", {}).get("settlement_source"),
|
||||
"settlement_source_label": data.get("current", {}).get("settlement_source_label"),
|
||||
},
|
||||
"deb": {"prediction": data.get("deb", {}).get("prediction")},
|
||||
"risk": {
|
||||
"level": data.get("risk", {}).get("level"),
|
||||
"warning": data.get("risk", {}).get("warning"),
|
||||
},
|
||||
"updated_at": data.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
def _build_city_detail_payload(
|
||||
data: Dict[str, Any],
|
||||
market_slug: Optional[str] = None,
|
||||
target_date: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
city = str(data.get("name") or "").strip().lower()
|
||||
local_date = str(data.get("local_date") or "").strip()
|
||||
requested_date = str(target_date or "").strip()
|
||||
selected_date = requested_date or local_date
|
||||
|
||||
multi_model_daily = data.get("multi_model_daily") or {}
|
||||
selected_daily = (
|
||||
multi_model_daily.get(selected_date)
|
||||
if isinstance(multi_model_daily, dict)
|
||||
else None
|
||||
)
|
||||
if not isinstance(selected_daily, dict):
|
||||
selected_daily = {}
|
||||
selected_date = local_date
|
||||
|
||||
distribution = selected_daily.get("probabilities")
|
||||
if not isinstance(distribution, list) or not distribution:
|
||||
distribution = data.get("probabilities", {}).get("distribution", []) or []
|
||||
|
||||
model_map = selected_daily.get("models") or data.get("multi_model") or {}
|
||||
if not isinstance(model_map, dict):
|
||||
model_map = {}
|
||||
|
||||
anchor_temp = None
|
||||
anchor_model = None
|
||||
for model_name, raw_value in model_map.items():
|
||||
value = _sf(raw_value)
|
||||
if value is None:
|
||||
continue
|
||||
if anchor_temp is None or value > anchor_temp:
|
||||
anchor_temp = value
|
||||
anchor_model = str(model_name or "").strip() or None
|
||||
|
||||
anchor_temp_c = anchor_temp
|
||||
temp_symbol = str(data.get("temp_symbol") or "")
|
||||
if anchor_temp_c is not None and "F" in temp_symbol.upper():
|
||||
anchor_temp_c = (anchor_temp_c - 32.0) * 5.0 / 9.0
|
||||
anchor_settlement = apply_city_settlement(city, anchor_temp_c) if anchor_temp_c is not None else None
|
||||
|
||||
primary_bucket = None
|
||||
if isinstance(distribution, list) and distribution:
|
||||
if anchor_temp is None:
|
||||
primary_bucket = distribution[0]
|
||||
else:
|
||||
ranked_buckets = []
|
||||
for idx, row in enumerate(distribution):
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
bucket_temp = _sf(row.get("value"))
|
||||
bucket_prob = _sf(row.get("probability"))
|
||||
if bucket_temp is None:
|
||||
continue
|
||||
prob_rank = bucket_prob if bucket_prob is not None else -1.0
|
||||
ranked_buckets.append((abs(bucket_temp - anchor_temp), -prob_rank, idx, row))
|
||||
if ranked_buckets:
|
||||
ranked_buckets.sort(key=lambda x: (x[0], x[1], x[2]))
|
||||
primary_bucket = ranked_buckets[0][3]
|
||||
else:
|
||||
primary_bucket = distribution[0]
|
||||
|
||||
model_probability = None
|
||||
if isinstance(primary_bucket, dict) and primary_bucket.get("probability") is not None:
|
||||
try:
|
||||
raw_probability = float(primary_bucket.get("probability"))
|
||||
model_probability = raw_probability / 100.0 if raw_probability > 1.0 else raw_probability
|
||||
except Exception:
|
||||
model_probability = None
|
||||
|
||||
fallback_sparkline = [
|
||||
p.get("probability", 0)
|
||||
for p in distribution[:8]
|
||||
if isinstance(p, dict)
|
||||
]
|
||||
market_scan = _market_layer.build_market_scan(
|
||||
city=data.get("name"),
|
||||
target_date=selected_date or data.get("local_date"),
|
||||
temperature_bucket=primary_bucket if isinstance(primary_bucket, dict) else None,
|
||||
model_probability=model_probability,
|
||||
fallback_sparkline=fallback_sparkline,
|
||||
forced_market_slug=market_slug,
|
||||
)
|
||||
if isinstance(market_scan, dict):
|
||||
market_scan["anchor_model"] = anchor_model
|
||||
market_scan["anchor_high"] = anchor_temp
|
||||
market_scan["anchor_settlement"] = anchor_settlement
|
||||
market_scan["open_meteo_settlement"] = anchor_settlement
|
||||
return {
|
||||
"city": data.get("name"),
|
||||
"fetched_at": data.get("updated_at"),
|
||||
"overview": {
|
||||
"name": data.get("name"),
|
||||
"display_name": data.get("display_name"),
|
||||
"icao": data.get("risk", {}).get("icao"),
|
||||
"airport": data.get("risk", {}).get("airport"),
|
||||
"lat": data.get("lat"),
|
||||
"lon": data.get("lon"),
|
||||
"local_time": data.get("local_time"),
|
||||
"local_date": data.get("local_date"),
|
||||
"temp_symbol": data.get("temp_symbol"),
|
||||
"current_temp": data.get("current", {}).get("temp"),
|
||||
"settlement_source": data.get("current", {}).get("settlement_source"),
|
||||
"settlement_source_label": data.get("current", {}).get("settlement_source_label"),
|
||||
"deb_prediction": data.get("deb", {}).get("prediction"),
|
||||
"risk_level": data.get("risk", {}).get("level"),
|
||||
"risk_warning": data.get("risk", {}).get("warning"),
|
||||
"updated_at": data.get("updated_at"),
|
||||
},
|
||||
"official": {
|
||||
"available": bool(data.get("current", {}).get("temp") is not None),
|
||||
"metar": {
|
||||
"observation_time": data.get("current", {}).get("obs_time"),
|
||||
"obs_age_min": data.get("current", {}).get("obs_age_min"),
|
||||
"report_time": data.get("current", {}).get("report_time"),
|
||||
"receipt_time": data.get("current", {}).get("receipt_time"),
|
||||
"raw_metar": data.get("current", {}).get("raw_metar"),
|
||||
"current": data.get("current"),
|
||||
},
|
||||
"weather_gov": {},
|
||||
"mgm": data.get("mgm") or {},
|
||||
"mgm_nearby": data.get("mgm_nearby") or [],
|
||||
"nearby_source": "mgm" if data.get("name") == "ankara" else "metar_cluster",
|
||||
},
|
||||
"timeseries": {
|
||||
"metar_recent_obs": data.get("metar_recent_obs") or [],
|
||||
"metar_today_obs": data.get("metar_today_obs") or [],
|
||||
"settlement_today_obs": data.get("settlement_today_obs") or [],
|
||||
"hourly": data.get("hourly") or {},
|
||||
"mgm_hourly": (data.get("mgm") or {}).get("hourly", []),
|
||||
"forecast_daily": (data.get("forecast") or {}).get("daily", []),
|
||||
},
|
||||
"models": {
|
||||
k: v
|
||||
for k, v in (data.get("multi_model") or {}).items()
|
||||
if not _is_excluded_model_name(k)
|
||||
},
|
||||
"probabilities": data.get("probabilities") or {"mu": None, "distribution": []},
|
||||
"market_scan": market_scan,
|
||||
"risk": data.get("risk"),
|
||||
"ai_analysis": data.get("ai_analysis") or "",
|
||||
"errors": {},
|
||||
}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# Routes
|
||||
# ──────────────────────────────────────────────────────────
|
||||
+4
-1445
File diff suppressed because it is too large
Load Diff
+260
@@ -0,0 +1,260 @@
|
||||
"""
|
||||
PolyWeather Web Core Context
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel, Field
|
||||
from loguru import logger
|
||||
|
||||
from src.utils.config_loader import load_config
|
||||
from src.data_collection.weather_sources import WeatherDataCollector
|
||||
from src.data_collection.city_risk_profiles import CITY_RISK_PROFILES # noqa: F401
|
||||
from src.data_collection.polymarket_readonly import PolymarketReadOnlyLayer
|
||||
from src.auth.supabase_entitlement import SUPABASE_ENTITLEMENT, extract_bearer_token
|
||||
from src.database.db_manager import DBManager
|
||||
from src.payments import PAYMENT_CHECKOUT, PaymentCheckoutError # noqa: F401
|
||||
from src.data_collection.city_registry import CITY_REGISTRY
|
||||
|
||||
app = FastAPI(title="PolyWeather Map", version="1.0")
|
||||
|
||||
_cors_origins = os.getenv(
|
||||
"WEB_CORS_ORIGINS",
|
||||
"http://localhost:3000,http://127.0.0.1:3000,https://polyweather-pro.vercel.app",
|
||||
)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[o.strip() for o in _cors_origins.split(",") if o.strip()],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
_config = load_config()
|
||||
_weather = WeatherDataCollector(_config)
|
||||
_market_layer = PolymarketReadOnlyLayer()
|
||||
_account_db = DBManager()
|
||||
|
||||
CITIES: Dict[str, Dict[str, Any]] = {
|
||||
cid: {
|
||||
"lat": info["lat"],
|
||||
"lon": info["lon"],
|
||||
"f": info["use_fahrenheit"],
|
||||
"tz": info["tz_offset"],
|
||||
"settlement_source": str(info.get("settlement_source") or "metar").strip().lower() or "metar",
|
||||
}
|
||||
for cid, info in CITY_REGISTRY.items()
|
||||
}
|
||||
|
||||
SETTLEMENT_SOURCE_LABELS: Dict[str, str] = {
|
||||
"metar": "METAR",
|
||||
"hko": "HKO",
|
||||
"cwa": "CWA",
|
||||
"mgm": "MGM",
|
||||
}
|
||||
|
||||
_cache: Dict[str, Dict] = {}
|
||||
CACHE_TTL = 300
|
||||
CACHE_TTL_ANKARA = 60
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool = False) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None:
|
||||
return default
|
||||
return raw.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
_ENTITLEMENT_GUARD_ENABLED = _env_bool("POLYWEATHER_REQUIRE_ENTITLEMENT", False)
|
||||
_ENTITLEMENT_HEADER = "x-polyweather-entitlement"
|
||||
_ENTITLEMENT_TOKEN = (os.getenv("POLYWEATHER_BACKEND_ENTITLEMENT_TOKEN") or "").strip()
|
||||
_FORWARDED_SUPABASE_USER_ID_HEADER = "x-polyweather-auth-user-id"
|
||||
_FORWARDED_SUPABASE_EMAIL_HEADER = "x-polyweather-auth-email"
|
||||
_SUPABASE_AUTH_REQUIRED = _env_bool(
|
||||
"POLYWEATHER_AUTH_REQUIRED",
|
||||
SUPABASE_ENTITLEMENT.enabled,
|
||||
)
|
||||
|
||||
|
||||
def _legacy_service_token_valid(request: Request) -> bool:
|
||||
token = request.headers.get(_ENTITLEMENT_HEADER)
|
||||
if not token:
|
||||
token = extract_bearer_token(request.headers.get("authorization"))
|
||||
return bool(_ENTITLEMENT_TOKEN and token == _ENTITLEMENT_TOKEN)
|
||||
|
||||
|
||||
def _bind_optional_supabase_identity(request: Request) -> None:
|
||||
if not SUPABASE_ENTITLEMENT.configured:
|
||||
return
|
||||
access_token = extract_bearer_token(request.headers.get("authorization"))
|
||||
if not access_token:
|
||||
return
|
||||
identity = SUPABASE_ENTITLEMENT.get_identity(access_token)
|
||||
if not identity:
|
||||
return
|
||||
request.state.auth_user_id = identity.user_id
|
||||
request.state.auth_email = identity.email
|
||||
request.state.auth_points = identity.points
|
||||
|
||||
|
||||
def _resolve_auth_points(request: Request) -> int:
|
||||
raw_points = getattr(request.state, "auth_points", 0)
|
||||
try:
|
||||
points = max(0, int(raw_points or 0))
|
||||
except Exception:
|
||||
points = 0
|
||||
if points > 0:
|
||||
return points
|
||||
|
||||
user_id = str(getattr(request.state, "auth_user_id", "") or "").strip()
|
||||
if not user_id:
|
||||
return points
|
||||
try:
|
||||
db_points = _account_db.get_points_by_supabase_user_id(user_id)
|
||||
if db_points > points:
|
||||
request.state.auth_points = db_points
|
||||
return db_points
|
||||
except Exception as exc:
|
||||
logger.warning(f"auth points fallback failed user_id={user_id}: {exc}")
|
||||
return points
|
||||
|
||||
|
||||
def _resolve_weekly_profile(request: Request) -> Dict[str, Any]:
|
||||
user_id = str(getattr(request.state, "auth_user_id", "") or "").strip()
|
||||
if not user_id:
|
||||
return {"weekly_points": 0, "weekly_rank": None}
|
||||
try:
|
||||
profile = _account_db.get_weekly_profile_by_supabase_user_id(user_id)
|
||||
return {
|
||||
"weekly_points": int(profile.get("weekly_points") or 0),
|
||||
"weekly_rank": profile.get("weekly_rank"),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.warning(f"auth weekly profile fallback failed user_id={user_id}: {exc}")
|
||||
return {"weekly_points": 0, "weekly_rank": None}
|
||||
|
||||
|
||||
def _assert_entitlement(request: Request) -> None:
|
||||
if SUPABASE_ENTITLEMENT.enabled:
|
||||
if _legacy_service_token_valid(request):
|
||||
return
|
||||
if not _SUPABASE_AUTH_REQUIRED:
|
||||
_bind_optional_supabase_identity(request)
|
||||
return
|
||||
if not SUPABASE_ENTITLEMENT.configured:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Supabase auth is enabled but SUPABASE_URL / SUPABASE_ANON_KEY is not configured",
|
||||
)
|
||||
|
||||
access_token = extract_bearer_token(request.headers.get("authorization"))
|
||||
if not access_token:
|
||||
raise HTTPException(status_code=401, detail="Unauthorized")
|
||||
|
||||
identity = SUPABASE_ENTITLEMENT.get_identity(access_token)
|
||||
if not identity:
|
||||
raise HTTPException(status_code=401, detail="Unauthorized")
|
||||
if not SUPABASE_ENTITLEMENT.has_active_subscription(identity.user_id):
|
||||
raise HTTPException(status_code=403, detail="Subscription required")
|
||||
|
||||
request.state.auth_user_id = identity.user_id
|
||||
request.state.auth_email = identity.email
|
||||
return
|
||||
|
||||
if not _ENTITLEMENT_GUARD_ENABLED:
|
||||
return
|
||||
|
||||
if not _ENTITLEMENT_TOKEN:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Entitlement guard is enabled but backend token is not configured",
|
||||
)
|
||||
|
||||
if not _legacy_service_token_valid(request):
|
||||
raise HTTPException(status_code=401, detail="Unauthorized")
|
||||
|
||||
|
||||
def _require_supabase_identity(request: Request) -> Dict[str, str]:
|
||||
if not SUPABASE_ENTITLEMENT.enabled:
|
||||
raise HTTPException(status_code=503, detail="payment requires POLYWEATHER_AUTH_ENABLED=true")
|
||||
if not SUPABASE_ENTITLEMENT.configured:
|
||||
raise HTTPException(status_code=503, detail="payment requires SUPABASE_URL and SUPABASE_ANON_KEY")
|
||||
|
||||
state_user_id = str(getattr(request.state, "auth_user_id", "") or "").strip()
|
||||
if state_user_id:
|
||||
state_email = str(getattr(request.state, "auth_email", "") or "").strip()
|
||||
return {"user_id": state_user_id, "email": state_email}
|
||||
|
||||
token = extract_bearer_token(request.headers.get("authorization"))
|
||||
if token:
|
||||
identity = SUPABASE_ENTITLEMENT.get_identity(token)
|
||||
if identity:
|
||||
return {"user_id": identity.user_id, "email": identity.email}
|
||||
|
||||
legacy_ok = _legacy_service_token_valid(request)
|
||||
if legacy_ok:
|
||||
forwarded_user_id = str(request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "").strip()
|
||||
if forwarded_user_id:
|
||||
forwarded_email = str(request.headers.get(_FORWARDED_SUPABASE_EMAIL_HEADER) or "").strip()
|
||||
return {"user_id": forwarded_user_id, "email": forwarded_email}
|
||||
|
||||
logger.warning(
|
||||
"payment auth identity missing state_user={} auth_bearer={} legacy_ok={} forwarded_user={}"
|
||||
.format(
|
||||
bool(state_user_id),
|
||||
bool(token),
|
||||
bool(legacy_ok),
|
||||
bool(str(request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "").strip()),
|
||||
)
|
||||
)
|
||||
raise HTTPException(status_code=401, detail="Unauthorized")
|
||||
|
||||
|
||||
class WalletChallengeRequest(BaseModel):
|
||||
address: str = Field(..., min_length=8)
|
||||
|
||||
|
||||
class WalletVerifyRequest(BaseModel):
|
||||
address: str = Field(..., min_length=8)
|
||||
nonce: str = Field(..., min_length=6)
|
||||
signature: str = Field(..., min_length=20)
|
||||
|
||||
|
||||
class WalletUnbindRequest(BaseModel):
|
||||
address: str = Field(..., min_length=8)
|
||||
|
||||
|
||||
class CreatePaymentIntentRequest(BaseModel):
|
||||
plan_code: str = Field(default="pro_monthly", min_length=2)
|
||||
payment_mode: str = Field(default="strict")
|
||||
allowed_wallet: Optional[str] = None
|
||||
token_address: Optional[str] = None
|
||||
use_points: bool = False
|
||||
points_to_consume: Optional[int] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SubmitPaymentTxRequest(BaseModel):
|
||||
tx_hash: str = Field(..., min_length=10)
|
||||
from_address: str = Field(..., min_length=8)
|
||||
|
||||
|
||||
class ConfirmPaymentTxRequest(BaseModel):
|
||||
tx_hash: Optional[str] = None
|
||||
|
||||
|
||||
def _sf(v) -> Optional[float]:
|
||||
if v is None:
|
||||
return None
|
||||
try:
|
||||
return float(v)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _is_excluded_model_name(model_name: str) -> bool:
|
||||
normalized = str(model_name or "").strip().lower().replace(" ", "").replace("_", "").replace("-", "")
|
||||
return "meteoblue" in normalized
|
||||
+359
@@ -0,0 +1,359 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from loguru import logger
|
||||
|
||||
from src.analysis.deb_algorithm import load_history
|
||||
from src.data_collection.city_registry import ALIASES
|
||||
from web.analysis_service import (
|
||||
_analyze,
|
||||
_build_city_detail_payload,
|
||||
_build_city_summary_payload,
|
||||
)
|
||||
from web.core import (
|
||||
CITIES,
|
||||
CITY_REGISTRY,
|
||||
CITY_RISK_PROFILES,
|
||||
PAYMENT_CHECKOUT,
|
||||
PaymentCheckoutError,
|
||||
SETTLEMENT_SOURCE_LABELS,
|
||||
SUPABASE_ENTITLEMENT,
|
||||
ConfirmPaymentTxRequest,
|
||||
CreatePaymentIntentRequest,
|
||||
SubmitPaymentTxRequest,
|
||||
WalletChallengeRequest,
|
||||
WalletUnbindRequest,
|
||||
WalletVerifyRequest,
|
||||
_ENTITLEMENT_GUARD_ENABLED,
|
||||
_SUPABASE_AUTH_REQUIRED,
|
||||
_assert_entitlement,
|
||||
_bind_optional_supabase_identity,
|
||||
_require_supabase_identity,
|
||||
_resolve_auth_points,
|
||||
_resolve_weekly_profile,
|
||||
_sf,
|
||||
_is_excluded_model_name,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _normalize_city_or_404(name: str) -> str:
|
||||
city = name.lower().strip().replace("-", " ")
|
||||
city = ALIASES.get(city, city)
|
||||
if city not in CITIES:
|
||||
raise HTTPException(404, detail=f"Unknown city: {city}")
|
||||
return city
|
||||
|
||||
|
||||
@router.get("/api/cities")
|
||||
async def list_cities(request: Request):
|
||||
_assert_entitlement(request)
|
||||
try:
|
||||
out = []
|
||||
for name, info in CITIES.items():
|
||||
risk = CITY_RISK_PROFILES.get(name, {})
|
||||
settlement_source = str(info.get("settlement_source") or "metar").strip().lower() or "metar"
|
||||
out.append(
|
||||
{
|
||||
"name": name,
|
||||
"display_name": name.title(),
|
||||
"lat": info["lat"],
|
||||
"lon": info["lon"],
|
||||
"risk_level": risk.get("risk_level", "low"),
|
||||
"risk_emoji": risk.get("risk_emoji", "🟢"),
|
||||
"airport": risk.get("airport_name", ""),
|
||||
"icao": risk.get("icao", ""),
|
||||
"temp_unit": "fahrenheit" if info["f"] else "celsius",
|
||||
"is_major": CITY_REGISTRY.get(name, {}).get("is_major", True),
|
||||
"settlement_source": settlement_source,
|
||||
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(
|
||||
settlement_source,
|
||||
settlement_source.upper(),
|
||||
),
|
||||
}
|
||||
)
|
||||
return {"cities": out}
|
||||
except Exception as exc:
|
||||
logger.error(f"Error in list_cities: {exc}")
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("/api/city/{name}")
|
||||
async def city_detail(request: Request, name: str, force_refresh: bool = False):
|
||||
_assert_entitlement(request)
|
||||
return _analyze(_normalize_city_or_404(name), force_refresh=force_refresh)
|
||||
|
||||
|
||||
@router.get("/api/history/{name}")
|
||||
async def city_history(request: Request, name: str):
|
||||
_assert_entitlement(request)
|
||||
city = _normalize_city_or_404(name)
|
||||
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
history_file = os.path.join(project_root, "data", "daily_records.json")
|
||||
data = load_history(history_file)
|
||||
|
||||
if city not in data:
|
||||
source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
|
||||
return {
|
||||
"history": [],
|
||||
"settlement_source": source,
|
||||
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
|
||||
}
|
||||
|
||||
city_data = data[city]
|
||||
out = []
|
||||
for day, rec in sorted(city_data.items()):
|
||||
act = rec.get("actual_high")
|
||||
deb = rec.get("deb_prediction")
|
||||
mu = rec.get("mu")
|
||||
forecasts_raw = rec.get("forecasts", {}) or {}
|
||||
forecasts = {}
|
||||
if isinstance(forecasts_raw, dict):
|
||||
for model_name, model_value in forecasts_raw.items():
|
||||
if _is_excluded_model_name(str(model_name)):
|
||||
continue
|
||||
fv = _sf(model_value)
|
||||
forecasts[str(model_name)] = fv if fv is not None else None
|
||||
mgm = forecasts.get("MGM")
|
||||
out.append(
|
||||
{
|
||||
"date": day,
|
||||
"actual": float(act) if act is not None else None,
|
||||
"deb": float(deb) if deb is not None else None,
|
||||
"mu": float(mu) if mu is not None else None,
|
||||
"mgm": float(mgm) if mgm is not None else None,
|
||||
"forecasts": forecasts,
|
||||
}
|
||||
)
|
||||
|
||||
source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
|
||||
return {
|
||||
"history": out,
|
||||
"settlement_source": source,
|
||||
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/auth/me")
|
||||
async def auth_me(request: Request):
|
||||
_assert_entitlement(request)
|
||||
_bind_optional_supabase_identity(request)
|
||||
|
||||
user_id = getattr(request.state, "auth_user_id", None)
|
||||
subscription_required = bool(
|
||||
SUPABASE_ENTITLEMENT.enabled
|
||||
and _SUPABASE_AUTH_REQUIRED
|
||||
and SUPABASE_ENTITLEMENT.require_subscription
|
||||
)
|
||||
subscription_active = None
|
||||
subscription_plan_code = None
|
||||
subscription_starts_at = None
|
||||
subscription_expires_at = None
|
||||
|
||||
if SUPABASE_ENTITLEMENT.enabled and user_id:
|
||||
try:
|
||||
latest_subscription = SUPABASE_ENTITLEMENT.get_latest_active_subscription(
|
||||
user_id,
|
||||
respect_requirement=False,
|
||||
)
|
||||
subscription_active = bool(latest_subscription)
|
||||
if isinstance(latest_subscription, dict):
|
||||
subscription_plan_code = latest_subscription.get("plan_code")
|
||||
subscription_starts_at = latest_subscription.get("starts_at")
|
||||
subscription_expires_at = latest_subscription.get("expires_at")
|
||||
except Exception:
|
||||
subscription_active = None
|
||||
subscription_plan_code = None
|
||||
subscription_starts_at = None
|
||||
subscription_expires_at = None
|
||||
|
||||
points = _resolve_auth_points(request)
|
||||
weekly_profile = _resolve_weekly_profile(request)
|
||||
|
||||
return {
|
||||
"authenticated": bool(user_id),
|
||||
"user_id": user_id,
|
||||
"email": getattr(request.state, "auth_email", None),
|
||||
"points": points,
|
||||
"weekly_points": weekly_profile["weekly_points"],
|
||||
"weekly_rank": weekly_profile["weekly_rank"],
|
||||
"entitlement_mode": (
|
||||
"supabase_required"
|
||||
if SUPABASE_ENTITLEMENT.enabled and _SUPABASE_AUTH_REQUIRED
|
||||
else "supabase_optional"
|
||||
if SUPABASE_ENTITLEMENT.enabled
|
||||
else "legacy_token"
|
||||
if _ENTITLEMENT_GUARD_ENABLED
|
||||
else "disabled"
|
||||
),
|
||||
"auth_required": bool(SUPABASE_ENTITLEMENT.enabled and _SUPABASE_AUTH_REQUIRED),
|
||||
"subscription_required": subscription_required,
|
||||
"subscription_active": subscription_active,
|
||||
"subscription_plan_code": subscription_plan_code,
|
||||
"subscription_starts_at": subscription_starts_at,
|
||||
"subscription_expires_at": subscription_expires_at,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/payments/config")
|
||||
async def payment_config(request: Request):
|
||||
_assert_entitlement(request)
|
||||
try:
|
||||
return PAYMENT_CHECKOUT.get_config_payload()
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.get("/api/payments/wallets")
|
||||
async def payment_wallets(request: Request):
|
||||
_assert_entitlement(request)
|
||||
identity = _require_supabase_identity(request)
|
||||
try:
|
||||
wallets = PAYMENT_CHECKOUT.list_wallets(identity["user_id"])
|
||||
return {
|
||||
"wallets": [wallet.__dict__ for wallet in wallets],
|
||||
"chain_id": PAYMENT_CHECKOUT.chain_id,
|
||||
}
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.delete("/api/payments/wallets")
|
||||
async def payment_wallet_unbind(request: Request, body: WalletUnbindRequest):
|
||||
_assert_entitlement(request)
|
||||
identity = _require_supabase_identity(request)
|
||||
try:
|
||||
return PAYMENT_CHECKOUT.unbind_wallet(
|
||||
user_id=identity["user_id"],
|
||||
address=body.address,
|
||||
)
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.post("/api/payments/wallets/challenge")
|
||||
async def payment_wallet_challenge(request: Request, body: WalletChallengeRequest):
|
||||
_assert_entitlement(request)
|
||||
identity = _require_supabase_identity(request)
|
||||
try:
|
||||
return PAYMENT_CHECKOUT.create_wallet_challenge(
|
||||
user_id=identity["user_id"],
|
||||
address=body.address,
|
||||
)
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.post("/api/payments/wallets/verify")
|
||||
async def payment_wallet_verify(request: Request, body: WalletVerifyRequest):
|
||||
_assert_entitlement(request)
|
||||
identity = _require_supabase_identity(request)
|
||||
try:
|
||||
bound = PAYMENT_CHECKOUT.verify_wallet_binding(
|
||||
user_id=identity["user_id"],
|
||||
address=body.address,
|
||||
nonce=body.nonce,
|
||||
signature=body.signature,
|
||||
)
|
||||
return {"wallet": bound.__dict__}
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.post("/api/payments/intents")
|
||||
async def payment_create_intent(request: Request, body: CreatePaymentIntentRequest):
|
||||
_assert_entitlement(request)
|
||||
identity = _require_supabase_identity(request)
|
||||
try:
|
||||
return PAYMENT_CHECKOUT.create_intent(
|
||||
user_id=identity["user_id"],
|
||||
plan_code=body.plan_code,
|
||||
payment_mode=body.payment_mode,
|
||||
allowed_wallet=body.allowed_wallet,
|
||||
token_address=body.token_address,
|
||||
use_points=body.use_points,
|
||||
points_to_consume=body.points_to_consume,
|
||||
metadata=body.metadata,
|
||||
)
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.get("/api/payments/intents/{intent_id}")
|
||||
async def payment_get_intent(request: Request, intent_id: str):
|
||||
_assert_entitlement(request)
|
||||
identity = _require_supabase_identity(request)
|
||||
try:
|
||||
intent = PAYMENT_CHECKOUT.get_intent(
|
||||
user_id=identity["user_id"],
|
||||
intent_id=intent_id,
|
||||
)
|
||||
return {"intent": intent.__dict__}
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.post("/api/payments/intents/{intent_id}/submit")
|
||||
async def payment_submit_tx(
|
||||
request: Request,
|
||||
intent_id: str,
|
||||
body: SubmitPaymentTxRequest,
|
||||
):
|
||||
_assert_entitlement(request)
|
||||
identity = _require_supabase_identity(request)
|
||||
try:
|
||||
return PAYMENT_CHECKOUT.submit_intent_tx(
|
||||
user_id=identity["user_id"],
|
||||
intent_id=intent_id,
|
||||
tx_hash=body.tx_hash,
|
||||
from_address=body.from_address,
|
||||
)
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.post("/api/payments/intents/{intent_id}/confirm")
|
||||
async def payment_confirm_tx(
|
||||
request: Request,
|
||||
intent_id: str,
|
||||
body: ConfirmPaymentTxRequest,
|
||||
):
|
||||
_assert_entitlement(request)
|
||||
identity = _require_supabase_identity(request)
|
||||
try:
|
||||
return PAYMENT_CHECKOUT.confirm_intent_tx(
|
||||
user_id=identity["user_id"],
|
||||
intent_id=intent_id,
|
||||
tx_hash=body.tx_hash,
|
||||
)
|
||||
except PaymentCheckoutError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
|
||||
|
||||
@router.get("/api/city/{name}/summary")
|
||||
async def city_summary(request: Request, name: str, force_refresh: bool = False):
|
||||
_assert_entitlement(request)
|
||||
data = _analyze(_normalize_city_or_404(name), force_refresh=force_refresh)
|
||||
return _build_city_summary_payload(data)
|
||||
|
||||
|
||||
@router.get("/api/city/{name}/detail")
|
||||
async def city_detail_aggregate(
|
||||
request: Request,
|
||||
name: str,
|
||||
force_refresh: bool = False,
|
||||
market_slug: Optional[str] = None,
|
||||
target_date: Optional[str] = None,
|
||||
):
|
||||
_assert_entitlement(request)
|
||||
data = _analyze(_normalize_city_or_404(name), force_refresh=force_refresh)
|
||||
return _build_city_detail_payload(
|
||||
data,
|
||||
market_slug=market_slug,
|
||||
target_date=target_date,
|
||||
)
|
||||
Reference in New Issue
Block a user