Add CI and shadow calibration reporting

This commit is contained in:
2569718930@qq.com
2026-03-20 20:59:30 +08:00
parent 25ab512371
commit 1c84893bed
39 changed files with 149223 additions and 3277 deletions
+61
View File
@@ -0,0 +1,61 @@
name: CI
on:
push:
branches:
- main
pull_request:
jobs:
python-quality:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt -r requirements-dev.txt
- name: Ruff
run: python -m ruff check .
- name: Pytest
run: python -m pytest
frontend-quality:
runs-on: ubuntu-latest
defaults:
run:
working-directory: frontend
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Node
uses: actions/setup-node@v4
with:
node-version: "20"
cache: "npm"
cache-dependency-path: frontend/package-lock.json
- name: Install dependencies
run: npm ci
- name: Build
run: npm run build
docker-build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build Docker image
run: docker build -t polyweather-ci .
+1
View File
@@ -33,6 +33,7 @@ Thumbs.db
frontend/node_modules/
frontend/.next/
frontend/.vercel/
frontend/*.tsbuildinfo
.npm-cache/
.env.local
@@ -0,0 +1,161 @@
{
"version": "emos-20260320123829",
"trained_at": "2026-03-20T12:38:29.985991+00:00",
"global": {
"mu": {
"intercept": -1.57406048,
"raw_mu_coef": 2.80583627,
"deb_coef": -0.06819634,
"ens_median_coef": -1.81560215,
"max_so_far_gap_coef": 0.0
},
"sigma": {
"intercept": 0.67509915,
"raw_sigma_coef": 0.14431833,
"spread_coef": 0.14431833,
"peak_flag_coef": 0.0,
"max_so_far_gap_coef": 0.0
}
},
"blending": {
"alpha_mu": 0.0,
"alpha_sigma": 1.0
},
"cities": {
"ankara": {
"samples": 7,
"mu_bias": 0.566273,
"sigma_scale": 2.0,
"confidence": 0.875
},
"london": {
"samples": 6,
"mu_bias": 0.489961,
"sigma_scale": 2.0,
"confidence": 0.75
},
"new york": {
"samples": 4,
"mu_bias": 1.852451,
"sigma_scale": 0.830444,
"confidence": 0.5
},
"paris": {
"samples": 7,
"mu_bias": 0.308599,
"sigma_scale": 2.0,
"confidence": 0.875
},
"seoul": {
"samples": 6,
"mu_bias": -1.486024,
"sigma_scale": 1.569585,
"confidence": 0.75
},
"toronto": {
"samples": 5,
"mu_bias": -0.734395,
"sigma_scale": 1.656751,
"confidence": 0.625
},
"buenos aires": {
"samples": 5,
"mu_bias": -1.753334,
"sigma_scale": 1.831739,
"confidence": 0.625
},
"wellington": {
"samples": 6,
"mu_bias": 0.350757,
"sigma_scale": 1.377974,
"confidence": 0.75
},
"chicago": {
"samples": 4,
"mu_bias": 3.01062,
"sigma_scale": 0.825575,
"confidence": 0.5
},
"sao paulo": {
"samples": 5,
"mu_bias": 1.632457,
"sigma_scale": 2.0,
"confidence": 0.625
},
"dallas": {
"samples": 4,
"mu_bias": 3.77714,
"sigma_scale": 0.796874,
"confidence": 0.5
},
"miami": {
"samples": 5,
"mu_bias": -4.868741,
"sigma_scale": 2.0,
"confidence": 0.625
},
"atlanta": {
"samples": 5,
"mu_bias": -7.648823,
"sigma_scale": 2.0,
"confidence": 0.625
},
"seattle": {
"samples": 4,
"mu_bias": 4.058619,
"sigma_scale": 2.0,
"confidence": 0.5
},
"lucknow": {
"samples": 4,
"mu_bias": 3.257609,
"sigma_scale": 2.0,
"confidence": 0.5
},
"munich": {
"samples": 6,
"mu_bias": -0.780811,
"sigma_scale": 2.0,
"confidence": 0.75
},
"hong kong": {
"samples": 3,
"mu_bias": -0.492675,
"sigma_scale": 2.0,
"confidence": 0.375
},
"taipei": {
"samples": 3,
"mu_bias": 1.462462,
"sigma_scale": 1.132834,
"confidence": 0.375
},
"milan": {
"samples": 3,
"mu_bias": -2.729203,
"sigma_scale": 2.0,
"confidence": 0.375
},
"warsaw": {
"samples": 3,
"mu_bias": 0.349319,
"sigma_scale": 1.337949,
"confidence": 0.375
}
},
"metrics": {
"sample_count": 105,
"mean_crps": 2.744961,
"legacy_mean_crps": 2.793938,
"legacy_mean_mae": 2.721143,
"legacy_bucket_hit_rate": 0.695238,
"selected_mean_crps": 2.747094,
"selected_mean_mae": 2.721143,
"selected_bucket_hit_rate": 0.704762,
"selected_score": 3.609685,
"legacy_score": 3.675576,
"filled_actual_from_history": 2,
"settlement_history_city_count": 30
},
"source": "artifacts\\probability_calibration\\default.json"
}
@@ -0,0 +1,248 @@
{
"summary": {
"sample_count": 105,
"filled_actual_from_history": 2,
"legacy": {
"mean_crps": 2.793938,
"mean_mae": 2.721143,
"bucket_hit_rate": 0.695238
},
"emos": {
"mean_crps": 2.747094,
"mean_mae": 2.721143,
"bucket_hit_rate": 0.685714
},
"delta": {
"crps": -0.046843,
"mae": 0.0,
"bucket_hit_rate": -0.009524
}
},
"by_city": {
"ankara": {
"samples": 7,
"legacy_mean_crps": 2.023242,
"emos_mean_crps": 2.259428,
"legacy_mean_mae": 1.984286,
"emos_mean_mae": 1.984286,
"legacy_bucket_hit_rate": 0.714286,
"emos_bucket_hit_rate": 0.714286
},
"atlanta": {
"samples": 5,
"legacy_mean_crps": 12.792034,
"emos_mean_crps": 12.537301,
"legacy_mean_mae": 12.806,
"emos_mean_mae": 12.806,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"buenos aires": {
"samples": 5,
"legacy_mean_crps": 3.846144,
"emos_mean_crps": 3.745234,
"legacy_mean_mae": 4.168,
"emos_mean_mae": 4.168,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"chicago": {
"samples": 4,
"legacy_mean_crps": 1.346667,
"emos_mean_crps": 0.66337,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"dallas": {
"samples": 4,
"legacy_mean_crps": 1.256111,
"emos_mean_crps": 0.651563,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"hong kong": {
"samples": 3,
"legacy_mean_crps": 0.261027,
"emos_mean_crps": 0.621432,
"legacy_mean_mae": 0.1,
"emos_mean_mae": 0.1,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.666667
},
"london": {
"samples": 6,
"legacy_mean_crps": 2.079624,
"emos_mean_crps": 1.986319,
"legacy_mean_mae": 2.451667,
"emos_mean_mae": 2.451667,
"legacy_bucket_hit_rate": 0.166667,
"emos_bucket_hit_rate": 0.166667
},
"lucknow": {
"samples": 4,
"legacy_mean_crps": 1.468528,
"emos_mean_crps": 1.523689,
"legacy_mean_mae": 1.6025,
"emos_mean_mae": 1.6025,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 0.5
},
"madrid": {
"samples": 2,
"legacy_mean_crps": 6.27726,
"emos_mean_crps": 6.028088,
"legacy_mean_mae": 7.33,
"emos_mean_mae": 7.33,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"miami": {
"samples": 5,
"legacy_mean_crps": 11.665378,
"emos_mean_crps": 11.614409,
"legacy_mean_mae": 12.07,
"emos_mean_mae": 12.07,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"milan": {
"samples": 3,
"legacy_mean_crps": 4.401392,
"emos_mean_crps": 3.905274,
"legacy_mean_mae": 4.06,
"emos_mean_mae": 4.06,
"legacy_bucket_hit_rate": 0.666667,
"emos_bucket_hit_rate": 0.666667
},
"munich": {
"samples": 6,
"legacy_mean_crps": 2.988583,
"emos_mean_crps": 2.892973,
"legacy_mean_mae": 3.143333,
"emos_mean_mae": 3.143333,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 0.5
},
"new york": {
"samples": 4,
"legacy_mean_crps": 1.861101,
"emos_mean_crps": 1.420965,
"legacy_mean_mae": 1.3725,
"emos_mean_mae": 1.3725,
"legacy_bucket_hit_rate": 0.75,
"emos_bucket_hit_rate": 0.75
},
"paris": {
"samples": 7,
"legacy_mean_crps": 2.430082,
"emos_mean_crps": 2.23397,
"legacy_mean_mae": 2.518571,
"emos_mean_mae": 2.518571,
"legacy_bucket_hit_rate": 0.571429,
"emos_bucket_hit_rate": 0.571429
},
"sao paulo": {
"samples": 5,
"legacy_mean_crps": 2.454756,
"emos_mean_crps": 2.351014,
"legacy_mean_mae": 2.628,
"emos_mean_mae": 2.628,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"seattle": {
"samples": 4,
"legacy_mean_crps": 0.531656,
"emos_mean_crps": 0.850792,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"seoul": {
"samples": 6,
"legacy_mean_crps": 0.328088,
"emos_mean_crps": 0.710725,
"legacy_mean_mae": 0.2,
"emos_mean_mae": 0.2,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"shanghai": {
"samples": 2,
"legacy_mean_crps": 0.250034,
"emos_mean_crps": 0.463888,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"singapore": {
"samples": 2,
"legacy_mean_crps": 0.281993,
"emos_mean_crps": 0.484792,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"taipei": {
"samples": 3,
"legacy_mean_crps": 0.356996,
"emos_mean_crps": 0.538714,
"legacy_mean_mae": 0.1,
"emos_mean_mae": 0.1,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.666667
},
"tel aviv": {
"samples": 2,
"legacy_mean_crps": 0.446758,
"emos_mean_crps": 0.55971,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 0.3,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"tokyo": {
"samples": 2,
"legacy_mean_crps": 0.450128,
"emos_mean_crps": 0.565212,
"legacy_mean_mae": 0.25,
"emos_mean_mae": 0.25,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 1.0
},
"toronto": {
"samples": 5,
"legacy_mean_crps": 2.647861,
"emos_mean_crps": 2.402627,
"legacy_mean_mae": 2.532,
"emos_mean_mae": 2.532,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"warsaw": {
"samples": 3,
"legacy_mean_crps": 1.618875,
"emos_mean_crps": 1.601375,
"legacy_mean_mae": 2.056667,
"emos_mean_mae": 2.056667,
"legacy_bucket_hit_rate": 0.333333,
"emos_bucket_hit_rate": 0.333333
},
"wellington": {
"samples": 6,
"legacy_mean_crps": 0.266349,
"emos_mean_crps": 0.592134,
"legacy_mean_mae": 0.2,
"emos_mean_mae": 0.2,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
}
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+2
View File
@@ -0,0 +1,2 @@
[pytest]
testpaths = tests
+2
View File
@@ -0,0 +1,2 @@
pytest
ruff
+210
View File
@@ -0,0 +1,210 @@
import argparse
import csv
import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import date, timedelta
from typing import Dict, Iterable, Tuple
import requests
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.data_collection.city_registry import CITY_REGISTRY # noqa: E402
HOURLY_FIELDS = (
"temperature_2m",
"relative_humidity_2m",
"wind_speed_10m",
"wind_direction_10m",
"cloud_cover",
"shortwave_radiation",
"precipitation",
"surface_pressure",
)
def _city_output_path(output_dir: str, city_name: str) -> str:
filename = f"{city_name.replace(' ', '_').lower()}_historical.csv"
return os.path.join(output_dir, filename)
def _fetch_city_history(
city_name: str,
city_info: Dict[str, object],
output_dir: str,
start_date: str,
end_date: str,
session: requests.Session,
overwrite: bool = False,
max_retries: int = 4,
) -> Tuple[str, str]:
output_path = _city_output_path(output_dir, city_name)
if os.path.exists(output_path) and not overwrite:
return city_name, "skipped_existing"
lat = city_info["lat"]
lon = city_info["lon"]
url = (
"https://archive-api.open-meteo.com/v1/archive"
f"?latitude={lat}&longitude={lon}"
f"&start_date={start_date}&end_date={end_date}"
f"&hourly={','.join(HOURLY_FIELDS)}"
"&timezone=auto"
)
last_error = None
payload = None
for attempt in range(max_retries + 1):
try:
response = session.get(url, timeout=90)
response.raise_for_status()
payload = response.json()
break
except requests.HTTPError as exc:
last_error = exc
response = getattr(exc, "response", None)
status_code = response.status_code if response is not None else None
if status_code == 429 and attempt < max_retries:
retry_after = 0
if response is not None:
try:
retry_after = int(response.headers.get("Retry-After") or "0")
except Exception:
retry_after = 0
sleep_sec = retry_after if retry_after > 0 else min(90, 5 * (attempt + 1))
time.sleep(sleep_sec)
continue
raise
except requests.RequestException as exc:
last_error = exc
if attempt < max_retries:
time.sleep(min(30, 3 * (attempt + 1)))
continue
raise
if payload is None:
raise RuntimeError(f"failed to fetch {city_name}: {last_error}")
hourly = payload.get("hourly") or {}
times = hourly.get("time") or []
if not times:
raise RuntimeError(f"missing hourly data for {city_name}")
fieldnames = ["time", *HOURLY_FIELDS]
rows = []
for idx, ts in enumerate(times):
row = {"time": ts}
for field in HOURLY_FIELDS:
values = hourly.get(field) or []
row[field] = values[idx] if idx < len(values) else None
rows.append(row)
os.makedirs(output_dir, exist_ok=True)
with open(output_path, "w", encoding="utf-8", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
return city_name, f"downloaded:{len(rows)}"
def _iter_target_cities(selected: Iterable[str]) -> Iterable[Tuple[str, Dict[str, object]]]:
if not selected:
for city_name, city_info in sorted(CITY_REGISTRY.items()):
yield city_name, city_info
return
normalized = {str(city).strip().lower() for city in selected if str(city).strip()}
for city_name, city_info in sorted(CITY_REGISTRY.items()):
if city_name in normalized:
yield city_name, city_info
def main():
parser = argparse.ArgumentParser(description="Backfill historical Open-Meteo CSVs for PolyWeather cities.")
parser.add_argument(
"--output-dir",
default=os.path.join(PROJECT_ROOT, "data", "historical"),
help="Output directory for per-city historical CSV files.",
)
parser.add_argument(
"--start-date",
default="2023-01-01",
help="Archive start date in YYYY-MM-DD.",
)
parser.add_argument(
"--end-date",
default=(date.today() - timedelta(days=1)).strftime("%Y-%m-%d"),
help="Archive end date in YYYY-MM-DD.",
)
parser.add_argument(
"--cities",
nargs="*",
default=[],
help="Optional subset of city registry keys to backfill.",
)
parser.add_argument(
"--overwrite",
action="store_true",
help="Re-download existing CSV files instead of skipping them.",
)
parser.add_argument(
"--workers",
type=int,
default=4,
help="Maximum concurrent download workers.",
)
args = parser.parse_args()
cities = list(_iter_target_cities(args.cities))
if not cities:
raise SystemExit("no matching cities found")
session = requests.Session()
session.headers.update({"User-Agent": "PolyWeather Historical Backfill/1.0"})
results = []
with ThreadPoolExecutor(max_workers=max(1, args.workers)) as executor:
futures = {
executor.submit(
_fetch_city_history,
city_name,
city_info,
args.output_dir,
args.start_date,
args.end_date,
session,
args.overwrite,
): city_name
for city_name, city_info in cities
}
for future in as_completed(futures):
city_name = futures[future]
try:
_, status = future.result()
print(f"{city_name}: {status}")
results.append((city_name, status))
except Exception as exc:
print(f"{city_name}: error:{exc}")
results.append((city_name, f"error:{exc}"))
time.sleep(0.1)
downloaded = sum(1 for _, status in results if status.startswith("downloaded:"))
skipped = sum(1 for _, status in results if status == "skipped_existing")
failed = [city for city, status in results if status.startswith("error:")]
print(
"summary downloaded={downloaded} skipped={skipped} failed={failed}".format(
downloaded=downloaded,
skipped=skipped,
failed=len(failed),
)
)
if failed:
print("failed_cities=" + ",".join(sorted(failed)))
if __name__ == "__main__":
main()
@@ -0,0 +1,152 @@
import argparse
import json
import os
import sys
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.deb_algorithm import load_history, save_history # noqa: E402
from src.analysis.probability_calibration import ( # noqa: E402
ENGINE_MODE_EMOS_SHADOW,
apply_probability_calibration,
build_probability_features,
)
def _sample_to_features(sample):
peak_flag = sample.get("peak_flag")
if peak_flag == 1.0:
peak_status = "past"
elif peak_flag == 0.5:
peak_status = "in_window"
else:
peak_status = "before"
return build_probability_features(
city_name=sample.get("city") or "",
raw_mu=sample.get("raw_mu"),
raw_sigma=sample.get("raw_sigma"),
deb_prediction=sample.get("deb_prediction"),
ens_data={
"median": sample.get("ens_median"),
"p10": None,
"p90": None,
},
current_forecasts={},
max_so_far=None,
peak_status=peak_status,
local_hour_frac=None,
)
def main():
parser = argparse.ArgumentParser(description="Backfill shadow probability snapshots into daily records.")
parser.add_argument(
"--history-file",
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
)
parser.add_argument(
"--training-samples",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"training_samples.json",
),
)
parser.add_argument(
"--calibration-file",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"default.json",
),
)
args = parser.parse_args()
history = load_history(args.history_file)
with open(args.training_samples, "r", encoding="utf-8") as fh:
training_payload = json.load(fh)
updated = 0
touched = 0
for sample in training_payload.get("samples") or []:
city = str(sample.get("city") or "").strip().lower()
date_str = str(sample.get("date") or "").strip()
if not city or not date_str:
continue
record = ((history.get(city) or {}).get(date_str) or {})
if not isinstance(record, dict) or not record:
continue
legacy_distribution = [
{"value": row.get("v"), "probability": row.get("p")}
for row in (record.get("prob_snapshot") or [])
if isinstance(row, dict) and row.get("v") is not None
]
if not legacy_distribution:
continue
calibration = apply_probability_calibration(
city_name=city,
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
raw_mu=sample.get("raw_mu"),
raw_sigma=sample.get("raw_sigma"),
max_so_far=None,
legacy_distribution=legacy_distribution,
features=_sample_to_features(sample),
calibration_path=args.calibration_file,
mode=ENGINE_MODE_EMOS_SHADOW,
)
shadow_distribution = calibration.get("shadow_distribution") or []
compact_shadow = [
{
"v": int(row.get("value")),
"p": round(float(row.get("probability") or 0.0), 3),
}
for row in shadow_distribution[:4]
if row.get("value") is not None
]
compact_calibration = {
"mode": calibration.get("mode"),
"engine": calibration.get("engine"),
"version": calibration.get("calibration_version"),
"source": calibration.get("calibration_source"),
"raw_mu": calibration.get("raw_mu"),
"raw_sigma": calibration.get("raw_sigma"),
"calibrated_mu": calibration.get("calibrated_mu"),
"calibrated_sigma": calibration.get("calibrated_sigma"),
}
touched += 1
if (
record.get("shadow_prob_snapshot") == compact_shadow
and record.get("probability_calibration") == compact_calibration
):
continue
record["shadow_prob_snapshot"] = compact_shadow
record["probability_calibration"] = compact_calibration
history[city][date_str] = record
updated += 1
save_history(args.history_file, history)
print(
json.dumps(
{
"samples_seen": len(training_payload.get("samples") or []),
"records_considered": touched,
"records_updated": updated,
},
ensure_ascii=False,
indent=2,
)
)
if __name__ == "__main__":
main()
+219
View File
@@ -0,0 +1,219 @@
import argparse
import json
import os
import sys
from collections import defaultdict
from datetime import datetime
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.deb_algorithm import load_history # noqa: E402
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
def _sf(value):
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _mean(values):
return round(sum(values) / len(values), 6) if values else None
def _top_bucket(snapshot):
if not isinstance(snapshot, list):
return None
best = None
best_prob = -1.0
for row in snapshot:
if not isinstance(row, dict):
continue
try:
prob = float(row.get("p") if "p" in row else row.get("probability"))
except Exception:
continue
value = row.get("v") if "v" in row else row.get("value")
if value is None:
continue
if prob > best_prob:
best = value
best_prob = prob
return best
def _bucket_probability(snapshot, target_bucket):
if not isinstance(snapshot, list):
return 0.0
for row in snapshot:
if not isinstance(row, dict):
continue
value = row.get("v") if "v" in row else row.get("value")
if value != target_bucket:
continue
try:
return float(row.get("p") if "p" in row else row.get("probability") or 0.0)
except Exception:
return 0.0
return 0.0
def _brier_from_snapshot(snapshot, target_bucket):
hit_prob = _bucket_probability(snapshot, target_bucket)
total = (1.0 - hit_prob) ** 2
if isinstance(snapshot, list):
for row in snapshot:
if not isinstance(row, dict):
continue
value = row.get("v") if "v" in row else row.get("value")
if value == target_bucket:
continue
try:
prob = float(row.get("p") if "p" in row else row.get("probability") or 0.0)
except Exception:
prob = 0.0
total += prob * prob
return round(total, 6)
def _blank_metrics():
return {
"samples": 0,
"legacy_mae": [],
"shadow_mae": [],
"legacy_bucket_hit": [],
"shadow_bucket_hit": [],
"legacy_bucket_brier": [],
"shadow_bucket_brier": [],
}
def _rollup(metrics):
return {
"samples": metrics["samples"],
"legacy_mean_mae": _mean(metrics["legacy_mae"]),
"shadow_mean_mae": _mean(metrics["shadow_mae"]),
"legacy_bucket_hit_rate": _mean(metrics["legacy_bucket_hit"]),
"shadow_bucket_hit_rate": _mean(metrics["shadow_bucket_hit"]),
"legacy_bucket_brier": _mean(metrics["legacy_bucket_brier"]),
"shadow_bucket_brier": _mean(metrics["shadow_bucket_brier"]),
"delta_mae": round((_mean(metrics["shadow_mae"]) or 0.0) - (_mean(metrics["legacy_mae"]) or 0.0), 6),
"delta_bucket_hit_rate": round((_mean(metrics["shadow_bucket_hit"]) or 0.0) - (_mean(metrics["legacy_bucket_hit"]) or 0.0), 6),
"delta_bucket_brier": round((_mean(metrics["shadow_bucket_brier"]) or 0.0) - (_mean(metrics["legacy_bucket_brier"]) or 0.0), 6),
}
def main():
parser = argparse.ArgumentParser(description="Build live shadow probability report from daily records.")
parser.add_argument(
"--history-file",
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
)
parser.add_argument(
"--output",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"shadow_report.json",
),
)
args = parser.parse_args()
history = load_history(args.history_file)
overall = _blank_metrics()
by_city = defaultdict(_blank_metrics)
by_date = defaultdict(_blank_metrics)
latest_observations = []
for city, city_records in sorted(history.items()):
if not isinstance(city_records, dict):
continue
for date_str, record in sorted(city_records.items()):
if not isinstance(record, dict):
continue
actual_high = _sf(record.get("actual_high"))
shadow_snapshot = record.get("shadow_prob_snapshot")
calibration = record.get("probability_calibration") or {}
if actual_high is None or not shadow_snapshot:
continue
legacy_mu = _sf(calibration.get("raw_mu"))
if legacy_mu is None:
legacy_mu = _sf(record.get("mu"))
shadow_mu = _sf(calibration.get("calibrated_mu"))
if shadow_mu is None:
continue
actual_bucket = apply_city_settlement(city, actual_high)
legacy_snapshot = record.get("prob_snapshot") or []
legacy_bucket = _top_bucket(legacy_snapshot)
shadow_bucket = _top_bucket(shadow_snapshot)
for metrics in (overall, by_city[city], by_date[date_str]):
metrics["samples"] += 1
metrics["legacy_mae"].append(abs(legacy_mu - actual_high))
metrics["shadow_mae"].append(abs(shadow_mu - actual_high))
metrics["legacy_bucket_hit"].append(1.0 if legacy_bucket == actual_bucket else 0.0)
metrics["shadow_bucket_hit"].append(1.0 if shadow_bucket == actual_bucket else 0.0)
metrics["legacy_bucket_brier"].append(_brier_from_snapshot(legacy_snapshot, actual_bucket))
metrics["shadow_bucket_brier"].append(_brier_from_snapshot(shadow_snapshot, actual_bucket))
latest_observations.append(
{
"city": city,
"date": date_str,
"actual_high": actual_high,
"actual_bucket": actual_bucket,
"legacy_mu": round(legacy_mu, 3),
"shadow_mu": round(shadow_mu, 3),
"legacy_top_bucket": legacy_bucket,
"shadow_top_bucket": shadow_bucket,
"calibration_version": calibration.get("version"),
"calibration_mode": calibration.get("mode"),
}
)
by_city_report = {
city: _rollup(metrics)
for city, metrics in sorted(by_city.items())
}
by_date_report = {
date_str: _rollup(metrics)
for date_str, metrics in sorted(
by_date.items(),
key=lambda item: datetime.strptime(item[0], "%Y-%m-%d"),
)
}
latest_observations = sorted(
latest_observations,
key=lambda row: (row["date"], row["city"]),
reverse=True,
)[:100]
payload = {
"generated_at": datetime.utcnow().isoformat() + "Z",
"summary": _rollup(overall),
"by_city": by_city_report,
"by_date": by_date_report,
"recent_observations": latest_observations,
}
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(payload, fh, ensure_ascii=False, indent=2)
print(json.dumps(payload["summary"], ensure_ascii=False, indent=2))
print(f"saved shadow report to {args.output}")
if __name__ == "__main__":
main()
@@ -0,0 +1,84 @@
import argparse
import csv
import json
import os
import sys
from collections import defaultdict
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
from src.data_collection.city_registry import CITY_REGISTRY # noqa: E402
def main():
parser = argparse.ArgumentParser(description="Build daily settlement history from historical weather CSV files.")
parser.add_argument(
"--history-dir",
default=os.path.join(PROJECT_ROOT, "data", "historical"),
help="Directory containing per-city *_historical.csv files.",
)
parser.add_argument(
"--output",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"settlement_history.json",
),
help="Output JSON path.",
)
args = parser.parse_args()
result = {}
for city in sorted(CITY_REGISTRY.keys()):
path = os.path.join(
args.history_dir,
f"{city.replace(' ', '_').lower()}_historical.csv",
)
if not os.path.exists(path):
continue
daily_max = defaultdict(lambda: None)
with open(path, "r", encoding="utf-8") as fh:
reader = csv.DictReader(fh)
for row in reader:
ts = str(row.get("time") or "")
raw_temp = row.get("temperature_2m")
if not ts or raw_temp in (None, ""):
continue
try:
temp = float(raw_temp)
except Exception:
continue
day = ts[:10]
prev = daily_max[day]
if prev is None or temp > prev:
daily_max[day] = temp
result[city] = {
day: {
"max_temp": round(max_temp, 3),
"settlement_value": apply_city_settlement(city, max_temp),
}
for day, max_temp in sorted(daily_max.items())
if max_temp is not None
}
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(result, fh, ensure_ascii=False, indent=2)
print(
"saved settlement history to {path} for {count} cities".format(
path=args.output,
count=len(result),
)
)
if __name__ == "__main__":
main()
+206
View File
@@ -0,0 +1,206 @@
import argparse
import json
import os
import sys
from collections import defaultdict
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.probability_calibration import ( # noqa: E402
ENGINE_MODE_EMOS_PRIMARY,
_gaussian_crps,
apply_probability_calibration,
build_probability_features,
)
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
from scripts.fit_probability_calibration import ( # noqa: E402
_extract_samples,
_load_json_if_exists,
)
from src.analysis.deb_algorithm import load_history # noqa: E402
def _mean(values):
return (sum(values) / len(values)) if values else None
def _sample_to_features(sample):
return build_probability_features(
city_name=sample.get("city") or "",
raw_mu=sample.get("raw_mu"),
raw_sigma=sample.get("raw_sigma"),
deb_prediction=sample.get("deb_prediction"),
ens_data={
"median": sample.get("ens_median"),
"p10": None,
"p90": None,
},
current_forecasts={},
max_so_far=None,
peak_status="in_window" if sample.get("peak_flag") == 0.5 else "past" if sample.get("peak_flag") == 1.0 else "before",
local_hour_frac=None,
)
def _top_bucket_value(distribution):
if not distribution:
return None
top = max(
(row for row in distribution if isinstance(row, dict)),
key=lambda row: float(row.get("probability") or 0.0),
default=None,
)
if not top:
return None
return top.get("value")
def main():
parser = argparse.ArgumentParser(description="Evaluate legacy vs EMOS probability calibration.")
parser.add_argument(
"--history-file",
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
)
parser.add_argument(
"--settlement-history",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"settlement_history.json",
),
)
parser.add_argument(
"--calibration-file",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"default.json",
),
)
parser.add_argument(
"--output",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"evaluation_report.json",
),
)
args = parser.parse_args()
history = load_history(args.history_file)
settlement_history = _load_json_if_exists(args.settlement_history)
samples, filled_actual_from_history = _extract_samples(
history,
settlement_history=settlement_history,
)
legacy_crps = []
emos_crps = []
legacy_mae = []
emos_mae = []
legacy_bucket_hits = []
emos_bucket_hits = []
by_city = defaultdict(lambda: {
"samples": 0,
"legacy_crps": [],
"emos_crps": [],
"legacy_mae": [],
"emos_mae": [],
"legacy_bucket_hits": [],
"emos_bucket_hits": [],
})
for sample in samples:
city = str(sample.get("city") or "").strip().lower()
actual_high = float(sample["actual_high"])
raw_mu = float(sample["raw_mu"])
raw_sigma = max(0.1, float(sample["raw_sigma"]))
legacy_crps.append(_gaussian_crps(actual_high, raw_mu, raw_sigma))
legacy_mae.append(abs(raw_mu - actual_high))
legacy_bucket = apply_city_settlement(city, raw_mu)
actual_bucket = apply_city_settlement(city, actual_high)
legacy_bucket_hits.append(1.0 if legacy_bucket == actual_bucket else 0.0)
calibration = apply_probability_calibration(
city_name=city,
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
raw_mu=raw_mu,
raw_sigma=raw_sigma,
max_so_far=None,
legacy_distribution=[],
features=_sample_to_features(sample),
calibration_path=args.calibration_file,
mode=ENGINE_MODE_EMOS_PRIMARY,
)
emos_mu = float(calibration.get("calibrated_mu") or raw_mu)
emos_sigma = max(0.1, float(calibration.get("calibrated_sigma") or raw_sigma))
emos_distribution = calibration.get("distribution") or []
emos_crps.append(_gaussian_crps(actual_high, emos_mu, emos_sigma))
emos_mae.append(abs(emos_mu - actual_high))
emos_bucket = _top_bucket_value(emos_distribution)
emos_bucket_hits.append(1.0 if emos_bucket == actual_bucket else 0.0)
row = by_city[city]
row["samples"] += 1
row["legacy_crps"].append(legacy_crps[-1])
row["emos_crps"].append(emos_crps[-1])
row["legacy_mae"].append(legacy_mae[-1])
row["emos_mae"].append(emos_mae[-1])
row["legacy_bucket_hits"].append(legacy_bucket_hits[-1])
row["emos_bucket_hits"].append(emos_bucket_hits[-1])
summary = {
"sample_count": len(samples),
"filled_actual_from_history": filled_actual_from_history,
"legacy": {
"mean_crps": round(_mean(legacy_crps), 6) if legacy_crps else None,
"mean_mae": round(_mean(legacy_mae), 6) if legacy_mae else None,
"bucket_hit_rate": round(_mean(legacy_bucket_hits), 6) if legacy_bucket_hits else None,
},
"emos": {
"mean_crps": round(_mean(emos_crps), 6) if emos_crps else None,
"mean_mae": round(_mean(emos_mae), 6) if emos_mae else None,
"bucket_hit_rate": round(_mean(emos_bucket_hits), 6) if emos_bucket_hits else None,
},
"delta": {
"crps": round((_mean(emos_crps) or 0.0) - (_mean(legacy_crps) or 0.0), 6),
"mae": round((_mean(emos_mae) or 0.0) - (_mean(legacy_mae) or 0.0), 6),
"bucket_hit_rate": round((_mean(emos_bucket_hits) or 0.0) - (_mean(legacy_bucket_hits) or 0.0), 6),
},
}
city_report = {}
for city, metrics in sorted(by_city.items()):
city_report[city] = {
"samples": metrics["samples"],
"legacy_mean_crps": round(_mean(metrics["legacy_crps"]), 6),
"emos_mean_crps": round(_mean(metrics["emos_crps"]), 6),
"legacy_mean_mae": round(_mean(metrics["legacy_mae"]), 6),
"emos_mean_mae": round(_mean(metrics["emos_mae"]), 6),
"legacy_bucket_hit_rate": round(_mean(metrics["legacy_bucket_hits"]), 6),
"emos_bucket_hit_rate": round(_mean(metrics["emos_bucket_hits"]), 6),
}
payload = {
"summary": summary,
"by_city": city_report,
}
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(payload, fh, ensure_ascii=False, indent=2)
print(json.dumps(summary, ensure_ascii=False, indent=2))
print(f"saved evaluation report to {args.output}")
if __name__ == "__main__":
main()
@@ -0,0 +1,65 @@
import argparse
import json
import os
import sys
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.deb_algorithm import load_history # noqa: E402
from scripts.fit_probability_calibration import ( # noqa: E402
_extract_samples,
_load_json_if_exists,
)
def main():
parser = argparse.ArgumentParser(description="Export normalized probability calibration training samples.")
parser.add_argument(
"--history-file",
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
)
parser.add_argument(
"--settlement-history",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"settlement_history.json",
),
)
parser.add_argument(
"--output",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"training_samples.json",
),
)
args = parser.parse_args()
history = load_history(args.history_file)
settlement_history = _load_json_if_exists(args.settlement_history)
samples, filled_actual_from_history = _extract_samples(
history,
settlement_history=settlement_history,
)
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
payload = {
"sample_count": len(samples),
"filled_actual_from_history": filled_actual_from_history,
"samples": samples,
}
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(payload, fh, ensure_ascii=False, indent=2)
print(f"exported {len(samples)} samples to {args.output}")
if __name__ == "__main__":
main()
+171
View File
@@ -0,0 +1,171 @@
import argparse
import json
import os
import sys
from datetime import datetime
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.probability_calibration import ( # noqa: E402
DEFAULT_CALIBRATION_FILE,
default_calibration_payload,
fit_calibration,
)
from src.analysis.deb_algorithm import load_history # noqa: E402
def _sf(value):
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _load_json_if_exists(path):
if not path or not os.path.exists(path):
return {}
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
return data if isinstance(data, dict) else {}
def _extract_samples(history, settlement_history=None):
samples = []
filled_actual_from_history = 0
today = datetime.utcnow().strftime("%Y-%m-%d")
settlement_history = settlement_history or {}
for city, city_rows in (history or {}).items():
if not isinstance(city_rows, dict):
continue
city_settlement = settlement_history.get(city) or {}
for date_str, record in city_rows.items():
if date_str == today or not isinstance(record, dict):
continue
actual_high = _sf(record.get("actual_high"))
if actual_high is None:
actual_high = _sf((city_settlement.get(date_str) or {}).get("max_temp"))
if actual_high is not None:
filled_actual_from_history += 1
deb_prediction = _sf(record.get("deb_prediction"))
raw_mu = _sf(record.get("mu")) or deb_prediction
forecasts = record.get("forecasts") or {}
if not isinstance(forecasts, dict):
forecasts = {}
forecast_values = [val for val in (_sf(v) for v in forecasts.values()) if val is not None]
forecast_values.sort()
forecast_median = (
forecast_values[len(forecast_values) // 2] if forecast_values else None
)
feature_snapshot = record.get("probability_features") or {}
if not isinstance(feature_snapshot, dict):
feature_snapshot = {}
ens_median = _sf(feature_snapshot.get("ens_median")) or forecast_median or raw_mu
ensemble_spread = _sf(feature_snapshot.get("ensemble_spread"))
if ensemble_spread is None:
if len(forecast_values) >= 2:
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
else:
ensemble_spread = 1.0
raw_sigma = _sf(feature_snapshot.get("raw_sigma")) or ensemble_spread or 1.0
peak_status = str(feature_snapshot.get("peak_status") or "before").strip().lower()
peak_flag = 0.0
if peak_status == "in_window":
peak_flag = 0.5
elif peak_status == "past":
peak_flag = 1.0
if actual_high is None or raw_mu is None:
continue
max_so_far = _sf(feature_snapshot.get("max_so_far"))
max_so_far_gap = _sf(feature_snapshot.get("max_so_far_gap"))
if max_so_far_gap is None and max_so_far is not None and deb_prediction is not None:
max_so_far_gap = deb_prediction - max_so_far
samples.append(
{
"city": city,
"date": date_str,
"actual_high": actual_high,
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"deb_prediction": deb_prediction,
"ens_median": ens_median,
"ensemble_spread": ensemble_spread,
"max_so_far_gap": max_so_far_gap,
"peak_flag": peak_flag,
}
)
return samples, filled_actual_from_history
def main():
parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.")
parser.add_argument(
"--history-file",
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
help="Path to the historical daily_records.json file.",
)
parser.add_argument(
"--output",
default=DEFAULT_CALIBRATION_FILE,
help="Output JSON file for fitted calibration parameters.",
)
parser.add_argument(
"--settlement-history",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"settlement_history.json",
),
help="Optional daily settlement history JSON built from historical CSV files.",
)
parser.add_argument(
"--version",
default=None,
help="Optional explicit calibration version.",
)
args = parser.parse_args()
history = load_history(args.history_file)
settlement_history = _load_json_if_exists(args.settlement_history)
samples, filled_actual_from_history = _extract_samples(
history,
settlement_history=settlement_history,
)
calibration = fit_calibration(samples, version=args.version)
if not samples:
calibration = default_calibration_payload(
version=args.version,
reason="no_samples",
)
calibration.setdefault("metrics", {})
calibration["metrics"]["filled_actual_from_history"] = filled_actual_from_history
calibration["metrics"]["settlement_history_city_count"] = len(settlement_history)
try:
calibration["source"] = os.path.relpath(args.output, PROJECT_ROOT)
except ValueError:
calibration["source"] = os.path.abspath(args.output)
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(calibration, fh, ensure_ascii=False, indent=2)
print(
"saved calibration to {path} with {count} samples".format(
path=args.output,
count=calibration.get("metrics", {}).get("sample_count", 0),
)
)
if __name__ == "__main__":
main()
+69 -3
View File
@@ -2,7 +2,7 @@ import os
import json
from datetime import datetime, timedelta
import requests
from src.analysis.settlement_rounding import wu_round, apply_city_settlement, is_exact_settlement_city
from src.analysis.settlement_rounding import apply_city_settlement
# Cross-platform file locking
import sys
@@ -38,6 +38,15 @@ _history_cache = {}
_history_mtime = 0
def _sf(value):
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _is_excluded_model_name(model_name: str) -> bool:
normalized = str(model_name or "").strip().lower().replace(" ", "").replace("_", "").replace("-", "")
return "meteoblue" in normalized
@@ -217,8 +226,16 @@ def reconcile_recent_actual_highs(city_name: str, lookback_days: int = 7):
def update_daily_record(
city_name, date_str, forecasts, actual_high, deb_prediction=None,
mu=None, probabilities=None
city_name,
date_str,
forecasts,
actual_high,
deb_prediction=None,
mu=None,
probabilities=None,
probability_features=None,
shadow_probabilities=None,
calibration_summary=None,
):
"""
保存/更新某城市某天的各个模型预报与最终实测值
@@ -253,6 +270,36 @@ def update_daily_record(
{"v": p["value"], "p": p["probability"]}
for p in probabilities[:4]
]
compact_features = None
if isinstance(probability_features, dict) and probability_features:
compact_features = {
"raw_mu": _sf(probability_features.get("raw_mu")),
"raw_sigma": _sf(probability_features.get("raw_sigma")),
"deb_prediction": _sf(probability_features.get("deb_prediction")),
"ens_median": _sf(probability_features.get("ens_median")),
"ensemble_spread": _sf(probability_features.get("ensemble_spread")),
"max_so_far": _sf(probability_features.get("max_so_far")),
"max_so_far_gap": _sf(probability_features.get("max_so_far_gap")),
"peak_status": probability_features.get("peak_status"),
}
compact_shadow_probs = None
if shadow_probabilities is not None:
compact_shadow_probs = [
{"v": p["value"], "p": p["probability"]}
for p in shadow_probabilities[:4]
]
compact_calibration = None
if isinstance(calibration_summary, dict) and calibration_summary:
compact_calibration = {
"mode": calibration_summary.get("mode"),
"engine": calibration_summary.get("engine"),
"version": calibration_summary.get("calibration_version"),
"source": calibration_summary.get("calibration_source"),
"raw_mu": _sf(calibration_summary.get("raw_mu")),
"raw_sigma": _sf(calibration_summary.get("raw_sigma")),
"calibrated_mu": _sf(calibration_summary.get("calibrated_mu")),
"calibrated_sigma": _sf(calibration_summary.get("calibrated_sigma")),
}
# 避免无意义的频繁磁盘写入
existing = data[city_name][date_str]
@@ -260,6 +307,7 @@ def update_daily_record(
old_deb = existing.get("deb_prediction")
old_mu = existing.get("mu")
old_probs = existing.get("prob_snapshot")
old_shadow_probs = existing.get("shadow_prob_snapshot")
next_mu = round(mu, 2) if mu is not None else None
if (
old_actual == actual_high
@@ -267,6 +315,18 @@ def update_daily_record(
and (deb_prediction is None or old_deb == deb_prediction)
and (mu is None or old_mu == next_mu)
and (compact_probs is None or old_probs == compact_probs)
and (
compact_shadow_probs is None
or old_shadow_probs == compact_shadow_probs
)
and (
compact_features is None
or existing.get("probability_features") == compact_features
)
and (
compact_calibration is None
or existing.get("probability_calibration") == compact_calibration
)
):
return
@@ -285,6 +345,12 @@ def update_daily_record(
existing["mu"] = next_mu
if probabilities is not None:
existing["prob_snapshot"] = compact_probs
if compact_features is not None:
existing["probability_features"] = compact_features
if shadow_probabilities is not None:
existing["shadow_prob_snapshot"] = compact_shadow_probs
if compact_calibration is not None:
existing["probability_calibration"] = compact_calibration
# 自动清理:只保留最近 14 天的记录(DEB 只用 7 天,14 天留足余量)
cutoff = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d")
+2 -1
View File
@@ -9,7 +9,7 @@ import re
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
from src.analysis.settlement_rounding import wu_round, apply_city_settlement, is_exact_settlement_city
from src.analysis.settlement_rounding import apply_city_settlement
def _sf(v: Any) -> Optional[float]:
@@ -652,6 +652,7 @@ def _pick_bucket_for_forecast(
def _extract_market_snapshot(city_weather: Dict[str, Any]) -> Dict[str, Any]:
scan = city_weather.get("market_scan") or {}
city = str(city_weather.get("name") or "").strip().lower()
if not isinstance(scan, dict):
return {"available": False}
if not scan.get("available"):
+643
View File
@@ -0,0 +1,643 @@
from __future__ import annotations
import json
import math
import os
from datetime import datetime, timezone
from typing import Any, Dict, Iterable, List, Optional, Tuple
import numpy as np
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
ENGINE_MODE_LEGACY = "legacy"
ENGINE_MODE_EMOS_SHADOW = "emos_shadow"
ENGINE_MODE_EMOS_PRIMARY = "emos_primary"
VALID_ENGINE_MODES = {
ENGINE_MODE_LEGACY,
ENGINE_MODE_EMOS_SHADOW,
ENGINE_MODE_EMOS_PRIMARY,
}
DEFAULT_CALIBRATION_FILE = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
"artifacts",
"probability_calibration",
"default.json",
)
_CALIBRATION_CACHE: Dict[str, Dict[str, Any]] = {}
_CALIBRATION_MTIME: Dict[str, float] = {}
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _coalesce_float(value: Any, default: float) -> float:
parsed = _sf(value)
return default if parsed is None else parsed
def _mean(values: Iterable[float]) -> Optional[float]:
values = list(values)
return (sum(values) / len(values)) if values else None
def resolve_probability_engine_mode(explicit_mode: Optional[str] = None) -> str:
mode = str(
explicit_mode
or os.getenv("POLYWEATHER_PROBABILITY_ENGINE")
or ENGINE_MODE_EMOS_SHADOW
).strip().lower()
if mode not in VALID_ENGINE_MODES:
return ENGINE_MODE_EMOS_SHADOW
return mode
def load_calibration(calibration_path: Optional[str] = None) -> Dict[str, Any]:
path = str(
calibration_path
or os.getenv("POLYWEATHER_PROBABILITY_CALIBRATION_FILE")
or DEFAULT_CALIBRATION_FILE
).strip()
if not path:
return {}
if not os.path.exists(path):
return {}
try:
mtime = os.path.getmtime(path)
cached = _CALIBRATION_CACHE.get(path)
if cached and _CALIBRATION_MTIME.get(path) == mtime:
return cached
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
if not isinstance(data, dict):
return {}
_CALIBRATION_CACHE[path] = data
_CALIBRATION_MTIME[path] = mtime
return data
except Exception:
return {}
def build_probability_features(
city_name: str,
raw_mu: Optional[float],
raw_sigma: Optional[float],
deb_prediction: Optional[float],
ens_data: Optional[Dict[str, Any]],
current_forecasts: Optional[Dict[str, Any]],
max_so_far: Optional[float],
peak_status: str,
local_hour_frac: Optional[float],
) -> Dict[str, Any]:
ens_data = ens_data or {}
current_forecasts = current_forecasts or {}
forecast_values = [
v for v in (_sf(val) for val in current_forecasts.values()) if v is not None
]
forecast_values.sort()
forecast_median = None
if forecast_values:
forecast_median = forecast_values[len(forecast_values) // 2]
ens_median = _sf(ens_data.get("median"))
ens_p10 = _sf(ens_data.get("p10"))
ens_p90 = _sf(ens_data.get("p90"))
ensemble_spread = None
if ens_p10 is not None and ens_p90 is not None and ens_p90 >= ens_p10:
ensemble_spread = max(0.1, (ens_p90 - ens_p10) / 2.56)
elif len(forecast_values) >= 2:
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
elif raw_sigma is not None:
ensemble_spread = max(0.1, raw_sigma)
baseline = deb_prediction if deb_prediction is not None else raw_mu
max_so_far_gap = None
if baseline is not None and max_so_far is not None:
max_so_far_gap = baseline - max_so_far
peak_flag = 0.0
if peak_status == "in_window":
peak_flag = 0.5
elif peak_status == "past":
peak_flag = 1.0
return {
"city": str(city_name or "").strip().lower(),
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"deb_prediction": deb_prediction,
"ens_median": ens_median,
"ens_p10": ens_p10,
"ens_p90": ens_p90,
"forecast_median": forecast_median,
"forecast_spread": forecast_values[-1] - forecast_values[0]
if len(forecast_values) >= 2
else None,
"ensemble_spread": ensemble_spread,
"max_so_far": max_so_far,
"max_so_far_gap": max_so_far_gap,
"peak_status": peak_status,
"peak_flag": peak_flag,
"local_hour_frac": local_hour_frac,
"model_count": len(forecast_values),
}
def _normal_cdf(x: float, mean: float, sigma: float) -> float:
return 0.5 * (1.0 + math.erf((x - mean) / (sigma * math.sqrt(2.0))))
def _normal_pdf(x: float) -> float:
return math.exp(-(x ** 2) / 2.0) / math.sqrt(2.0 * math.pi)
def _bucket_probabilities(
mu: float,
sigma: float,
max_so_far: Optional[float],
city_name: str,
) -> Tuple[List[Dict[str, Any]], List[Tuple[int, float]]]:
sigma = max(0.1, float(sigma))
min_possible = (
apply_city_settlement(city_name, max_so_far) if max_so_far is not None else -999
)
probs: Dict[int, float] = {}
search_range = max(2, int(sigma * 2.5))
is_exact = is_exact_settlement_city(city_name)
target_mu = apply_city_settlement(city_name, mu)
if is_exact:
target_mu = int(math.floor(mu))
for value in range(target_mu - search_range, target_mu + search_range + 1):
if value < min_possible:
continue
if is_exact:
prob = _normal_cdf(value + 1.0, mu, sigma) - _normal_cdf(value, mu, sigma)
else:
prob = _normal_cdf(value + 0.5, mu, sigma) - _normal_cdf(value - 0.5, mu, sigma)
if prob > 0.01:
probs[value] = prob
total = sum(probs.values())
if total <= 0:
return [], []
normalized = {key: val / total for key, val in probs.items()}
sorted_probs = sorted(normalized.items(), key=lambda item: item[1], reverse=True)
distribution = []
for value, prob in sorted_probs[:4]:
if is_exact:
bucket_range = "[{0}.0~{1}.0)".format(value, value + 1)
else:
bucket_range = "[{0}~{1})".format(value - 0.5, value + 0.5)
distribution.append(
{
"value": int(value),
"range": bucket_range,
"probability": round(prob, 3),
}
)
return distribution, sorted_probs
def _top_bucket_value(distribution: Optional[List[Dict[str, Any]]]) -> Optional[int]:
if not distribution:
return None
top = max(
(row for row in distribution if isinstance(row, dict)),
key=lambda row: float(row.get("probability") or 0.0),
default=None,
)
if not top:
return None
value = top.get("value")
return int(value) if value is not None else None
def _composite_score(mean_crps: float, mean_mae: float, bucket_hit_rate: float) -> float:
return mean_crps + 0.1 * mean_mae + 2.0 * (1.0 - bucket_hit_rate)
def _blend_value(raw_value: float, calibrated_value: float, alpha: float) -> float:
return (1.0 - alpha) * raw_value + alpha * calibrated_value
def apply_probability_calibration(
city_name: str,
temp_symbol: str,
raw_mu: Optional[float],
raw_sigma: Optional[float],
max_so_far: Optional[float],
legacy_distribution: Optional[List[Dict[str, Any]]],
features: Optional[Dict[str, Any]] = None,
calibration_path: Optional[str] = None,
mode: Optional[str] = None,
) -> Dict[str, Any]:
selected_mode = resolve_probability_engine_mode(mode)
if raw_mu is None or raw_sigma is None:
return {
"mode": selected_mode,
"engine": ENGINE_MODE_LEGACY,
"distribution": legacy_distribution or [],
"shadow_distribution": [],
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"calibrated_mu": None,
"calibrated_sigma": None,
"calibration_version": None,
"calibration_source": None,
}
calibration = load_calibration(calibration_path)
if not calibration:
return {
"mode": selected_mode,
"engine": ENGINE_MODE_LEGACY,
"distribution": legacy_distribution or [],
"shadow_distribution": [],
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"calibrated_mu": None,
"calibrated_sigma": None,
"calibration_version": None,
"calibration_source": None,
}
features = features or {}
city_key = str(city_name or "").strip().lower()
global_params = calibration.get("global", {}) or {}
city_params = (calibration.get("cities", {}) or {}).get(city_key, {}) or {}
blending_cfg = calibration.get("blending", {}) or {}
mu_cfg = global_params.get("mu", {}) or {}
sigma_cfg = global_params.get("sigma", {}) or {}
city_confidence = max(0.0, min(1.0, _coalesce_float(city_params.get("confidence"), 1.0)))
city_mu_bias = _coalesce_float(city_params.get("mu_bias"), 0.0) * city_confidence
city_sigma_scale = 1.0 + (
(_coalesce_float(city_params.get("sigma_scale"), 1.0) - 1.0) * city_confidence
)
deb_prediction = _sf(features.get("deb_prediction"))
ens_median = _sf(features.get("ens_median"))
max_so_far_gap = _sf(features.get("max_so_far_gap"))
peak_flag = _sf(features.get("peak_flag")) or 0.0
ensemble_spread = _sf(features.get("ensemble_spread"))
mu_intercept = _coalesce_float(mu_cfg.get("intercept"), 0.0)
mu_raw_coef = _coalesce_float(mu_cfg.get("raw_mu_coef"), 1.0)
mu_deb_coef = _coalesce_float(mu_cfg.get("deb_coef"), 0.0)
mu_ens_coef = _coalesce_float(mu_cfg.get("ens_median_coef"), 0.0)
mu_gap_coef = _coalesce_float(mu_cfg.get("max_so_far_gap_coef"), 0.0)
sigma_intercept = _coalesce_float(
sigma_cfg.get("intercept"),
math.log(max(raw_sigma, 0.1)),
)
sigma_raw_coef = _coalesce_float(sigma_cfg.get("raw_sigma_coef"), 1.0)
sigma_spread_coef = _coalesce_float(sigma_cfg.get("spread_coef"), 0.0)
sigma_peak_coef = _coalesce_float(sigma_cfg.get("peak_flag_coef"), 0.0)
sigma_gap_coef = _coalesce_float(sigma_cfg.get("max_so_far_gap_coef"), 0.0)
calibrated_mu = (
mu_intercept
+ mu_raw_coef * raw_mu
+ mu_deb_coef * (deb_prediction if deb_prediction is not None else raw_mu)
+ mu_ens_coef * (ens_median if ens_median is not None else raw_mu)
+ mu_gap_coef * (max_so_far_gap if max_so_far_gap is not None else 0.0)
+ city_mu_bias
)
sigma_log = (
sigma_intercept
+ sigma_raw_coef * math.log(max(raw_sigma, 0.1))
+ sigma_spread_coef * math.log(max(ensemble_spread or raw_sigma, 0.1))
+ sigma_peak_coef * peak_flag
+ sigma_gap_coef * (max_so_far_gap if max_so_far_gap is not None else 0.0)
)
calibrated_sigma = max(0.1, math.exp(sigma_log) * city_sigma_scale)
blend_alpha_mu = max(0.0, min(1.0, _coalesce_float(blending_cfg.get("alpha_mu"), 1.0)))
blend_alpha_sigma = max(0.0, min(1.0, _coalesce_float(blending_cfg.get("alpha_sigma"), 1.0)))
calibrated_mu = _blend_value(raw_mu, calibrated_mu, blend_alpha_mu)
calibrated_sigma = max(0.1, _blend_value(raw_sigma, calibrated_sigma, blend_alpha_sigma))
calibrated_distribution, calibrated_sorted = _bucket_probabilities(
calibrated_mu,
calibrated_sigma,
max_so_far=max_so_far,
city_name=city_key,
)
engine = ENGINE_MODE_LEGACY
selected_distribution = legacy_distribution or []
selected_sorted: List[Tuple[int, float]] = []
shadow_distribution: List[Dict[str, Any]] = []
shadow_sorted: List[Tuple[int, float]] = []
if selected_mode == ENGINE_MODE_EMOS_PRIMARY:
engine = "emos"
selected_distribution = calibrated_distribution
selected_sorted = calibrated_sorted
elif selected_mode == ENGINE_MODE_EMOS_SHADOW:
shadow_distribution = calibrated_distribution
shadow_sorted = calibrated_sorted
return {
"mode": selected_mode,
"engine": engine,
"distribution": selected_distribution,
"selected_sorted_probs": selected_sorted,
"shadow_distribution": shadow_distribution,
"shadow_sorted_probs": shadow_sorted,
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"calibrated_mu": calibrated_mu,
"calibrated_sigma": calibrated_sigma,
"blend_alpha_mu": blend_alpha_mu,
"blend_alpha_sigma": blend_alpha_sigma,
"calibration_version": calibration.get("version"),
"calibration_source": calibration.get("source")
or os.path.relpath(
calibration_path or DEFAULT_CALIBRATION_FILE,
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
),
}
def _gaussian_crps(observation: float, mean: float, sigma: float) -> float:
sigma = max(0.1, float(sigma))
z = (observation - mean) / sigma
return sigma * (
z * (2.0 * _normal_cdf(z, 0.0, 1.0) - 1.0)
+ 2.0 * _normal_pdf(z)
- 1.0 / math.sqrt(math.pi)
)
def _fit_linear(xs: np.ndarray, ys: np.ndarray) -> np.ndarray:
if len(xs) == 0:
return np.zeros(xs.shape[1], dtype=float)
coeffs, _, _, _ = np.linalg.lstsq(xs, ys, rcond=None)
return coeffs
def fit_calibration(
samples: Iterable[Dict[str, Any]],
version: Optional[str] = None,
) -> Dict[str, Any]:
normalized_samples: List[Dict[str, Any]] = []
city_residuals: Dict[str, List[float]] = {}
city_sigma_ratios: Dict[str, List[float]] = {}
for raw_sample in samples:
actual = _sf(raw_sample.get("actual_high"))
raw_mu = _sf(raw_sample.get("raw_mu"))
raw_sigma = _sf(raw_sample.get("raw_sigma"))
if actual is None or raw_mu is None or raw_sigma is None:
continue
feature_row = {
"city": str(raw_sample.get("city") or "").strip().lower(),
"actual_high": actual,
"raw_mu": raw_mu,
"raw_sigma": max(0.1, raw_sigma),
"deb_prediction": _sf(raw_sample.get("deb_prediction")),
"ens_median": _sf(raw_sample.get("ens_median")),
"ensemble_spread": _sf(raw_sample.get("ensemble_spread")),
"max_so_far_gap": _sf(raw_sample.get("max_so_far_gap")),
"peak_flag": _sf(raw_sample.get("peak_flag")) or 0.0,
}
normalized_samples.append(feature_row)
if len(normalized_samples) < 3:
return default_calibration_payload(version=version, reason="insufficient_samples")
mu_rows = []
mu_targets = []
for sample in normalized_samples:
deb = sample["deb_prediction"] if sample["deb_prediction"] is not None else sample["raw_mu"]
ens_median = sample["ens_median"] if sample["ens_median"] is not None else sample["raw_mu"]
gap = sample["max_so_far_gap"] if sample["max_so_far_gap"] is not None else 0.0
mu_rows.append([1.0, sample["raw_mu"], deb, ens_median, gap])
mu_targets.append(sample["actual_high"])
mu_coeffs = _fit_linear(np.array(mu_rows, dtype=float), np.array(mu_targets, dtype=float))
sigma_rows = []
sigma_targets = []
mu_predictions = []
for idx, sample in enumerate(normalized_samples):
predicted_mu = float(np.dot(mu_coeffs, np.array(mu_rows[idx], dtype=float)))
mu_predictions.append(predicted_mu)
residual = max(abs(sample["actual_high"] - predicted_mu), 0.1)
spread = max(sample["ensemble_spread"] or sample["raw_sigma"], 0.1)
gap = sample["max_so_far_gap"] if sample["max_so_far_gap"] is not None else 0.0
sigma_rows.append([1.0, math.log(sample["raw_sigma"]), math.log(spread), sample["peak_flag"], gap])
sigma_targets.append(math.log(residual))
city_residuals.setdefault(sample["city"], []).append(sample["actual_high"] - predicted_mu)
city_sigma_ratios.setdefault(sample["city"], []).append(residual / max(sample["raw_sigma"], 0.1))
sigma_coeffs = _fit_linear(np.array(sigma_rows, dtype=float), np.array(sigma_targets, dtype=float))
crps_values = []
for idx, sample in enumerate(normalized_samples):
predicted_mu = mu_predictions[idx]
sigma_log = float(np.dot(sigma_coeffs, np.array(sigma_rows[idx], dtype=float)))
predicted_sigma = max(0.1, math.exp(sigma_log))
crps_values.append(_gaussian_crps(sample["actual_high"], predicted_mu, predicted_sigma))
city_params: Dict[str, Dict[str, Any]] = {}
for city, residuals in city_residuals.items():
if len(residuals) < 3:
continue
sigma_ratios = city_sigma_ratios.get(city) or [1.0]
confidence = max(0.25, min(1.0, len(residuals) / 8.0))
city_params[city] = {
"samples": len(residuals),
"mu_bias": round(sum(residuals) / len(residuals), 6),
"sigma_scale": round(
max(0.5, min(2.0, sum(sigma_ratios) / len(sigma_ratios))),
6,
),
"confidence": round(confidence, 6),
}
legacy_crps_values = []
legacy_mae_values = []
legacy_bucket_hits = []
candidate_predictions = []
for idx, sample in enumerate(normalized_samples):
city = sample["city"]
city_meta = city_params.get(city, {})
city_confidence = max(
0.0,
min(1.0, _coalesce_float(city_meta.get("confidence"), 1.0)),
)
city_mu_bias = _coalesce_float(city_meta.get("mu_bias"), 0.0) * city_confidence
city_sigma_scale = 1.0 + (
(_coalesce_float(city_meta.get("sigma_scale"), 1.0) - 1.0) * city_confidence
)
legacy_mu = sample["raw_mu"]
legacy_sigma = sample["raw_sigma"]
actual_high = sample["actual_high"]
legacy_crps_values.append(_gaussian_crps(actual_high, legacy_mu, legacy_sigma))
legacy_mae_values.append(abs(legacy_mu - actual_high))
legacy_bucket_hits.append(
1.0
if apply_city_settlement(city, legacy_mu)
== apply_city_settlement(city, actual_high)
else 0.0
)
calibrated_mu = mu_predictions[idx] + city_mu_bias
sigma_log = float(np.dot(sigma_coeffs, np.array(sigma_rows[idx], dtype=float)))
calibrated_sigma = max(0.1, math.exp(sigma_log) * city_sigma_scale)
candidate_predictions.append(
{
"city": city,
"actual_high": actual_high,
"raw_mu": legacy_mu,
"raw_sigma": legacy_sigma,
"calibrated_mu": calibrated_mu,
"calibrated_sigma": calibrated_sigma,
}
)
legacy_mean_crps = _mean(legacy_crps_values) or 0.0
legacy_mean_mae = _mean(legacy_mae_values) or 0.0
legacy_bucket_hit_rate = _mean(legacy_bucket_hits) or 0.0
legacy_score = _composite_score(
legacy_mean_crps,
legacy_mean_mae,
legacy_bucket_hit_rate,
)
best_alpha_mu = 0.0
best_alpha_sigma = 0.0
best_score = legacy_score
best_metrics = {
"mean_crps": legacy_mean_crps,
"mean_mae": legacy_mean_mae,
"bucket_hit_rate": legacy_bucket_hit_rate,
}
alpha_grid = [step / 20.0 for step in range(21)]
for alpha_mu in alpha_grid:
for alpha_sigma in alpha_grid:
crps_values = []
mae_values = []
bucket_hits = []
for row in candidate_predictions:
mu_hat = _blend_value(row["raw_mu"], row["calibrated_mu"], alpha_mu)
sigma_hat = max(
0.1,
_blend_value(row["raw_sigma"], row["calibrated_sigma"], alpha_sigma),
)
actual_high = row["actual_high"]
city = row["city"]
crps_values.append(_gaussian_crps(actual_high, mu_hat, sigma_hat))
mae_values.append(abs(mu_hat - actual_high))
distribution, _ = _bucket_probabilities(
mu_hat,
sigma_hat,
max_so_far=None,
city_name=city,
)
predicted_bucket = _top_bucket_value(distribution)
actual_bucket = apply_city_settlement(city, actual_high)
bucket_hits.append(1.0 if predicted_bucket == actual_bucket else 0.0)
mean_crps = _mean(crps_values) or 0.0
mean_mae = _mean(mae_values) or 0.0
bucket_hit_rate = _mean(bucket_hits) or 0.0
score = _composite_score(mean_crps, mean_mae, bucket_hit_rate)
if score + 1e-9 < best_score:
best_score = score
best_alpha_mu = alpha_mu
best_alpha_sigma = alpha_sigma
best_metrics = {
"mean_crps": mean_crps,
"mean_mae": mean_mae,
"bucket_hit_rate": bucket_hit_rate,
}
return {
"version": version or datetime.now(timezone.utc).strftime("emos-%Y%m%d%H%M%S"),
"trained_at": datetime.now(timezone.utc).isoformat(),
"global": {
"mu": {
"intercept": round(float(mu_coeffs[0]), 8),
"raw_mu_coef": round(float(mu_coeffs[1]), 8),
"deb_coef": round(float(mu_coeffs[2]), 8),
"ens_median_coef": round(float(mu_coeffs[3]), 8),
"max_so_far_gap_coef": round(float(mu_coeffs[4]), 8),
},
"sigma": {
"intercept": round(float(sigma_coeffs[0]), 8),
"raw_sigma_coef": round(float(sigma_coeffs[1]), 8),
"spread_coef": round(float(sigma_coeffs[2]), 8),
"peak_flag_coef": round(float(sigma_coeffs[3]), 8),
"max_so_far_gap_coef": round(float(sigma_coeffs[4]), 8),
},
},
"blending": {
"alpha_mu": round(best_alpha_mu, 6),
"alpha_sigma": round(best_alpha_sigma, 6),
},
"cities": city_params,
"metrics": {
"sample_count": len(normalized_samples),
"mean_crps": round(sum(crps_values) / len(crps_values), 6),
"legacy_mean_crps": round(legacy_mean_crps, 6),
"legacy_mean_mae": round(legacy_mean_mae, 6),
"legacy_bucket_hit_rate": round(legacy_bucket_hit_rate, 6),
"selected_mean_crps": round(best_metrics["mean_crps"], 6),
"selected_mean_mae": round(best_metrics["mean_mae"], 6),
"selected_bucket_hit_rate": round(best_metrics["bucket_hit_rate"], 6),
"selected_score": round(best_score, 6),
"legacy_score": round(legacy_score, 6),
},
}
def default_calibration_payload(
version: Optional[str] = None,
reason: str = "bootstrap",
) -> Dict[str, Any]:
return {
"version": version or "emos-bootstrap-v1",
"trained_at": datetime.now(timezone.utc).isoformat(),
"global": {
"mu": {
"intercept": 0.0,
"raw_mu_coef": 1.0,
"deb_coef": 0.0,
"ens_median_coef": 0.0,
"max_so_far_gap_coef": 0.0,
},
"sigma": {
"intercept": 0.0,
"raw_sigma_coef": 1.0,
"spread_coef": 0.0,
"peak_flag_coef": 0.0,
"max_so_far_gap_coef": 0.0,
},
},
"blending": {
"alpha_mu": 1.0,
"alpha_sigma": 1.0,
},
"cities": {},
"metrics": {
"sample_count": 0,
"mean_crps": None,
"reason": reason,
},
}
+74 -2
View File
@@ -15,7 +15,11 @@ from src.analysis.deb_algorithm import (
update_daily_record,
_is_excluded_model_name,
)
from src.analysis.settlement_rounding import wu_round, apply_city_settlement, is_exact_settlement_city
from src.analysis.probability_calibration import (
apply_probability_calibration,
build_probability_features,
)
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
from src.data_collection.city_registry import CITY_REGISTRY
from src.data_collection.city_risk_profiles import get_city_risk_profile
@@ -431,7 +435,19 @@ def analyze_weather_trend(
# === Probability Engine ===
probabilities: List[Dict[str, Any]] = []
shadow_probabilities: List[Dict[str, Any]] = []
forecast_miss_deg = 0.0
probability_features = None
calibration_summary = {
"mode": "legacy",
"engine": "legacy",
"raw_mu": None,
"raw_sigma": sigma,
"calibrated_mu": None,
"calibrated_sigma": None,
"calibration_version": None,
"calibration_source": None,
}
if is_dead_market:
settled_wu = apply_city_settlement(city_name, max_so_far) if max_so_far is not None else 0
@@ -492,6 +508,43 @@ def analyze_weather_trend(
probabilities = probs_result.get("probabilities", [])
sorted_probs = probs_result.get("sorted_probs", [])
probability_features = build_probability_features(
city_name=city_name or "",
raw_mu=mu,
raw_sigma=sigma,
deb_prediction=deb_prediction,
ens_data=ens_data,
current_forecasts=current_forecasts,
max_so_far=max_so_far,
peak_status=peak_status,
local_hour_frac=local_hour_frac,
)
calibration_result = apply_probability_calibration(
city_name=city_name or "",
temp_symbol=temp_symbol,
raw_mu=mu,
raw_sigma=sigma,
max_so_far=max_so_far,
legacy_distribution=probabilities,
features=probability_features,
)
calibration_summary = {
"mode": calibration_result.get("mode", "legacy"),
"engine": calibration_result.get("engine", "legacy"),
"raw_mu": calibration_result.get("raw_mu"),
"raw_sigma": calibration_result.get("raw_sigma"),
"calibrated_mu": calibration_result.get("calibrated_mu"),
"calibrated_sigma": calibration_result.get("calibrated_sigma"),
"calibration_version": calibration_result.get("calibration_version"),
"calibration_source": calibration_result.get("calibration_source"),
}
shadow_probabilities = calibration_result.get("shadow_distribution") or []
if calibration_result.get("engine") == "emos":
mu = calibration_result.get("calibrated_mu", mu)
sigma = calibration_result.get("calibrated_sigma", sigma)
probabilities = calibration_result.get("distribution") or probabilities
sorted_probs = calibration_result.get("selected_sorted_probs") or sorted_probs
if sorted_probs:
prob_parts = [
f"{int(t)}{temp_symbol} [{t - 0.5}~{t + 0.5}) {p * 100:.0f}%"
@@ -650,6 +703,7 @@ def analyze_weather_trend(
# === Save daily record (with μ + prob snapshot) ===
try:
_prob_list = None
_shadow_prob_list = None
if sorted_probs:
_prob_list = [
{"value": int(t), "probability": round(p, 3)}
@@ -657,6 +711,12 @@ def analyze_weather_trend(
]
elif is_dead_market and max_so_far is not None:
_prob_list = [{"value": apply_city_settlement(city_name, max_so_far), "probability": 1.0}]
if shadow_probabilities:
_shadow_prob_list = [
{"value": int(row.get("value")), "probability": round(float(row.get("probability") or 0.0), 3)}
for row in shadow_probabilities[:4]
if row.get("value") is not None
]
update_daily_record(
city_name,
@@ -666,6 +726,9 @@ def analyze_weather_trend(
deb_prediction=_deb_to_save,
mu=mu,
probabilities=_prob_list,
probability_features=probability_features,
shadow_probabilities=_shadow_prob_list,
calibration_summary=calibration_summary,
)
except Exception:
pass
@@ -679,6 +742,15 @@ def analyze_weather_trend(
structured = {
"mu": mu,
"probabilities": probabilities,
"shadow_probabilities": shadow_probabilities,
"probability_engine": calibration_summary["engine"],
"probability_calibration_mode": calibration_summary["mode"],
"probability_calibration_version": calibration_summary["calibration_version"],
"probability_calibration_source": calibration_summary["calibration_source"],
"probability_raw_mu": calibration_summary["raw_mu"],
"probability_raw_sigma": calibration_summary["raw_sigma"],
"probability_calibrated_mu": calibration_summary["calibrated_mu"],
"probability_calibrated_sigma": calibration_summary["calibrated_sigma"],
"trend_info": {
"direction": trend_direction if 'trend_direction' in dir() else "unknown",
"recent": recent_list,
@@ -711,7 +783,7 @@ def calculate_prob_distribution(
def _norm_cdf(x, m, s):
# 0.5 * (1 + erf( (x-m)/(s*sqrt(2)) ))
return 0.5 * (1 + math.erf((x - m) / (sigma * math.sqrt(2))))
return 0.5 * (1 + math.erf((x - m) / (s * math.sqrt(2))))
min_possible_wu = apply_city_settlement(city_name, max_so_far) if max_so_far is not None else -999
probs = {}
+268
View File
@@ -0,0 +1,268 @@
from __future__ import annotations
import re
import time
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional
import requests
from loguru import logger
class MetarSourceMixin:
def get_icao_code(self, city: str) -> Optional[str]:
"""根据城市名获取对应的 ICAO 机场代码"""
normalized = city.lower().strip()
if normalized in self.CITY_TO_ICAO:
return self.CITY_TO_ICAO[normalized]
for key, icao in self.CITY_TO_ICAO.items():
if key in normalized or normalized in key:
return icao
return None
def fetch_metar(
self, city: str, use_fahrenheit: bool = False, utc_offset: int = 0
) -> Optional[Dict]:
"""从 NOAA Aviation Weather Center 获取 METAR 航空气象数据。"""
icao = self.get_icao_code(city)
if not icao:
logger.warning(f"未找到城市 {city} 对应的 ICAO 代码")
return None
cache_key = f"{icao}:{utc_offset}:{use_fahrenheit}"
now_ts = time.time()
with self._metar_cache_lock:
cached = self._metar_cache.get(cache_key)
if cached and now_ts - cached["t"] < self.metar_cache_ttl_sec:
logger.debug(f"METAR cache hit {icao} age={int(now_ts - cached['t'])}s")
return cached["d"]
try:
url = "https://aviationweather.gov/api/data/metar"
params = {
"ids": icao,
"format": "json",
"hours": 24,
"_t": int(time.time()),
}
response = self.session.get(url, params=params, timeout=self.timeout)
response.raise_for_status()
data = response.json()
if not data:
return None
latest = data[0]
temp_c = latest.get("temp")
dewp_c = latest.get("dewp")
def _parse_rawob_time(obs):
raw = obs.get("rawOb", "")
match = re.search(r"(\d{2})(\d{2})(\d{2})Z", raw)
if match:
_day, hour, minute = (
int(match.group(1)),
int(match.group(2)),
int(match.group(3)),
)
fallback = obs.get("reportTime", "")
try:
clean = fallback.replace(" ", "T")
if not clean.endswith("Z"):
clean += "Z"
base_dt = datetime.fromisoformat(clean.replace("Z", "+00:00"))
result = base_dt.replace(hour=hour, minute=minute, second=0)
if result > base_dt + timedelta(hours=2):
result -= timedelta(days=1)
return result
except Exception:
pass
fallback = obs.get("reportTime", "")
try:
clean = fallback.replace(" ", "T")
if not clean.endswith("Z"):
clean += "Z"
return datetime.fromisoformat(clean.replace("Z", "+00:00"))
except Exception:
return None
obs_dt = _parse_rawob_time(latest)
obs_time = (
obs_dt.strftime("%Y-%m-%dT%H:%M:%S.000Z")
if obs_dt
else latest.get("reportTime", "")
)
now_utc = datetime.now(timezone.utc)
local_now = now_utc + timedelta(seconds=utc_offset)
local_midnight = local_now.replace(hour=0, minute=0, second=0, microsecond=0)
utc_midnight = local_midnight - timedelta(seconds=utc_offset)
max_so_far_c = -999
max_temp_time = None
for obs in data:
obs_dt_iter = _parse_rawob_time(obs)
if obs_dt_iter is None:
continue
try:
if obs_dt_iter >= utc_midnight:
temp_value = obs.get("temp")
if temp_value is not None and temp_value > max_so_far_c:
max_so_far_c = temp_value
local_report = obs_dt_iter + timedelta(seconds=utc_offset)
max_temp_time = local_report.strftime("%H:%M")
except Exception:
continue
recent_temps_raw = []
recent_obs_raw = []
today_obs_raw = []
cloud_rank_map = {
"CLR": 0,
"SKC": 0,
"FEW": 1,
"SCT": 2,
"BKN": 3,
"OVC": 4,
}
for index, obs in enumerate(data):
obs_temp = obs.get("temp")
obs_dt_iter = _parse_rawob_time(obs)
if obs_temp is not None and obs_dt_iter:
local_rt = obs_dt_iter + timedelta(seconds=utc_offset)
time_str = local_rt.strftime("%H:%M")
if obs_dt_iter >= utc_midnight:
today_obs_raw.append((time_str, obs_temp))
if index < 4:
recent_temps_raw.append((time_str, obs_temp))
clouds = obs.get("clouds", [])
max_cloud_rank = 0
for cloud in clouds:
rank = cloud_rank_map.get(cloud.get("cover", ""), 0)
if rank > max_cloud_rank:
max_cloud_rank = rank
recent_obs_raw.append(
{
"time": time_str,
"temp": obs_temp,
"wdir": obs.get("wdir"),
"wspd": obs.get("wspd"),
"cloud_rank": max_cloud_rank,
"altim": obs.get("altim"),
}
)
if use_fahrenheit:
temp = temp_c * 9 / 5 + 32 if temp_c is not None else None
max_so_far = max_so_far_c * 9 / 5 + 32 if max_so_far_c > -900 else None
dewp = dewp_c * 9 / 5 + 32 if dewp_c is not None else None
unit = "fahrenheit"
recent_temps = [(t, round(v * 9 / 5 + 32, 1)) for t, v in recent_temps_raw]
today_obs = [(t, round(v * 9 / 5 + 32, 1)) for t, v in today_obs_raw]
else:
temp = temp_c
max_so_far = max_so_far_c if max_so_far_c > -900 else None
dewp = dewp_c
unit = "celsius"
recent_temps = [(t, v) for t, v in recent_temps_raw]
today_obs = [(t, v) for t, v in today_obs_raw]
result = {
"source": "metar",
"icao": icao,
"station_name": latest.get("name", icao),
"timestamp": datetime.utcnow().isoformat(),
"observation_time": obs_time,
"report_time": latest.get("reportTime"),
"receipt_time": latest.get("receiptTime"),
"obs_time_epoch": latest.get("obsTime"),
"current": {
"temp": round(temp, 1) if temp is not None else None,
"max_temp_so_far": round(max_so_far, 1) if max_so_far is not None else None,
"max_temp_time": max_temp_time,
"dewpoint": round(dewp, 1) if dewp is not None else None,
"humidity": latest.get("rh"),
"wind_speed_kt": latest.get("wspd"),
"wind_dir": latest.get("wdir"),
"visibility_mi": latest.get("visib"),
"wx_desc": latest.get("wxString"),
"altimeter": latest.get("altim"),
"raw_metar": latest.get("rawOb"),
"clouds": latest.get("clouds", []),
},
"recent_temps": recent_temps,
"today_obs": today_obs,
"recent_obs": recent_obs_raw,
"unit": unit,
}
logger.info(
f"✈️ METAR {icao}: {temp:.1f}°{'F' if use_fahrenheit else 'C'} (obs: {obs_time})"
)
with self._metar_cache_lock:
self._metar_cache[cache_key] = {"d": result, "t": now_ts}
return result
except requests.exceptions.RequestException as exc:
logger.error(f"METAR 请求失败 ({icao}): {exc}")
with self._metar_cache_lock:
stale = self._metar_cache.get(cache_key)
if stale:
logger.warning(f"METAR {icao} 请求失败,使用缓存回退")
return stale["d"]
return None
except (KeyError, IndexError, TypeError) as exc:
logger.error(f"METAR 数据解析失败 ({icao}): {exc}")
return None
def fetch_metar_nearby_cluster(self, icaos: List[str], use_fahrenheit: bool = False) -> list:
"""批量获取一组 ICAO 站点的 METAR 数据,用于地图周边显示。"""
if not icaos:
return []
results = []
try:
ids_str = ",".join(icaos)
url = f"https://aviationweather.gov/api/data/metar?ids={ids_str}&format=json"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}
resp = self.session.get(url, headers=headers, timeout=self.timeout)
if resp.status_code != 200:
logger.warning(f"METAR cluster fetch HTTP {resp.status_code} for {icaos}")
return []
data = resp.json()
if not isinstance(data, list):
return []
for obs in data:
icao = obs.get("icaoId")
lat = obs.get("lat")
lon = obs.get("lon")
temp_c = obs.get("temp")
if icao and lat and lon and temp_c is not None:
display_temp = (temp_c * 9 / 5) + 32 if use_fahrenheit else temp_c
name = obs.get("name") or icao
name = name.split(" Airport")[0].split(" Intl")[0].split(" International")[0].split(" Arpt")[0].split(",")[0].strip()
results.append(
{
"name": name,
"lat": lat,
"lon": lon,
"temp": round(display_temp, 1),
"istNo": icao,
"icao": icao,
"wind_dir": obs.get("wdir"),
"wind_speed": obs.get("wspd"),
"wind_speed_kt": obs.get("wspd"),
"raw_metar": obs.get("rawOb"),
}
)
if results:
logger.info(f"📍 METAR 集群: 成功抓取 {len(results)} 个参考站数据")
return results
except Exception as exc:
logger.error(f"Failed to fetch METAR cluster {icaos}: {exc}")
return []
+318
View File
@@ -0,0 +1,318 @@
from __future__ import annotations
from datetime import datetime, timedelta
from typing import Dict, Optional
from loguru import logger
class MgmSourceMixin:
def fetch_from_mgm(self, istno: str) -> Optional[Dict]:
"""
从土耳其气象局 (MGM) 获取实时数据和预测 (由用户提供其内部 API)
"""
base_url = "https://servis.mgm.gov.tr/web"
# 必须带 Origin,否则会被反爬拦截
headers = {
"Origin": "https://www.mgm.gov.tr",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}
results = {}
try:
# 1. 实时数据 (添加时间戳防止 CDN 缓存)
import time
obs_resp = self.session.get(
f"{base_url}/sondurumlar?istno={istno}&_={int(time.time() * 1000)}",
headers=headers,
timeout=self.timeout,
)
if obs_resp.status_code == 200:
data = obs_resp.json()
if data:
latest = data[0] if isinstance(data, list) else data
# MGM 数据字段映射
# ruzgarHiz 实测为 km/h,转为 m/s 需要除以 3.6
ruz_hiz_kmh = latest.get("ruzgarHiz", 0)
# MGM 返回 -9999 表示数据缺失,需要过滤
def _valid(v):
return v is not None and v > -9000
results["current"] = {
"temp": latest.get("sicaklik")
if _valid(latest.get("sicaklik"))
else None,
"feels_like": latest.get("hissedilenSicaklik")
if _valid(latest.get("hissedilenSicaklik"))
else None,
"humidity": latest.get("nem")
if _valid(latest.get("nem"))
else None,
"wind_speed_ms": round(ruz_hiz_kmh / 3.6, 1)
if _valid(ruz_hiz_kmh)
else None,
"wind_speed_kt": round(ruz_hiz_kmh / 1.852, 1)
if _valid(ruz_hiz_kmh)
else None,
"wind_dir": latest.get("ruzgarYon")
if _valid(latest.get("ruzgarYon"))
else None,
"rain_24h": latest.get("toplamYagis")
if _valid(latest.get("toplamYagis"))
else None,
"pressure": latest.get("aktuelBasinc")
if _valid(latest.get("aktuelBasinc"))
else None,
"cloud_cover": latest.get("kapalilik"), # 0-8 八分位云量
"mgm_max_temp": latest.get("maxSicaklik")
if _valid(latest.get("maxSicaklik"))
else None,
"time": latest.get("veriZamani"),
"station_name": latest.get("istasyonAd")
or latest.get("adi")
or latest.get("merkezAd")
or "Ankara Bölge",
}
# 2. 每日预报(尝试两个可能的 API 路径)
forecast_urls = [
f"{base_url}/tahminler/gunluk?istno={istno}",
f"https://servis.mgm.gov.tr/api/tahminler/gunluk?istno={istno}",
]
for forecast_url in forecast_urls:
try:
daily_resp = self.session.get(
forecast_url, headers=headers, timeout=self.timeout
)
if daily_resp.status_code == 200:
forecasts = daily_resp.json()
if forecasts and isinstance(forecasts, list):
# Store today extra clearly
today = forecasts[0]
high_val = today.get("enYuksekGun1")
low_val = today.get("enDusukGun1")
if high_val is not None:
results["today_high"] = high_val
results["today_low"] = low_val
logger.info(f"📋 MGM 每日预报: 今天的最高温 {high_val}°C")
# Store all 5 days for multi_model_daily
results["daily_forecasts"] = {}
for i, day in enumerate(forecasts[:5]):
d_high = day.get("enYuksekGun1")
if d_high is not None:
# Calculate date (today + offset)
target_date = (datetime.now() + timedelta(days=i)).strftime("%Y-%m-%d")
results["daily_forecasts"][target_date] = d_high
break
else:
logger.debug(
f"MGM forecast URL {forecast_url} returned {daily_resp.status_code}"
)
except Exception as e:
logger.debug(f"MGM forecast URL {forecast_url} failed: {e}")
# 3. 小时预报
try:
hourly_resp = self.session.get(
f"{base_url}/tahminler/saatlik?istno={istno}",
headers=headers,
timeout=self.timeout
)
if hourly_resp.status_code == 200:
h_data = hourly_resp.json()
if h_data and isinstance(h_data, list):
tahmin_list = h_data[0].get("tahmin", [])
results["hourly"] = []
for t_data in tahmin_list:
if "tarih" in t_data and "sicaklik" in t_data:
results["hourly"].append({
"time": t_data["tarih"],
"temp": t_data["sicaklik"]
})
except Exception as e:
logger.debug(f"MGM hourly failed: {e}")
# 4. Fallback for today_high (if daily forecast is missing it)
if "today_high" not in results:
# Try from current max
cur_max = results.get("current", {}).get("mgm_max_temp")
if cur_max is not None:
results["today_high"] = cur_max
logger.info(f"📋 MGM 每日预报: 使用当前测站最高温作为今日预报回退: {cur_max}°C")
elif "hourly" in results and results["hourly"]:
# Try from hourly
h_max = max((h["temp"] for h in results["hourly"] if h["temp"] is not None), default=None)
if h_max is not None:
results["today_high"] = h_max
logger.info(f"📋 MGM 每日预报: 使用小时预报最高温作为今日预报回退: {h_max}°C")
# 5. Fallback for daily_forecasts from hourly data
if not results.get("daily_forecasts") and results.get("hourly"):
# Guardrail: avoid treating short intraday snippets as full-day highs.
hourly_rows = results.get("hourly") or []
parsed_times = []
for h in hourly_rows:
t = str(h.get("time") or "")
if "T" not in t:
continue
try:
parsed_times.append(datetime.fromisoformat(t.replace("Z", "+00:00")))
except Exception:
continue
horizon_hours = 0.0
if len(parsed_times) >= 2:
parsed_times.sort()
horizon_hours = (
parsed_times[-1] - parsed_times[0]
).total_seconds() / 3600.0
if len(hourly_rows) >= 24 or horizon_hours >= 30:
from collections import defaultdict
daily_max = defaultdict(list)
for h in hourly_rows:
t = h.get("time", "")
temp = h.get("temp")
if t and temp is not None:
# Extract date from ISO timestamp like "2026-03-05T12:00:00.000Z"
date_str = t[:10]
daily_max[date_str].append(temp)
if daily_max:
results["daily_forecasts"] = {}
for d, temps in sorted(daily_max.items()):
results["daily_forecasts"][d] = max(temps)
logger.info(
f"📋 MGM daily_forecasts (from hourly fallback): "
f"{dict(results['daily_forecasts'])}"
)
else:
logger.info(
"📋 Skip MGM daily_forecasts hourly fallback: "
f"hourly points={len(hourly_rows)}, horizon={horizon_hours:.1f}h"
)
return results if "current" in results else None
except Exception as e:
logger.error(f"MGM API 请求失败 ({istno}): {e}")
return None
def fetch_mgm_nearby_stations(self, province: str, root_ist_no: str = None) -> list:
"""
获取一个土耳其省份内所有气象站的当前温度及经纬度
使用多线程辅助抓取因为直接通过 il={province} 往往只返回 1 个站
"""
base_url = "https://servis.mgm.gov.tr/web"
headers = {
"Origin": "https://www.mgm.gov.tr",
"User-Agent": "Mozilla/5.0",
}
import time
from concurrent.futures import ThreadPoolExecutor
results = []
try:
# 1. 加载测站元数据 (缓存到实例中),用于过滤属于该省份的站点
if not getattr(self, "mgm_stations_meta", None):
meta_resp = self.session.get(f"{base_url}/istasyonlar", headers=headers, timeout=self.timeout)
if meta_resp.status_code == 200:
meta_json = meta_resp.json()
if isinstance(meta_json, list):
self.mgm_stations_meta = {s["istNo"]: s for s in meta_json if "istNo" in s}
else:
self.mgm_stations_meta = {}
metadata = getattr(self, "mgm_stations_meta", {})
# 2. 找出属于该省份的所有站点 istNo
province_upper = province.upper()
province_ist_nos = [
ist_no for ist_no, s in metadata.items()
if (s.get("il") or "").upper() == province_upper
]
if not province_ist_nos:
logger.warning(f"MGM 找不到省份 {province} 的站点元数据")
return []
# 同时确保我们关心的几个核心站一定在里面
target_ist_nos = [str(i) for i in province_ist_nos[:25]]
# 17130: 安卡拉总站 (市区核心)
if 17130 in province_ist_nos or "17130" in province_ist_nos:
if "17130" not in target_ist_nos:
target_ist_nos.append("17130")
# 17128: 机场官方站
if 17128 in province_ist_nos or "17128" in province_ist_nos:
if "17128" not in target_ist_nos:
target_ist_nos.append("17128")
if root_ist_no:
rs = str(root_ist_no)
if rs not in target_ist_nos:
target_ist_nos.append(rs)
# 3. 多线程获取每个站点的最新观测 (sondurumlar)
def fetch_single_station(ist_no):
try:
# sondurumlar?istno={ist_no} 是目前最稳的获取多站数据的办法
url = f"{base_url}/sondurumlar?istno={ist_no}&_={int(time.time() * 1000)}"
resp = self.session.get(url, headers=headers, timeout=5)
if resp.status_code == 200:
obs_list = resp.json()
if obs_list:
obs = obs_list[0] if isinstance(obs_list, list) else obs_list
temp = obs.get("sicaklik")
wind_speed = obs.get("ruzgarHiz")
wind_dir = obs.get("ruzgarYon")
if temp is not None and temp > -9000:
return ist_no, {"temp": temp, "wind_speed": wind_speed, "wind_dir": wind_dir}
except Exception:
pass
return None, None
# 并发抓取
station_temps = {}
with ThreadPoolExecutor(max_workers=10) as executor:
fetch_results = list(executor.map(fetch_single_station, target_ist_nos))
for ist_no, data in fetch_results:
if ist_no is not None:
station_temps[ist_no] = data
# 4. 组装最终结果
for ist_no, temp in station_temps.items():
sid = str(ist_no)
# metadata 可能使用 int 或 str 作为 key
meta = metadata.get(sid) or metadata.get(int(sid))
if not meta:
continue
lat = meta.get("enlem")
lon = meta.get("boylam")
# 优先显示区县名,地图更清晰
display_name = (meta.get("ilce") or meta.get("istAd") or f"Station {ist_no}").title()
# 特殊处理核心站点的显示名称
sid = str(ist_no)
if sid == "17130":
display_name = "Ankara (Bölge/Center)"
elif sid == "17128":
display_name = "Airport (MGM/17128)"
results.append({
"name": display_name,
"lat": lat,
"lon": lon,
"temp": temp.get("temp") if isinstance(temp, dict) else temp,
"wind_speed": temp.get("wind_speed") if isinstance(temp, dict) else None,
"wind_dir": temp.get("wind_dir") if isinstance(temp, dict) else None,
"istNo": ist_no
})
logger.info(f"📍 MGM 周边测站: 成功并发抓取 {len(results)}{province} 站点的实时气温")
return results
except Exception as e:
logger.error(f"Failed to fetch MGM nearby stations for {province}: {e}")
return []
@@ -0,0 +1,578 @@
from __future__ import annotations
import time
from datetime import datetime, timedelta
from typing import Dict, Optional
from loguru import logger
class NwsOpenMeteoSourceMixin:
def fetch_nws(self, lat: float, lon: float) -> Optional[Dict]:
"""
NWS (美国国家气象局) 获取高精度预报
仅适用于美国城市全球 VPS 均可访问
"""
try:
# 1. 获取网格点
points_url = f"https://api.weather.gov/points/{lat},{lon}"
headers = {"User-Agent": "PolyWeather/1.0 (weather-bot)"}
points_resp = self.session.get(
points_url, headers=headers, timeout=self.timeout
)
points_resp.raise_for_status()
points_data = points_resp.json()
properties = points_data.get("properties", {})
forecast_url = properties.get("forecast")
hourly_url = properties.get("forecastHourly")
if not forecast_url:
return None
# 2. 获取预报
forecast_resp = self.session.get(
forecast_url, headers=headers, timeout=self.timeout
)
forecast_resp.raise_for_status()
forecast_data = forecast_resp.json()
periods = forecast_data.get("properties", {}).get("periods", [])
if not periods:
return None
hourly_periods = []
if hourly_url:
hourly_resp = self.session.get(
hourly_url, headers=headers, timeout=self.timeout
)
hourly_resp.raise_for_status()
hourly_data = hourly_resp.json()
hourly_periods = hourly_data.get("properties", {}).get("periods", [])[:48]
active_alerts = []
try:
alerts_resp = self.session.get(
"https://api.weather.gov/alerts/active",
params={"point": f"{lat},{lon}"},
headers=headers,
timeout=self.timeout,
)
alerts_resp.raise_for_status()
alerts_data = alerts_resp.json()
for feature in alerts_data.get("features", [])[:8]:
ap = feature.get("properties", {})
active_alerts.append(
{
"event": ap.get("event"),
"headline": ap.get("headline"),
"severity": ap.get("severity"),
"certainty": ap.get("certainty"),
"urgency": ap.get("urgency"),
"effective": ap.get("effective"),
"ends": ap.get("ends"),
}
)
except Exception:
active_alerts = []
# 3. 提取今日最高温(找 isDaytime=True 的第一个)
today_high = None
for p in periods:
if p.get("isDaytime") and "High" in p.get("name", ""):
today_high = p.get("temperature")
break
# 如果没有明确的 High,取第一个 daytime 的温度
if today_high is None:
for p in periods:
if p.get("isDaytime"):
today_high = p.get("temperature")
break
return {
"source": "nws",
"today_high": today_high,
"unit": "fahrenheit",
"forecast_periods": [
{
"name": p.get("name"),
"start_time": p.get("startTime"),
"end_time": p.get("endTime"),
"is_daytime": p.get("isDaytime"),
"temperature": p.get("temperature"),
"temperature_trend": p.get("temperatureTrend"),
"wind_speed": p.get("windSpeed"),
"wind_direction": p.get("windDirection"),
"short_forecast": p.get("shortForecast"),
"detailed_forecast": p.get("detailedForecast"),
"precipitation_probability": (p.get("probabilityOfPrecipitation") or {}).get("value"),
}
for p in periods[:14]
],
"hourly_periods": [
{
"start_time": p.get("startTime"),
"end_time": p.get("endTime"),
"temperature": p.get("temperature"),
"temperature_unit": p.get("temperatureUnit"),
"wind_speed": p.get("windSpeed"),
"wind_direction": p.get("windDirection"),
"short_forecast": p.get("shortForecast"),
"precipitation_probability": (p.get("probabilityOfPrecipitation") or {}).get("value"),
}
for p in hourly_periods
],
"active_alerts": active_alerts,
}
except Exception as e:
logger.warning(f"NWS 请求失败: {e}")
return None
def fetch_from_open_meteo(
self,
lat: float,
lon: float,
forecast_days: int = 14,
use_fahrenheit: bool = False,
) -> Optional[Dict]:
"""
Fetch weather from Open-Meteo with forecast data
Args:
lat: Latitude
lon: Longitude
forecast_days: Number of forecast days to fetch (default 14 to cover all market dates)
use_fahrenheit: Whether to return temperatures in Fahrenheit (for US markets)
"""
cache_key = (
f"{round(float(lat), 4)}:{round(float(lon), 4)}:"
f"{forecast_days}:{'f' if use_fahrenheit else 'c'}"
)
self._maybe_reload_open_meteo_disk_cache()
now_ts = time.time()
# ── 429 冷却期检查(所有 Open-Meteo 端点共享)─────────────────
with self._open_meteo_rl_lock:
if now_ts < self._open_meteo_rate_limit_until:
remaining = int(self._open_meteo_rate_limit_until - now_ts)
logger.debug(f"Open-Meteo 冷却期中,跳过请求,还需 {remaining}s")
with self._open_meteo_cache_lock:
stale = self._open_meteo_cache.get(cache_key)
if stale and isinstance(stale.get("data"), dict):
return dict(stale["data"])
return None
with self._open_meteo_cache_lock:
cached = self._open_meteo_cache.get(cache_key)
if (
cached
and now_ts - float(cached.get("t", 0)) < self.open_meteo_cache_ttl_sec
):
cached_data = cached.get("data")
if isinstance(cached_data, dict):
return dict(cached_data)
try:
url = "https://api.open-meteo.com/v1/forecast"
params = {
"latitude": lat,
"longitude": lon,
"current_weather": "true",
"hourly": "temperature_2m,shortwave_radiation,dew_point_2m,pressure_msl,wind_speed_10m,wind_direction_10m,precipitation_probability,cloud_cover",
"daily": "temperature_2m_max,apparent_temperature_max,sunrise,sunset,sunshine_duration",
"timezone": "auto",
"forecast_days": forecast_days,
}
# 显式指定单位,防止 API 默认行为漂移
if use_fahrenheit:
params["temperature_unit"] = "fahrenheit"
else:
params["temperature_unit"] = "celsius"
self._wait_open_meteo_slot("forecast")
response = self.session.get(
url,
params=params,
timeout=self.timeout,
)
response.raise_for_status()
data = response.json()
current = data.get("current_weather", {})
utc_offset = data.get("utc_offset_seconds", 0)
timezone_name = data.get("timezone", "UTC")
# 处理多模型数据 (如果请求了 models 参数,返回结构会变化)
daily_data = data.get("daily", {})
if "temperature_2m_max_ecmwf_ifs04" in daily_data:
ecmwf_max = daily_data.get("temperature_2m_max_ecmwf_ifs04", [])
hrrr_max = daily_data.get("temperature_2m_max_ncep_hrrr_conus", [])
# 记录今日模型分歧
daily_data["model_split"] = {
"ecmwf": ecmwf_max[0] if ecmwf_max else None,
"hrrr": hrrr_max[0] if hrrr_max else None,
}
# 智能合并:HRRR 仅覆盖 48 小时,远期用 ECMWF 补全
merged_max = []
for i in range(len(ecmwf_max)):
hrrr_val = hrrr_max[i] if i < len(hrrr_max) else None
ecmwf_val = ecmwf_max[i] if i < len(ecmwf_max) else None
# 优先 HRRR,其次 ECMWF,都没有就跳过
if hrrr_val is not None:
merged_max.append(hrrr_val)
elif ecmwf_val is not None:
merged_max.append(ecmwf_val)
else:
# 两个都没有,用占位符 (理论上不应该发生)
merged_max.append(ecmwf_val) # None
daily_data["temperature_2m_max"] = merged_max
# 映射逐小时数据
hourly_data = data.get("hourly", {})
if "temperature_2m_ncep_hrrr_conus" in hourly_data:
hourly_data["temperature_2m"] = hourly_data[
"temperature_2m_ncep_hrrr_conus"
]
# 计算精确的当地时间
now_utc = datetime.utcnow()
local_now = now_utc + timedelta(seconds=utc_offset)
local_time_str = local_now.strftime("%Y-%m-%d %H:%M")
result = {
"source": "open-meteo",
"timestamp": now_utc.isoformat(),
"timezone": timezone_name,
"utc_offset": utc_offset,
"current": {
"temp": current.get("temperature"),
"local_time": local_time_str,
},
"hourly": hourly_data,
"daily": daily_data,
"unit": "fahrenheit" if use_fahrenheit else "celsius",
}
with self._open_meteo_cache_lock:
self._open_meteo_cache[cache_key] = {
"t": time.time(),
"data": dict(result),
}
self._flush_open_meteo_disk_cache()
return result
except Exception as e:
status_code = getattr(getattr(e, "response", None), "status_code", None)
if status_code == 429:
retry_after_str = getattr(e.response, "headers", {}).get("Retry-After")
cooldown_to_use = self._open_meteo_rl_cooldown
if retry_after_str:
try:
parsed = int(retry_after_str)
if parsed > 0:
cooldown_to_use = min(parsed + 60, 3600) # Add 60s buffer, max 1 hour
logger.info(f"Open-Meteo 响应包含 Retry-After: {retry_after_str}s")
except ValueError:
pass
logger.warning(
f"Open-Meteo rate limited (429), fallback to cache if available: lat={lat}, lon={lon}"
)
# 设置全局冷却期,避免短时内重复触发 429
with self._open_meteo_rl_lock:
self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
logger.warning(f"Open-Meteo 触发限流,设置 {cooldown_to_use}s 冷却期")
else:
logger.error(f"Open-Meteo forecast failed: {e}")
with self._open_meteo_cache_lock:
stale = self._open_meteo_cache.get(cache_key)
if stale and isinstance(stale.get("data"), dict):
fallback = dict(stale["data"])
fallback["stale_cache"] = True
return fallback
return None
def fetch_ensemble(
self,
lat: float,
lon: float,
use_fahrenheit: bool = False,
) -> Optional[Dict]:
"""
Open-Meteo Ensemble API 获取 51 成员集合预报
用于计算预报不确定性范围散度
"""
cache_key = (
f"{round(float(lat), 4)}:{round(float(lon), 4)}:"
f"{'f' if use_fahrenheit else 'c'}"
)
self._maybe_reload_open_meteo_disk_cache()
now_ts = time.time()
# ── 429 冷却期检查(所有 Open-Meteo 端点共享)─────────────────
with self._open_meteo_rl_lock:
if now_ts < self._open_meteo_rate_limit_until:
remaining = int(self._open_meteo_rate_limit_until - now_ts)
logger.debug(f"Open-Meteo Ensemble 冷却期中,跳过请求,还需 {remaining}s")
with self._ensemble_cache_lock:
stale = self._ensemble_cache.get(cache_key)
if stale and isinstance(stale.get("data"), dict):
return dict(stale["data"])
return None
with self._ensemble_cache_lock:
cached = self._ensemble_cache.get(cache_key)
if (
cached
and now_ts - float(cached.get("t", 0))
< self.open_meteo_ensemble_cache_ttl_sec
):
cached_data = cached.get("data")
if isinstance(cached_data, dict):
return dict(cached_data)
try:
url = "https://ensemble-api.open-meteo.com/v1/ensemble"
params = {
"latitude": lat,
"longitude": lon,
"daily": "temperature_2m_max",
"timezone": "auto",
"forecast_days": 3,
}
if use_fahrenheit:
params["temperature_unit"] = "fahrenheit"
else:
params["temperature_unit"] = "celsius"
self._wait_open_meteo_slot("ensemble")
response = self.session.get(
url,
params=params,
timeout=self.timeout,
)
response.raise_for_status()
data = response.json()
daily = data.get("daily", {})
# 每个成员都会返回一组 temperature_2m_max
# 格式: {"time": [...], "temperature_2m_max_member01": [...], ...}
today_highs = []
for key, values in daily.items():
if key.startswith("temperature_2m_max") and key != "temperature_2m_max":
if values and values[0] is not None:
today_highs.append(values[0])
# 也检查非成员键(有些返回格式不同)
if not today_highs:
raw_max = daily.get("temperature_2m_max", [])
if isinstance(raw_max, list) and raw_max:
if isinstance(raw_max[0], list):
# 嵌套列表格式: [[member1_day1, member1_day2], [member2_day1, ...]]
today_highs = [m[0] for m in raw_max if m and m[0] is not None]
elif raw_max[0] is not None:
today_highs = [raw_max[0]]
if len(today_highs) < 3:
logger.warning(f"Ensemble 数据不足: 仅获取 {len(today_highs)} 个成员")
return None
today_highs.sort()
n = len(today_highs)
median = today_highs[n // 2]
p10 = today_highs[max(0, int(n * 0.1))]
p90 = today_highs[min(n - 1, int(n * 0.9))]
result = {
"source": "ensemble",
"members": n,
"median": round(median, 1),
"p10": round(p10, 1),
"p90": round(p90, 1),
"min": round(today_highs[0], 1),
"max": round(today_highs[-1], 1),
"unit": "fahrenheit" if use_fahrenheit else "celsius",
}
logger.info(
f"📊 Ensemble ({n} members): median={median:.1f}, "
f"p10={p10:.1f}, p90={p90:.1f}"
)
with self._ensemble_cache_lock:
self._ensemble_cache[cache_key] = {
"t": time.time(),
"data": dict(result),
}
self._flush_open_meteo_disk_cache()
return result
except Exception as e:
status_code = getattr(getattr(e, "response", None), "status_code", None)
if status_code == 429:
retry_after_str = getattr(e.response, "headers", {}).get("Retry-After")
cooldown_to_use = self._open_meteo_rl_cooldown
if retry_after_str:
try:
parsed = int(retry_after_str)
if parsed > 0:
cooldown_to_use = min(parsed + 60, 3600)
except ValueError:
pass
logger.warning(
f"Ensemble API rate limited (429), fallback to cache if available: lat={lat}, lon={lon}"
)
with self._open_meteo_rl_lock:
self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
else:
logger.warning(f"Ensemble API 请求失败: {e}")
with self._ensemble_cache_lock:
stale = self._ensemble_cache.get(cache_key)
if stale and isinstance(stale.get("data"), dict):
fallback = dict(stale["data"])
fallback["stale_cache"] = True
return fallback
return None
def fetch_multi_model(
self,
lat: float,
lon: float,
city: str = "",
use_fahrenheit: bool = False,
) -> Optional[Dict]:
"""
Open-Meteo 获取多个独立 NWP 模型的预报
用于真正的多模型共识评分
模型列表:
- ECMWF IFS (欧洲中期天气预报中心)
- GFS (美国 NOAA)
- ICON (德国气象局 DWD)
- GEM (加拿大气象局)
- JMA (日本气象厅)
返回 3 天的预报数据支持今日+明日共识分析
"""
cache_city = str(city or "").strip().lower()
cache_key = (
f"{round(float(lat), 4)}:{round(float(lon), 4)}:{cache_city}:"
f"{'f' if use_fahrenheit else 'c'}:{self.multi_model_cache_version}"
)
self._maybe_reload_open_meteo_disk_cache()
now_ts = time.time()
# ── 429 冷却期检查(所有 Open-Meteo 端点共享)─────────────────
with self._open_meteo_rl_lock:
if now_ts < self._open_meteo_rate_limit_until:
remaining = int(self._open_meteo_rate_limit_until - now_ts)
logger.debug(f"Open-Meteo Multi-model 冷却期中,跳过请求,还需 {remaining}s")
with self._multi_model_cache_lock:
stale = self._multi_model_cache.get(cache_key)
if stale and isinstance(stale.get("data"), dict):
return dict(stale["data"])
return None
with self._multi_model_cache_lock:
cached = self._multi_model_cache.get(cache_key)
if (
cached
and now_ts - float(cached.get("t", 0))
< self.open_meteo_multi_model_cache_ttl_sec
):
cached_data = cached.get("data")
if isinstance(cached_data, dict):
return dict(cached_data)
try:
url = "https://api.open-meteo.com/v1/forecast"
models = "ecmwf_ifs025,gfs_seamless,icon_seamless,gem_seamless,jma_seamless"
params = {
"latitude": lat,
"longitude": lon,
"daily": "temperature_2m_max",
"models": models,
"timezone": "auto",
"forecast_days": 3,
}
if use_fahrenheit:
params["temperature_unit"] = "fahrenheit"
self._wait_open_meteo_slot("multi-model")
response = self.session.get(
url,
params=params,
timeout=self.timeout,
)
response.raise_for_status()
data = response.json()
daily = data.get("daily", {})
dates = daily.get("time", [])
model_labels = {
"ecmwf_ifs025": "ECMWF",
"gfs_seamless": "GFS",
"icon_seamless": "ICON",
"gem_seamless": "GEM",
"jma_seamless": "JMA",
}
# 按天提取每个模型的预报
daily_forecasts = {} # {"2026-02-23": {"ECMWF": 7.9, "GFS": 6.5, ...}, ...}
for day_idx, date_str in enumerate(dates):
day_data = {}
for model_key, label in model_labels.items():
key = f"temperature_2m_max_{model_key}"
values = daily.get(key, [])
if day_idx < len(values) and values[day_idx] is not None:
day_data[label] = round(values[day_idx], 1)
if day_data:
daily_forecasts[date_str] = day_data
if not daily_forecasts:
logger.warning("Multi-model: 无有效模型数据")
return None
# 今天的预报 (向后兼容)
today_date = dates[0] if dates else None
forecasts = daily_forecasts.get(today_date, {})
labels_str = ", ".join([f"{k}={v}" for k, v in forecasts.items()])
logger.info(
f"🔬 Multi-model ({len(forecasts)}个, {len(daily_forecasts)}天): {labels_str}"
)
result = {
"source": "multi_model",
"forecasts": forecasts, # 今天 {"ECMWF": 12.3, "GFS": 11.8, ...} (向后兼容)
"daily_forecasts": daily_forecasts, # 按天 {"2026-02-23": {...}, "2026-02-24": {...}}
"dates": dates,
"unit": "fahrenheit" if use_fahrenheit else "celsius",
}
with self._multi_model_cache_lock:
self._multi_model_cache[cache_key] = {
"t": time.time(),
"data": dict(result),
}
self._flush_open_meteo_disk_cache()
return result
except Exception as e:
status_code = getattr(getattr(e, "response", None), "status_code", None)
if status_code == 429:
retry_after_str = getattr(e.response, "headers", {}).get("Retry-After")
cooldown_to_use = self._open_meteo_rl_cooldown
if retry_after_str:
try:
parsed = int(retry_after_str)
if parsed > 0:
cooldown_to_use = min(parsed + 60, 3600)
except ValueError:
pass
logger.warning(
f"Multi-model API rate limited (429), fallback to cache if available: lat={lat}, lon={lon}"
)
with self._open_meteo_rl_lock:
self._open_meteo_rate_limit_until = time.time() + cooldown_to_use
else:
logger.warning(f"Multi-model API 请求失败: {e}")
with self._multi_model_cache_lock:
stale = self._multi_model_cache.get(cache_key)
if stale and isinstance(stale.get("data"), dict):
fallback = dict(stale["data"])
fallback["stale_cache"] = True
return fallback
return None
+116
View File
@@ -0,0 +1,116 @@
from __future__ import annotations
import json
import os
import time
from loguru import logger
class OpenMeteoCacheMixin:
def _load_open_meteo_disk_cache(self) -> None:
"""启动时从磁盘加载 Open-Meteo 三类缓存,避免重启后冷启动打爆 API"""
try:
path = self._disk_cache_path
if not os.path.exists(path):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(
{
"forecast": {},
"ensemble": {},
"multi_model": {},
"saved_at": time.time(),
},
f,
)
self._disk_cache_last_mtime = os.path.getmtime(path)
return
current_mtime = os.path.getmtime(path)
if current_mtime <= self._disk_cache_last_mtime:
return
with open(path, "r", encoding="utf-8") as f:
saved = json.load(f)
now = time.time()
max_age = max(600, self._disk_cache_max_age_sec)
loaded = 0
with self._open_meteo_cache_lock:
for key, entry in saved.get("forecast", {}).items():
if now - float(entry.get("t", 0)) < max_age:
old = self._open_meteo_cache.get(key)
if old is None or float(entry.get("t", 0)) >= float(old.get("t", 0)):
self._open_meteo_cache[key] = entry
loaded += 1
with self._ensemble_cache_lock:
for key, entry in saved.get("ensemble", {}).items():
if now - float(entry.get("t", 0)) < max_age:
old = self._ensemble_cache.get(key)
if old is None or float(entry.get("t", 0)) >= float(old.get("t", 0)):
self._ensemble_cache[key] = entry
loaded += 1
with self._multi_model_cache_lock:
for key, entry in saved.get("multi_model", {}).items():
if now - float(entry.get("t", 0)) < max_age:
old = self._multi_model_cache.get(key)
if old is None or float(entry.get("t", 0)) >= float(old.get("t", 0)):
self._multi_model_cache[key] = entry
loaded += 1
self._disk_cache_last_mtime = current_mtime
if loaded:
logger.info(f"✅ 从磁盘加载 Open-Meteo 缓存 {loaded} 条 ({self._disk_cache_path})")
except Exception as exc:
logger.warning(f"磁盘缓存加载失败(首次启动不影响运行): {exc}")
def _maybe_reload_open_meteo_disk_cache(self) -> None:
"""跨进程共享缓存:当缓存文件有更新时增量重载到当前进程内存"""
try:
path = self._disk_cache_path
if not os.path.exists(path):
return
current_mtime = os.path.getmtime(path)
if current_mtime <= self._disk_cache_last_mtime:
return
self._load_open_meteo_disk_cache()
except Exception:
pass
def _flush_open_meteo_disk_cache(self) -> None:
"""将三类 Open-Meteo 内存缓存持久化到磁盘"""
try:
os.makedirs(os.path.dirname(self._disk_cache_path), exist_ok=True)
with self._open_meteo_cache_lock:
forecast_snapshot = dict(self._open_meteo_cache)
with self._ensemble_cache_lock:
ensemble_snapshot = dict(self._ensemble_cache)
with self._multi_model_cache_lock:
multi_model_snapshot = dict(self._multi_model_cache)
payload = {
"forecast": forecast_snapshot,
"ensemble": ensemble_snapshot,
"multi_model": multi_model_snapshot,
"saved_at": time.time(),
}
with self._disk_cache_lock:
tmp_path = self._disk_cache_path + ".tmp"
with open(tmp_path, "w", encoding="utf-8") as f:
json.dump(payload, f)
os.replace(tmp_path, self._disk_cache_path)
self._disk_cache_last_mtime = os.path.getmtime(self._disk_cache_path)
except Exception as exc:
logger.warning(f"磁盘缓存写入失败: {exc}")
def _wait_open_meteo_slot(self, endpoint: str) -> None:
"""Simple per-process rate gate for Open-Meteo endpoints."""
min_interval = self._open_meteo_min_interval_sec
if min_interval <= 0:
return
with self._open_meteo_call_lock:
now_ts = time.time()
wait_for = min_interval - (now_ts - self._open_meteo_last_call_ts)
if wait_for > 0:
logger.debug(
f"Open-Meteo {endpoint} 限流保护:sleep {wait_for:.2f}s (min_interval={min_interval:.2f}s)"
)
time.sleep(wait_for)
now_ts = time.time()
self._open_meteo_last_call_ts = now_ts
+269
View File
@@ -0,0 +1,269 @@
from __future__ import annotations
import csv
import time
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from loguru import logger
class SettlementSourceMixin:
def _get_settlement_cache(self, key: str) -> Optional[Dict[str, Any]]:
now_ts = time.time()
with self._settlement_cache_lock:
cached = self._settlement_cache.get(key)
if cached and now_ts - float(cached.get("t", 0)) < self.settlement_cache_ttl_sec:
return cached.get("d")
return None
def _set_settlement_cache(self, key: str, payload: Dict[str, Any]) -> None:
with self._settlement_cache_lock:
self._settlement_cache[key] = {"t": time.time(), "d": payload}
@staticmethod
def _csv_rows(text: str) -> List[Dict[str, str]]:
normalized = str(text or "").lstrip("")
if not normalized.strip():
return []
return [row for row in csv.DictReader(normalized.splitlines()) if isinstance(row, dict)]
@staticmethod
def _hko_parse_local_iso(raw_yyyymmddhhmm: Optional[str]) -> Optional[str]:
raw = str(raw_yyyymmddhhmm or "").strip()
if len(raw) != 12 or not raw.isdigit():
return None
try:
dt = datetime.strptime(raw, "%Y%m%d%H%M").replace(
tzinfo=timezone(timedelta(hours=8))
)
return dt.isoformat()
except Exception:
return None
@staticmethod
def _hko_compass_to_deg(compass: Optional[str]) -> Optional[float]:
value = str(compass or "").strip().upper()
if not value:
return None
mapping = {
"N": 0.0,
"NNE": 22.5,
"NE": 45.0,
"ENE": 67.5,
"E": 90.0,
"ESE": 112.5,
"SE": 135.0,
"SSE": 157.5,
"S": 180.0,
"SSW": 202.5,
"SW": 225.0,
"WSW": 247.5,
"W": 270.0,
"WNW": 292.5,
"NW": 315.0,
"NNW": 337.5,
}
return mapping.get(value)
@staticmethod
def _safe_float(value: Any) -> Optional[float]:
if value is None:
return None
text = str(value).strip()
if not text or text in {"***", "N/A", "NA"}:
return None
try:
return float(text)
except Exception:
return None
@staticmethod
def _pick_station_row(
rows: List[Dict[str, str]], candidates: List[str]
) -> Optional[Dict[str, str]]:
if not rows:
return None
normalized_map = {
str(name).strip().lower(): row
for row in rows
for name in [row.get("Automatic Weather Station")]
if isinstance(row, dict) and name
}
for name in candidates:
hit = normalized_map.get(str(name).strip().lower())
if hit:
return hit
for row in rows:
station = str(row.get("Automatic Weather Station") or "").strip().lower()
if "observatory" in station:
return row
return rows[0] if rows else None
def fetch_hko_settlement_current(self) -> Optional[Dict[str, Any]]:
cache_key = "hko:hong_kong"
cached = self._get_settlement_cache(cache_key)
if cached:
return cached
try:
base = "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
temp_csv = self.session.get(f"{base}/latest_1min_temperature.csv", timeout=self.timeout)
temp_csv.raise_for_status()
maxmin_csv = self.session.get(f"{base}/latest_since_midnight_maxmin.csv", timeout=self.timeout)
maxmin_csv.raise_for_status()
humidity_csv = self.session.get(f"{base}/latest_1min_humidity.csv", timeout=self.timeout)
humidity_csv.raise_for_status()
wind_csv = self.session.get(f"{base}/latest_10min_wind.csv", timeout=self.timeout)
wind_csv.raise_for_status()
temp_rows = self._csv_rows(temp_csv.text)
maxmin_rows = self._csv_rows(maxmin_csv.text)
humidity_rows = self._csv_rows(humidity_csv.text)
wind_rows = self._csv_rows(wind_csv.text)
station_candidates = ["HK Observatory", "Hong Kong Observatory"]
temp_row = self._pick_station_row(temp_rows, station_candidates)
maxmin_row = self._pick_station_row(maxmin_rows, station_candidates)
humidity_row = self._pick_station_row(humidity_rows, station_candidates)
wind_row = self._pick_station_row(wind_rows, station_candidates)
if not temp_row or not maxmin_row:
return None
obs_raw = temp_row.get("Date time") or maxmin_row.get("Date time")
obs_iso = self._hko_parse_local_iso(obs_raw)
current_temp = self._safe_float(temp_row.get("Air Temperature(degree Celsius)"))
max_so_far = self._safe_float(maxmin_row.get("Maximum Air Temperature Since Midnight(degree Celsius)"))
min_so_far = self._safe_float(maxmin_row.get("Minimum Air Temperature Since Midnight(degree Celsius)"))
humidity = self._safe_float(humidity_row.get("Relative Humidity(percent)")) if humidity_row else None
wind_speed_kmh = self._safe_float(wind_row.get("10-Minute Mean Speed(km/hour)")) if wind_row else None
wind_speed_kt = round(float(wind_speed_kmh) / 1.852, 1) if wind_speed_kmh is not None else None
wind_dir = self._hko_compass_to_deg(
wind_row.get("10-Minute Mean Wind Direction(Compass points)") if wind_row else None
)
payload: Dict[str, Any] = {
"source": "hko",
"source_label": "HKO",
"station_code": "HKO",
"station_name": "HK Observatory",
"observation_time": obs_iso,
"current": {
"temp": round(current_temp, 1) if current_temp is not None else None,
"max_temp_so_far": round(max_so_far, 1) if max_so_far is not None else None,
"max_temp_time": None,
"today_low": round(min_so_far, 1) if min_so_far is not None else None,
"humidity": round(humidity, 1) if humidity is not None else None,
"wind_speed_kt": wind_speed_kt,
"wind_dir": wind_dir,
},
"unit": "celsius",
}
self._set_settlement_cache(cache_key, payload)
return payload
except Exception as exc:
logger.warning(f"HKO settlement fetch failed: {exc}")
return None
def fetch_cwa_taipei_settlement_current(self) -> Optional[Dict[str, Any]]:
cache_key = "cwa:taipei:466920"
cached = self._get_settlement_cache(cache_key)
if cached:
return cached
try:
url = "https://opendata.cwa.gov.tw/api/v1/rest/datastore/O-A0003-001"
response = self.session.get(
url,
params={"Authorization": self.cwa_open_data_auth, "format": "JSON", "StationId": "466920"},
timeout=self.timeout,
)
response.raise_for_status()
data = response.json() if response.content else {}
stations = (data.get("records") or {}).get("Station") or []
if isinstance(stations, dict):
station = stations
elif isinstance(stations, list) and stations:
station = stations[0]
else:
station = None
if not isinstance(station, dict):
return None
wx = station.get("WeatherElement") or {}
if not isinstance(wx, dict):
wx = {}
daily_extreme = wx.get("DailyExtreme") or {}
high_info = (((daily_extreme.get("DailyHigh") or {}).get("TemperatureInfo") or {}))
low_info = (((daily_extreme.get("DailyLow") or {}).get("TemperatureInfo") or {}))
high_time_raw = (((high_info.get("Occurred_at") or {}).get("DateTime")))
high_hhmm = None
if high_time_raw and "T" in str(high_time_raw):
try:
high_hhmm = datetime.fromisoformat(str(high_time_raw)).strftime("%H:%M")
except Exception:
high_hhmm = str(high_time_raw).split("T")[1][:5]
obs_time_raw = (station.get("ObsTime") or {}).get("DateTime")
wind_speed_ms = self._safe_float(wx.get("WindSpeed"))
payload: Dict[str, Any] = {
"source": "cwa",
"source_label": "CWA",
"station_code": str(station.get("StationId") or "466920"),
"station_name": str(station.get("StationName") or "臺北"),
"observation_time": str(obs_time_raw or "").strip() or None,
"current": {
"temp": self._safe_float(wx.get("AirTemperature")),
"max_temp_so_far": self._safe_float(high_info.get("AirTemperature")),
"max_temp_time": high_hhmm,
"today_low": self._safe_float(low_info.get("AirTemperature")),
"humidity": self._safe_float(wx.get("RelativeHumidity")),
"wind_speed_kt": round(float(wind_speed_ms) * 1.943844, 1) if wind_speed_ms is not None else None,
"wind_dir": self._safe_float(wx.get("WindDirection")),
},
"unit": "celsius",
}
self._set_settlement_cache(cache_key, payload)
return payload
except Exception as exc:
logger.warning(f"CWA settlement fetch failed: {exc}")
return None
def fetch_hko_forecast(self) -> Optional[float]:
try:
url = "https://data.weather.gov.hk/weatherAPI/opendata/weather.php?dataType=fnd&lang=tc"
res = self.session.get(url, timeout=self.timeout).json()
return float(res["weatherForecast"][0]["forecastMaxtemp"]["value"])
except Exception as exc:
logger.warning(f"HKO Forecast request failed: {exc}")
return None
def fetch_cwa_taipei_forecast(self) -> Optional[float]:
try:
if not self.cwa_open_data_auth:
return None
url = "https://opendata.cwa.gov.tw/api/v1/rest/datastore/F-D0047-061"
res = self.session.get(
url,
params={"Authorization": self.cwa_open_data_auth, "format": "JSON", "elementName": "MaxT"},
timeout=self.timeout,
).json()
locs = res.get("records", {}).get("Locations", [])[0].get("Location", [])
if not locs:
return None
loc = locs[0]
for weather_element in loc.get("WeatherElement", []):
if weather_element.get("ElementName") == "MaxT":
return float(weather_element["Time"][0]["ElementValue"][0]["Temperature"])
return None
except Exception as exc:
logger.warning(f"CWA Forecast request failed: {exc}")
return None
def fetch_settlement_current(self, city: str) -> Optional[Dict[str, Any]]:
normalized = str(city or "").strip().lower()
if normalized == "hong kong":
return self.fetch_hko_settlement_current()
if normalized == "taipei":
return self.fetch_cwa_taipei_settlement_current()
return None
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -33,8 +33,9 @@ def test_basic_handler_diag_returns_html():
runtime = RuntimeStatus(
started_at="2026-03-12 00:00:00 UTC",
loops=[],
entitlement_enabled=False,
command_access_mode="group_member",
protected_commands=["/city", "/deb"],
required_group_chat_id="-1001234567890",
)
bot = DummyBot()
io_layer = SimpleNamespace(
@@ -52,4 +53,3 @@ def test_basic_handler_diag_returns_html():
assert len(bot.replies) == 1
assert bot.replies[0]["parse_mode"] == "HTML"
assert "Bot 启动诊断" in bot.replies[0]["text"]
+12 -21
View File
@@ -2,15 +2,6 @@ from types import SimpleNamespace
from unittest.mock import Mock
from src.bot.command_guard import CommandGuard
from src.bot.services.entitlement_service import BotEntitlementService
class _FakeDB:
def __init__(self, user):
self._user = user
def get_user(self, _user_id):
return self._user
def _message():
@@ -20,11 +11,13 @@ def _message():
)
def test_guard_blocks_non_premium_when_entitlement_enabled():
fake_bot = SimpleNamespace(reply_to=Mock())
def test_guard_blocks_non_member_before_points_charge():
fake_bot = SimpleNamespace(
reply_to=Mock(),
get_chat_member=Mock(return_value=SimpleNamespace(status="left")),
)
io_layer = SimpleNamespace(bot=fake_bot, ensure_query_points=Mock(return_value=True))
entitlement = BotEntitlementService(db=_FakeDB(user=None), enabled=True)
guard = CommandGuard(io_layer=io_layer, entitlement_service=entitlement)
guard = CommandGuard(io_layer=io_layer, group_chat_id="-100123")
ok = guard.ensure_access_and_points(_message(), 1, "/city")
@@ -33,18 +26,16 @@ def test_guard_blocks_non_premium_when_entitlement_enabled():
assert fake_bot.reply_to.call_count == 1
def test_guard_allows_premium_then_charges_points():
fake_bot = SimpleNamespace(reply_to=Mock())
io_layer = SimpleNamespace(bot=fake_bot, ensure_query_points=Mock(return_value=True))
entitlement = BotEntitlementService(
db=_FakeDB(user={"is_web_premium": 1, "is_group_premium": 0}),
enabled=True,
def test_guard_allows_member_then_charges_points():
fake_bot = SimpleNamespace(
reply_to=Mock(),
get_chat_member=Mock(return_value=SimpleNamespace(status="member")),
)
guard = CommandGuard(io_layer=io_layer, entitlement_service=entitlement)
io_layer = SimpleNamespace(bot=fake_bot, ensure_query_points=Mock(return_value=True))
guard = CommandGuard(io_layer=io_layer, group_chat_id="-100123")
ok = guard.ensure_access_and_points(_message(), 1, "/city")
assert ok is True
assert io_layer.ensure_query_points.call_count == 1
assert fake_bot.reply_to.call_count == 0
+40 -49
View File
@@ -7,92 +7,83 @@ from src.bot.services.city_command_service import CityReportResult, CityResolveR
from src.bot.services.deb_command_service import DebReportResult
class DummyBot:
def __init__(self):
self.replies = []
self.sent = []
def reply_to(self, message, text, parse_mode=None):
self.replies.append(
{
"chat_id": message.chat.id,
"text": text,
"parse_mode": parse_mode,
}
)
def send_message(self, chat_id, text, parse_mode=None):
self.sent.append(
{
"chat_id": chat_id,
"text": text,
"parse_mode": parse_mode,
}
)
def message_handler(self, *args, **kwargs): # pragma: no cover - decorator stub
def _decorator(func):
return func
return _decorator
def _message(text: str):
return SimpleNamespace(
text=text,
from_user=SimpleNamespace(id=123, username="tester", first_name="Tester"),
chat=SimpleNamespace(id=999),
entities=[],
)
def test_city_handler_missing_city_shows_usage():
bot = DummyBot()
bot = SimpleNamespace()
guard = SimpleNamespace(ensure_access_and_points=Mock(return_value=True))
city_service = SimpleNamespace(
resolve_city=Mock(),
build_report=Mock(),
city_service = SimpleNamespace(resolve_city=Mock(), build_report=Mock())
io_layer = SimpleNamespace(send_query_message=Mock())
handler = CityCommandHandler(
bot=bot,
guard=guard,
city_service=city_service,
io_layer=io_layer,
)
handler = CityCommandHandler(bot=bot, guard=guard, city_service=city_service)
handler.handle(_message("/city"))
assert len(bot.replies) == 1
assert "请输入城市名称" in bot.replies[0]["text"]
assert io_layer.send_query_message.call_count == 1
assert "请输入城市名称" in io_layer.send_query_message.call_args[0][1]
assert guard.ensure_access_and_points.call_count == 0
def test_city_handler_happy_path_pushes_progress_and_report():
bot = DummyBot()
bot = SimpleNamespace()
guard = SimpleNamespace(ensure_access_and_points=Mock(return_value=True))
city_service = SimpleNamespace(
resolve_city=Mock(
return_value=CityResolveResult(ok=True, city_name="london", supported_cities=["london"])
return_value=CityResolveResult(
ok=True,
city_name="london",
supported_cities=["london"],
)
),
build_report=Mock(return_value=CityReportResult(ok=True, report="CITY_REPORT")),
)
handler = CityCommandHandler(bot=bot, guard=guard, city_service=city_service)
io_layer = SimpleNamespace(send_query_message=Mock())
handler = CityCommandHandler(
bot=bot,
guard=guard,
city_service=city_service,
io_layer=io_layer,
)
handler.handle(_message("/city london"))
assert len(bot.sent) == 2
assert "正在查询 London 的天气数据" in bot.sent[0]["text"]
assert bot.sent[1]["text"] == "CITY_REPORT"
assert bot.sent[1]["parse_mode"] == "HTML"
assert io_layer.send_query_message.call_count == 2
first_call = io_layer.send_query_message.call_args_list[0]
second_call = io_layer.send_query_message.call_args_list[1]
assert "正在查询 London 的天气数据" in first_call.args[1]
assert second_call.args[1] == "CITY_REPORT"
assert second_call.kwargs["parse_mode"] == "HTML"
def test_deb_handler_history_missing_returns_hint():
bot = DummyBot()
bot = SimpleNamespace()
guard = SimpleNamespace(ensure_access_and_points=Mock(return_value=True))
deb_service = SimpleNamespace(
resolve_city=Mock(return_value="ankara"),
has_history=Mock(return_value=False),
build_report=Mock(return_value=DebReportResult(ok=True, report="DEB_REPORT")),
)
handler = DebCommandHandler(bot=bot, guard=guard, deb_service=deb_service)
io_layer = SimpleNamespace(send_query_message=Mock())
handler = DebCommandHandler(
bot=bot,
guard=guard,
deb_service=deb_service,
io_layer=io_layer,
)
handler.handle(_message("/deb ankara"))
assert len(bot.replies) == 1
assert "暂无 ankara 的历史数据" in bot.replies[0]["text"]
assert io_layer.send_query_message.call_count == 1
assert "暂无 ankara 的历史数据" in io_layer.send_query_message.call_args[0][1]
assert guard.ensure_access_and_points.call_count == 0
+5 -4
View File
@@ -14,8 +14,9 @@ def test_startup_coordinator_respects_disable_flags(monkeypatch):
coordinator = StartupCoordinator(
bot=DummyBot(),
config={},
entitlement_enabled=False,
command_access_mode="group_member",
protected_commands=["/city", "/deb"],
required_group_chat_id="-1001234567890",
)
runtime = coordinator.start_all()
loop_map = runtime.loop_map()
@@ -30,13 +31,13 @@ def test_startup_coordinator_respects_disable_flags(monkeypatch):
def test_render_runtime_status_html_contains_key_fields():
runtime = RuntimeStatus(
started_at="2026-03-12 00:00:00 UTC",
entitlement_enabled=True,
command_access_mode="group_member",
protected_commands=["/city", "/deb"],
required_group_chat_id="-1001234567890",
loops=[],
)
html = render_runtime_status_html(runtime)
assert "Bot 启动诊断" in html
assert "Entitlement" in html
assert "命令准入" in html
assert "/city, /deb" in html
+10 -10
View File
@@ -34,15 +34,15 @@ def _sample_weather_payload():
],
"mgm_nearby": [
{
"name": "Ankara (Bölge/Center)",
"istNo": "17130",
"name": "Airport (MGM/17128)",
"istNo": "17128",
"lat": 39.95,
"lon": 32.97,
"temp": 12.4,
},
{
"name": "Airport (MGM/17128)",
"istNo": "17128",
"name": "Ankara (Bölge/Center)",
"istNo": "17130",
"lat": 40.1281,
"lon": 32.9951,
"temp": 11.2,
@@ -57,7 +57,7 @@ def test_trading_alerts_all_core_rules_trigger():
map_url="https://example.com/map",
)
assert out["trigger_count"] >= 3
assert out["trigger_count"] >= 2
assert out["rules"]["momentum_spike"]["triggered"] is True
assert out["rules"]["forecast_breakthrough"]["triggered"] is True
assert out["rules"]["advection"]["triggered"] is True
@@ -113,8 +113,8 @@ def test_ankara_center_signal_only_uses_official_center_station():
"temp": 12.6,
},
{
"name": "Ankara (Bölge/Center)",
"istNo": "17130",
"name": "Airport (MGM/17128)",
"istNo": "17128",
"lat": 39.95,
"lon": 32.97,
"temp": 11.3,
@@ -125,9 +125,9 @@ def test_ankara_center_signal_only_uses_official_center_station():
center_rule = out["rules"]["ankara_center_deb_hit"]
assert center_rule["triggered"] is True
assert center_rule["center_station"]["istNo"] == "17130"
assert center_rule["center_station"]["name"] == "Ankara (Bölge/Center)"
assert "Ankara (Bölge/Center)" in out["telegram"]["zh"]
assert center_rule["center_station"]["istNo"] == "17128"
assert center_rule["center_station"]["name"] == "Airport (MGM/17128)"
assert "Airport (MGM/17128)" in out["telegram"]["zh"]
assert "Etimesgut" not in out["telegram"]["zh"]
+155
View File
@@ -0,0 +1,155 @@
import json
from pathlib import Path
from src.analysis.probability_calibration import (
ENGINE_MODE_EMOS_PRIMARY,
ENGINE_MODE_EMOS_SHADOW,
ENGINE_MODE_LEGACY,
apply_probability_calibration,
build_probability_features,
fit_calibration,
)
def _write_calibration(tmp_path: Path):
payload = {
"version": "test-emos-v1",
"source": "tmp/test-emos-v1.json",
"global": {
"mu": {
"intercept": 0.0,
"raw_mu_coef": 0.0,
"deb_coef": 1.0,
"ens_median_coef": 0.0,
"max_so_far_gap_coef": 0.0,
},
"sigma": {
"intercept": 0.0,
"raw_sigma_coef": 1.0,
"spread_coef": 0.0,
"peak_flag_coef": 0.0,
"max_so_far_gap_coef": 0.0,
},
},
"cities": {
"ankara": {
"mu_bias": 0.5,
"sigma_scale": 1.0,
}
},
"metrics": {"sample_count": 10, "mean_crps": 0.4},
}
path = tmp_path / "calibration.json"
path.write_text(json.dumps(payload), encoding="utf-8")
return path
def test_shadow_mode_keeps_legacy_distribution(tmp_path):
calibration_path = _write_calibration(tmp_path)
features = build_probability_features(
city_name="ankara",
raw_mu=9.0,
raw_sigma=1.0,
deb_prediction=10.0,
ens_data={"median": 9.5, "p10": 8.0, "p90": 11.0},
current_forecasts={"Open-Meteo": 9.0, "MGM": 10.0},
max_so_far=8.8,
peak_status="before",
local_hour_frac=11.0,
)
legacy_distribution = [{"value": 9, "range": "[8.5~9.5)", "probability": 0.7}]
result = apply_probability_calibration(
city_name="ankara",
temp_symbol="°C",
raw_mu=9.0,
raw_sigma=1.0,
max_so_far=8.8,
legacy_distribution=legacy_distribution,
features=features,
calibration_path=str(calibration_path),
mode=ENGINE_MODE_EMOS_SHADOW,
)
assert result["mode"] == ENGINE_MODE_EMOS_SHADOW
assert result["engine"] == ENGINE_MODE_LEGACY
assert result["distribution"] == legacy_distribution
assert result["shadow_distribution"]
assert result["calibrated_mu"] == 10.5
def test_primary_mode_switches_to_calibrated_distribution(tmp_path):
calibration_path = _write_calibration(tmp_path)
features = build_probability_features(
city_name="ankara",
raw_mu=9.0,
raw_sigma=1.0,
deb_prediction=10.0,
ens_data={"median": 9.5, "p10": 8.0, "p90": 11.0},
current_forecasts={"Open-Meteo": 9.0, "MGM": 10.0},
max_so_far=8.8,
peak_status="before",
local_hour_frac=11.0,
)
result = apply_probability_calibration(
city_name="ankara",
temp_symbol="°C",
raw_mu=9.0,
raw_sigma=1.0,
max_so_far=8.8,
legacy_distribution=[{"value": 9, "range": "[8.5~9.5)", "probability": 0.7}],
features=features,
calibration_path=str(calibration_path),
mode=ENGINE_MODE_EMOS_PRIMARY,
)
assert result["mode"] == ENGINE_MODE_EMOS_PRIMARY
assert result["engine"] == "emos"
assert result["calibrated_mu"] == 10.5
assert result["distribution"]
assert result["distribution"][0]["value"] >= 10
def test_fit_calibration_returns_metrics():
samples = [
{
"city": "ankara",
"actual_high": 11.0,
"raw_mu": 10.2,
"raw_sigma": 1.0,
"deb_prediction": 10.5,
"ens_median": 10.6,
"ensemble_spread": 0.9,
"max_so_far_gap": 0.5,
"peak_flag": 0.0,
},
{
"city": "ankara",
"actual_high": 12.0,
"raw_mu": 11.1,
"raw_sigma": 1.0,
"deb_prediction": 11.3,
"ens_median": 11.2,
"ensemble_spread": 1.0,
"max_so_far_gap": 0.4,
"peak_flag": 0.5,
},
{
"city": "new york",
"actual_high": 19.0,
"raw_mu": 18.2,
"raw_sigma": 1.4,
"deb_prediction": 18.4,
"ens_median": 18.3,
"ensemble_spread": 1.2,
"max_so_far_gap": 0.6,
"peak_flag": 1.0,
},
]
result = fit_calibration(samples, version="unit-test-v1")
assert result["version"] == "unit-test-v1"
assert result["metrics"]["sample_count"] == 3
assert "mean_crps" in result["metrics"]
+61
View File
@@ -0,0 +1,61 @@
import json
from pathlib import Path
from scripts.build_probability_shadow_report import main
def test_shadow_report_builds_summary(monkeypatch, tmp_path: Path):
history_file = tmp_path / "daily_records.json"
output_file = tmp_path / "shadow_report.json"
history_file.write_text(
json.dumps(
{
"ankara": {
"2026-03-18": {
"actual_high": 10.0,
"mu": 9.6,
"prob_snapshot": [
{"v": 10, "p": 0.48},
{"v": 9, "p": 0.41},
],
"shadow_prob_snapshot": [
{"v": 10, "p": 0.55},
{"v": 9, "p": 0.28},
],
"probability_calibration": {
"mode": "emos_shadow",
"engine": "legacy",
"version": "emos-test",
"raw_mu": 9.6,
"calibrated_mu": 9.9,
},
}
}
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
monkeypatch.setattr(
"sys.argv",
[
"build_probability_shadow_report.py",
"--history-file",
str(history_file),
"--output",
str(output_file),
],
)
main()
payload = json.loads(output_file.read_text(encoding="utf-8"))
assert payload["summary"]["samples"] == 1
assert payload["summary"]["legacy_mean_mae"] == 0.4
assert payload["summary"]["shadow_mean_mae"] == 0.1
assert payload["summary"]["delta_mae"] == -0.3
assert payload["summary"]["legacy_bucket_hit_rate"] == 1.0
assert payload["summary"]["shadow_bucket_hit_rate"] == 1.0
assert payload["recent_observations"][0]["calibration_version"] == "emos-test"
-1
View File
@@ -9,7 +9,6 @@ import os
# Ensure project root is in path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import pytest
from unittest.mock import patch
from src.analysis.trend_engine import analyze_weather_trend, _sf
+3
View File
@@ -0,0 +1,3 @@
from web.core import app
__all__ = ["app"]
+862
View File
@@ -0,0 +1,862 @@
from __future__ import annotations
import time as _time
from datetime import datetime, timezone, timedelta
from typing import Dict, Any, Optional
from fastapi import HTTPException
from loguru import logger
from web.core import (
_cache,
CACHE_TTL,
CACHE_TTL_ANKARA,
CITIES,
CITY_RISK_PROFILES,
SETTLEMENT_SOURCE_LABELS,
_is_excluded_model_name,
_market_layer,
_sf,
_weather,
)
from src.analysis.deb_algorithm import calculate_dynamic_weights
from src.analysis.settlement_rounding import apply_city_settlement
from src.analysis.metar_narrator import describe_metar_report
from src.data_collection.city_registry import ALIASES
def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]:
"""Fetch, analyse, and return structured weather data for one city."""
# Check cache
ttl = CACHE_TTL_ANKARA if city.lower() == "ankara" else CACHE_TTL
if not force_refresh:
cached = _cache.get(city)
if cached and _time.time() - cached["t"] < ttl:
return cached["d"]
info = CITIES[city]
lat, lon, is_f = info["lat"], info["lon"], info["f"]
sym = "°F" if is_f else "°C"
settlement_source = str(info.get("settlement_source") or "metar").strip().lower() or "metar"
settlement_source_label = SETTLEMENT_SOURCE_LABELS.get(
settlement_source,
settlement_source.upper(),
)
# ── 1. Fetch raw data ──
raw = _weather.fetch_all_sources(
city,
lat=lat,
lon=lon,
force_refresh=force_refresh,
)
om = raw.get("open-meteo", {})
metar = raw.get("metar", {})
mgm = raw.get("mgm") or {}
settlement_current = raw.get("settlement_current") or {}
ens_raw = raw.get("ensemble", {})
mm = raw.get("multi_model", {})
if not isinstance(om, dict):
om = {}
if not isinstance(metar, dict):
metar = {}
if not isinstance(mgm, dict):
mgm = {}
if not isinstance(settlement_current, dict):
settlement_current = {}
if not isinstance(ens_raw, dict):
ens_raw = {}
if not isinstance(mm, dict):
mm = {}
risk = CITY_RISK_PROFILES.get(city, {})
# ── 2. Current conditions (city-specific settlement source first, then METAR/MGM fallback) ──
mc = metar.get("current", {}) if metar else {}
mg_cur = mgm.get("current", {}) if mgm else {}
sc_cur = settlement_current.get("current", {}) if settlement_current else {}
use_settlement_current = settlement_source in {"hko", "cwa"} and bool(sc_cur)
primary_current = sc_cur if use_settlement_current else mc
cur_temp = _sf(primary_current.get("temp"))
if cur_temp is None:
cur_temp = _sf(mc.get("temp"))
if cur_temp is None:
cur_temp = _sf(mg_cur.get("temp"))
max_so_far = _sf(primary_current.get("max_temp_so_far"))
if max_so_far is None:
max_so_far = _sf(mc.get("max_temp_so_far"))
if max_so_far is None:
max_so_far = _sf(mg_cur.get("mgm_max_temp"))
max_temp_time = primary_current.get("max_temp_time")
if not max_temp_time and not use_settlement_current:
max_temp_time = mc.get("max_temp_time")
if not max_temp_time:
max_temp_time = mg_cur.get("time", "")
if " " in max_temp_time:
max_temp_time = max_temp_time.split(" ")[1][:5]
if max_temp_time == "":
max_temp_time = None
wu_settle = apply_city_settlement(city.lower(), max_so_far) if max_so_far is not None else None
# Observation time → local
obs_time_str = ""
metar_age_min = None
obs_t = ""
if use_settlement_current:
obs_t = str(settlement_current.get("observation_time") or "").strip()
if not obs_t:
obs_t = metar.get("observation_time", "") if metar else ""
# 优先从 API 获取偏移;若缺失则尝试 NWS 动态偏移;最后回退静态配置
utc_offset = om.get("utc_offset")
if utc_offset is None:
try:
nws_periods = (raw.get("nws", {}) or {}).get("forecast_periods", []) or []
if nws_periods:
first_start = nws_periods[0].get("start_time")
if first_start:
maybe_dt = datetime.fromisoformat(str(first_start))
if maybe_dt.utcoffset() is not None:
utc_offset = int(maybe_dt.utcoffset().total_seconds())
except Exception:
utc_offset = None
if utc_offset is None:
utc_offset = info.get("tz", 0)
if obs_t and "T" in obs_t:
try:
dt = datetime.fromisoformat(str(obs_t).replace("Z", "+00:00"))
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
local_dt = dt.astimezone(timezone(timedelta(seconds=utc_offset)))
obs_time_str = local_dt.strftime("%H:%M")
metar_age_min = int(
(datetime.now(timezone.utc) - dt.astimezone(timezone.utc)).total_seconds() / 60
)
except Exception:
obs_time_str = str(obs_t)[:16]
settlement_today_obs = []
if use_settlement_current:
if obs_time_str and cur_temp is not None:
settlement_today_obs.append({"time": obs_time_str, "temp": cur_temp})
if (
max_temp_time
and max_so_far is not None
and str(max_temp_time) != str(obs_time_str)
):
settlement_today_obs.append({"time": str(max_temp_time), "temp": max_so_far})
metar_today_obs_payload = (
[]
if use_settlement_current
else [
{"time": t, "temp": v}
for t, v in (metar.get("today_obs", []) if metar else [])
]
)
metar_recent_obs_payload = (
[] if use_settlement_current else (metar.get("recent_obs", []) if metar else [])
)
# ── 3. Local time parsing ──
local_time_full = om.get("current", {}).get("local_time", "")
local_hour, local_minute = 12, 0
now_utc = datetime.now(timezone.utc)
local_now = now_utc + timedelta(seconds=utc_offset)
local_date_str = local_now.strftime("%Y-%m-%d")
try:
if local_time_full:
local_date_str = local_time_full.split(" ")[0]
tp = local_time_full.split(" ")[1].split(":")
local_hour = int(tp[0])
local_minute = int(tp[1]) if len(tp) > 1 else 0
else:
local_hour = local_now.hour
local_minute = local_now.minute
except Exception:
local_hour = local_now.hour
local_minute = local_now.minute
local_time_str = f"{local_hour:02d}:{local_minute:02d}"
local_hour_frac = local_hour + local_minute / 60
# ── 4. Daily forecast ──
daily = om.get("daily", {})
dates = daily.get("time", [])[:5]
maxtemps = daily.get("temperature_2m_max", [])[:5]
sunrises = daily.get("sunrise", [])
sunsets = daily.get("sunset", [])
sunshine = daily.get("sunshine_duration", [])
om_today = _sf(maxtemps[0]) if maxtemps else None
forecast_daily = [{"date": d, "max_temp": t} for d, t in zip(dates, maxtemps)]
if om_today is None:
nws_high = _sf(raw.get("nws", {}).get("today_high"))
mgm_high = _sf(mgm.get("today_high")) if mgm else None
fallback_high = (
nws_high
if nws_high is not None
else mgm_high
if mgm_high is not None
else max_so_far
if max_so_far is not None
else cur_temp
)
if fallback_high is not None:
om_today = float(fallback_high)
if not forecast_daily:
forecast_daily = [{"date": local_date_str, "max_temp": om_today}]
sunrise = (
sunrises[0].split("T")[1][:5]
if sunrises and "T" in str(sunrises[0])
else ""
)
sunset = (
sunsets[0].split("T")[1][:5]
if sunsets and "T" in str(sunsets[0])
else ""
)
sunshine_h = round(sunshine[0] / 3600, 1) if sunshine else 0
# ── 5. Multi-model forecasts ──
current_forecasts: Dict[str, float] = {}
if om_today is not None:
current_forecasts["Open-Meteo"] = om_today
for m, v in mm.get("forecasts", {}).items():
if v is not None and not _is_excluded_model_name(m):
current_forecasts[m] = _sf(v)
nws_high = _sf(raw.get("nws", {}).get("today_high"))
if nws_high is not None:
current_forecasts["NWS"] = nws_high
mgm_high = _sf(mgm.get("today_high")) if mgm else None
if mgm_high is not None:
current_forecasts["MGM"] = mgm_high
# ── 6. DEB fusion ──
deb_val, deb_weights = None, ""
if current_forecasts:
blended, winfo = calculate_dynamic_weights(city, current_forecasts)
if blended is not None:
deb_val = blended
deb_weights = winfo
# ── 7. Ensemble stats ──
ens_data = {
"median": _sf(ens_raw.get("median")),
"p10": _sf(ens_raw.get("p10")),
"p90": _sf(ens_raw.get("p90")),
}
# ── 8. METAR trend ──
recent_temps = metar.get("recent_temps", []) if metar else []
trend_info = {
"direction": "unknown",
"recent": [{"time": t, "temp": v} for t, v in recent_temps[:6]],
"is_cooling": False,
"is_dead_market": False,
}
if len(recent_temps) >= 2:
t_only = [t for _, t in recent_temps]
latest, prev = t_only[0], t_only[1]
diff = latest - prev
if len(t_only) >= 3:
n = min(3, len(t_only))
all_same = all(t == latest for t in t_only[:n])
all_rising = all(t_only[i] >= t_only[i + 1] for i in range(n - 1))
all_falling = all(t_only[i] <= t_only[i + 1] for i in range(n - 1))
if all_same:
trend_info["direction"] = "stagnant"
elif all_rising and diff > 0:
trend_info["direction"] = "rising"
elif all_falling and diff < 0:
trend_info["direction"] = "falling"
else:
trend_info["direction"] = "mixed"
elif diff > 0:
trend_info["direction"] = "rising"
elif diff < 0:
trend_info["direction"] = "falling"
else:
trend_info["direction"] = "stagnant"
trend_info["is_cooling"] = trend_info["direction"] in ("falling", "stagnant")
# ── 9. Peak hour detection ──
hourly = om.get("hourly", {})
h_times = hourly.get("time", [])
h_temps = hourly.get("temperature_2m", [])
h_rad = hourly.get("shortwave_radiation", [])
h_dew = hourly.get("dew_point_2m", [])
h_pressure = hourly.get("pressure_msl", [])
h_wspd = hourly.get("wind_speed_10m", [])
h_wdir = hourly.get("wind_direction_10m", [])
h_precip_prob = hourly.get("precipitation_probability", [])
h_cloud_cover = hourly.get("cloud_cover", [])
if (not h_times or not h_temps) and metar:
metar_today_obs = metar.get("today_obs", []) or []
parsed_obs = []
for item in metar_today_obs:
try:
t_str, t_val = item
if t_str is None or t_val is None:
continue
hh, minute_part = str(t_str).split(":")
parsed_obs.append((int(hh), int(minute_part), float(t_val)))
except Exception:
continue
if parsed_obs:
parsed_obs.sort(key=lambda x: (x[0], x[1]))
h_times = [f"{local_date_str}T{hh:02d}:{mm:02d}" for hh, mm, _ in parsed_obs]
h_temps = [v for _, _, v in parsed_obs]
h_rad = [0 for _ in parsed_obs]
h_dew = [None for _ in parsed_obs]
h_pressure = [None for _ in parsed_obs]
h_wspd = [None for _ in parsed_obs]
h_wdir = [None for _ in parsed_obs]
h_precip_prob = [None for _ in parsed_obs]
h_cloud_cover = [None for _ in parsed_obs]
peak_hours = []
if h_times and h_temps and om_today is not None:
for ts, tmp in zip(h_times, h_temps):
if ts.startswith(local_date_str) and abs(tmp - om_today) <= 0.2:
hr = int(ts.split("T")[1][:2])
if 8 <= hr <= 19:
peak_hours.append(ts.split("T")[1][:5])
first_peak_h = int(peak_hours[0].split(":")[0]) if peak_hours else 13
last_peak_h = int(peak_hours[-1].split(":")[0]) if peak_hours else 15
if local_hour_frac > last_peak_h:
peak_status = "past"
elif first_peak_h <= local_hour_frac <= last_peak_h:
peak_status = "in_window"
else:
peak_status = "before"
# ── 10. Shared analysis (probability, trend, AI) via trend_engine ──
# This single call replaces the duplicate probability engine, dead market
# detection, forecast bust grading, and AI context building.
from src.analysis.trend_engine import analyze_weather_trend as _trend_analyze, calculate_prob_distribution
probabilities = []
shadow_probabilities = []
mu = None
probability_engine = "legacy"
probability_calibration_mode = "legacy"
probability_calibration_version = None
probability_raw_mu = None
probability_raw_sigma = None
probability_calibrated_mu = None
probability_calibrated_sigma = None
ai_text = ""
try:
_, _ai_context, sd = _trend_analyze(raw, sym, city)
# Use structured data from shared engine
mu = sd.get("mu")
probabilities = sd.get("probabilities", [])
shadow_probabilities = sd.get("shadow_probabilities", [])
probability_engine = sd.get("probability_engine", "legacy")
probability_calibration_mode = sd.get("probability_calibration_mode", "legacy")
probability_calibration_version = sd.get("probability_calibration_version")
probability_raw_mu = sd.get("probability_raw_mu")
probability_raw_sigma = sd.get("probability_raw_sigma")
probability_calibrated_mu = sd.get("probability_calibrated_mu")
probability_calibrated_sigma = sd.get("probability_calibrated_sigma")
trend_info["is_dead_market"] = sd.get("trend_info", {}).get("is_dead_market", False)
trend_info["direction"] = sd.get("trend_info", {}).get("direction", trend_info.get("direction", "unknown"))
trend_info["is_cooling"] = sd.get("trend_info", {}).get("is_cooling", False)
peak_status = sd.get("peak_status", peak_status)
# Use shared DEB if not already set
if deb_val is None and sd.get("deb_prediction") is not None:
deb_val = sd["deb_prediction"]
deb_weights = sd.get("deb_weights", "")
except Exception as e:
logger.warning(f"Structured analysis skipped for {city}: {e}")
ai_text = describe_metar_report(
raw_metar=str(primary_current.get("raw_metar") or mc.get("raw_metar") or ""),
temp_symbol=sym,
fallback={
"icao": metar.get("icao"),
"station_name": metar.get("station_name"),
"temp": cur_temp,
"wind_speed_kt": _sf(primary_current.get("wind_speed_kt")),
"wind_dir": _sf(primary_current.get("wind_dir")),
"altimeter": _sf(primary_current.get("altimeter")),
"wx_desc": primary_current.get("wx_desc"),
"clouds": primary_current.get("clouds", []) or mc.get("clouds", []),
},
)
# ── 12. Hourly data (today only, for chart) ──
today_hourly: Dict[str, list] = {"times": [], "temps": [], "radiation": []}
for i, ts in enumerate(h_times):
if ts.startswith(local_date_str):
today_hourly["times"].append(ts.split("T")[1][:5])
today_hourly["temps"].append(h_temps[i] if i < len(h_temps) else None)
today_hourly["radiation"].append(h_rad[i] if i < len(h_rad) else None)
# ── 12b. Next 48h hourly block for future-date analysis modal ──
next_48h_hourly = {
"times": [],
"temps": [],
"radiation": [],
"dew_point": [],
"pressure_msl": [],
"wind_speed_10m": [],
"wind_direction_10m": [],
"precipitation_probability": [],
"cloud_cover": [],
}
try:
local_anchor = datetime.strptime(
f"{local_date_str} {local_time_str}", "%Y-%m-%d %H:%M"
)
except Exception:
local_anchor = None
if local_anchor is not None:
horizon = local_anchor + timedelta(hours=48)
for i, ts in enumerate(h_times):
try:
ts_dt = datetime.fromisoformat(ts)
except Exception:
continue
if ts_dt < local_anchor or ts_dt > horizon:
continue
next_48h_hourly["times"].append(ts)
next_48h_hourly["temps"].append(h_temps[i] if i < len(h_temps) else None)
next_48h_hourly["radiation"].append(h_rad[i] if i < len(h_rad) else None)
next_48h_hourly["dew_point"].append(h_dew[i] if i < len(h_dew) else None)
next_48h_hourly["pressure_msl"].append(
h_pressure[i] if i < len(h_pressure) else None
)
next_48h_hourly["wind_speed_10m"].append(
h_wspd[i] if i < len(h_wspd) else None
)
next_48h_hourly["wind_direction_10m"].append(
h_wdir[i] if i < len(h_wdir) else None
)
next_48h_hourly["precipitation_probability"].append(
h_precip_prob[i] if i < len(h_precip_prob) else None
)
next_48h_hourly["cloud_cover"].append(
h_cloud_cover[i] if i < len(h_cloud_cover) else None
)
# ── 13. Cloud description (METAR primary, MGM fallback) ──
clouds = mc.get("clouds", [])
cloud_desc = ""
if clouds:
c_map = {
"BKN": "多云",
"OVC": "阴天",
"FEW": "少云",
"SCT": "散云",
"SKC": "",
"CLR": "",
}
main = clouds[-1]
cloud_desc = c_map.get(main.get("cover"), main.get("cover", ""))
if not cloud_desc and mgm:
mgc_cover = mgm.get("current", {}).get("cloud_cover")
if mgc_cover is not None:
cloud_desc_map = {
0: "晴朗",
1: "少云",
2: "少云",
3: "散云",
4: "散云",
5: "多云",
6: "多云",
7: "阴天",
8: "阴天",
}
cloud_desc = cloud_desc_map.get(mgc_cover, "")
# Final fallback: If we have ANY actual observation but no cloud info, it's usually clear.
if not cloud_desc:
if mc.get("temp") is not None or (mgm and mgm.get("current", {}).get("temp") is not None):
# If weather phenomenon exists (e.g. rain), we'll let app.js handle wx_desc priority.
# Otherwise, clear skies.
if not mc.get("wx_desc"):
cloud_desc = "晴朗"
# ── 14. MGM data (Ankara-specific) ──
mgm_data = {}
if mgm:
mgc = mgm.get("current", {})
mgm_time_str = mgc.get("time", "")
# MGM time is usually "2026-03-04T10:40:00.000Z" (UTC)
if mgm_time_str and "T" in mgm_time_str:
try:
# Handle ISO format with Z or +00:00
ts = mgm_time_str.replace("Z", "+00:00")
if "+" in ts:
base, offset_part = ts.split("+", 1)
if "." in base:
base = base.split(".")[0]
ts = base + "+" + offset_part
dt = datetime.fromisoformat(ts)
local_dt = dt.astimezone(timezone(timedelta(seconds=utc_offset or 0)))
mgm_time_str = local_dt.strftime("%H:%M")
except Exception as e:
logger.debug(f"MGM time conversion failed: {e}")
pass
mgm_data = {
"temp": _sf(mgc.get("temp")),
"time": mgm_time_str,
"feels_like": _sf(mgc.get("feels_like")),
"humidity": _sf(mgc.get("humidity")),
"wind_dir": _sf(mgc.get("wind_dir")),
"wind_speed_ms": _sf(mgc.get("wind_speed_ms")),
"pressure": _sf(mgc.get("pressure")),
"cloud_cover": mgc.get("cloud_cover"),
"rain_24h": _sf(mgc.get("rain_24h")),
"today_high": _sf(mgm.get("today_high")),
"today_low": _sf(mgm.get("today_low")),
"hourly": [],
}
mgm_hourly = mgm.get("hourly", [])
for h in mgm_hourly:
dt_str = h.get("time")
val = _sf(h.get("temp"))
if dt_str and "T" in dt_str and val is not None:
try:
dt = datetime.fromisoformat(dt_str.replace("Z", "+00:00"))
local_dt = dt.astimezone(timezone(timedelta(seconds=utc_offset)))
mgm_data["hourly"].append({
"time": local_dt.strftime("%Y-%m-%dT%H:%M"),
"temp": val
})
except Exception:
pass
# ── 15. Extended Multi-Model Daily ──
multi_model_daily = {}
mm_daily_raw = mm.get("daily_forecasts", {})
for i, d_str in enumerate(dates):
if i == 0:
day_m = current_forecasts.copy()
d_val, d_winfo = deb_val, deb_weights
else:
day_m = mm_daily_raw.get(d_str, {}).copy()
if i < len(maxtemps) and maxtemps[i] is not None:
day_m["Open-Meteo"] = _sf(maxtemps[i])
# Add MGM per-day forecast
mgm_daily = mgm.get("daily_forecasts", {})
if d_str in mgm_daily:
day_m["MGM"] = _sf(mgm_daily[d_str])
day_m = {
m: v for m, v in day_m.items() if not _is_excluded_model_name(m)
}
d_val, d_winfo = None, ""
d_probs = []
if day_m:
try:
blended, winfo = calculate_dynamic_weights(city, day_m)
if blended is not None:
d_val = blended
d_winfo = winfo
# Calculate future probability based on model divergence
m_vals = [v for v in day_m.values() if v is not None]
if len(m_vals) > 1:
# Use spread as a proxy for sigma.
# sigma = (max-min)/2 with a floor of 0.6
d_sigma = max(0.6, (max(m_vals) - min(m_vals)) / 2.0)
else:
d_sigma = 1.0
prob_obj = calculate_prob_distribution(d_val, d_sigma, None, sym)
d_probs = prob_obj.get("probabilities", [])
except Exception:
pass
if day_m:
multi_model_daily[d_str] = {
"models": day_m,
"deb": {"prediction": d_val, "weights_info": d_winfo},
"probabilities": d_probs if i > 0 else probabilities # Use today's real prob for today
}
# ── Assemble result ──
result = {
"name": city,
"display_name": city.title(),
"lat": lat,
"lon": lon,
"temp_symbol": sym,
"local_time": local_time_str,
"local_date": local_date_str,
"risk": {
"level": risk.get("risk_level", "low"),
"emoji": risk.get("risk_emoji", "🟢"),
"airport": risk.get("airport_name", ""),
"icao": risk.get("icao", ""),
"distance_km": risk.get("distance_km", 0),
"warning": risk.get("warning", ""),
},
"current": {
"temp": cur_temp,
"max_so_far": max_so_far,
"max_temp_time": max_temp_time,
"wu_settlement": wu_settle,
"settlement_source": settlement_source,
"settlement_source_label": settlement_source_label,
"obs_time": obs_time_str,
"obs_age_min": metar_age_min,
"report_time": metar.get("report_time") if metar else None,
"receipt_time": metar.get("receipt_time") if metar else None,
"obs_time_epoch": metar.get("obs_time_epoch") if metar else None,
"wind_speed_kt": _sf(primary_current.get("wind_speed_kt")),
"wind_dir": _sf(primary_current.get("wind_dir")),
"humidity": _sf(primary_current.get("humidity")),
"cloud_desc": cloud_desc,
"clouds_raw": [
{"cover": c.get("cover"), "base": c.get("base")} for c in clouds
],
"visibility_mi": _sf(primary_current.get("visibility_mi")),
"wx_desc": primary_current.get("wx_desc"),
"raw_metar": primary_current.get("raw_metar"),
},
"mgm": mgm_data,
"mgm_nearby": raw.get("mgm_nearby", []),
"forecast": {
"today_high": om_today,
"daily": forecast_daily,
"sunrise": sunrise,
"sunset": sunset,
"sunshine_hours": sunshine_h,
},
"source_forecasts": {
"weather_gov": raw.get("nws") or {},
},
"multi_model": {k: v for k, v in current_forecasts.items() if v is not None},
"multi_model_daily": multi_model_daily,
"deb": {"prediction": deb_val, "weights_info": deb_weights},
"ensemble": ens_data,
"probabilities": {
"mu": round(mu, 1) if mu is not None else None,
"distribution": probabilities,
"engine": probability_engine,
"calibration_mode": probability_calibration_mode,
"calibration_version": probability_calibration_version,
"raw_mu": probability_raw_mu,
"raw_sigma": probability_raw_sigma,
"calibrated_mu": probability_calibrated_mu,
"calibrated_sigma": probability_calibrated_sigma,
"shadow_distribution": shadow_probabilities,
},
"trend": trend_info,
"peak": {
"hours": peak_hours,
"first_h": first_peak_h,
"last_h": last_peak_h,
"status": peak_status,
},
"hourly": today_hourly,
"hourly_next_48h": next_48h_hourly,
"metar_today_obs": metar_today_obs_payload,
"metar_recent_obs": metar_recent_obs_payload,
"settlement_today_obs": settlement_today_obs,
"ai_analysis": ai_text,
"updated_at": datetime.now(timezone.utc).isoformat(),
}
_cache[city] = {"t": _time.time(), "d": result}
return result
def _normalize_city_or_404(name: str) -> str:
city = name.lower().strip().replace("-", " ")
city = ALIASES.get(city, city)
if city not in CITIES:
raise HTTPException(404, detail=f"Unknown city: {city}")
return city
def _build_city_summary_payload(data: Dict[str, Any]) -> Dict[str, Any]:
return {
"name": data.get("name"),
"display_name": data.get("display_name"),
"icao": data.get("risk", {}).get("icao"),
"local_time": data.get("local_time"),
"temp_symbol": data.get("temp_symbol"),
"current": {
"temp": data.get("current", {}).get("temp"),
"obs_time": data.get("current", {}).get("obs_time"),
"settlement_source": data.get("current", {}).get("settlement_source"),
"settlement_source_label": data.get("current", {}).get("settlement_source_label"),
},
"deb": {"prediction": data.get("deb", {}).get("prediction")},
"risk": {
"level": data.get("risk", {}).get("level"),
"warning": data.get("risk", {}).get("warning"),
},
"updated_at": data.get("updated_at"),
}
def _build_city_detail_payload(
data: Dict[str, Any],
market_slug: Optional[str] = None,
target_date: Optional[str] = None,
) -> Dict[str, Any]:
city = str(data.get("name") or "").strip().lower()
local_date = str(data.get("local_date") or "").strip()
requested_date = str(target_date or "").strip()
selected_date = requested_date or local_date
multi_model_daily = data.get("multi_model_daily") or {}
selected_daily = (
multi_model_daily.get(selected_date)
if isinstance(multi_model_daily, dict)
else None
)
if not isinstance(selected_daily, dict):
selected_daily = {}
selected_date = local_date
distribution = selected_daily.get("probabilities")
if not isinstance(distribution, list) or not distribution:
distribution = data.get("probabilities", {}).get("distribution", []) or []
model_map = selected_daily.get("models") or data.get("multi_model") or {}
if not isinstance(model_map, dict):
model_map = {}
anchor_temp = None
anchor_model = None
for model_name, raw_value in model_map.items():
value = _sf(raw_value)
if value is None:
continue
if anchor_temp is None or value > anchor_temp:
anchor_temp = value
anchor_model = str(model_name or "").strip() or None
anchor_temp_c = anchor_temp
temp_symbol = str(data.get("temp_symbol") or "")
if anchor_temp_c is not None and "F" in temp_symbol.upper():
anchor_temp_c = (anchor_temp_c - 32.0) * 5.0 / 9.0
anchor_settlement = apply_city_settlement(city, anchor_temp_c) if anchor_temp_c is not None else None
primary_bucket = None
if isinstance(distribution, list) and distribution:
if anchor_temp is None:
primary_bucket = distribution[0]
else:
ranked_buckets = []
for idx, row in enumerate(distribution):
if not isinstance(row, dict):
continue
bucket_temp = _sf(row.get("value"))
bucket_prob = _sf(row.get("probability"))
if bucket_temp is None:
continue
prob_rank = bucket_prob if bucket_prob is not None else -1.0
ranked_buckets.append((abs(bucket_temp - anchor_temp), -prob_rank, idx, row))
if ranked_buckets:
ranked_buckets.sort(key=lambda x: (x[0], x[1], x[2]))
primary_bucket = ranked_buckets[0][3]
else:
primary_bucket = distribution[0]
model_probability = None
if isinstance(primary_bucket, dict) and primary_bucket.get("probability") is not None:
try:
raw_probability = float(primary_bucket.get("probability"))
model_probability = raw_probability / 100.0 if raw_probability > 1.0 else raw_probability
except Exception:
model_probability = None
fallback_sparkline = [
p.get("probability", 0)
for p in distribution[:8]
if isinstance(p, dict)
]
market_scan = _market_layer.build_market_scan(
city=data.get("name"),
target_date=selected_date or data.get("local_date"),
temperature_bucket=primary_bucket if isinstance(primary_bucket, dict) else None,
model_probability=model_probability,
fallback_sparkline=fallback_sparkline,
forced_market_slug=market_slug,
)
if isinstance(market_scan, dict):
market_scan["anchor_model"] = anchor_model
market_scan["anchor_high"] = anchor_temp
market_scan["anchor_settlement"] = anchor_settlement
market_scan["open_meteo_settlement"] = anchor_settlement
return {
"city": data.get("name"),
"fetched_at": data.get("updated_at"),
"overview": {
"name": data.get("name"),
"display_name": data.get("display_name"),
"icao": data.get("risk", {}).get("icao"),
"airport": data.get("risk", {}).get("airport"),
"lat": data.get("lat"),
"lon": data.get("lon"),
"local_time": data.get("local_time"),
"local_date": data.get("local_date"),
"temp_symbol": data.get("temp_symbol"),
"current_temp": data.get("current", {}).get("temp"),
"settlement_source": data.get("current", {}).get("settlement_source"),
"settlement_source_label": data.get("current", {}).get("settlement_source_label"),
"deb_prediction": data.get("deb", {}).get("prediction"),
"risk_level": data.get("risk", {}).get("level"),
"risk_warning": data.get("risk", {}).get("warning"),
"updated_at": data.get("updated_at"),
},
"official": {
"available": bool(data.get("current", {}).get("temp") is not None),
"metar": {
"observation_time": data.get("current", {}).get("obs_time"),
"obs_age_min": data.get("current", {}).get("obs_age_min"),
"report_time": data.get("current", {}).get("report_time"),
"receipt_time": data.get("current", {}).get("receipt_time"),
"raw_metar": data.get("current", {}).get("raw_metar"),
"current": data.get("current"),
},
"weather_gov": {},
"mgm": data.get("mgm") or {},
"mgm_nearby": data.get("mgm_nearby") or [],
"nearby_source": "mgm" if data.get("name") == "ankara" else "metar_cluster",
},
"timeseries": {
"metar_recent_obs": data.get("metar_recent_obs") or [],
"metar_today_obs": data.get("metar_today_obs") or [],
"settlement_today_obs": data.get("settlement_today_obs") or [],
"hourly": data.get("hourly") or {},
"mgm_hourly": (data.get("mgm") or {}).get("hourly", []),
"forecast_daily": (data.get("forecast") or {}).get("daily", []),
},
"models": {
k: v
for k, v in (data.get("multi_model") or {}).items()
if not _is_excluded_model_name(k)
},
"probabilities": data.get("probabilities") or {"mu": None, "distribution": []},
"market_scan": market_scan,
"risk": data.get("risk"),
"ai_analysis": data.get("ai_analysis") or "",
"errors": {},
}
# ──────────────────────────────────────────────────────────
# Routes
# ──────────────────────────────────────────────────────────
+4 -1445
View File
File diff suppressed because it is too large Load Diff
+260
View File
@@ -0,0 +1,260 @@
"""
PolyWeather Web Core Context
"""
import os
from typing import Dict, Any, Optional
from fastapi import FastAPI, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel, Field
from loguru import logger
from src.utils.config_loader import load_config
from src.data_collection.weather_sources import WeatherDataCollector
from src.data_collection.city_risk_profiles import CITY_RISK_PROFILES # noqa: F401
from src.data_collection.polymarket_readonly import PolymarketReadOnlyLayer
from src.auth.supabase_entitlement import SUPABASE_ENTITLEMENT, extract_bearer_token
from src.database.db_manager import DBManager
from src.payments import PAYMENT_CHECKOUT, PaymentCheckoutError # noqa: F401
from src.data_collection.city_registry import CITY_REGISTRY
app = FastAPI(title="PolyWeather Map", version="1.0")
_cors_origins = os.getenv(
"WEB_CORS_ORIGINS",
"http://localhost:3000,http://127.0.0.1:3000,https://polyweather-pro.vercel.app",
)
app.add_middleware(
CORSMiddleware,
allow_origins=[o.strip() for o in _cors_origins.split(",") if o.strip()],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
_config = load_config()
_weather = WeatherDataCollector(_config)
_market_layer = PolymarketReadOnlyLayer()
_account_db = DBManager()
CITIES: Dict[str, Dict[str, Any]] = {
cid: {
"lat": info["lat"],
"lon": info["lon"],
"f": info["use_fahrenheit"],
"tz": info["tz_offset"],
"settlement_source": str(info.get("settlement_source") or "metar").strip().lower() or "metar",
}
for cid, info in CITY_REGISTRY.items()
}
SETTLEMENT_SOURCE_LABELS: Dict[str, str] = {
"metar": "METAR",
"hko": "HKO",
"cwa": "CWA",
"mgm": "MGM",
}
_cache: Dict[str, Dict] = {}
CACHE_TTL = 300
CACHE_TTL_ANKARA = 60
def _env_bool(name: str, default: bool = False) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
_ENTITLEMENT_GUARD_ENABLED = _env_bool("POLYWEATHER_REQUIRE_ENTITLEMENT", False)
_ENTITLEMENT_HEADER = "x-polyweather-entitlement"
_ENTITLEMENT_TOKEN = (os.getenv("POLYWEATHER_BACKEND_ENTITLEMENT_TOKEN") or "").strip()
_FORWARDED_SUPABASE_USER_ID_HEADER = "x-polyweather-auth-user-id"
_FORWARDED_SUPABASE_EMAIL_HEADER = "x-polyweather-auth-email"
_SUPABASE_AUTH_REQUIRED = _env_bool(
"POLYWEATHER_AUTH_REQUIRED",
SUPABASE_ENTITLEMENT.enabled,
)
def _legacy_service_token_valid(request: Request) -> bool:
token = request.headers.get(_ENTITLEMENT_HEADER)
if not token:
token = extract_bearer_token(request.headers.get("authorization"))
return bool(_ENTITLEMENT_TOKEN and token == _ENTITLEMENT_TOKEN)
def _bind_optional_supabase_identity(request: Request) -> None:
if not SUPABASE_ENTITLEMENT.configured:
return
access_token = extract_bearer_token(request.headers.get("authorization"))
if not access_token:
return
identity = SUPABASE_ENTITLEMENT.get_identity(access_token)
if not identity:
return
request.state.auth_user_id = identity.user_id
request.state.auth_email = identity.email
request.state.auth_points = identity.points
def _resolve_auth_points(request: Request) -> int:
raw_points = getattr(request.state, "auth_points", 0)
try:
points = max(0, int(raw_points or 0))
except Exception:
points = 0
if points > 0:
return points
user_id = str(getattr(request.state, "auth_user_id", "") or "").strip()
if not user_id:
return points
try:
db_points = _account_db.get_points_by_supabase_user_id(user_id)
if db_points > points:
request.state.auth_points = db_points
return db_points
except Exception as exc:
logger.warning(f"auth points fallback failed user_id={user_id}: {exc}")
return points
def _resolve_weekly_profile(request: Request) -> Dict[str, Any]:
user_id = str(getattr(request.state, "auth_user_id", "") or "").strip()
if not user_id:
return {"weekly_points": 0, "weekly_rank": None}
try:
profile = _account_db.get_weekly_profile_by_supabase_user_id(user_id)
return {
"weekly_points": int(profile.get("weekly_points") or 0),
"weekly_rank": profile.get("weekly_rank"),
}
except Exception as exc:
logger.warning(f"auth weekly profile fallback failed user_id={user_id}: {exc}")
return {"weekly_points": 0, "weekly_rank": None}
def _assert_entitlement(request: Request) -> None:
if SUPABASE_ENTITLEMENT.enabled:
if _legacy_service_token_valid(request):
return
if not _SUPABASE_AUTH_REQUIRED:
_bind_optional_supabase_identity(request)
return
if not SUPABASE_ENTITLEMENT.configured:
raise HTTPException(
status_code=503,
detail="Supabase auth is enabled but SUPABASE_URL / SUPABASE_ANON_KEY is not configured",
)
access_token = extract_bearer_token(request.headers.get("authorization"))
if not access_token:
raise HTTPException(status_code=401, detail="Unauthorized")
identity = SUPABASE_ENTITLEMENT.get_identity(access_token)
if not identity:
raise HTTPException(status_code=401, detail="Unauthorized")
if not SUPABASE_ENTITLEMENT.has_active_subscription(identity.user_id):
raise HTTPException(status_code=403, detail="Subscription required")
request.state.auth_user_id = identity.user_id
request.state.auth_email = identity.email
return
if not _ENTITLEMENT_GUARD_ENABLED:
return
if not _ENTITLEMENT_TOKEN:
raise HTTPException(
status_code=503,
detail="Entitlement guard is enabled but backend token is not configured",
)
if not _legacy_service_token_valid(request):
raise HTTPException(status_code=401, detail="Unauthorized")
def _require_supabase_identity(request: Request) -> Dict[str, str]:
if not SUPABASE_ENTITLEMENT.enabled:
raise HTTPException(status_code=503, detail="payment requires POLYWEATHER_AUTH_ENABLED=true")
if not SUPABASE_ENTITLEMENT.configured:
raise HTTPException(status_code=503, detail="payment requires SUPABASE_URL and SUPABASE_ANON_KEY")
state_user_id = str(getattr(request.state, "auth_user_id", "") or "").strip()
if state_user_id:
state_email = str(getattr(request.state, "auth_email", "") or "").strip()
return {"user_id": state_user_id, "email": state_email}
token = extract_bearer_token(request.headers.get("authorization"))
if token:
identity = SUPABASE_ENTITLEMENT.get_identity(token)
if identity:
return {"user_id": identity.user_id, "email": identity.email}
legacy_ok = _legacy_service_token_valid(request)
if legacy_ok:
forwarded_user_id = str(request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "").strip()
if forwarded_user_id:
forwarded_email = str(request.headers.get(_FORWARDED_SUPABASE_EMAIL_HEADER) or "").strip()
return {"user_id": forwarded_user_id, "email": forwarded_email}
logger.warning(
"payment auth identity missing state_user={} auth_bearer={} legacy_ok={} forwarded_user={}"
.format(
bool(state_user_id),
bool(token),
bool(legacy_ok),
bool(str(request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "").strip()),
)
)
raise HTTPException(status_code=401, detail="Unauthorized")
class WalletChallengeRequest(BaseModel):
address: str = Field(..., min_length=8)
class WalletVerifyRequest(BaseModel):
address: str = Field(..., min_length=8)
nonce: str = Field(..., min_length=6)
signature: str = Field(..., min_length=20)
class WalletUnbindRequest(BaseModel):
address: str = Field(..., min_length=8)
class CreatePaymentIntentRequest(BaseModel):
plan_code: str = Field(default="pro_monthly", min_length=2)
payment_mode: str = Field(default="strict")
allowed_wallet: Optional[str] = None
token_address: Optional[str] = None
use_points: bool = False
points_to_consume: Optional[int] = None
metadata: Dict[str, Any] = Field(default_factory=dict)
class SubmitPaymentTxRequest(BaseModel):
tx_hash: str = Field(..., min_length=10)
from_address: str = Field(..., min_length=8)
class ConfirmPaymentTxRequest(BaseModel):
tx_hash: Optional[str] = None
def _sf(v) -> Optional[float]:
if v is None:
return None
try:
return float(v)
except Exception:
return None
def _is_excluded_model_name(model_name: str) -> bool:
normalized = str(model_name or "").strip().lower().replace(" ", "").replace("_", "").replace("-", "")
return "meteoblue" in normalized
+359
View File
@@ -0,0 +1,359 @@
from __future__ import annotations
import os
from typing import Optional
from fastapi import APIRouter, HTTPException, Request
from loguru import logger
from src.analysis.deb_algorithm import load_history
from src.data_collection.city_registry import ALIASES
from web.analysis_service import (
_analyze,
_build_city_detail_payload,
_build_city_summary_payload,
)
from web.core import (
CITIES,
CITY_REGISTRY,
CITY_RISK_PROFILES,
PAYMENT_CHECKOUT,
PaymentCheckoutError,
SETTLEMENT_SOURCE_LABELS,
SUPABASE_ENTITLEMENT,
ConfirmPaymentTxRequest,
CreatePaymentIntentRequest,
SubmitPaymentTxRequest,
WalletChallengeRequest,
WalletUnbindRequest,
WalletVerifyRequest,
_ENTITLEMENT_GUARD_ENABLED,
_SUPABASE_AUTH_REQUIRED,
_assert_entitlement,
_bind_optional_supabase_identity,
_require_supabase_identity,
_resolve_auth_points,
_resolve_weekly_profile,
_sf,
_is_excluded_model_name,
)
router = APIRouter()
def _normalize_city_or_404(name: str) -> str:
city = name.lower().strip().replace("-", " ")
city = ALIASES.get(city, city)
if city not in CITIES:
raise HTTPException(404, detail=f"Unknown city: {city}")
return city
@router.get("/api/cities")
async def list_cities(request: Request):
_assert_entitlement(request)
try:
out = []
for name, info in CITIES.items():
risk = CITY_RISK_PROFILES.get(name, {})
settlement_source = str(info.get("settlement_source") or "metar").strip().lower() or "metar"
out.append(
{
"name": name,
"display_name": name.title(),
"lat": info["lat"],
"lon": info["lon"],
"risk_level": risk.get("risk_level", "low"),
"risk_emoji": risk.get("risk_emoji", "🟢"),
"airport": risk.get("airport_name", ""),
"icao": risk.get("icao", ""),
"temp_unit": "fahrenheit" if info["f"] else "celsius",
"is_major": CITY_REGISTRY.get(name, {}).get("is_major", True),
"settlement_source": settlement_source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(
settlement_source,
settlement_source.upper(),
),
}
)
return {"cities": out}
except Exception as exc:
logger.error(f"Error in list_cities: {exc}")
raise HTTPException(status_code=500, detail=str(exc)) from exc
@router.get("/api/city/{name}")
async def city_detail(request: Request, name: str, force_refresh: bool = False):
_assert_entitlement(request)
return _analyze(_normalize_city_or_404(name), force_refresh=force_refresh)
@router.get("/api/history/{name}")
async def city_history(request: Request, name: str):
_assert_entitlement(request)
city = _normalize_city_or_404(name)
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
history_file = os.path.join(project_root, "data", "daily_records.json")
data = load_history(history_file)
if city not in data:
source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
return {
"history": [],
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
}
city_data = data[city]
out = []
for day, rec in sorted(city_data.items()):
act = rec.get("actual_high")
deb = rec.get("deb_prediction")
mu = rec.get("mu")
forecasts_raw = rec.get("forecasts", {}) or {}
forecasts = {}
if isinstance(forecasts_raw, dict):
for model_name, model_value in forecasts_raw.items():
if _is_excluded_model_name(str(model_name)):
continue
fv = _sf(model_value)
forecasts[str(model_name)] = fv if fv is not None else None
mgm = forecasts.get("MGM")
out.append(
{
"date": day,
"actual": float(act) if act is not None else None,
"deb": float(deb) if deb is not None else None,
"mu": float(mu) if mu is not None else None,
"mgm": float(mgm) if mgm is not None else None,
"forecasts": forecasts,
}
)
source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
return {
"history": out,
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
}
@router.get("/api/auth/me")
async def auth_me(request: Request):
_assert_entitlement(request)
_bind_optional_supabase_identity(request)
user_id = getattr(request.state, "auth_user_id", None)
subscription_required = bool(
SUPABASE_ENTITLEMENT.enabled
and _SUPABASE_AUTH_REQUIRED
and SUPABASE_ENTITLEMENT.require_subscription
)
subscription_active = None
subscription_plan_code = None
subscription_starts_at = None
subscription_expires_at = None
if SUPABASE_ENTITLEMENT.enabled and user_id:
try:
latest_subscription = SUPABASE_ENTITLEMENT.get_latest_active_subscription(
user_id,
respect_requirement=False,
)
subscription_active = bool(latest_subscription)
if isinstance(latest_subscription, dict):
subscription_plan_code = latest_subscription.get("plan_code")
subscription_starts_at = latest_subscription.get("starts_at")
subscription_expires_at = latest_subscription.get("expires_at")
except Exception:
subscription_active = None
subscription_plan_code = None
subscription_starts_at = None
subscription_expires_at = None
points = _resolve_auth_points(request)
weekly_profile = _resolve_weekly_profile(request)
return {
"authenticated": bool(user_id),
"user_id": user_id,
"email": getattr(request.state, "auth_email", None),
"points": points,
"weekly_points": weekly_profile["weekly_points"],
"weekly_rank": weekly_profile["weekly_rank"],
"entitlement_mode": (
"supabase_required"
if SUPABASE_ENTITLEMENT.enabled and _SUPABASE_AUTH_REQUIRED
else "supabase_optional"
if SUPABASE_ENTITLEMENT.enabled
else "legacy_token"
if _ENTITLEMENT_GUARD_ENABLED
else "disabled"
),
"auth_required": bool(SUPABASE_ENTITLEMENT.enabled and _SUPABASE_AUTH_REQUIRED),
"subscription_required": subscription_required,
"subscription_active": subscription_active,
"subscription_plan_code": subscription_plan_code,
"subscription_starts_at": subscription_starts_at,
"subscription_expires_at": subscription_expires_at,
}
@router.get("/api/payments/config")
async def payment_config(request: Request):
_assert_entitlement(request)
try:
return PAYMENT_CHECKOUT.get_config_payload()
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.get("/api/payments/wallets")
async def payment_wallets(request: Request):
_assert_entitlement(request)
identity = _require_supabase_identity(request)
try:
wallets = PAYMENT_CHECKOUT.list_wallets(identity["user_id"])
return {
"wallets": [wallet.__dict__ for wallet in wallets],
"chain_id": PAYMENT_CHECKOUT.chain_id,
}
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.delete("/api/payments/wallets")
async def payment_wallet_unbind(request: Request, body: WalletUnbindRequest):
_assert_entitlement(request)
identity = _require_supabase_identity(request)
try:
return PAYMENT_CHECKOUT.unbind_wallet(
user_id=identity["user_id"],
address=body.address,
)
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.post("/api/payments/wallets/challenge")
async def payment_wallet_challenge(request: Request, body: WalletChallengeRequest):
_assert_entitlement(request)
identity = _require_supabase_identity(request)
try:
return PAYMENT_CHECKOUT.create_wallet_challenge(
user_id=identity["user_id"],
address=body.address,
)
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.post("/api/payments/wallets/verify")
async def payment_wallet_verify(request: Request, body: WalletVerifyRequest):
_assert_entitlement(request)
identity = _require_supabase_identity(request)
try:
bound = PAYMENT_CHECKOUT.verify_wallet_binding(
user_id=identity["user_id"],
address=body.address,
nonce=body.nonce,
signature=body.signature,
)
return {"wallet": bound.__dict__}
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.post("/api/payments/intents")
async def payment_create_intent(request: Request, body: CreatePaymentIntentRequest):
_assert_entitlement(request)
identity = _require_supabase_identity(request)
try:
return PAYMENT_CHECKOUT.create_intent(
user_id=identity["user_id"],
plan_code=body.plan_code,
payment_mode=body.payment_mode,
allowed_wallet=body.allowed_wallet,
token_address=body.token_address,
use_points=body.use_points,
points_to_consume=body.points_to_consume,
metadata=body.metadata,
)
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.get("/api/payments/intents/{intent_id}")
async def payment_get_intent(request: Request, intent_id: str):
_assert_entitlement(request)
identity = _require_supabase_identity(request)
try:
intent = PAYMENT_CHECKOUT.get_intent(
user_id=identity["user_id"],
intent_id=intent_id,
)
return {"intent": intent.__dict__}
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.post("/api/payments/intents/{intent_id}/submit")
async def payment_submit_tx(
request: Request,
intent_id: str,
body: SubmitPaymentTxRequest,
):
_assert_entitlement(request)
identity = _require_supabase_identity(request)
try:
return PAYMENT_CHECKOUT.submit_intent_tx(
user_id=identity["user_id"],
intent_id=intent_id,
tx_hash=body.tx_hash,
from_address=body.from_address,
)
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.post("/api/payments/intents/{intent_id}/confirm")
async def payment_confirm_tx(
request: Request,
intent_id: str,
body: ConfirmPaymentTxRequest,
):
_assert_entitlement(request)
identity = _require_supabase_identity(request)
try:
return PAYMENT_CHECKOUT.confirm_intent_tx(
user_id=identity["user_id"],
intent_id=intent_id,
tx_hash=body.tx_hash,
)
except PaymentCheckoutError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
@router.get("/api/city/{name}/summary")
async def city_summary(request: Request, name: str, force_refresh: bool = False):
_assert_entitlement(request)
data = _analyze(_normalize_city_or_404(name), force_refresh=force_refresh)
return _build_city_summary_payload(data)
@router.get("/api/city/{name}/detail")
async def city_detail_aggregate(
request: Request,
name: str,
force_refresh: bool = False,
market_slug: Optional[str] = None,
target_date: Optional[str] = None,
):
_assert_entitlement(request)
data = _analyze(_normalize_city_or_404(name), force_refresh=force_refresh)
return _build_city_detail_payload(
data,
market_slug=market_slug,
target_date=target_date,
)