Add CI and shadow calibration reporting
This commit is contained in:
2569718930@qq.com
@@ -0,0 +1,210 @@
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import date, timedelta
|
||||
from typing import Dict, Iterable, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.data_collection.city_registry import CITY_REGISTRY # noqa: E402
|
||||
|
||||
|
||||
HOURLY_FIELDS = (
|
||||
"temperature_2m",
|
||||
"relative_humidity_2m",
|
||||
"wind_speed_10m",
|
||||
"wind_direction_10m",
|
||||
"cloud_cover",
|
||||
"shortwave_radiation",
|
||||
"precipitation",
|
||||
"surface_pressure",
|
||||
)
|
||||
|
||||
|
||||
def _city_output_path(output_dir: str, city_name: str) -> str:
|
||||
filename = f"{city_name.replace(' ', '_').lower()}_historical.csv"
|
||||
return os.path.join(output_dir, filename)
|
||||
|
||||
|
||||
def _fetch_city_history(
|
||||
city_name: str,
|
||||
city_info: Dict[str, object],
|
||||
output_dir: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
session: requests.Session,
|
||||
overwrite: bool = False,
|
||||
max_retries: int = 4,
|
||||
) -> Tuple[str, str]:
|
||||
output_path = _city_output_path(output_dir, city_name)
|
||||
if os.path.exists(output_path) and not overwrite:
|
||||
return city_name, "skipped_existing"
|
||||
|
||||
lat = city_info["lat"]
|
||||
lon = city_info["lon"]
|
||||
url = (
|
||||
"https://archive-api.open-meteo.com/v1/archive"
|
||||
f"?latitude={lat}&longitude={lon}"
|
||||
f"&start_date={start_date}&end_date={end_date}"
|
||||
f"&hourly={','.join(HOURLY_FIELDS)}"
|
||||
"&timezone=auto"
|
||||
)
|
||||
last_error = None
|
||||
payload = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
response = session.get(url, timeout=90)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
break
|
||||
except requests.HTTPError as exc:
|
||||
last_error = exc
|
||||
response = getattr(exc, "response", None)
|
||||
status_code = response.status_code if response is not None else None
|
||||
if status_code == 429 and attempt < max_retries:
|
||||
retry_after = 0
|
||||
if response is not None:
|
||||
try:
|
||||
retry_after = int(response.headers.get("Retry-After") or "0")
|
||||
except Exception:
|
||||
retry_after = 0
|
||||
sleep_sec = retry_after if retry_after > 0 else min(90, 5 * (attempt + 1))
|
||||
time.sleep(sleep_sec)
|
||||
continue
|
||||
raise
|
||||
except requests.RequestException as exc:
|
||||
last_error = exc
|
||||
if attempt < max_retries:
|
||||
time.sleep(min(30, 3 * (attempt + 1)))
|
||||
continue
|
||||
raise
|
||||
|
||||
if payload is None:
|
||||
raise RuntimeError(f"failed to fetch {city_name}: {last_error}")
|
||||
|
||||
hourly = payload.get("hourly") or {}
|
||||
times = hourly.get("time") or []
|
||||
if not times:
|
||||
raise RuntimeError(f"missing hourly data for {city_name}")
|
||||
|
||||
fieldnames = ["time", *HOURLY_FIELDS]
|
||||
rows = []
|
||||
for idx, ts in enumerate(times):
|
||||
row = {"time": ts}
|
||||
for field in HOURLY_FIELDS:
|
||||
values = hourly.get(field) or []
|
||||
row[field] = values[idx] if idx < len(values) else None
|
||||
rows.append(row)
|
||||
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(output_path, "w", encoding="utf-8", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
return city_name, f"downloaded:{len(rows)}"
|
||||
|
||||
|
||||
def _iter_target_cities(selected: Iterable[str]) -> Iterable[Tuple[str, Dict[str, object]]]:
|
||||
if not selected:
|
||||
for city_name, city_info in sorted(CITY_REGISTRY.items()):
|
||||
yield city_name, city_info
|
||||
return
|
||||
|
||||
normalized = {str(city).strip().lower() for city in selected if str(city).strip()}
|
||||
for city_name, city_info in sorted(CITY_REGISTRY.items()):
|
||||
if city_name in normalized:
|
||||
yield city_name, city_info
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Backfill historical Open-Meteo CSVs for PolyWeather cities.")
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "historical"),
|
||||
help="Output directory for per-city historical CSV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--start-date",
|
||||
default="2023-01-01",
|
||||
help="Archive start date in YYYY-MM-DD.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--end-date",
|
||||
default=(date.today() - timedelta(days=1)).strftime("%Y-%m-%d"),
|
||||
help="Archive end date in YYYY-MM-DD.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cities",
|
||||
nargs="*",
|
||||
default=[],
|
||||
help="Optional subset of city registry keys to backfill.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--overwrite",
|
||||
action="store_true",
|
||||
help="Re-download existing CSV files instead of skipping them.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workers",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Maximum concurrent download workers.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
cities = list(_iter_target_cities(args.cities))
|
||||
if not cities:
|
||||
raise SystemExit("no matching cities found")
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "PolyWeather Historical Backfill/1.0"})
|
||||
|
||||
results = []
|
||||
with ThreadPoolExecutor(max_workers=max(1, args.workers)) as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
_fetch_city_history,
|
||||
city_name,
|
||||
city_info,
|
||||
args.output_dir,
|
||||
args.start_date,
|
||||
args.end_date,
|
||||
session,
|
||||
args.overwrite,
|
||||
): city_name
|
||||
for city_name, city_info in cities
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
city_name = futures[future]
|
||||
try:
|
||||
_, status = future.result()
|
||||
print(f"{city_name}: {status}")
|
||||
results.append((city_name, status))
|
||||
except Exception as exc:
|
||||
print(f"{city_name}: error:{exc}")
|
||||
results.append((city_name, f"error:{exc}"))
|
||||
time.sleep(0.1)
|
||||
|
||||
downloaded = sum(1 for _, status in results if status.startswith("downloaded:"))
|
||||
skipped = sum(1 for _, status in results if status == "skipped_existing")
|
||||
failed = [city for city, status in results if status.startswith("error:")]
|
||||
print(
|
||||
"summary downloaded={downloaded} skipped={skipped} failed={failed}".format(
|
||||
downloaded=downloaded,
|
||||
skipped=skipped,
|
||||
failed=len(failed),
|
||||
)
|
||||
)
|
||||
if failed:
|
||||
print("failed_cities=" + ",".join(sorted(failed)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,152 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.deb_algorithm import load_history, save_history # noqa: E402
|
||||
from src.analysis.probability_calibration import ( # noqa: E402
|
||||
ENGINE_MODE_EMOS_SHADOW,
|
||||
apply_probability_calibration,
|
||||
build_probability_features,
|
||||
)
|
||||
|
||||
|
||||
def _sample_to_features(sample):
|
||||
peak_flag = sample.get("peak_flag")
|
||||
if peak_flag == 1.0:
|
||||
peak_status = "past"
|
||||
elif peak_flag == 0.5:
|
||||
peak_status = "in_window"
|
||||
else:
|
||||
peak_status = "before"
|
||||
return build_probability_features(
|
||||
city_name=sample.get("city") or "",
|
||||
raw_mu=sample.get("raw_mu"),
|
||||
raw_sigma=sample.get("raw_sigma"),
|
||||
deb_prediction=sample.get("deb_prediction"),
|
||||
ens_data={
|
||||
"median": sample.get("ens_median"),
|
||||
"p10": None,
|
||||
"p90": None,
|
||||
},
|
||||
current_forecasts={},
|
||||
max_so_far=None,
|
||||
peak_status=peak_status,
|
||||
local_hour_frac=None,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Backfill shadow probability snapshots into daily records.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--training-samples",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"training_samples.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-file",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"default.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
with open(args.training_samples, "r", encoding="utf-8") as fh:
|
||||
training_payload = json.load(fh)
|
||||
|
||||
updated = 0
|
||||
touched = 0
|
||||
|
||||
for sample in training_payload.get("samples") or []:
|
||||
city = str(sample.get("city") or "").strip().lower()
|
||||
date_str = str(sample.get("date") or "").strip()
|
||||
if not city or not date_str:
|
||||
continue
|
||||
record = ((history.get(city) or {}).get(date_str) or {})
|
||||
if not isinstance(record, dict) or not record:
|
||||
continue
|
||||
|
||||
legacy_distribution = [
|
||||
{"value": row.get("v"), "probability": row.get("p")}
|
||||
for row in (record.get("prob_snapshot") or [])
|
||||
if isinstance(row, dict) and row.get("v") is not None
|
||||
]
|
||||
if not legacy_distribution:
|
||||
continue
|
||||
|
||||
calibration = apply_probability_calibration(
|
||||
city_name=city,
|
||||
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
|
||||
raw_mu=sample.get("raw_mu"),
|
||||
raw_sigma=sample.get("raw_sigma"),
|
||||
max_so_far=None,
|
||||
legacy_distribution=legacy_distribution,
|
||||
features=_sample_to_features(sample),
|
||||
calibration_path=args.calibration_file,
|
||||
mode=ENGINE_MODE_EMOS_SHADOW,
|
||||
)
|
||||
|
||||
shadow_distribution = calibration.get("shadow_distribution") or []
|
||||
compact_shadow = [
|
||||
{
|
||||
"v": int(row.get("value")),
|
||||
"p": round(float(row.get("probability") or 0.0), 3),
|
||||
}
|
||||
for row in shadow_distribution[:4]
|
||||
if row.get("value") is not None
|
||||
]
|
||||
compact_calibration = {
|
||||
"mode": calibration.get("mode"),
|
||||
"engine": calibration.get("engine"),
|
||||
"version": calibration.get("calibration_version"),
|
||||
"source": calibration.get("calibration_source"),
|
||||
"raw_mu": calibration.get("raw_mu"),
|
||||
"raw_sigma": calibration.get("raw_sigma"),
|
||||
"calibrated_mu": calibration.get("calibrated_mu"),
|
||||
"calibrated_sigma": calibration.get("calibrated_sigma"),
|
||||
}
|
||||
|
||||
touched += 1
|
||||
if (
|
||||
record.get("shadow_prob_snapshot") == compact_shadow
|
||||
and record.get("probability_calibration") == compact_calibration
|
||||
):
|
||||
continue
|
||||
|
||||
record["shadow_prob_snapshot"] = compact_shadow
|
||||
record["probability_calibration"] = compact_calibration
|
||||
history[city][date_str] = record
|
||||
updated += 1
|
||||
|
||||
save_history(args.history_file, history)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"samples_seen": len(training_payload.get("samples") or []),
|
||||
"records_considered": touched,
|
||||
"records_updated": updated,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,219 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
|
||||
|
||||
|
||||
def _sf(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _mean(values):
|
||||
return round(sum(values) / len(values), 6) if values else None
|
||||
|
||||
|
||||
def _top_bucket(snapshot):
|
||||
if not isinstance(snapshot, list):
|
||||
return None
|
||||
best = None
|
||||
best_prob = -1.0
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
try:
|
||||
prob = float(row.get("p") if "p" in row else row.get("probability"))
|
||||
except Exception:
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value is None:
|
||||
continue
|
||||
if prob > best_prob:
|
||||
best = value
|
||||
best_prob = prob
|
||||
return best
|
||||
|
||||
|
||||
def _bucket_probability(snapshot, target_bucket):
|
||||
if not isinstance(snapshot, list):
|
||||
return 0.0
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value != target_bucket:
|
||||
continue
|
||||
try:
|
||||
return float(row.get("p") if "p" in row else row.get("probability") or 0.0)
|
||||
except Exception:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
|
||||
def _brier_from_snapshot(snapshot, target_bucket):
|
||||
hit_prob = _bucket_probability(snapshot, target_bucket)
|
||||
total = (1.0 - hit_prob) ** 2
|
||||
if isinstance(snapshot, list):
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value == target_bucket:
|
||||
continue
|
||||
try:
|
||||
prob = float(row.get("p") if "p" in row else row.get("probability") or 0.0)
|
||||
except Exception:
|
||||
prob = 0.0
|
||||
total += prob * prob
|
||||
return round(total, 6)
|
||||
|
||||
|
||||
def _blank_metrics():
|
||||
return {
|
||||
"samples": 0,
|
||||
"legacy_mae": [],
|
||||
"shadow_mae": [],
|
||||
"legacy_bucket_hit": [],
|
||||
"shadow_bucket_hit": [],
|
||||
"legacy_bucket_brier": [],
|
||||
"shadow_bucket_brier": [],
|
||||
}
|
||||
|
||||
|
||||
def _rollup(metrics):
|
||||
return {
|
||||
"samples": metrics["samples"],
|
||||
"legacy_mean_mae": _mean(metrics["legacy_mae"]),
|
||||
"shadow_mean_mae": _mean(metrics["shadow_mae"]),
|
||||
"legacy_bucket_hit_rate": _mean(metrics["legacy_bucket_hit"]),
|
||||
"shadow_bucket_hit_rate": _mean(metrics["shadow_bucket_hit"]),
|
||||
"legacy_bucket_brier": _mean(metrics["legacy_bucket_brier"]),
|
||||
"shadow_bucket_brier": _mean(metrics["shadow_bucket_brier"]),
|
||||
"delta_mae": round((_mean(metrics["shadow_mae"]) or 0.0) - (_mean(metrics["legacy_mae"]) or 0.0), 6),
|
||||
"delta_bucket_hit_rate": round((_mean(metrics["shadow_bucket_hit"]) or 0.0) - (_mean(metrics["legacy_bucket_hit"]) or 0.0), 6),
|
||||
"delta_bucket_brier": round((_mean(metrics["shadow_bucket_brier"]) or 0.0) - (_mean(metrics["legacy_bucket_brier"]) or 0.0), 6),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Build live shadow probability report from daily records.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"shadow_report.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
overall = _blank_metrics()
|
||||
by_city = defaultdict(_blank_metrics)
|
||||
by_date = defaultdict(_blank_metrics)
|
||||
latest_observations = []
|
||||
|
||||
for city, city_records in sorted(history.items()):
|
||||
if not isinstance(city_records, dict):
|
||||
continue
|
||||
for date_str, record in sorted(city_records.items()):
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
actual_high = _sf(record.get("actual_high"))
|
||||
shadow_snapshot = record.get("shadow_prob_snapshot")
|
||||
calibration = record.get("probability_calibration") or {}
|
||||
if actual_high is None or not shadow_snapshot:
|
||||
continue
|
||||
|
||||
legacy_mu = _sf(calibration.get("raw_mu"))
|
||||
if legacy_mu is None:
|
||||
legacy_mu = _sf(record.get("mu"))
|
||||
shadow_mu = _sf(calibration.get("calibrated_mu"))
|
||||
if shadow_mu is None:
|
||||
continue
|
||||
|
||||
actual_bucket = apply_city_settlement(city, actual_high)
|
||||
legacy_snapshot = record.get("prob_snapshot") or []
|
||||
legacy_bucket = _top_bucket(legacy_snapshot)
|
||||
shadow_bucket = _top_bucket(shadow_snapshot)
|
||||
|
||||
for metrics in (overall, by_city[city], by_date[date_str]):
|
||||
metrics["samples"] += 1
|
||||
metrics["legacy_mae"].append(abs(legacy_mu - actual_high))
|
||||
metrics["shadow_mae"].append(abs(shadow_mu - actual_high))
|
||||
metrics["legacy_bucket_hit"].append(1.0 if legacy_bucket == actual_bucket else 0.0)
|
||||
metrics["shadow_bucket_hit"].append(1.0 if shadow_bucket == actual_bucket else 0.0)
|
||||
metrics["legacy_bucket_brier"].append(_brier_from_snapshot(legacy_snapshot, actual_bucket))
|
||||
metrics["shadow_bucket_brier"].append(_brier_from_snapshot(shadow_snapshot, actual_bucket))
|
||||
|
||||
latest_observations.append(
|
||||
{
|
||||
"city": city,
|
||||
"date": date_str,
|
||||
"actual_high": actual_high,
|
||||
"actual_bucket": actual_bucket,
|
||||
"legacy_mu": round(legacy_mu, 3),
|
||||
"shadow_mu": round(shadow_mu, 3),
|
||||
"legacy_top_bucket": legacy_bucket,
|
||||
"shadow_top_bucket": shadow_bucket,
|
||||
"calibration_version": calibration.get("version"),
|
||||
"calibration_mode": calibration.get("mode"),
|
||||
}
|
||||
)
|
||||
|
||||
by_city_report = {
|
||||
city: _rollup(metrics)
|
||||
for city, metrics in sorted(by_city.items())
|
||||
}
|
||||
by_date_report = {
|
||||
date_str: _rollup(metrics)
|
||||
for date_str, metrics in sorted(
|
||||
by_date.items(),
|
||||
key=lambda item: datetime.strptime(item[0], "%Y-%m-%d"),
|
||||
)
|
||||
}
|
||||
|
||||
latest_observations = sorted(
|
||||
latest_observations,
|
||||
key=lambda row: (row["date"], row["city"]),
|
||||
reverse=True,
|
||||
)[:100]
|
||||
|
||||
payload = {
|
||||
"generated_at": datetime.utcnow().isoformat() + "Z",
|
||||
"summary": _rollup(overall),
|
||||
"by_city": by_city_report,
|
||||
"by_date": by_date_report,
|
||||
"recent_observations": latest_observations,
|
||||
}
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps(payload["summary"], ensure_ascii=False, indent=2))
|
||||
print(f"saved shadow report to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,84 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
|
||||
from src.data_collection.city_registry import CITY_REGISTRY # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Build daily settlement history from historical weather CSV files.")
|
||||
parser.add_argument(
|
||||
"--history-dir",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "historical"),
|
||||
help="Directory containing per-city *_historical.csv files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
help="Output JSON path.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = {}
|
||||
for city in sorted(CITY_REGISTRY.keys()):
|
||||
path = os.path.join(
|
||||
args.history_dir,
|
||||
f"{city.replace(' ', '_').lower()}_historical.csv",
|
||||
)
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
|
||||
daily_max = defaultdict(lambda: None)
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
for row in reader:
|
||||
ts = str(row.get("time") or "")
|
||||
raw_temp = row.get("temperature_2m")
|
||||
if not ts or raw_temp in (None, ""):
|
||||
continue
|
||||
try:
|
||||
temp = float(raw_temp)
|
||||
except Exception:
|
||||
continue
|
||||
day = ts[:10]
|
||||
prev = daily_max[day]
|
||||
if prev is None or temp > prev:
|
||||
daily_max[day] = temp
|
||||
|
||||
result[city] = {
|
||||
day: {
|
||||
"max_temp": round(max_temp, 3),
|
||||
"settlement_value": apply_city_settlement(city, max_temp),
|
||||
}
|
||||
for day, max_temp in sorted(daily_max.items())
|
||||
if max_temp is not None
|
||||
}
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(result, fh, ensure_ascii=False, indent=2)
|
||||
print(
|
||||
"saved settlement history to {path} for {count} cities".format(
|
||||
path=args.output,
|
||||
count=len(result),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,206 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.probability_calibration import ( # noqa: E402
|
||||
ENGINE_MODE_EMOS_PRIMARY,
|
||||
_gaussian_crps,
|
||||
apply_probability_calibration,
|
||||
build_probability_features,
|
||||
)
|
||||
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
|
||||
from scripts.fit_probability_calibration import ( # noqa: E402
|
||||
_extract_samples,
|
||||
_load_json_if_exists,
|
||||
)
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
|
||||
|
||||
def _mean(values):
|
||||
return (sum(values) / len(values)) if values else None
|
||||
|
||||
|
||||
def _sample_to_features(sample):
|
||||
return build_probability_features(
|
||||
city_name=sample.get("city") or "",
|
||||
raw_mu=sample.get("raw_mu"),
|
||||
raw_sigma=sample.get("raw_sigma"),
|
||||
deb_prediction=sample.get("deb_prediction"),
|
||||
ens_data={
|
||||
"median": sample.get("ens_median"),
|
||||
"p10": None,
|
||||
"p90": None,
|
||||
},
|
||||
current_forecasts={},
|
||||
max_so_far=None,
|
||||
peak_status="in_window" if sample.get("peak_flag") == 0.5 else "past" if sample.get("peak_flag") == 1.0 else "before",
|
||||
local_hour_frac=None,
|
||||
)
|
||||
|
||||
|
||||
def _top_bucket_value(distribution):
|
||||
if not distribution:
|
||||
return None
|
||||
top = max(
|
||||
(row for row in distribution if isinstance(row, dict)),
|
||||
key=lambda row: float(row.get("probability") or 0.0),
|
||||
default=None,
|
||||
)
|
||||
if not top:
|
||||
return None
|
||||
return top.get("value")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Evaluate legacy vs EMOS probability calibration.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-file",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"default.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"evaluation_report.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
settlement_history=settlement_history,
|
||||
)
|
||||
|
||||
legacy_crps = []
|
||||
emos_crps = []
|
||||
legacy_mae = []
|
||||
emos_mae = []
|
||||
legacy_bucket_hits = []
|
||||
emos_bucket_hits = []
|
||||
by_city = defaultdict(lambda: {
|
||||
"samples": 0,
|
||||
"legacy_crps": [],
|
||||
"emos_crps": [],
|
||||
"legacy_mae": [],
|
||||
"emos_mae": [],
|
||||
"legacy_bucket_hits": [],
|
||||
"emos_bucket_hits": [],
|
||||
})
|
||||
|
||||
for sample in samples:
|
||||
city = str(sample.get("city") or "").strip().lower()
|
||||
actual_high = float(sample["actual_high"])
|
||||
raw_mu = float(sample["raw_mu"])
|
||||
raw_sigma = max(0.1, float(sample["raw_sigma"]))
|
||||
legacy_crps.append(_gaussian_crps(actual_high, raw_mu, raw_sigma))
|
||||
legacy_mae.append(abs(raw_mu - actual_high))
|
||||
|
||||
legacy_bucket = apply_city_settlement(city, raw_mu)
|
||||
actual_bucket = apply_city_settlement(city, actual_high)
|
||||
legacy_bucket_hits.append(1.0 if legacy_bucket == actual_bucket else 0.0)
|
||||
|
||||
calibration = apply_probability_calibration(
|
||||
city_name=city,
|
||||
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
|
||||
raw_mu=raw_mu,
|
||||
raw_sigma=raw_sigma,
|
||||
max_so_far=None,
|
||||
legacy_distribution=[],
|
||||
features=_sample_to_features(sample),
|
||||
calibration_path=args.calibration_file,
|
||||
mode=ENGINE_MODE_EMOS_PRIMARY,
|
||||
)
|
||||
emos_mu = float(calibration.get("calibrated_mu") or raw_mu)
|
||||
emos_sigma = max(0.1, float(calibration.get("calibrated_sigma") or raw_sigma))
|
||||
emos_distribution = calibration.get("distribution") or []
|
||||
emos_crps.append(_gaussian_crps(actual_high, emos_mu, emos_sigma))
|
||||
emos_mae.append(abs(emos_mu - actual_high))
|
||||
emos_bucket = _top_bucket_value(emos_distribution)
|
||||
emos_bucket_hits.append(1.0 if emos_bucket == actual_bucket else 0.0)
|
||||
|
||||
row = by_city[city]
|
||||
row["samples"] += 1
|
||||
row["legacy_crps"].append(legacy_crps[-1])
|
||||
row["emos_crps"].append(emos_crps[-1])
|
||||
row["legacy_mae"].append(legacy_mae[-1])
|
||||
row["emos_mae"].append(emos_mae[-1])
|
||||
row["legacy_bucket_hits"].append(legacy_bucket_hits[-1])
|
||||
row["emos_bucket_hits"].append(emos_bucket_hits[-1])
|
||||
|
||||
summary = {
|
||||
"sample_count": len(samples),
|
||||
"filled_actual_from_history": filled_actual_from_history,
|
||||
"legacy": {
|
||||
"mean_crps": round(_mean(legacy_crps), 6) if legacy_crps else None,
|
||||
"mean_mae": round(_mean(legacy_mae), 6) if legacy_mae else None,
|
||||
"bucket_hit_rate": round(_mean(legacy_bucket_hits), 6) if legacy_bucket_hits else None,
|
||||
},
|
||||
"emos": {
|
||||
"mean_crps": round(_mean(emos_crps), 6) if emos_crps else None,
|
||||
"mean_mae": round(_mean(emos_mae), 6) if emos_mae else None,
|
||||
"bucket_hit_rate": round(_mean(emos_bucket_hits), 6) if emos_bucket_hits else None,
|
||||
},
|
||||
"delta": {
|
||||
"crps": round((_mean(emos_crps) or 0.0) - (_mean(legacy_crps) or 0.0), 6),
|
||||
"mae": round((_mean(emos_mae) or 0.0) - (_mean(legacy_mae) or 0.0), 6),
|
||||
"bucket_hit_rate": round((_mean(emos_bucket_hits) or 0.0) - (_mean(legacy_bucket_hits) or 0.0), 6),
|
||||
},
|
||||
}
|
||||
|
||||
city_report = {}
|
||||
for city, metrics in sorted(by_city.items()):
|
||||
city_report[city] = {
|
||||
"samples": metrics["samples"],
|
||||
"legacy_mean_crps": round(_mean(metrics["legacy_crps"]), 6),
|
||||
"emos_mean_crps": round(_mean(metrics["emos_crps"]), 6),
|
||||
"legacy_mean_mae": round(_mean(metrics["legacy_mae"]), 6),
|
||||
"emos_mean_mae": round(_mean(metrics["emos_mae"]), 6),
|
||||
"legacy_bucket_hit_rate": round(_mean(metrics["legacy_bucket_hits"]), 6),
|
||||
"emos_bucket_hit_rate": round(_mean(metrics["emos_bucket_hits"]), 6),
|
||||
}
|
||||
|
||||
payload = {
|
||||
"summary": summary,
|
||||
"by_city": city_report,
|
||||
}
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
print(f"saved evaluation report to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
from scripts.fit_probability_calibration import ( # noqa: E402
|
||||
_extract_samples,
|
||||
_load_json_if_exists,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Export normalized probability calibration training samples.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"training_samples.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
settlement_history=settlement_history,
|
||||
)
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
payload = {
|
||||
"sample_count": len(samples),
|
||||
"filled_actual_from_history": filled_actual_from_history,
|
||||
"samples": samples,
|
||||
}
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"exported {len(samples)} samples to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.probability_calibration import ( # noqa: E402
|
||||
DEFAULT_CALIBRATION_FILE,
|
||||
default_calibration_payload,
|
||||
fit_calibration,
|
||||
)
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
|
||||
|
||||
def _sf(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _load_json_if_exists(path):
|
||||
if not path or not os.path.exists(path):
|
||||
return {}
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _extract_samples(history, settlement_history=None):
|
||||
samples = []
|
||||
filled_actual_from_history = 0
|
||||
today = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
settlement_history = settlement_history or {}
|
||||
for city, city_rows in (history or {}).items():
|
||||
if not isinstance(city_rows, dict):
|
||||
continue
|
||||
city_settlement = settlement_history.get(city) or {}
|
||||
for date_str, record in city_rows.items():
|
||||
if date_str == today or not isinstance(record, dict):
|
||||
continue
|
||||
actual_high = _sf(record.get("actual_high"))
|
||||
if actual_high is None:
|
||||
actual_high = _sf((city_settlement.get(date_str) or {}).get("max_temp"))
|
||||
if actual_high is not None:
|
||||
filled_actual_from_history += 1
|
||||
deb_prediction = _sf(record.get("deb_prediction"))
|
||||
raw_mu = _sf(record.get("mu")) or deb_prediction
|
||||
forecasts = record.get("forecasts") or {}
|
||||
if not isinstance(forecasts, dict):
|
||||
forecasts = {}
|
||||
forecast_values = [val for val in (_sf(v) for v in forecasts.values()) if val is not None]
|
||||
forecast_values.sort()
|
||||
forecast_median = (
|
||||
forecast_values[len(forecast_values) // 2] if forecast_values else None
|
||||
)
|
||||
feature_snapshot = record.get("probability_features") or {}
|
||||
if not isinstance(feature_snapshot, dict):
|
||||
feature_snapshot = {}
|
||||
|
||||
ens_median = _sf(feature_snapshot.get("ens_median")) or forecast_median or raw_mu
|
||||
ensemble_spread = _sf(feature_snapshot.get("ensemble_spread"))
|
||||
if ensemble_spread is None:
|
||||
if len(forecast_values) >= 2:
|
||||
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
|
||||
else:
|
||||
ensemble_spread = 1.0
|
||||
raw_sigma = _sf(feature_snapshot.get("raw_sigma")) or ensemble_spread or 1.0
|
||||
peak_status = str(feature_snapshot.get("peak_status") or "before").strip().lower()
|
||||
peak_flag = 0.0
|
||||
if peak_status == "in_window":
|
||||
peak_flag = 0.5
|
||||
elif peak_status == "past":
|
||||
peak_flag = 1.0
|
||||
|
||||
if actual_high is None or raw_mu is None:
|
||||
continue
|
||||
|
||||
max_so_far = _sf(feature_snapshot.get("max_so_far"))
|
||||
max_so_far_gap = _sf(feature_snapshot.get("max_so_far_gap"))
|
||||
if max_so_far_gap is None and max_so_far is not None and deb_prediction is not None:
|
||||
max_so_far_gap = deb_prediction - max_so_far
|
||||
|
||||
samples.append(
|
||||
{
|
||||
"city": city,
|
||||
"date": date_str,
|
||||
"actual_high": actual_high,
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": raw_sigma,
|
||||
"deb_prediction": deb_prediction,
|
||||
"ens_median": ens_median,
|
||||
"ensemble_spread": ensemble_spread,
|
||||
"max_so_far_gap": max_so_far_gap,
|
||||
"peak_flag": peak_flag,
|
||||
}
|
||||
)
|
||||
return samples, filled_actual_from_history
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=os.path.join(PROJECT_ROOT, "data", "daily_records.json"),
|
||||
help="Path to the historical daily_records.json file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=DEFAULT_CALIBRATION_FILE,
|
||||
help="Output JSON file for fitted calibration parameters.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
help="Optional daily settlement history JSON built from historical CSV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
default=None,
|
||||
help="Optional explicit calibration version.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = load_history(args.history_file)
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
settlement_history=settlement_history,
|
||||
)
|
||||
calibration = fit_calibration(samples, version=args.version)
|
||||
if not samples:
|
||||
calibration = default_calibration_payload(
|
||||
version=args.version,
|
||||
reason="no_samples",
|
||||
)
|
||||
calibration.setdefault("metrics", {})
|
||||
calibration["metrics"]["filled_actual_from_history"] = filled_actual_from_history
|
||||
calibration["metrics"]["settlement_history_city_count"] = len(settlement_history)
|
||||
try:
|
||||
calibration["source"] = os.path.relpath(args.output, PROJECT_ROOT)
|
||||
except ValueError:
|
||||
calibration["source"] = os.path.abspath(args.output)
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(calibration, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(
|
||||
"saved calibration to {path} with {count} samples".format(
|
||||
path=args.output,
|
||||
count=calibration.get("metrics", {}).get("sample_count", 0),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user