gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refine EMOS calibration and add Chinese training report

Browse Source
This commit is contained in:
2569718930@qq.com
2026-03-20 21:17:37 +08:00
parent 1c84893bed
commit 03dcb4329b
6 changed files with 879 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
artifacts/probability_calibration/default.json
+21 -8
View File
@@ -1,6 +1,6 @@
{
"version": "emos-20260320123829",
"trained_at": "2026-03-20T12:38:29.985991+00:00",
"version": "emos-20260320130245",
"trained_at": "2026-03-20T13:02:45.903772+00:00",
"global": {
"mu": {
"intercept": -1.57406048,
@@ -17,9 +17,20 @@
"max_so_far_gap_coef": 0.0
}
},
"sigma_constraints": {
"min_ratio": 0.85,
"max_ratio": 1.35,
"absolute_min": 0.25,
"absolute_max": 3.0
},
"selection_guardrails": {
"max_mae_increase": 0.02,
"max_bucket_hit_drop": 0.01,
"max_bucket_brier_increase": 0.05
},
"blending": {
"alpha_mu": 0.0,
"alpha_sigma": 1.0
"alpha_sigma": 0.0
},
"cities": {
"ankara": {
@@ -145,15 +156,17 @@
},
"metrics": {
"sample_count": 105,
"mean_crps": 2.744961,
"mean_crps": 2.923823,
"legacy_mean_crps": 2.793938,
"legacy_mean_mae": 2.721143,
"legacy_bucket_hit_rate": 0.695238,
"selected_mean_crps": 2.747094,
"legacy_bucket_brier": 0.775463,
"selected_mean_crps": 2.700275,
"selected_mean_mae": 2.721143,
"selected_bucket_hit_rate": 0.704762,
"selected_score": 3.609685,
"legacy_score": 3.675576,
"selected_bucket_hit_rate": 0.695238,
"selected_bucket_brier": 0.765459,
"selected_score": 4.003626,
"legacy_score": 4.104792,
"filled_actual_from_history": 2,
"settlement_history_city_count": 30
},
artifacts/probability_calibration/evaluation_report.json
+59 -59
View File
@@ -8,239 +8,239 @@
"bucket_hit_rate": 0.695238
},
"emos": {
"mean_crps": 2.747094,
"mean_mae": 2.721143,
"bucket_hit_rate": 0.685714
"mean_crps": 2.650216,
"mean_mae": 2.722829,
"bucket_hit_rate": 0.666667
},
"delta": {
"crps": -0.046843,
"mae": 0.0,
"bucket_hit_rate": -0.009524
"crps": -0.143722,
"mae": 0.001686,
"bucket_hit_rate": -0.028571
}
},
"by_city": {
"ankara": {
"samples": 7,
"legacy_mean_crps": 2.023242,
"emos_mean_crps": 2.259428,
"emos_mean_crps": 2.015458,
"legacy_mean_mae": 1.984286,
"emos_mean_mae": 1.984286,
"emos_mean_mae": 1.977721,
"legacy_bucket_hit_rate": 0.714286,
"emos_bucket_hit_rate": 0.714286
},
"atlanta": {
"samples": 5,
"legacy_mean_crps": 12.792034,
"emos_mean_crps": 12.537301,
"emos_mean_crps": 12.526108,
"legacy_mean_mae": 12.806,
"emos_mean_mae": 12.806,
"emos_mean_mae": 12.919677,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
"emos_bucket_hit_rate": 0.4
},
"buenos aires": {
"samples": 5,
"legacy_mean_crps": 3.846144,
"emos_mean_crps": 3.745234,
"emos_mean_crps": 3.743967,
"legacy_mean_mae": 4.168,
"emos_mean_mae": 4.168,
"emos_mean_mae": 4.086663,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"chicago": {
"samples": 4,
"legacy_mean_crps": 1.346667,
"emos_mean_crps": 0.66337,
"emos_mean_crps": 0.603128,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.0,
"emos_mean_mae": 0.075265,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"dallas": {
"samples": 4,
"legacy_mean_crps": 1.256111,
"emos_mean_crps": 0.651563,
"emos_mean_crps": 0.65514,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.0,
"emos_mean_mae": 0.114807,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"hong kong": {
"samples": 3,
"legacy_mean_crps": 0.261027,
"emos_mean_crps": 0.621432,
"emos_mean_crps": 0.269856,
"legacy_mean_mae": 0.1,
"emos_mean_mae": 0.1,
"emos_mean_mae": 0.174879,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.666667
},
"london": {
"samples": 6,
"legacy_mean_crps": 2.079624,
"emos_mean_crps": 1.986319,
"emos_mean_crps": 2.005311,
"legacy_mean_mae": 2.451667,
"emos_mean_mae": 2.451667,
"emos_mean_mae": 2.327304,
"legacy_bucket_hit_rate": 0.166667,
"emos_bucket_hit_rate": 0.166667
"emos_bucket_hit_rate": 0.333333
},
"lucknow": {
"samples": 4,
"legacy_mean_crps": 1.468528,
"emos_mean_crps": 1.523689,
"emos_mean_crps": 1.443159,
"legacy_mean_mae": 1.6025,
"emos_mean_mae": 1.6025,
"emos_mean_mae": 1.682697,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 0.5
},
"madrid": {
"samples": 2,
"legacy_mean_crps": 6.27726,
"emos_mean_crps": 6.028088,
"emos_mean_crps": 6.090563,
"legacy_mean_mae": 7.33,
"emos_mean_mae": 7.33,
"emos_mean_mae": 7.141649,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"miami": {
"samples": 5,
"legacy_mean_crps": 11.665378,
"emos_mean_crps": 11.614409,
"emos_mean_crps": 11.493094,
"legacy_mean_mae": 12.07,
"emos_mean_mae": 12.07,
"emos_mean_mae": 12.157623,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
"emos_bucket_hit_rate": 0.2
},
"milan": {
"samples": 3,
"legacy_mean_crps": 4.401392,
"emos_mean_crps": 3.905274,
"emos_mean_crps": 3.837715,
"legacy_mean_mae": 4.06,
"emos_mean_mae": 4.06,
"emos_mean_mae": 3.942288,
"legacy_bucket_hit_rate": 0.666667,
"emos_bucket_hit_rate": 0.666667
},
"munich": {
"samples": 6,
"legacy_mean_crps": 2.988583,
"emos_mean_crps": 2.892973,
"emos_mean_crps": 2.918299,
"legacy_mean_mae": 3.143333,
"emos_mean_mae": 3.143333,
"emos_mean_mae": 3.126482,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 0.5
},
"new york": {
"samples": 4,
"legacy_mean_crps": 1.861101,
"emos_mean_crps": 1.420965,
"emos_mean_crps": 1.363203,
"legacy_mean_mae": 1.3725,
"emos_mean_mae": 1.3725,
"emos_mean_mae": 1.369301,
"legacy_bucket_hit_rate": 0.75,
"emos_bucket_hit_rate": 0.75
},
"paris": {
"samples": 7,
"legacy_mean_crps": 2.430082,
"emos_mean_crps": 2.23397,
"emos_mean_crps": 2.365416,
"legacy_mean_mae": 2.518571,
"emos_mean_mae": 2.518571,
"emos_mean_mae": 2.48437,
"legacy_bucket_hit_rate": 0.571429,
"emos_bucket_hit_rate": 0.571429
},
"sao paulo": {
"samples": 5,
"legacy_mean_crps": 2.454756,
"emos_mean_crps": 2.351014,
"emos_mean_crps": 2.410624,
"legacy_mean_mae": 2.628,
"emos_mean_mae": 2.628,
"emos_mean_mae": 2.689909,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"seattle": {
"samples": 4,
"legacy_mean_crps": 0.531656,
"emos_mean_crps": 0.850792,
"emos_mean_crps": 0.457257,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.0,
"emos_mean_mae": 0.11262,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"seoul": {
"samples": 6,
"legacy_mean_crps": 0.328088,
"emos_mean_crps": 0.710725,
"emos_mean_crps": 0.332722,
"legacy_mean_mae": 0.2,
"emos_mean_mae": 0.2,
"emos_mean_mae": 0.246659,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"shanghai": {
"samples": 2,
"legacy_mean_crps": 0.250034,
"emos_mean_crps": 0.463888,
"emos_mean_crps": 0.249952,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 0.15,
"emos_mean_mae": 0.204362,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"singapore": {
"samples": 2,
"legacy_mean_crps": 0.281993,
"emos_mean_crps": 0.484792,
"emos_mean_crps": 0.270875,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 0.15,
"emos_mean_mae": 0.070547,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"taipei": {
"samples": 3,
"legacy_mean_crps": 0.356996,
"emos_mean_crps": 0.538714,
"emos_mean_crps": 0.351432,
"legacy_mean_mae": 0.1,
"emos_mean_mae": 0.1,
"emos_mean_mae": 0.084784,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.666667
},
"tel aviv": {
"samples": 2,
"legacy_mean_crps": 0.446758,
"emos_mean_crps": 0.55971,
"emos_mean_crps": 0.434936,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 0.3,
"emos_mean_mae": 0.196589,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"tokyo": {
"samples": 2,
"legacy_mean_crps": 0.450128,
"emos_mean_crps": 0.565212,
"emos_mean_crps": 0.429389,
"legacy_mean_mae": 0.25,
"emos_mean_mae": 0.25,
"emos_mean_mae": 0.100782,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 1.0
},
"toronto": {
"samples": 5,
"legacy_mean_crps": 2.647861,
"emos_mean_crps": 2.402627,
"emos_mean_crps": 2.512236,
"legacy_mean_mae": 2.532,
"emos_mean_mae": 2.532,
"emos_mean_mae": 2.535078,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"warsaw": {
"samples": 3,
"legacy_mean_crps": 1.618875,
"emos_mean_crps": 1.601375,
"emos_mean_crps": 1.545129,
"legacy_mean_mae": 2.056667,
"emos_mean_mae": 2.056667,
"emos_mean_mae": 1.942917,
"legacy_bucket_hit_rate": 0.333333,
"emos_bucket_hit_rate": 0.333333
},
"wellington": {
"samples": 6,
"legacy_mean_crps": 0.266349,
"emos_mean_crps": 0.592134,
"emos_mean_crps": 0.264506,
"legacy_mean_mae": 0.2,
"emos_mean_mae": 0.2,
"emos_mean_mae": 0.205623,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
}
artifacts/probability_calibration/shadow_report.json
+368 -368
View File
File diff suppressed because it is too large Load Diff
docs/EMOS_TRAINING_REPORT_ZH.md
+311
View File
@@ -0,0 +1,311 @@
# EMOS 训练报告（2026-03-20）
## 1. 报告目的
本文档用于记录当前 PolyWeather 概率校准引擎（EMOS）的训练结果、离线评估结果、线上 shadow 观测结果，以及是否具备切换为主路径的条件。
当前结论先写在前面：
- `EMOS` 已完成接入、训练、离线评估、shadow 落盘与滚动报表。
- 当前默认运行模式应继续保持 `emos_shadow`。
- 现阶段 **不建议切换到 `emos_primary`**。
## 2. 本次训练版本
- 校准版本：`emos-20260320130245`
- 训练时间：`2026-03-20T13:02:45.903772+00:00`
- 参数文件：[default.json](/E:/web/PolyWeather/artifacts/probability_calibration/default.json)
- 离线评估报告：[evaluation_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/evaluation_report.json)
- 线上 shadow 报表：[shadow_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/shadow_report.json)
## 3. 训练数据概况
### 3.1 数据来源
当前训练主要使用两类数据：
1. 项目历史日记录
文件：[daily_records.json](/E:/web/PolyWeather/data/daily_records.json)
2. 历史天气 CSV 构建出的结算标签
文件：[settlement_history.json](/E:/web/PolyWeather/artifacts/probability_calibration/settlement_history.json)
### 3.2 样本规模
- 总训练样本数：`105`
- 通过历史天气 CSV 补回的缺失 `actual_high`：`2`
- 历史结算标签覆盖城市数：`30`
说明：
- 当前样本已覆盖 30 个城市，但有效监督样本量仍偏小。
- 部分城市样本数只有 `2-7` 条，城市级参数容易波动。
## 4. 模型结构
### 4.1 当前实现
EMOS 属于统计后处理层，不是数值天气模型本身。当前结构位于：
- [probability_calibration.py](/E:/web/PolyWeather/src/analysis/probability_calibration.py)
当前目标是对原有概率引擎输出进行校准：
- 输入：`raw_mu`、`raw_sigma`、`DEB`、`ensemble median/spread`、`peak_status` 等特征
- 输出：校准后的 `mu / sigma / distribution`
### 4.2 当前运行模式
支持三种模式：
- `legacy`
- `emos_shadow`
- `emos_primary`
当前建议默认模式：
- `emos_shadow`
即：
- 对外仍展示 legacy 结果
- 后台并行计算 EMOS 结果
- 用于持续评估，不直接影响用户
## 5. 本次训练参数摘要
### 5.1 全局约束
本次训练已加入两类约束：
1. `sigma_constraints`
- `min_ratio = 0.85`
- `max_ratio = 1.35`
- `absolute_min = 0.25`
- `absolute_max = 3.0`
2. `selection_guardrails`
- `max_mae_increase = 0.02`
- `max_bucket_hit_drop = 0.01`
- `max_bucket_brier_increase = 0.05`
这两类约束的目的不是追求“更激进的拟合”，而是防止 EMOS 为了降低 CRPS 而把分布摊得过平，导致业务上更关键的顶桶命中和概率质量变差。
### 5.2 当前选中的 blending
本次训练产物中最终选择：
- `alpha_mu = 0.0`
- `alpha_sigma = 0.0`
含义是：
- 训练器在护栏约束下，没有找到足够安全的候选方案可以替代 legacy 主路径
- 因此当前正式选中的可用结果，本质上仍然锚定在 legacy
这是一种正确的保护行为，不是失败。说明门禁已经起作用，避免了坏校准进入主路径。
## 6. 离线评估结果
评估报告来源：
- [evaluation_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/evaluation_report.json)
### 6.1 总体结果
Legacy：
- `mean_crps = 2.793938`
- `mean_mae = 2.721143`
- `bucket_hit_rate = 0.695238`
EMOS（强制 primary 评估）：
- `mean_crps = 2.650216`
- `mean_mae = 2.722829`
- `bucket_hit_rate = 0.666667`
Delta：
- `CRPS = -0.143722`
- `MAE = +0.001686`
- `bucket_hit_rate = -0.028571`
### 6.2 解读
这组结果说明：
1. `CRPS` 有改善
说明从“分布整体平滑度”角度看，EMOS 有一定价值。
2. `MAE` 基本持平但略差
不是大问题，但也不能算改善。
3. `bucket_hit_rate` 明显下降
这是当前最大阻塞项。对 PolyWeather 这种结算桶业务来说，顶桶命中率比单纯 CRPS 更关键。
因此，离线结论是：
- `EMOS` 有研究价值
- 但 **离线强切 primary 仍然不合格**
## 7. 线上 Shadow 观测结果
线上 shadow 报表来源：
- [shadow_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/shadow_report.json)
### 7.1 总体结果
- `samples = 103`
- `legacy_mean_mae = 1.839223`
- `shadow_mean_mae = 1.851931`
- `delta_mae = +0.012708`
- `legacy_bucket_hit_rate = 0.669903`
- `shadow_bucket_hit_rate = 0.679612`
- `delta_bucket_hit_rate = +0.009709`
- `legacy_bucket_brier = 0.462814`
- `shadow_bucket_brier = 0.756649`
- `delta_bucket_brier = +0.293835`
### 7.2 解读
线上 shadow 结果和离线强制 primary 结果不完全相同，这是正常的。原因是：
- `shadow_report` 反映的是历史记录中实际落盘的 shadow 输出
- `evaluation_report` 反映的是离线脚本在强制 `emos_primary` 下重新计算的效果
当前线上 shadow 的含义是：
1. 顶桶命中率略有提升
`+0.97%`
2. 但 `MAE` 轻微变差
虽然幅度不大，但没有形成明确优势
3. `bucket_brier` 明显更差
说明 shadow 分布仍然偏“摊平”，概率质量不足
这是当前最重要的信号：
- EMOS 在“顶桶命中”上偶尔能赢
- 但在“概率质量”上还不够好
## 8. 城市级观察
从当前城市级结果看，EMOS 并不是“全城市统一改善”，而是明显分化：
### 8.1 相对改善较明显的城市
- `London`
- `Hong Kong`
- `Tokyo`
- `New York`
这些城市在部分指标上看到一定改善，说明当前校准特征在这些城市上更有效。
### 8.2 风险较高的城市
- `Atlanta`
- `Miami`
- `Chicago`
- `Dallas`
- `Seattle`
这些城市常见现象是：
- 顶桶命中没有显著提高
- 或 `bucket_brier` 明显恶化
- 或者 `MAE` 出现不必要抬升
这说明当前 EMOS 还没有形成稳定的全局校准能力，城市间异质性很强。
## 9. 当前判断
### 9.1 能不能上线为主路径
当前答案：
- **不能**
原因：
1. 离线强制 primary 时，`bucket_hit_rate` 下降
2. 线上 shadow 时，`bucket_brier` 明显变差
3. 样本量依然偏小，城市样本不均衡
4. 城市级表现分化明显
### 9.2 当前应该怎么运行
当前最合理的运行方式：
1. 保持 `emos_shadow`
2. 继续落盘 `shadow_prob_snapshot`
3. 继续维护滚动报表
4. 不修改机器人和网页的正式对外概率展示
## 10. 已完成的工程能力
目前已经具备以下能力：
1. 可离线训练
脚本：[fit_probability_calibration.py](/E:/web/PolyWeather/scripts/fit_probability_calibration.py)
2. 可离线评估
脚本：[evaluate_probability_calibration.py](/E:/web/PolyWeather/scripts/evaluate_probability_calibration.py)
3. 可导出训练样本
脚本：[export_probability_training_dataset.py](/E:/web/PolyWeather/scripts/export_probability_training_dataset.py)
4. 可历史回填 shadow 结果
脚本：[backfill_probability_shadow_history.py](/E:/web/PolyWeather/scripts/backfill_probability_shadow_history.py)
5. 可生成滚动 shadow 报表
脚本：[build_probability_shadow_report.py](/E:/web/PolyWeather/scripts/build_probability_shadow_report.py)
6. CI 已接入
包含 `ruff / pytest / frontend build / docker build workflow`
## 11. 下一步建议
### 11.1 必做
1. 扩大监督样本量
重点不是继续堆原始天气 CSV，而是补更多带 forecast snapshot 的历史样本。
2. 继续按版本沉淀训练报告
每次重训后都更新本报告或新增版本报告，避免只看单次结果。
3. 保持 `shadow` 连续观测
至少持续一段时间观察滚动指标是否稳定。
### 11.2 再做
1. 细分城市组建模
比如按气候区、结算规则、温度单位分组，而不是完全全局一套参数。
2. 优化训练目标
目前已经把 `bucket_brier` 纳入目标，但仍需进一步靠近 PolyWeather 的业务目标。
3. 补更严格的切换门槛
只有在同时满足以下条件时，才考虑切 `emos_primary`：
- `CRPS` 下降
- `MAE` 不上升
- `bucket_hit_rate` 不下降
- `bucket_brier` 不上升
## 12. 结论
当前 EMOS 状态可以概括为：
- 工程上：已经完整接入，具备训练、评估、shadow 观测能力
- 模型上：有一定价值，但还不稳定
- 产品上：适合继续做 shadow，不适合切主路径
最终结论：
- **继续使用 `emos_shadow`**
- **暂不切 `emos_primary`**
- **继续积累样本并按版本跟踪训练结果**
src/analysis/probability_calibration.py
+110 -6
View File
@@ -226,14 +226,61 @@ def _top_bucket_value(distribution: Optional[List[Dict[str, Any]]]) -> Optional[
return int(value) if value is not None else None
def _composite_score(mean_crps: float, mean_mae: float, bucket_hit_rate: float) -> float:
return mean_crps + 0.1 * mean_mae + 2.0 * (1.0 - bucket_hit_rate)
def _bucket_brier_score(
distribution: Optional[List[Dict[str, Any]]],
city_name: str,
actual_high: float,
) -> float:
actual_bucket = apply_city_settlement(city_name, actual_high)
hit_prob = 0.0
total = 0.0
for row in distribution or []:
if not isinstance(row, dict):
continue
value = row.get("value")
try:
prob = float(row.get("probability") or 0.0)
except Exception:
prob = 0.0
if value == actual_bucket:
hit_prob = prob
else:
total += prob * prob
total += (1.0 - hit_prob) ** 2
return total
def _composite_score(
mean_crps: float,
mean_mae: float,
bucket_hit_rate: float,
bucket_brier: float,
) -> float:
return mean_crps + 0.1 * mean_mae + 1.5 * (1.0 - bucket_hit_rate) + 0.75 * bucket_brier
def _blend_value(raw_value: float, calibrated_value: float, alpha: float) -> float:
return (1.0 - alpha) * raw_value + alpha * calibrated_value
def _clamp_sigma(
raw_sigma: float,
calibrated_sigma: float,
constraints: Optional[Dict[str, Any]],
) -> float:
constraints = constraints or {}
min_ratio = max(0.25, _coalesce_float(constraints.get("min_ratio"), 0.85))
max_ratio = max(min_ratio, _coalesce_float(constraints.get("max_ratio"), 1.35))
absolute_min = max(0.1, _coalesce_float(constraints.get("absolute_min"), 0.25))
absolute_max = max(
absolute_min,
_coalesce_float(constraints.get("absolute_max"), raw_sigma * max_ratio),
)
sigma_floor = max(absolute_min, raw_sigma * min_ratio)
sigma_cap = min(absolute_max, raw_sigma * max_ratio)
return min(max(calibrated_sigma, sigma_floor), sigma_cap)
def apply_probability_calibration(
city_name: str,
temp_symbol: str,
@@ -280,6 +327,7 @@ def apply_probability_calibration(
global_params = calibration.get("global", {}) or {}
city_params = (calibration.get("cities", {}) or {}).get(city_key, {}) or {}
blending_cfg = calibration.get("blending", {}) or {}
sigma_constraints = calibration.get("sigma_constraints", {}) or {}
mu_cfg = global_params.get("mu", {}) or {}
sigma_cfg = global_params.get("sigma", {}) or {}
@@ -330,6 +378,7 @@ def apply_probability_calibration(
blend_alpha_sigma = max(0.0, min(1.0, _coalesce_float(blending_cfg.get("alpha_sigma"), 1.0)))
calibrated_mu = _blend_value(raw_mu, calibrated_mu, blend_alpha_mu)
calibrated_sigma = max(0.1, _blend_value(raw_sigma, calibrated_sigma, blend_alpha_sigma))
calibrated_sigma = _clamp_sigma(raw_sigma, calibrated_sigma, sigma_constraints)
calibrated_distribution, calibrated_sorted = _bucket_probabilities(
calibrated_mu,
calibrated_sigma,
@@ -472,6 +521,7 @@ def fit_calibration(
legacy_crps_values = []
legacy_mae_values = []
legacy_bucket_hits = []
legacy_bucket_briers = []
candidate_predictions = []
for idx, sample in enumerate(normalized_samples):
city = sample["city"]
@@ -496,6 +546,15 @@ def fit_calibration(
== apply_city_settlement(city, actual_high)
else 0.0
)
legacy_distribution, _ = _bucket_probabilities(
legacy_mu,
legacy_sigma,
max_so_far=None,
city_name=city,
)
legacy_bucket_briers.append(
_bucket_brier_score(legacy_distribution, city, actual_high)
)
calibrated_mu = mu_predictions[idx] + city_mu_bias
sigma_log = float(np.dot(sigma_coeffs, np.array(sigma_rows[idx], dtype=float)))
@@ -514,11 +573,24 @@ def fit_calibration(
legacy_mean_crps = _mean(legacy_crps_values) or 0.0
legacy_mean_mae = _mean(legacy_mae_values) or 0.0
legacy_bucket_hit_rate = _mean(legacy_bucket_hits) or 0.0
legacy_bucket_brier = _mean(legacy_bucket_briers) or 0.0
sigma_constraints = {
"min_ratio": 0.85,
"max_ratio": 1.35,
"absolute_min": 0.25,
"absolute_max": 3.0,
}
legacy_score = _composite_score(
legacy_mean_crps,
legacy_mean_mae,
legacy_bucket_hit_rate,
legacy_bucket_brier,
)
guardrails = {
"max_mae_increase": 0.02,
"max_bucket_hit_drop": 0.01,
"max_bucket_brier_increase": 0.05,
}
best_alpha_mu = 0.0
best_alpha_sigma = 0.0
@@ -527,6 +599,7 @@ def fit_calibration(
"mean_crps": legacy_mean_crps,
"mean_mae": legacy_mean_mae,
"bucket_hit_rate": legacy_bucket_hit_rate,
"bucket_brier": legacy_bucket_brier,
}
alpha_grid = [step / 20.0 for step in range(21)]
for alpha_mu in alpha_grid:
@@ -534,11 +607,16 @@ def fit_calibration(
crps_values = []
mae_values = []
bucket_hits = []
bucket_briers = []
for row in candidate_predictions:
mu_hat = _blend_value(row["raw_mu"], row["calibrated_mu"], alpha_mu)
sigma_hat = max(
0.1,
_blend_value(row["raw_sigma"], row["calibrated_sigma"], alpha_sigma),
sigma_hat = _clamp_sigma(
row["raw_sigma"],
max(
0.1,
_blend_value(row["raw_sigma"], row["calibrated_sigma"], alpha_sigma),
),
sigma_constraints,
)
actual_high = row["actual_high"]
city = row["city"]
@@ -553,11 +631,21 @@ def fit_calibration(
predicted_bucket = _top_bucket_value(distribution)
actual_bucket = apply_city_settlement(city, actual_high)
bucket_hits.append(1.0 if predicted_bucket == actual_bucket else 0.0)
bucket_briers.append(
_bucket_brier_score(distribution, city, actual_high)
)
mean_crps = _mean(crps_values) or 0.0
mean_mae = _mean(mae_values) or 0.0
bucket_hit_rate = _mean(bucket_hits) or 0.0
score = _composite_score(mean_crps, mean_mae, bucket_hit_rate)
bucket_brier = _mean(bucket_briers) or 0.0
if mean_mae > legacy_mean_mae + guardrails["max_mae_increase"]:
continue
if bucket_hit_rate + guardrails["max_bucket_hit_drop"] < legacy_bucket_hit_rate:
continue
if bucket_brier > legacy_bucket_brier + guardrails["max_bucket_brier_increase"]:
continue
score = _composite_score(mean_crps, mean_mae, bucket_hit_rate, bucket_brier)
if score + 1e-9 < best_score:
best_score = score
best_alpha_mu = alpha_mu
@@ -566,6 +654,7 @@ def fit_calibration(
"mean_crps": mean_crps,
"mean_mae": mean_mae,
"bucket_hit_rate": bucket_hit_rate,
"bucket_brier": bucket_brier,
}
return {
@@ -587,6 +676,8 @@ def fit_calibration(
"max_so_far_gap_coef": round(float(sigma_coeffs[4]), 8),
},
},
"sigma_constraints": sigma_constraints,
"selection_guardrails": guardrails,
"blending": {
"alpha_mu": round(best_alpha_mu, 6),
"alpha_sigma": round(best_alpha_sigma, 6),
@@ -598,9 +689,11 @@ def fit_calibration(
"legacy_mean_crps": round(legacy_mean_crps, 6),
"legacy_mean_mae": round(legacy_mean_mae, 6),
"legacy_bucket_hit_rate": round(legacy_bucket_hit_rate, 6),
"legacy_bucket_brier": round(legacy_bucket_brier, 6),
"selected_mean_crps": round(best_metrics["mean_crps"], 6),
"selected_mean_mae": round(best_metrics["mean_mae"], 6),
"selected_bucket_hit_rate": round(best_metrics["bucket_hit_rate"], 6),
"selected_bucket_brier": round(best_metrics["bucket_brier"], 6),
"selected_score": round(best_score, 6),
"legacy_score": round(legacy_score, 6),
},
@@ -630,6 +723,17 @@ def default_calibration_payload(
"max_so_far_gap_coef": 0.0,
},
},
"sigma_constraints": {
"min_ratio": 0.85,
"max_ratio": 1.35,
"absolute_min": 0.25,
"absolute_max": 3.0,
},
"selection_guardrails": {
"max_mae_increase": 0.02,
"max_bucket_hit_drop": 0.01,
"max_bucket_brier_increase": 0.05,
},
"blending": {
"alpha_mu": 1.0,
"alpha_sigma": 1.0,
tests/test_probability_calibration.py
+10 -1
View File
@@ -31,10 +31,17 @@ def _write_calibration(tmp_path: Path):
"max_so_far_gap_coef": 0.0,
},
},
"sigma_constraints": {
"min_ratio": 0.85,
"max_ratio": 1.2,
"absolute_min": 0.25,
"absolute_max": 2.0,
},
"cities": {
"ankara": {
"mu_bias": 0.5,
"sigma_scale": 1.0,
"sigma_scale": 2.0,
"confidence": 1.0,
}
},
"metrics": {"sample_count": 10, "mean_crps": 0.4},
@@ -76,6 +83,7 @@ def test_shadow_mode_keeps_legacy_distribution(tmp_path):
assert result["distribution"] == legacy_distribution
assert result["shadow_distribution"]
assert result["calibrated_mu"] == 10.5
assert result["calibrated_sigma"] == 1.2
def test_primary_mode_switches_to_calibrated_distribution(tmp_path):
@@ -107,6 +115,7 @@ def test_primary_mode_switches_to_calibrated_distribution(tmp_path):
assert result["mode"] == ENGINE_MODE_EMOS_PRIMARY
assert result["engine"] == "emos"
assert result["calibrated_mu"] == 10.5
assert result["calibrated_sigma"] == 1.2
assert result["distribution"]
assert result["distribution"][0]["value"] >= 10