gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

删除 LGBM 全部代码和模型文件，EMOS 简化为纯 legacy 高斯分桶模式

Browse Source
This commit is contained in:
2569718930@qq.com
2026-05-18 22:05:55 +08:00
parent aec47adda1
commit 0e0aad3171
26 changed files with 11 additions and 149413 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
-2
View File
@@ -63,8 +63,6 @@ POLYWEATHER_METAR_CLUSTER_TIMEOUT_SEC=3.5
METAR_CACHE_TTL_SEC=600
JMA_AMEDAS_CACHE_TTL_SEC=120
METEOBLUE_CACHE_TTL_SEC=7200
# Probability engine modes:
# - legacy: production-safe primary path.
# - emos_shadow: user-facing probability stays legacy, EMOS is generated for comparison.
# - emos_primary: only after offline evaluation passes and manual rollout is approved.
POLYWEATHER_PROBABILITY_ENGINE=legacy
artifacts/models/lgbm_daily_high.txt
-2142
View File
File diff suppressed because it is too large Load Diff
artifacts/models/lgbm_daily_high_schema.json
-66
View File
@@ -1,66 +0,0 @@
{
"model_type": "LightGBMRegressor",
"target": "actual_high",
"horizon": "D0",
"feature_names": [
"actual_high_lag_1",
"actual_high_lag_2",
"actual_high_lag_3",
"actual_high_lag_7",
"actual_high_mean_7",
"actual_high_mean_14",
"actual_high_trend_3",
"open_meteo",
"ecmwf",
"gfs",
"gem",
"jma",
"icon",
"mgm",
"nws",
"deb_prediction",
"model_median",
"model_spread",
"current_temp",
"max_so_far",
"humidity",
"wind_speed_kt",
"visibility_mi",
"local_hour",
"month",
"weekday",
"peak_status_code"
],
"base_model_columns": [
"open_meteo",
"ecmwf",
"gfs",
"gem",
"jma",
"icon",
"mgm",
"nws"
],
"model_path": "artifacts\\models\\lgbm_daily_high.txt",
"sample_count": 1247,
"train_count": 998,
"validation_count": 249,
"metrics": {
"validation": {
"sample_count": 249,
"lgbm_mae": 2.626,
"deb_mae": 2.745,
"best_single_mae": 1.733,
"median_mae": 2.866
},
"full_sample": {
"sample_count": 1247,
"lgbm_mae": 0.953,
"deb_mae": 1.872,
"best_single_mae": 1.052,
"median_mae": 1.952
}
},
"generated_at": "2026-05-06T10:45:04.319587Z",
"trained_at": "2026-05-06T10:45:04.319587Z"
}
artifacts/probability_calibration/default.json
-361
View File
@@ -1,361 +0,0 @@
{
"version": "emos-auto-20260421122743",
"trained_at": "2026-04-21T12:28:05.031165+00:00",
"global": {
"mu": {
"intercept": 0.34330438,
"raw_mu_coef": 0.48749629,
"deb_coef": 0.45733479,
"ens_median_coef": 0.05955297,
"max_so_far_gap_coef": -0.104935
},
"sigma": {
"intercept": -0.62765582,
"raw_sigma_coef": 0.38492261,
"spread_coef": 0.29095921,
"peak_flag_coef": 0.33966327,
"max_so_far_gap_coef": 0.08357376
}
},
"sigma_constraints": {
"min_ratio": 0.85,
"max_ratio": 1.35,
"absolute_min": 0.25,
"absolute_max": 3.0
},
"selection_guardrails": {
"max_mae_increase": 0.02,
"max_bucket_hit_drop": 0.01,
"max_bucket_brier_increase": 0.05
},
"blending": {
"alpha_mu": 0.25,
"alpha_sigma": 1.0
},
"cities": {
"istanbul": {
"samples": 20,
"mu_bias": -0.438949,
"sigma_scale": 0.553793,
"confidence": 1.0
},
"buenos aires": {
"samples": 22,
"mu_bias": -1.044971,
"sigma_scale": 1.389753,
"confidence": 1.0
},
"ankara": {
"samples": 23,
"mu_bias": 0.067406,
"sigma_scale": 1.270298,
"confidence": 1.0
},
"london": {
"samples": 22,
"mu_bias": -0.010833,
"sigma_scale": 2.0,
"confidence": 1.0
},
"paris": {
"samples": 22,
"mu_bias": -0.21677,
"sigma_scale": 1.095669,
"confidence": 1.0
},
"new york": {
"samples": 21,
"mu_bias": 1.072189,
"sigma_scale": 2.0,
"confidence": 1.0
},
"lagos": {
"samples": 10,
"mu_bias": -0.092732,
"sigma_scale": 1.558646,
"confidence": 1.0
},
"madrid": {
"samples": 22,
"mu_bias": -0.627862,
"sigma_scale": 1.761239,
"confidence": 1.0
},
"toronto": {
"samples": 22,
"mu_bias": -0.424101,
"sigma_scale": 1.326675,
"confidence": 1.0
},
"wellington": {
"samples": 22,
"mu_bias": -0.347796,
"sigma_scale": 0.893459,
"confidence": 1.0
},
"los angeles": {
"samples": 20,
"mu_bias": -0.659983,
"sigma_scale": 2.0,
"confidence": 1.0
},
"san francisco": {
"samples": 20,
"mu_bias": 0.941818,
"sigma_scale": 1.979403,
"confidence": 1.0
},
"aurora": {
"samples": 20,
"mu_bias": 2.040009,
"sigma_scale": 0.982492,
"confidence": 1.0
},
"austin": {
"samples": 20,
"mu_bias": 0.990023,
"sigma_scale": 2.0,
"confidence": 1.0
},
"houston": {
"samples": 20,
"mu_bias": 0.188787,
"sigma_scale": 1.819594,
"confidence": 1.0
},
"chicago": {
"samples": 21,
"mu_bias": 2.285098,
"sigma_scale": 2.0,
"confidence": 1.0
},
"dallas": {
"samples": 21,
"mu_bias": 0.258095,
"sigma_scale": 2.0,
"confidence": 1.0
},
"atlanta": {
"samples": 22,
"mu_bias": -2.093105,
"sigma_scale": 2.0,
"confidence": 1.0
},
"seattle": {
"samples": 21,
"mu_bias": -0.982188,
"sigma_scale": 1.695172,
"confidence": 1.0
},
"sao paulo": {
"samples": 22,
"mu_bias": -0.786242,
"sigma_scale": 1.427037,
"confidence": 1.0
},
"munich": {
"samples": 22,
"mu_bias": 0.096559,
"sigma_scale": 2.0,
"confidence": 1.0
},
"milan": {
"samples": 23,
"mu_bias": -0.045038,
"sigma_scale": 1.626331,
"confidence": 1.0
},
"warsaw": {
"samples": 23,
"mu_bias": -0.50853,
"sigma_scale": 1.239146,
"confidence": 1.0
},
"helsinki": {
"samples": 17,
"mu_bias": 0.442247,
"sigma_scale": 2.0,
"confidence": 1.0
},
"amsterdam": {
"samples": 17,
"mu_bias": 0.129889,
"sigma_scale": 2.0,
"confidence": 1.0
},
"panama city": {
"samples": 17,
"mu_bias": -0.040792,
"sigma_scale": 1.393259,
"confidence": 1.0
},
"cape town": {
"samples": 10,
"mu_bias": 0.507293,
"sigma_scale": 1.420004,
"confidence": 1.0
},
"miami": {
"samples": 22,
"mu_bias": -2.736316,
"sigma_scale": 2.0,
"confidence": 1.0
},
"manila": {
"samples": 5,
"mu_bias": 1.772229,
"sigma_scale": 2.0,
"confidence": 0.625
},
"singapore": {
"samples": 22,
"mu_bias": 0.289639,
"sigma_scale": 1.124006,
"confidence": 1.0
},
"tokyo": {
"samples": 22,
"mu_bias": -0.332129,
"sigma_scale": 0.729483,
"confidence": 1.0
},
"seoul": {
"samples": 22,
"mu_bias": 1.6533,
"sigma_scale": 0.82648,
"confidence": 1.0
},
"busan": {
"samples": 16,
"mu_bias": 0.343239,
"sigma_scale": 1.886868,
"confidence": 1.0
},
"mexico city": {
"samples": 20,
"mu_bias": 0.087122,
"sigma_scale": 1.108884,
"confidence": 1.0
},
"shanghai": {
"samples": 23,
"mu_bias": 0.348022,
"sigma_scale": 0.810527,
"confidence": 1.0
},
"beijing": {
"samples": 20,
"mu_bias": -0.288176,
"sigma_scale": 0.952972,
"confidence": 1.0
},
"shenzhen": {
"samples": 21,
"mu_bias": -0.006114,
"sigma_scale": 0.728494,
"confidence": 1.0
},
"wuhan": {
"samples": 20,
"mu_bias": -0.771154,
"sigma_scale": 1.30329,
"confidence": 1.0
},
"chengdu": {
"samples": 20,
"mu_bias": 0.445906,
"sigma_scale": 0.796989,
"confidence": 1.0
},
"chongqing": {
"samples": 20,
"mu_bias": -0.021301,
"sigma_scale": 1.106566,
"confidence": 1.0
},
"hong kong": {
"samples": 24,
"mu_bias": 0.37527,
"sigma_scale": 0.517756,
"confidence": 1.0
},
"taipei": {
"samples": 23,
"mu_bias": 0.600671,
"sigma_scale": 0.5,
"confidence": 1.0
},
"guangzhou": {
"samples": 6,
"mu_bias": 0.7082,
"sigma_scale": 1.975103,
"confidence": 0.75
},
"lau fau shan": {
"samples": 19,
"mu_bias": 0.41371,
"sigma_scale": 0.656578,
"confidence": 1.0
},
"kuala lumpur": {
"samples": 16,
"mu_bias": 0.499197,
"sigma_scale": 1.361171,
"confidence": 1.0
},
"jakarta": {
"samples": 16,
"mu_bias": 0.293537,
"sigma_scale": 0.751498,
"confidence": 1.0
},
"lucknow": {
"samples": 22,
"mu_bias": -1.593437,
"sigma_scale": 2.0,
"confidence": 1.0
},
"karachi": {
"samples": 5,
"mu_bias": -0.035595,
"sigma_scale": 0.588612,
"confidence": 0.625
},
"jeddah": {
"samples": 10,
"mu_bias": 0.516879,
"sigma_scale": 2.0,
"confidence": 1.0
},
"moscow": {
"samples": 20,
"mu_bias": -0.609602,
"sigma_scale": 0.783723,
"confidence": 1.0
},
"tel aviv": {
"samples": 22,
"mu_bias": 0.436837,
"sigma_scale": 1.46524,
"confidence": 1.0
}
},
"metrics": {
"sample_count": 978,
"mean_crps": 1.330733,
"legacy_mean_crps": 1.376936,
"legacy_mean_mae": 1.589438,
"legacy_bucket_hit_rate": 0.43865,
"legacy_bucket_brier": 0.824571,
"selected_mean_crps": 1.311763,
"selected_mean_mae": 1.553658,
"selected_bucket_hit_rate": 0.432515,
"selected_bucket_brier": 0.789817,
"selected_score": 2.910718,
"legacy_score": 2.996333,
"filled_actual_from_history": 0,
"settlement_history_city_count": 30,
"legacy_archive_samples": 54
},
"source": "artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json"
}
artifacts/probability_calibration/evaluation_report.json
-293
View File
@@ -1,293 +0,0 @@
{
"summary": {
"sample_count": 74,
"filled_actual_from_history": 0,
"legacy": {
"mean_crps": 3.474108,
"mean_mae": 3.679324,
"bucket_hit_rate": 0.5
},
"emos": {
"mean_crps": 3.33124,
"mean_mae": 3.622584,
"bucket_hit_rate": 0.5
},
"delta": {
"crps": -0.142868,
"mae": -0.056741,
"bucket_hit_rate": 0.0
}
},
"by_city": {
"ankara": {
"samples": 6,
"legacy_mean_crps": 1.365135,
"emos_mean_crps": 1.392829,
"legacy_mean_mae": 1.466667,
"emos_mean_mae": 1.444819,
"legacy_bucket_hit_rate": 0.666667,
"emos_bucket_hit_rate": 0.666667
},
"atlanta": {
"samples": 2,
"legacy_mean_crps": 30.449382,
"emos_mean_crps": 29.886236,
"legacy_mean_mae": 32.015,
"emos_mean_mae": 31.322804,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"buenos aires": {
"samples": 2,
"legacy_mean_crps": 9.113412,
"emos_mean_crps": 8.463522,
"legacy_mean_mae": 10.27,
"emos_mean_mae": 10.024533,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"busan": {
"samples": 3,
"legacy_mean_crps": 0.276202,
"emos_mean_crps": 0.326485,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 0.239636,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"chengdu": {
"samples": 1,
"legacy_mean_crps": 0.306469,
"emos_mean_crps": 0.378812,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 0.153224,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"chicago": {
"samples": 1,
"legacy_mean_crps": 1.250268,
"emos_mean_crps": 0.714669,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.319756,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"dallas": {
"samples": 1,
"legacy_mean_crps": 2.173363,
"emos_mean_crps": 0.840297,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 1.028146,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.0
},
"hong kong": {
"samples": 7,
"legacy_mean_crps": 0.706099,
"emos_mean_crps": 0.731926,
"legacy_mean_mae": 0.657143,
"emos_mean_mae": 0.669911,
"legacy_bucket_hit_rate": 0.714286,
"emos_bucket_hit_rate": 0.571429
},
"istanbul": {
"samples": 3,
"legacy_mean_crps": 1.264103,
"emos_mean_crps": 1.214967,
"legacy_mean_mae": 1.5,
"emos_mean_mae": 1.469837,
"legacy_bucket_hit_rate": 0.333333,
"emos_bucket_hit_rate": 0.666667
},
"london": {
"samples": 2,
"legacy_mean_crps": 3.885033,
"emos_mean_crps": 3.799841,
"legacy_mean_mae": 4.135,
"emos_mean_mae": 4.027503,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.5
},
"lucknow": {
"samples": 2,
"legacy_mean_crps": 2.487193,
"emos_mean_crps": 2.107441,
"legacy_mean_mae": 3.205,
"emos_mean_mae": 2.876943,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"madrid": {
"samples": 2,
"legacy_mean_crps": 6.27726,
"emos_mean_crps": 5.810228,
"legacy_mean_mae": 7.33,
"emos_mean_mae": 7.195133,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"miami": {
"samples": 2,
"legacy_mean_crps": 28.637631,
"emos_mean_crps": 27.592664,
"legacy_mean_mae": 30.175,
"emos_mean_mae": 29.284979,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"milan": {
"samples": 3,
"legacy_mean_crps": 4.401392,
"emos_mean_crps": 3.782613,
"legacy_mean_mae": 4.06,
"emos_mean_mae": 3.989808,
"legacy_bucket_hit_rate": 0.666667,
"emos_bucket_hit_rate": 0.666667
},
"munich": {
"samples": 2,
"legacy_mean_crps": 3.145192,
"emos_mean_crps": 2.990712,
"legacy_mean_mae": 3.64,
"emos_mean_mae": 3.61502,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"new york": {
"samples": 1,
"legacy_mean_crps": 3.692845,
"emos_mean_crps": 3.021798,
"legacy_mean_mae": 4.94,
"emos_mean_mae": 4.54437,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"paris": {
"samples": 2,
"legacy_mean_crps": 4.013782,
"emos_mean_crps": 3.915303,
"legacy_mean_mae": 4.265,
"emos_mean_mae": 4.179694,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 0.5
},
"sao paulo": {
"samples": 2,
"legacy_mean_crps": 5.540967,
"emos_mean_crps": 5.018605,
"legacy_mean_mae": 6.57,
"emos_mean_mae": 6.314389,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"seattle": {
"samples": 1,
"legacy_mean_crps": 0.315488,
"emos_mean_crps": 0.524119,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.673609,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.0
},
"seoul": {
"samples": 3,
"legacy_mean_crps": 0.508331,
"emos_mean_crps": 0.520762,
"legacy_mean_mae": 0.2,
"emos_mean_mae": 0.247862,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"shanghai": {
"samples": 3,
"legacy_mean_crps": 0.299116,
"emos_mean_crps": 0.402453,
"legacy_mean_mae": 0.1,
"emos_mean_mae": 0.189437,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"shenzhen": {
"samples": 1,
"legacy_mean_crps": 1.198351,
"emos_mean_crps": 1.490963,
"legacy_mean_mae": 1.3,
"emos_mean_mae": 1.628189,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"singapore": {
"samples": 2,
"legacy_mean_crps": 0.281993,
"emos_mean_crps": 0.370374,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 0.118005,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"taipei": {
"samples": 5,
"legacy_mean_crps": 0.950739,
"emos_mean_crps": 1.042332,
"legacy_mean_mae": 0.94,
"emos_mean_mae": 0.951241,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"tel aviv": {
"samples": 2,
"legacy_mean_crps": 0.446758,
"emos_mean_crps": 0.578691,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 0.116966,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"tokyo": {
"samples": 5,
"legacy_mean_crps": 0.879366,
"emos_mean_crps": 0.876608,
"legacy_mean_mae": 1.022,
"emos_mean_mae": 1.008317,
"legacy_bucket_hit_rate": 0.2,
"emos_bucket_hit_rate": 0.4
},
"toronto": {
"samples": 2,
"legacy_mean_crps": 5.497916,
"emos_mean_crps": 5.268783,
"legacy_mean_mae": 6.33,
"emos_mean_mae": 6.388522,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"warsaw": {
"samples": 3,
"legacy_mean_crps": 1.618875,
"emos_mean_crps": 1.524219,
"legacy_mean_mae": 2.056667,
"emos_mean_mae": 2.006524,
"legacy_bucket_hit_rate": 0.333333,
"emos_bucket_hit_rate": 0.333333
},
"wellington": {
"samples": 2,
"legacy_mean_crps": 0.364919,
"emos_mean_crps": 0.484124,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 0.123335,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"wuhan": {
"samples": 1,
"legacy_mean_crps": 0.476225,
"emos_mean_crps": 0.578405,
"legacy_mean_mae": 0.8,
"emos_mean_mae": 0.962852,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
}
}
}
artifacts/probability_calibration/rollout_report.json
-74
View File
@@ -1,74 +0,0 @@
{
"evaluation_report_path": "E:\\web\\PolyWeather\\artifacts\\probability_calibration\\evaluation_report.json",
"shadow_report_path": "E:\\web\\PolyWeather\\artifacts\\probability_calibration\\shadow_report.json",
"evaluation_report_exists": true,
"shadow_report_exists": true,
"decision": {
"decision": "hold",
"ready_for_primary": false,
"summary": "当前指标不足以切换 emos_primary，应继续保持 shadow。",
"thresholds": {
"evaluation_min_samples": 80,
"shadow_min_samples": 50,
"max_delta_mae": 0.05,
"min_delta_crps": -0.02,
"min_delta_bucket_hit_rate": 0.0,
"max_delta_bucket_brier_promote": 0.02,
"max_delta_bucket_brier_observe": 0.15
},
"evaluation": {
"sample_count": 54,
"delta_crps": -0.086732,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0
},
"shadow": {
"sample_count": 48,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.041666,
"delta_bucket_brier": 0.123252
},
"blocking_reasons": [
"离线评估样本不足：54 < 80",
"shadow 样本不足：48 < 50",
"shadow bucket brier 退化超限：delta=0.123252"
],
"worst_shadow_regressions": [
{
"city": "dallas",
"samples": 1,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.792585
},
{
"city": "chicago",
"samples": 1,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.791878
},
{
"city": "seattle",
"samples": 1,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.61609
},
{
"city": "wellington",
"samples": 2,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.509203
},
{
"city": "tel aviv",
"samples": 2,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.439879
}
]
}
}
artifacts/probability_calibration/settlement_history.json
-140502
View File
File diff suppressed because it is too large Load Diff
artifacts/probability_calibration/shadow_report.json
-933
View File
@@ -1,933 +0,0 @@
{
"generated_at": "2026-04-02T16:23:24.376528Z",
"summary": {
"samples": 48,
"legacy_mean_mae": 3.04125,
"shadow_mean_mae": 3.04125,
"legacy_bucket_hit_rate": 0.5,
"shadow_bucket_hit_rate": 0.5,
"legacy_bucket_brier": 0.68666,
"shadow_bucket_brier": 0.814079,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.127419
},
"by_city": {
"ankara": {
"samples": 2,
"legacy_mean_mae": 0.1,
"shadow_mean_mae": 0.1,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.494847,
"shadow_bucket_brier": 0.654648,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.159801
},
"atlanta": {
"samples": 1,
"legacy_mean_mae": 17.06,
"shadow_mean_mae": 17.06,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 1.029097,
"shadow_bucket_brier": 1.064101,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.035004
},
"buenos aires": {
"samples": 2,
"legacy_mean_mae": 10.27,
"shadow_mean_mae": 10.27,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 1.117726,
"shadow_bucket_brier": 1.116732,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": -0.000994
},
"chicago": {
"samples": 1,
"legacy_mean_mae": 0.0,
"shadow_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.0,
"shadow_bucket_brier": 0.791878,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.791878
},
"dallas": {
"samples": 1,
"legacy_mean_mae": 0.0,
"shadow_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.0,
"shadow_bucket_brier": 0.792585,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.792585
},
"hong kong": {
"samples": 3,
"legacy_mean_mae": 0.1,
"shadow_mean_mae": 0.1,
"legacy_bucket_hit_rate": 0.333333,
"shadow_bucket_hit_rate": 0.666667,
"legacy_bucket_brier": 0.882203,
"shadow_bucket_brier": 0.437187,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.333334,
"delta_bucket_brier": -0.445016
},
"london": {
"samples": 2,
"legacy_mean_mae": 4.135,
"shadow_mean_mae": 4.135,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 0.929652,
"shadow_bucket_brier": 1.039551,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.109899
},
"lucknow": {
"samples": 2,
"legacy_mean_mae": 3.205,
"shadow_mean_mae": 3.205,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 1.673607,
"shadow_bucket_brier": 0.96272,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": -0.710887
},
"madrid": {
"samples": 2,
"legacy_mean_mae": 7.33,
"shadow_mean_mae": 7.33,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 1.148213,
"shadow_bucket_brier": 1.133888,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": -0.014325
},
"miami": {
"samples": 1,
"legacy_mean_mae": 11.04,
"shadow_mean_mae": 11.04,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 1.182605,
"shadow_bucket_brier": 1.067257,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": -0.115348
},
"milan": {
"samples": 3,
"legacy_mean_mae": 4.06,
"shadow_mean_mae": 4.06,
"legacy_bucket_hit_rate": 0.666667,
"shadow_bucket_hit_rate": 0.666667,
"legacy_bucket_brier": 0.587548,
"shadow_bucket_brier": 0.78616,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.198612
},
"munich": {
"samples": 2,
"legacy_mean_mae": 3.64,
"shadow_mean_mae": 3.64,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 0.918378,
"shadow_bucket_brier": 0.945234,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.026856
},
"new york": {
"samples": 1,
"legacy_mean_mae": 4.94,
"shadow_mean_mae": 4.94,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 1.111711,
"shadow_bucket_brier": 1.099556,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": -0.012155
},
"paris": {
"samples": 2,
"legacy_mean_mae": 4.265,
"shadow_mean_mae": 4.265,
"legacy_bucket_hit_rate": 0.5,
"shadow_bucket_hit_rate": 0.5,
"legacy_bucket_brier": 0.90186,
"shadow_bucket_brier": 0.9967,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.09484
},
"sao paulo": {
"samples": 2,
"legacy_mean_mae": 6.57,
"shadow_mean_mae": 6.57,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 1.263988,
"shadow_bucket_brier": 1.159352,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": -0.104636
},
"seattle": {
"samples": 1,
"legacy_mean_mae": 0.0,
"shadow_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.0,
"shadow_bucket_brier": 0.61609,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.61609
},
"seoul": {
"samples": 2,
"legacy_mean_mae": 0.15,
"shadow_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.258977,
"shadow_bucket_brier": 0.548485,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.289508
},
"shanghai": {
"samples": 2,
"legacy_mean_mae": 0.15,
"shadow_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.1156,
"shadow_bucket_brier": 0.501323,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.385723
},
"singapore": {
"samples": 2,
"legacy_mean_mae": 0.15,
"shadow_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.306527,
"shadow_bucket_brier": 0.553647,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.24712
},
"taipei": {
"samples": 3,
"legacy_mean_mae": 0.1,
"shadow_mean_mae": 0.1,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 0.333333,
"legacy_bucket_brier": 0.195261,
"shadow_bucket_brier": 0.684998,
"delta_mae": 0.0,
"delta_bucket_hit_rate": -0.666667,
"delta_bucket_brier": 0.489737
},
"tel aviv": {
"samples": 2,
"legacy_mean_mae": 0.3,
"shadow_mean_mae": 0.3,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.257895,
"shadow_bucket_brier": 0.697774,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.439879
},
"tokyo": {
"samples": 2,
"legacy_mean_mae": 0.25,
"shadow_mean_mae": 0.25,
"legacy_bucket_hit_rate": 0.5,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.284759,
"shadow_bucket_brier": 0.69701,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.5,
"delta_bucket_brier": 0.412251
},
"toronto": {
"samples": 2,
"legacy_mean_mae": 6.33,
"shadow_mean_mae": 6.33,
"legacy_bucket_hit_rate": 0.0,
"shadow_bucket_hit_rate": 0.0,
"legacy_bucket_brier": 1.256736,
"shadow_bucket_brier": 1.221422,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": -0.035314
},
"warsaw": {
"samples": 3,
"legacy_mean_mae": 2.056667,
"shadow_mean_mae": 2.056667,
"legacy_bucket_hit_rate": 0.333333,
"shadow_bucket_hit_rate": 0.333333,
"legacy_bucket_brier": 0.86425,
"shadow_bucket_brier": 0.750985,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": -0.113265
},
"wellington": {
"samples": 2,
"legacy_mean_mae": 0.15,
"shadow_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 1.0,
"legacy_bucket_brier": 0.095481,
"shadow_bucket_brier": 0.604684,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.0,
"delta_bucket_brier": 0.509203
}
},
"by_date": {
"2026-03-17": {
"samples": 4,
"legacy_mean_mae": 0.15,
"shadow_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"shadow_bucket_hit_rate": 0.5,
"legacy_bucket_brier": 0.101543,
"shadow_bucket_brier": 0.6636,
"delta_mae": 0.0,
"delta_bucket_hit_rate": -0.5,
"delta_bucket_brier": 0.562057
},
"2026-03-18": {
"samples": 25,
"legacy_mean_mae": 2.5476,
"shadow_mean_mae": 2.5476,
"legacy_bucket_hit_rate": 0.52,
"shadow_bucket_hit_rate": 0.56,
"legacy_bucket_brier": 0.657516,
"shadow_bucket_brier": 0.74574,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.04,
"delta_bucket_brier": 0.088224
},
"2026-03-19": {
"samples": 19,
"legacy_mean_mae": 4.299474,
"shadow_mean_mae": 4.299474,
"legacy_bucket_hit_rate": 0.368421,
"shadow_bucket_hit_rate": 0.421053,
"legacy_bucket_brier": 0.848191,
"shadow_bucket_brier": 0.935678,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.052632,
"delta_bucket_brier": 0.087487
}
},
"recent_observations": [
{
"city": "wellington",
"date": "2026-03-19",
"actual_high": 18.0,
"actual_bucket": 18,
"legacy_mu": 18.3,
"shadow_mu": 18.3,
"legacy_top_bucket": 18,
"shadow_top_bucket": 18,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "warsaw",
"date": "2026-03-19",
"actual_high": 7.0,
"actual_bucket": 7,
"legacy_mu": 12.03,
"shadow_mu": 12.03,
"legacy_top_bucket": 12,
"shadow_top_bucket": 12,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "toronto",
"date": "2026-03-19",
"actual_high": -2.0,
"actual_bucket": -2,
"legacy_mu": 5.67,
"shadow_mu": 5.67,
"legacy_top_bucket": 6,
"shadow_top_bucket": 6,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "tokyo",
"date": "2026-03-19",
"actual_high": 16.0,
"actual_bucket": 16,
"legacy_mu": 16.5,
"shadow_mu": 16.5,
"legacy_top_bucket": 17,
"shadow_top_bucket": 16,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "tel aviv",
"date": "2026-03-19",
"actual_high": 21.0,
"actual_bucket": 21,
"legacy_mu": 21.3,
"shadow_mu": 21.3,
"legacy_top_bucket": 21,
"shadow_top_bucket": 21,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "taipei",
"date": "2026-03-19",
"actual_high": 22.0,
"actual_bucket": 22,
"legacy_mu": 21.7,
"shadow_mu": 21.7,
"legacy_top_bucket": 22,
"shadow_top_bucket": 21,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "singapore",
"date": "2026-03-19",
"actual_high": 32.0,
"actual_bucket": 32,
"legacy_mu": 32.3,
"shadow_mu": 32.3,
"legacy_top_bucket": 32,
"shadow_top_bucket": 32,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "shanghai",
"date": "2026-03-19",
"actual_high": 12.0,
"actual_bucket": 12,
"legacy_mu": 12.3,
"shadow_mu": 12.3,
"legacy_top_bucket": 12,
"shadow_top_bucket": 12,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "seoul",
"date": "2026-03-19",
"actual_high": 10.0,
"actual_bucket": 10,
"legacy_mu": 10.3,
"shadow_mu": 10.3,
"legacy_top_bucket": 10,
"shadow_top_bucket": 10,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "sao paulo",
"date": "2026-03-19",
"actual_high": 21.0,
"actual_bucket": 21,
"legacy_mu": 26.5,
"shadow_mu": 26.5,
"legacy_top_bucket": 26,
"shadow_top_bucket": 26,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "paris",
"date": "2026-03-19",
"actual_high": 8.0,
"actual_bucket": 8,
"legacy_mu": 16.06,
"shadow_mu": 16.06,
"legacy_top_bucket": 16,
"shadow_top_bucket": 16,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "munich",
"date": "2026-03-19",
"actual_high": 5.0,
"actual_bucket": 5,
"legacy_mu": 11.62,
"shadow_mu": 11.62,
"legacy_top_bucket": 12,
"shadow_top_bucket": 12,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "milan",
"date": "2026-03-19",
"actual_high": 5.0,
"actual_bucket": 5,
"legacy_mu": 16.58,
"shadow_mu": 16.58,
"legacy_top_bucket": 17,
"shadow_top_bucket": 17,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "madrid",
"date": "2026-03-19",
"actual_high": 8.0,
"actual_bucket": 8,
"legacy_mu": 18.23,
"shadow_mu": 18.23,
"legacy_top_bucket": 18,
"shadow_top_bucket": 18,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "lucknow",
"date": "2026-03-19",
"actual_high": 30.0,
"actual_bucket": 30,
"legacy_mu": 35.3,
"shadow_mu": 35.3,
"legacy_top_bucket": 35,
"shadow_top_bucket": 35,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "london",
"date": "2026-03-19",
"actual_high": 8.0,
"actual_bucket": 8,
"legacy_mu": 15.71,
"shadow_mu": 15.71,
"legacy_top_bucket": 16,
"shadow_top_bucket": 16,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "hong kong",
"date": "2026-03-19",
"actual_high": 27.3,
"actual_bucket": 27,
"legacy_mu": 27.6,
"shadow_mu": 27.6,
"legacy_top_bucket": 28,
"shadow_top_bucket": 27,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "buenos aires",
"date": "2026-03-19",
"actual_high": 16.0,
"actual_bucket": 16,
"legacy_mu": 27.39,
"shadow_mu": 27.39,
"legacy_top_bucket": 27,
"shadow_top_bucket": 27,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "ankara",
"date": "2026-03-19",
"actual_high": 11.0,
"actual_bucket": 11,
"legacy_mu": 11.0,
"shadow_mu": 11.0,
"legacy_top_bucket": 11,
"shadow_top_bucket": 11,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "wellington",
"date": "2026-03-18",
"actual_high": 21.0,
"actual_bucket": 21,
"legacy_mu": 21.0,
"shadow_mu": 21.0,
"legacy_top_bucket": 21,
"shadow_top_bucket": 21,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "warsaw",
"date": "2026-03-18",
"actual_high": 13.0,
"actual_bucket": 13,
"legacy_mu": 13.84,
"shadow_mu": 13.84,
"legacy_top_bucket": 14,
"shadow_top_bucket": 14,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "toronto",
"date": "2026-03-18",
"actual_high": -6.0,
"actual_bucket": -6,
"legacy_mu": -1.01,
"shadow_mu": -1.01,
"legacy_top_bucket": -1,
"shadow_top_bucket": -1,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "tokyo",
"date": "2026-03-18",
"actual_high": 17.0,
"actual_bucket": 17,
"legacy_mu": 17.0,
"shadow_mu": 17.0,
"legacy_top_bucket": 17,
"shadow_top_bucket": 17,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "tel aviv",
"date": "2026-03-18",
"actual_high": 30.0,
"actual_bucket": 30,
"legacy_mu": 30.3,
"shadow_mu": 30.3,
"legacy_top_bucket": 30,
"shadow_top_bucket": 30,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "taipei",
"date": "2026-03-18",
"actual_high": 29.0,
"actual_bucket": 29,
"legacy_mu": 29.0,
"shadow_mu": 29.0,
"legacy_top_bucket": 29,
"shadow_top_bucket": 29,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "singapore",
"date": "2026-03-18",
"actual_high": 32.0,
"actual_bucket": 32,
"legacy_mu": 32.0,
"shadow_mu": 32.0,
"legacy_top_bucket": 32,
"shadow_top_bucket": 32,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "shanghai",
"date": "2026-03-18",
"actual_high": 13.0,
"actual_bucket": 13,
"legacy_mu": 13.0,
"shadow_mu": 13.0,
"legacy_top_bucket": 13,
"shadow_top_bucket": 13,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "seoul",
"date": "2026-03-18",
"actual_high": 8.0,
"actual_bucket": 8,
"legacy_mu": 8.0,
"shadow_mu": 8.0,
"legacy_top_bucket": 8,
"shadow_top_bucket": 8,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "seattle",
"date": "2026-03-18",
"actual_high": 55.9,
"actual_bucket": 56,
"legacy_mu": 55.9,
"shadow_mu": 55.9,
"legacy_top_bucket": 56,
"shadow_top_bucket": 56,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "sao paulo",
"date": "2026-03-18",
"actual_high": 22.0,
"actual_bucket": 22,
"legacy_mu": 29.64,
"shadow_mu": 29.64,
"legacy_top_bucket": 30,
"shadow_top_bucket": 30,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "paris",
"date": "2026-03-18",
"actual_high": 14.0,
"actual_bucket": 14,
"legacy_mu": 14.47,
"shadow_mu": 14.47,
"legacy_top_bucket": 14,
"shadow_top_bucket": 14,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "new york",
"date": "2026-03-18",
"actual_high": 32.0,
"actual_bucket": 32,
"legacy_mu": 36.94,
"shadow_mu": 36.94,
"legacy_top_bucket": 37,
"shadow_top_bucket": 37,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "munich",
"date": "2026-03-18",
"actual_high": 10.0,
"actual_bucket": 10,
"legacy_mu": 10.66,
"shadow_mu": 10.66,
"legacy_top_bucket": 11,
"shadow_top_bucket": 11,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "milan",
"date": "2026-03-18",
"actual_high": 14.0,
"actual_bucket": 14,
"legacy_mu": 14.3,
"shadow_mu": 14.3,
"legacy_top_bucket": 14,
"shadow_top_bucket": 14,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "miami",
"date": "2026-03-18",
"actual_high": 59.0,
"actual_bucket": 59,
"legacy_mu": 70.04,
"shadow_mu": 70.04,
"legacy_top_bucket": 70,
"shadow_top_bucket": 70,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "madrid",
"date": "2026-03-18",
"actual_high": 14.0,
"actual_bucket": 14,
"legacy_mu": 18.43,
"shadow_mu": 18.43,
"legacy_top_bucket": 18,
"shadow_top_bucket": 18,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "lucknow",
"date": "2026-03-18",
"actual_high": 34.0,
"actual_bucket": 34,
"legacy_mu": 35.11,
"shadow_mu": 35.11,
"legacy_top_bucket": 35,
"shadow_top_bucket": 35,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "london",
"date": "2026-03-18",
"actual_high": 16.0,
"actual_bucket": 16,
"legacy_mu": 16.56,
"shadow_mu": 16.56,
"legacy_top_bucket": 17,
"shadow_top_bucket": 17,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "hong kong",
"date": "2026-03-18",
"actual_high": 27.8,
"actual_bucket": 27,
"legacy_mu": 27.8,
"shadow_mu": 27.8,
"legacy_top_bucket": 28,
"shadow_top_bucket": 27,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "dallas",
"date": "2026-03-18",
"actual_high": 73.9,
"actual_bucket": 74,
"legacy_mu": 73.9,
"shadow_mu": 73.9,
"legacy_top_bucket": 74,
"shadow_top_bucket": 74,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "chicago",
"date": "2026-03-18",
"actual_high": 46.0,
"actual_bucket": 46,
"legacy_mu": 46.0,
"shadow_mu": 46.0,
"legacy_top_bucket": 46,
"shadow_top_bucket": 46,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "buenos aires",
"date": "2026-03-18",
"actual_high": 17.0,
"actual_bucket": 17,
"legacy_mu": 26.15,
"shadow_mu": 26.15,
"legacy_top_bucket": 26,
"shadow_top_bucket": 26,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "atlanta",
"date": "2026-03-18",
"actual_high": 37.0,
"actual_bucket": 37,
"legacy_mu": 54.06,
"shadow_mu": 54.06,
"legacy_top_bucket": 54,
"shadow_top_bucket": 54,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "ankara",
"date": "2026-03-18",
"actual_high": 15.0,
"actual_bucket": 15,
"legacy_mu": 15.2,
"shadow_mu": 15.2,
"legacy_top_bucket": 15,
"shadow_top_bucket": 15,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "warsaw",
"date": "2026-03-17",
"actual_high": 11.0,
"actual_bucket": 11,
"legacy_mu": 11.3,
"shadow_mu": 11.3,
"legacy_top_bucket": 11,
"shadow_top_bucket": 11,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "taipei",
"date": "2026-03-17",
"actual_high": 26.7,
"actual_bucket": 27,
"legacy_mu": 26.7,
"shadow_mu": 26.7,
"legacy_top_bucket": 27,
"shadow_top_bucket": 26,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "milan",
"date": "2026-03-17",
"actual_high": 17.0,
"actual_bucket": 17,
"legacy_mu": 17.3,
"shadow_mu": 17.3,
"legacy_top_bucket": 17,
"shadow_top_bucket": 17,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
},
{
"city": "hong kong",
"date": "2026-03-17",
"actual_high": 24.0,
"actual_bucket": 24,
"legacy_mu": 24.0,
"shadow_mu": 24.0,
"legacy_top_bucket": 24,
"shadow_top_bucket": 23,
"calibration_version": "emos-20260320130245",
"calibration_mode": "emos_shadow"
}
]
}
artifacts/probability_calibration/training_samples.json
-981
View File
@@ -1,981 +0,0 @@
{
"sample_count": 54,
"snapshot_sample_count": 1,
"daily_record_sample_count": 53,
"filled_actual_from_history": 2,
"samples": [
{
"city": "shenzhen",
"date": "2026-03-25",
"timestamp": "2026-03-25T08:57:11.783182+00:00",
"actual_high": 28.0,
"raw_mu": 26.7,
"raw_sigma": 0.18016764322916676,
"deb_prediction": 28.1,
"ens_median": 31.4,
"ensemble_spread": 0.5078125000000002,
"max_so_far_gap": 1.4000000000000021,
"peak_flag": 1.0,
"sample_source": "snapshot",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "ankara",
"date": "2026-03-18",
"actual_high": 15.0,
"raw_mu": 15.2,
"raw_sigma": 1.2000000000000002,
"deb_prediction": 15.4,
"ens_median": 15.8,
"ensemble_spread": 1.2000000000000002,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "ankara",
"date": "2026-03-19",
"actual_high": 11.0,
"raw_mu": 11.0,
"raw_sigma": 2.05,
"deb_prediction": 9.8,
"ens_median": 10.1,
"ensemble_spread": 2.05,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "ankara",
"date": "2026-03-29",
"actual_high": 10.0,
"raw_mu": 10.0,
"raw_sigma": 0.9000000000000004,
"deb_prediction": 9.7,
"ens_median": 10.0,
"ensemble_spread": 0.9000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "atlanta",
"date": "2026-03-18",
"actual_high": 37.0,
"raw_mu": 54.06,
"raw_sigma": 4.0,
"deb_prediction": 52.5,
"ens_median": 52.2,
"ensemble_spread": 4.0,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "atlanta",
"date": "2026-03-19",
"actual_high": 18.6,
"raw_mu": 65.57,
"raw_sigma": 1.5500000000000007,
"deb_prediction": 65.7,
"ens_median": 65.9,
"ensemble_spread": 1.5500000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "buenos aires",
"date": "2026-03-18",
"actual_high": 17.0,
"raw_mu": 26.15,
"raw_sigma": 2.0,
"deb_prediction": 26.4,
"ens_median": 26.0,
"ensemble_spread": 2.0,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "buenos aires",
"date": "2026-03-19",
"actual_high": 16.0,
"raw_mu": 27.39,
"raw_sigma": 2.0999999999999996,
"deb_prediction": 26.8,
"ens_median": 27.9,
"ensemble_spread": 2.0999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "chicago",
"date": "2026-03-18",
"actual_high": 46.0,
"raw_mu": 46.0,
"raw_sigma": 5.350000000000001,
"deb_prediction": 42.7,
"ens_median": 44.2,
"ensemble_spread": 5.350000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "dallas",
"date": "2026-03-18",
"actual_high": 73.9,
"raw_mu": 73.9,
"raw_sigma": 9.299999999999997,
"deb_prediction": 75.2,
"ens_median": 74.4,
"ensemble_spread": 9.299999999999997,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "hong kong",
"date": "2026-03-17",
"actual_high": 24.0,
"raw_mu": 24.0,
"raw_sigma": 1.9000000000000004,
"deb_prediction": 23.1,
"ens_median": 23.1,
"ensemble_spread": 1.9000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "hong kong",
"date": "2026-03-18",
"actual_high": 27.8,
"raw_mu": 27.8,
"raw_sigma": 0.6,
"deb_prediction": 24.5,
"ens_median": 24.9,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "hong kong",
"date": "2026-03-19",
"actual_high": 27.3,
"raw_mu": 27.6,
"raw_sigma": 0.6,
"deb_prediction": 24.9,
"ens_median": 25.0,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "hong kong",
"date": "2026-03-23",
"actual_high": 27.4,
"raw_mu": 27.4,
"raw_sigma": 1.6999999999999993,
"deb_prediction": 25.2,
"ens_median": 25.1,
"ensemble_spread": 1.6999999999999993,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "london",
"date": "2026-03-18",
"actual_high": 16.0,
"raw_mu": 16.56,
"raw_sigma": 1.299999999999999,
"deb_prediction": 16.5,
"ens_median": 16.8,
"ensemble_spread": 1.299999999999999,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "london",
"date": "2026-03-19",
"actual_high": 8.0,
"raw_mu": 15.71,
"raw_sigma": 0.6,
"deb_prediction": 15.9,
"ens_median": 15.8,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "lucknow",
"date": "2026-03-18",
"actual_high": 34.0,
"raw_mu": 35.11,
"raw_sigma": 1.3000000000000007,
"deb_prediction": 34.3,
"ens_median": 34.6,
"ensemble_spread": 1.3000000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "lucknow",
"date": "2026-03-19",
"actual_high": 30.0,
"raw_mu": 35.3,
"raw_sigma": 1.75,
"deb_prediction": 34.5,
"ens_median": 34.7,
"ensemble_spread": 1.75,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "madrid",
"date": "2026-03-18",
"actual_high": 14.0,
"raw_mu": 18.43,
"raw_sigma": 1.8499999999999996,
"deb_prediction": 18.3,
"ens_median": 18.7,
"ensemble_spread": 1.8499999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "madrid",
"date": "2026-03-19",
"actual_high": 8.0,
"raw_mu": 18.23,
"raw_sigma": 1.8999999999999995,
"deb_prediction": 18.4,
"ens_median": 18.8,
"ensemble_spread": 1.8999999999999995,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "miami",
"date": "2026-03-18",
"actual_high": 59.0,
"raw_mu": 70.04,
"raw_sigma": 2.8999999999999986,
"deb_prediction": 69.1,
"ens_median": 70.4,
"ensemble_spread": 2.8999999999999986,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "miami",
"date": "2026-03-19",
"actual_high": 24.6,
"raw_mu": 73.91,
"raw_sigma": 2.5500000000000043,
"deb_prediction": 74.2,
"ens_median": 74.0,
"ensemble_spread": 2.5500000000000043,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "milan",
"date": "2026-03-17",
"actual_high": 17.0,
"raw_mu": 17.3,
"raw_sigma": 9.1,
"deb_prediction": 12.5,
"ens_median": 15.1,
"ensemble_spread": 9.1,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "milan",
"date": "2026-03-18",
"actual_high": 14.0,
"raw_mu": 14.3,
"raw_sigma": 0.6,
"deb_prediction": 13.7,
"ens_median": 13.8,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "milan",
"date": "2026-03-19",
"actual_high": 5.0,
"raw_mu": 16.58,
"raw_sigma": 1.25,
"deb_prediction": 16.9,
"ens_median": 17.0,
"ensemble_spread": 1.25,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "munich",
"date": "2026-03-18",
"actual_high": 10.0,
"raw_mu": 10.66,
"raw_sigma": 0.6,
"deb_prediction": 10.6,
"ens_median": 10.6,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "munich",
"date": "2026-03-19",
"actual_high": 5.0,
"raw_mu": 11.62,
"raw_sigma": 1.3000000000000007,
"deb_prediction": 11.5,
"ens_median": 11.8,
"ensemble_spread": 1.3000000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "new york",
"date": "2026-03-18",
"actual_high": 32.0,
"raw_mu": 36.94,
"raw_sigma": 2.25,
"deb_prediction": 36.8,
"ens_median": 37.0,
"ensemble_spread": 2.25,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "paris",
"date": "2026-03-18",
"actual_high": 14.0,
"raw_mu": 14.47,
"raw_sigma": 0.9000000000000004,
"deb_prediction": 14.2,
"ens_median": 14.8,
"ensemble_spread": 0.9000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "paris",
"date": "2026-03-19",
"actual_high": 8.0,
"raw_mu": 16.06,
"raw_sigma": 0.6,
"deb_prediction": 16.2,
"ens_median": 16.3,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "sao paulo",
"date": "2026-03-18",
"actual_high": 22.0,
"raw_mu": 29.64,
"raw_sigma": 2.450000000000001,
"deb_prediction": 30.0,
"ens_median": 29.4,
"ensemble_spread": 2.450000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "sao paulo",
"date": "2026-03-19",
"actual_high": 21.0,
"raw_mu": 26.5,
"raw_sigma": 1.200000000000001,
"deb_prediction": 25.5,
"ens_median": 26.5,
"ensemble_spread": 1.200000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "seattle",
"date": "2026-03-18",
"actual_high": 55.9,
"raw_mu": 55.9,
"raw_sigma": 1.3500000000000014,
"deb_prediction": 55.8,
"ens_median": 55.9,
"ensemble_spread": 1.3500000000000014,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "seoul",
"date": "2026-03-18",
"actual_high": 8.0,
"raw_mu": 8.0,
"raw_sigma": 0.7999999999999998,
"deb_prediction": 7.4,
"ens_median": 7.4,
"ensemble_spread": 0.7999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "seoul",
"date": "2026-03-19",
"actual_high": 10.0,
"raw_mu": 10.3,
"raw_sigma": 1.7999999999999998,
"deb_prediction": 7.6,
"ens_median": 6.3,
"ensemble_spread": 1.7999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "shanghai",
"date": "2026-03-18",
"actual_high": 13.0,
"raw_mu": 13.0,
"raw_sigma": 1.1500000000000004,
"deb_prediction": 13.2,
"ens_median": 13.0,
"ensemble_spread": 1.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "shanghai",
"date": "2026-03-19",
"actual_high": 12.0,
"raw_mu": 12.3,
"raw_sigma": 0.7999999999999998,
"deb_prediction": 10.7,
"ens_median": 11.2,
"ensemble_spread": 0.7999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "shanghai",
"date": "2026-03-29",
"actual_high": 18.0,
"raw_mu": 18.0,
"raw_sigma": 1.6999999999999993,
"deb_prediction": 18.4,
"ens_median": 17.6,
"ensemble_spread": 1.6999999999999993,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "singapore",
"date": "2026-03-18",
"actual_high": 32.0,
"raw_mu": 32.0,
"raw_sigma": 0.9500000000000011,
"deb_prediction": 30.2,
"ens_median": 30.1,
"ensemble_spread": 0.9500000000000011,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "singapore",
"date": "2026-03-19",
"actual_high": 32.0,
"raw_mu": 32.3,
"raw_sigma": 1.3499999999999996,
"deb_prediction": 31.5,
"ens_median": 32.1,
"ensemble_spread": 1.3499999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "taipei",
"date": "2026-03-17",
"actual_high": 26.7,
"raw_mu": 26.7,
"raw_sigma": 2.25,
"deb_prediction": 24.9,
"ens_median": 25.4,
"ensemble_spread": 2.25,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "taipei",
"date": "2026-03-18",
"actual_high": 29.0,
"raw_mu": 29.0,
"raw_sigma": 1.0500000000000007,
"deb_prediction": 27.2,
"ens_median": 27.5,
"ensemble_spread": 1.0500000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "taipei",
"date": "2026-03-19",
"actual_high": 22.0,
"raw_mu": 21.7,
"raw_sigma": 1.1500000000000004,
"deb_prediction": 21.5,
"ens_median": 21.3,
"ensemble_spread": 1.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "tel aviv",
"date": "2026-03-18",
"actual_high": 30.0,
"raw_mu": 30.3,
"raw_sigma": 2.1500000000000004,
"deb_prediction": 29.0,
"ens_median": 28.9,
"ensemble_spread": 2.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "tel aviv",
"date": "2026-03-19",
"actual_high": 21.0,
"raw_mu": 21.3,
"raw_sigma": 1.5,
"deb_prediction": 20.7,
"ens_median": 21.1,
"ensemble_spread": 1.5,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "tokyo",
"date": "2026-03-18",
"actual_high": 17.0,
"raw_mu": 17.0,
"raw_sigma": 1.5499999999999998,
"deb_prediction": 15.4,
"ens_median": 15.8,
"ensemble_spread": 1.5499999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "tokyo",
"date": "2026-03-19",
"actual_high": 16.0,
"raw_mu": 16.5,
"raw_sigma": 2.0999999999999996,
"deb_prediction": 17.8,
"ens_median": 18.5,
"ensemble_spread": 2.0999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "toronto",
"date": "2026-03-18",
"actual_high": -6.0,
"raw_mu": -1.01,
"raw_sigma": 0.8,
"deb_prediction": -0.6,
"ens_median": -1.1,
"ensemble_spread": 0.8,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "toronto",
"date": "2026-03-19",
"actual_high": -2.0,
"raw_mu": 5.67,
"raw_sigma": 2.1500000000000004,
"deb_prediction": 6.4,
"ens_median": 6.3,
"ensemble_spread": 2.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "warsaw",
"date": "2026-03-17",
"actual_high": 11.0,
"raw_mu": 11.3,
"raw_sigma": 0.6499999999999995,
"deb_prediction": 10.4,
"ens_median": 10.5,
"ensemble_spread": 0.6499999999999995,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "warsaw",
"date": "2026-03-18",
"actual_high": 13.0,
"raw_mu": 13.84,
"raw_sigma": 1.4000000000000004,
"deb_prediction": 13.6,
"ens_median": 14.2,
"ensemble_spread": 1.4000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "warsaw",
"date": "2026-03-19",
"actual_high": 7.0,
"raw_mu": 12.03,
"raw_sigma": 1.5999999999999996,
"deb_prediction": 11.8,
"ens_median": 12.3,
"ensemble_spread": 1.5999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "wellington",
"date": "2026-03-18",
"actual_high": 21.0,
"raw_mu": 21.0,
"raw_sigma": 0.9500000000000011,
"deb_prediction": 19.2,
"ens_median": 19.1,
"ensemble_spread": 0.9500000000000011,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
},
{
"city": "wellington",
"date": "2026-03-19",
"actual_high": 18.0,
"raw_mu": 18.3,
"raw_sigma": 2.0999999999999996,
"deb_prediction": 17.9,
"ens_median": 17.1,
"ensemble_spread": 2.0999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0,
"sample_source": "daily_record",
"settlement_source": null,
"settlement_station_code": null,
"truth_version": null,
"truth_updated_by": null,
"truth_updated_at": null
}
]
}
frontend/components/dashboard/ProbabilityDistribution.tsx
+6 -6
View File
@@ -841,17 +841,17 @@ export function ProbabilityDistribution({
<div className="prob-distribution-head">
<span>
{locale === "en-US"
? "EMOS probability distribution"
: "EMOS 概率分布"}
? "Probability distribution"
: "概率分布"}
</span>
<em>
{marketContractRows.length > 0
? locale === "en-US"
? "market buckets are aggregated from single-degree EMOS buckets"
: "市场合约桶由单点 EMOS 概率聚合"
? "market buckets are aggregated from single-degree probability buckets"
: "市场合约桶由单点概率分布聚合"
: locale === "en-US"
? "calibrated temperature buckets"
: "校准后的温度桶"}
? "temperature probability buckets"
: "温度概率桶"}
</em>
</div>
{probabilityRows.length === 0 ? (
scripts/auto_retrain_probability_calibration.py
-359
View File
@@ -1,359 +0,0 @@
import argparse
import json
import os
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.probability_calibration import DEFAULT_CALIBRATION_FILE # noqa: E402
ARTIFACT_DIR = os.path.join(PROJECT_ROOT, "artifacts", "probability_calibration")
def _runtime_calibration_dir() -> str:
runtime_dir = str(os.getenv("POLYWEATHER_RUNTIME_DATA_DIR") or "").strip()
if runtime_dir:
return os.path.join(runtime_dir, "probability_calibration")
return ARTIFACT_DIR
DEFAULT_CANDIDATE_ROOT = os.path.join(_runtime_calibration_dir(), "candidates")
DEFAULT_DECISION_REPORT = os.path.join(
_runtime_calibration_dir(),
"auto_retrain_report.json",
)
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _env_float(name: str, default: float) -> float:
value = _sf(os.getenv(name))
return value if value is not None else default
def _env_int(name: str, default: int) -> int:
value = _sf(os.getenv(name))
return int(value) if value is not None else default
def _load_json(path: str) -> Dict[str, Any]:
try:
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
return data if isinstance(data, dict) else {}
except Exception:
return {}
def _write_json(path: str, payload: Dict[str, Any]) -> None:
output_dir = os.path.dirname(os.path.abspath(path))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(path, "w", encoding="utf-8") as fh:
json.dump(payload, fh, ensure_ascii=False, indent=2)
def _run_python(args: List[str], *, stream: bool = False) -> Dict[str, Any]:
command = [sys.executable, *args]
if stream:
completed = subprocess.run(
command,
cwd=PROJECT_ROOT,
text=True,
check=False,
)
return {
"command": command,
"returncode": completed.returncode,
"stdout": "",
"stderr": "",
}
completed = subprocess.run(
command,
cwd=PROJECT_ROOT,
text=True,
capture_output=True,
check=False,
)
return {
"command": command,
"returncode": completed.returncode,
"stdout": completed.stdout,
"stderr": completed.stderr,
}
def _append_blocker(blockers: List[str], condition: bool, message: str) -> None:
if condition:
blockers.append(message)
def judge_candidate(
evaluation_report: Dict[str, Any],
*,
min_samples: int,
max_delta_crps: float,
max_delta_mae: float,
min_delta_bucket_hit_rate: float,
) -> Dict[str, Any]:
summary = evaluation_report.get("summary") or {}
delta = summary.get("delta") or {}
sample_count = int(summary.get("sample_count") or 0)
delta_crps = _sf(delta.get("crps"))
delta_mae = _sf(delta.get("mae"))
delta_hit = _sf(delta.get("bucket_hit_rate"))
blockers: List[str] = []
_append_blocker(
blockers,
sample_count < min_samples,
f"sample_count {sample_count} < {min_samples}",
)
_append_blocker(
blockers,
delta_crps is None or delta_crps > max_delta_crps,
f"delta_crps {delta_crps} > {max_delta_crps}",
)
_append_blocker(
blockers,
delta_mae is None or delta_mae > max_delta_mae,
f"delta_mae {delta_mae} > {max_delta_mae}",
)
_append_blocker(
blockers,
delta_hit is None or delta_hit < min_delta_bucket_hit_rate,
f"delta_bucket_hit_rate {delta_hit} < {min_delta_bucket_hit_rate}",
)
return {
"decision": "promote" if not blockers else "hold",
"ready_for_promotion": not blockers,
"blocking_reasons": blockers,
"thresholds": {
"min_samples": min_samples,
"max_delta_crps": max_delta_crps,
"max_delta_mae": max_delta_mae,
"min_delta_bucket_hit_rate": min_delta_bucket_hit_rate,
},
"metrics": {
"sample_count": sample_count,
"delta_crps": delta_crps,
"delta_mae": delta_mae,
"delta_bucket_hit_rate": delta_hit,
},
}
def _promote(candidate_path: str, target_path: str) -> str:
target_dir = os.path.dirname(os.path.abspath(target_path))
if target_dir:
os.makedirs(target_dir, exist_ok=True)
backup_path = os.path.join(
target_dir,
"default.backup-{ts}.json".format(
ts=datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
),
)
if os.path.exists(target_path):
shutil.copy2(target_path, backup_path)
shutil.copy2(candidate_path, target_path)
return backup_path
def main() -> int:
parser = argparse.ArgumentParser(
description="Train an EMOS candidate, evaluate it, and optionally promote it behind gates."
)
parser.add_argument("--candidate-root", default=DEFAULT_CANDIDATE_ROOT)
parser.add_argument("--target", default=DEFAULT_CALIBRATION_FILE)
parser.add_argument("--decision-output", default=DEFAULT_DECISION_REPORT)
parser.add_argument(
"--promote-if-passed",
action="store_true",
help="Copy the candidate over the active calibration file only if gates pass.",
)
parser.add_argument(
"--run-tests",
action="store_true",
help="Run focused probability tests before promotion.",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Print child script progress while training and evaluating.",
)
parser.add_argument(
"--snapshot-limit",
type=int,
default=_env_int("POLYWEATHER_EMOS_TRAINING_SNAPSHOT_LIMIT", 0),
help="Optional max number of recent probability snapshots to load from SQLite.",
)
parser.add_argument(
"--min-samples",
type=int,
default=_env_int("POLYWEATHER_EMOS_AUTO_MIN_SAMPLES", 50),
)
parser.add_argument(
"--max-delta-crps",
type=float,
default=_env_float("POLYWEATHER_EMOS_AUTO_MAX_DELTA_CRPS", 0.0),
help="Candidate EMOS CRPS may not be worse than legacy by more than this.",
)
parser.add_argument(
"--max-delta-mae",
type=float,
default=_env_float("POLYWEATHER_EMOS_AUTO_MAX_DELTA_MAE", 0.05),
)
parser.add_argument(
"--min-delta-bucket-hit-rate",
type=float,
default=_env_float("POLYWEATHER_EMOS_AUTO_MIN_DELTA_BUCKET_HIT_RATE", -0.05),
help="Soft guard only; bucket hit rate is boundary-sensitive.",
)
args = parser.parse_args()
version = "emos-auto-{ts}".format(
ts=datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
)
candidate_dir = os.path.join(args.candidate_root, version)
os.makedirs(candidate_dir, exist_ok=True)
candidate_path = os.path.join(candidate_dir, "default.json")
evaluation_path = os.path.join(candidate_dir, "evaluation_report.json")
decision_path = os.path.join(candidate_dir, "decision_report.json")
fit_args = [
"scripts/fit_probability_calibration.py",
"--output",
candidate_path,
"--version",
version,
]
if args.verbose:
fit_args.append("--verbose")
if args.snapshot_limit and args.snapshot_limit > 0:
fit_args.extend(["--snapshot-limit", str(args.snapshot_limit)])
fit_result = _run_python(
fit_args,
stream=args.verbose,
)
if fit_result["returncode"] != 0:
payload = {
"ok": False,
"version": version,
"stage": "fit",
"fit": fit_result,
}
_write_json(decision_path, payload)
_write_json(args.decision_output, payload)
print(json.dumps(payload, ensure_ascii=False, indent=2))
return fit_result["returncode"] or 1
eval_args = [
"scripts/evaluate_probability_calibration.py",
"--calibration-file",
candidate_path,
"--output",
evaluation_path,
]
if args.verbose:
eval_args.append("--verbose")
if args.snapshot_limit and args.snapshot_limit > 0:
eval_args.extend(["--snapshot-limit", str(args.snapshot_limit)])
eval_result = _run_python(
eval_args,
stream=args.verbose,
)
if eval_result["returncode"] != 0:
payload = {
"ok": False,
"version": version,
"stage": "evaluate",
"candidate_path": candidate_path,
"fit": fit_result,
"evaluate": eval_result,
}
_write_json(decision_path, payload)
_write_json(args.decision_output, payload)
print(json.dumps(payload, ensure_ascii=False, indent=2))
return eval_result["returncode"] or 1
evaluation_report = _load_json(evaluation_path)
decision = judge_candidate(
evaluation_report,
min_samples=args.min_samples,
max_delta_crps=args.max_delta_crps,
max_delta_mae=args.max_delta_mae,
min_delta_bucket_hit_rate=args.min_delta_bucket_hit_rate,
)
test_result = None
if args.run_tests and decision["ready_for_promotion"]:
test_result = _run_python(
[
"-m",
"pytest",
"tests/test_probability_calibration.py",
"tests/test_probability_rollout.py",
"tests/test_trend_engine.py",
]
)
if test_result["returncode"] != 0:
decision["decision"] = "hold"
decision["ready_for_promotion"] = False
decision.setdefault("blocking_reasons", []).append(
"focused tests failed"
)
promoted = False
backup_path = None
if args.promote_if_passed and decision["ready_for_promotion"]:
backup_path = _promote(candidate_path, args.target)
promoted = True
payload = {
"ok": True,
"version": version,
"generated_at": datetime.now(timezone.utc).isoformat(),
"candidate_dir": candidate_dir,
"candidate_path": candidate_path,
"evaluation_path": evaluation_path,
"target_path": args.target,
"promote_requested": bool(args.promote_if_passed),
"promoted": promoted,
"backup_path": backup_path,
"decision": decision,
"fit": fit_result,
"evaluate": eval_result,
"tests": test_result,
}
_write_json(decision_path, payload)
_write_json(args.decision_output, payload)
print(json.dumps(payload["decision"], ensure_ascii=False, indent=2))
print(f"candidate: {candidate_path}")
print(f"evaluation: {evaluation_path}")
print(f"decision: {decision_path}")
if promoted:
print(f"promoted to {args.target}; backup: {backup_path}")
elif args.promote_if_passed:
print("not promoted; gates did not pass")
else:
print("not promoted; run with --promote-if-passed to allow gated promotion")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/build_probability_shadow_report.py
-222
View File
@@ -1,222 +0,0 @@
import argparse
import json
import os
import sys
from collections import defaultdict
from datetime import datetime
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
from scripts.fit_probability_calibration import ( # noqa: E402
_default_history_arg,
_load_history_with_fallback,
)
def _sf(value):
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _mean(values):
return round(sum(values) / len(values), 6) if values else None
def _top_bucket(snapshot):
if not isinstance(snapshot, list):
return None
best = None
best_prob = -1.0
for row in snapshot:
if not isinstance(row, dict):
continue
try:
prob = float(row.get("p") if "p" in row else row.get("probability"))
except Exception:
continue
value = row.get("v") if "v" in row else row.get("value")
if value is None:
continue
if prob > best_prob:
best = value
best_prob = prob
return best
def _bucket_probability(snapshot, target_bucket):
if not isinstance(snapshot, list):
return 0.0
for row in snapshot:
if not isinstance(row, dict):
continue
value = row.get("v") if "v" in row else row.get("value")
if value != target_bucket:
continue
try:
return float(row.get("p") if "p" in row else row.get("probability") or 0.0)
except Exception:
return 0.0
return 0.0
def _brier_from_snapshot(snapshot, target_bucket):
hit_prob = _bucket_probability(snapshot, target_bucket)
total = (1.0 - hit_prob) ** 2
if isinstance(snapshot, list):
for row in snapshot:
if not isinstance(row, dict):
continue
value = row.get("v") if "v" in row else row.get("value")
if value == target_bucket:
continue
try:
prob = float(row.get("p") if "p" in row else row.get("probability") or 0.0)
except Exception:
prob = 0.0
total += prob * prob
return round(total, 6)
def _blank_metrics():
return {
"samples": 0,
"legacy_mae": [],
"shadow_mae": [],
"legacy_bucket_hit": [],
"shadow_bucket_hit": [],
"legacy_bucket_brier": [],
"shadow_bucket_brier": [],
}
def _rollup(metrics):
return {
"samples": metrics["samples"],
"legacy_mean_mae": _mean(metrics["legacy_mae"]),
"shadow_mean_mae": _mean(metrics["shadow_mae"]),
"legacy_bucket_hit_rate": _mean(metrics["legacy_bucket_hit"]),
"shadow_bucket_hit_rate": _mean(metrics["shadow_bucket_hit"]),
"legacy_bucket_brier": _mean(metrics["legacy_bucket_brier"]),
"shadow_bucket_brier": _mean(metrics["shadow_bucket_brier"]),
"delta_mae": round((_mean(metrics["shadow_mae"]) or 0.0) - (_mean(metrics["legacy_mae"]) or 0.0), 6),
"delta_bucket_hit_rate": round((_mean(metrics["shadow_bucket_hit"]) or 0.0) - (_mean(metrics["legacy_bucket_hit"]) or 0.0), 6),
"delta_bucket_brier": round((_mean(metrics["shadow_bucket_brier"]) or 0.0) - (_mean(metrics["legacy_bucket_brier"]) or 0.0), 6),
}
def main():
parser = argparse.ArgumentParser(description="Build live shadow probability report from daily records.")
parser.add_argument(
"--history-file",
default=_default_history_arg(),
)
parser.add_argument(
"--output",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"shadow_report.json",
),
)
args = parser.parse_args()
history = _load_history_with_fallback(args.history_file)
overall = _blank_metrics()
by_city = defaultdict(_blank_metrics)
by_date = defaultdict(_blank_metrics)
latest_observations = []
for city, city_records in sorted(history.items()):
if not isinstance(city_records, dict):
continue
for date_str, record in sorted(city_records.items()):
if not isinstance(record, dict):
continue
actual_high = _sf(record.get("actual_high"))
shadow_snapshot = record.get("shadow_prob_snapshot")
calibration = record.get("probability_calibration") or {}
if actual_high is None or not shadow_snapshot:
continue
legacy_mu = _sf(calibration.get("raw_mu"))
if legacy_mu is None:
legacy_mu = _sf(record.get("mu"))
shadow_mu = _sf(calibration.get("calibrated_mu"))
if shadow_mu is None:
continue
actual_bucket = apply_city_settlement(city, actual_high)
legacy_snapshot = record.get("prob_snapshot") or []
legacy_bucket = _top_bucket(legacy_snapshot)
shadow_bucket = _top_bucket(shadow_snapshot)
for metrics in (overall, by_city[city], by_date[date_str]):
metrics["samples"] += 1
metrics["legacy_mae"].append(abs(legacy_mu - actual_high))
metrics["shadow_mae"].append(abs(shadow_mu - actual_high))
metrics["legacy_bucket_hit"].append(1.0 if legacy_bucket == actual_bucket else 0.0)
metrics["shadow_bucket_hit"].append(1.0 if shadow_bucket == actual_bucket else 0.0)
metrics["legacy_bucket_brier"].append(_brier_from_snapshot(legacy_snapshot, actual_bucket))
metrics["shadow_bucket_brier"].append(_brier_from_snapshot(shadow_snapshot, actual_bucket))
latest_observations.append(
{
"city": city,
"date": date_str,
"actual_high": actual_high,
"actual_bucket": actual_bucket,
"legacy_mu": round(legacy_mu, 3),
"shadow_mu": round(shadow_mu, 3),
"legacy_top_bucket": legacy_bucket,
"shadow_top_bucket": shadow_bucket,
"calibration_version": calibration.get("version"),
"calibration_mode": calibration.get("mode"),
}
)
by_city_report = {
city: _rollup(metrics)
for city, metrics in sorted(by_city.items())
}
by_date_report = {
date_str: _rollup(metrics)
for date_str, metrics in sorted(
by_date.items(),
key=lambda item: datetime.strptime(item[0], "%Y-%m-%d"),
)
}
latest_observations = sorted(
latest_observations,
key=lambda row: (row["date"], row["city"]),
reverse=True,
)[:100]
payload = {
"generated_at": datetime.utcnow().isoformat() + "Z",
"summary": _rollup(overall),
"by_city": by_city_report,
"by_date": by_date_report,
"recent_observations": latest_observations,
}
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(payload, fh, ensure_ascii=False, indent=2)
print(json.dumps(payload["summary"], ensure_ascii=False, indent=2))
print(f"saved shadow report to {args.output}")
if __name__ == "__main__":
main()
scripts/evaluate_probability_calibration.py
-270
View File
@@ -1,270 +0,0 @@
import argparse
import json
import os
import sys
from collections import defaultdict
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.probability_calibration import ( # noqa: E402
ENGINE_MODE_EMOS_PRIMARY,
_gaussian_crps,
apply_probability_calibration,
build_probability_features,
)
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
from scripts.fit_probability_calibration import ( # noqa: E402
_default_history_arg,
_extract_samples,
_load_history_with_fallback,
_load_json_if_exists,
_load_legacy_training_samples,
_load_snapshot_rows,
_load_training_feature_history,
_load_truth_history,
merge_samples_with_legacy_archive,
)
def _env_int(name, default=None):
try:
value = os.getenv(name)
if value is None or str(value).strip() == "":
return default
return int(value)
except Exception:
return default
def _log(enabled, message):
if enabled:
print(f"[evaluate_probability_calibration] {message}", flush=True)
def _mean(values):
return (sum(values) / len(values)) if values else None
def _sample_to_features(sample):
return build_probability_features(
city_name=sample.get("city") or "",
raw_mu=sample.get("raw_mu"),
raw_sigma=sample.get("raw_sigma"),
deb_prediction=sample.get("deb_prediction"),
ens_data={
"median": sample.get("ens_median"),
"p10": None,
"p90": None,
},
current_forecasts={},
max_so_far=None,
peak_status="in_window" if sample.get("peak_flag") == 0.5 else "past" if sample.get("peak_flag") == 1.0 else "before",
local_hour_frac=None,
)
def _top_bucket_value(distribution):
if not distribution:
return None
top = max(
(row for row in distribution if isinstance(row, dict)),
key=lambda row: float(row.get("probability") or 0.0),
default=None,
)
if not top:
return None
return top.get("value")
def main():
parser = argparse.ArgumentParser(description="Evaluate legacy vs EMOS probability calibration.")
parser.add_argument(
"--history-file",
default=_default_history_arg(),
)
parser.add_argument(
"--settlement-history",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"settlement_history.json",
),
)
parser.add_argument(
"--calibration-file",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"default.json",
),
)
parser.add_argument(
"--output",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"evaluation_report.json",
),
)
parser.add_argument(
"--snapshot-file",
default=None,
help="Optional legacy JSONL snapshot archive path. In sqlite mode this defaults to the runtime database.",
)
parser.add_argument(
"--snapshot-limit",
type=int,
default=_env_int("POLYWEATHER_EMOS_TRAINING_SNAPSHOT_LIMIT"),
help="Optional max number of recent probability snapshots to load from SQLite.",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Print data loading and evaluation progress.",
)
args = parser.parse_args()
_log(args.verbose, "loading daily records")
history = _load_history_with_fallback(args.history_file)
_log(args.verbose, f"loaded daily record cities={len(history or {})}")
_log(args.verbose, "loading training feature history")
training_feature_history = _load_training_feature_history()
_log(args.verbose, f"loaded training feature cities={len(training_feature_history or {})}")
_log(args.verbose, "loading truth history")
truth_history = _load_truth_history()
_log(args.verbose, f"loaded truth cities={len(truth_history or {})}")
_log(args.verbose, "loading settlement history")
settlement_history = _load_json_if_exists(args.settlement_history)
_log(args.verbose, f"loaded settlement history cities={len(settlement_history or {})}")
_log(
args.verbose,
"loading probability snapshots"
+ (f" limit={args.snapshot_limit}" if args.snapshot_limit else ""),
)
snapshot_rows = _load_snapshot_rows(args.snapshot_file, limit=args.snapshot_limit)
_log(args.verbose, f"loaded probability snapshots={len(snapshot_rows or [])}")
_log(args.verbose, "loading legacy training archive")
legacy_training_samples = _load_legacy_training_samples()
_log(args.verbose, f"loaded legacy training samples={len(legacy_training_samples or [])}")
_log(args.verbose, "extracting evaluation samples")
samples, filled_actual_from_history = _extract_samples(
history,
training_feature_history=training_feature_history,
truth_history=truth_history,
settlement_history=settlement_history,
snapshot_rows=snapshot_rows,
)
samples = merge_samples_with_legacy_archive(samples, legacy_training_samples)
_log(args.verbose, f"evaluating samples={len(samples or [])}")
legacy_crps = []
emos_crps = []
legacy_mae = []
emos_mae = []
legacy_bucket_hits = []
emos_bucket_hits = []
by_city = defaultdict(lambda: {
"samples": 0,
"legacy_crps": [],
"emos_crps": [],
"legacy_mae": [],
"emos_mae": [],
"legacy_bucket_hits": [],
"emos_bucket_hits": [],
})
for sample in samples:
city = str(sample.get("city") or "").strip().lower()
actual_high = float(sample["actual_high"])
raw_mu = float(sample["raw_mu"])
raw_sigma = max(0.1, float(sample["raw_sigma"]))
legacy_crps.append(_gaussian_crps(actual_high, raw_mu, raw_sigma))
legacy_mae.append(abs(raw_mu - actual_high))
legacy_bucket = apply_city_settlement(city, raw_mu)
actual_bucket = apply_city_settlement(city, actual_high)
legacy_bucket_hits.append(1.0 if legacy_bucket == actual_bucket else 0.0)
calibration = apply_probability_calibration(
city_name=city,
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
raw_mu=raw_mu,
raw_sigma=raw_sigma,
max_so_far=None,
legacy_distribution=[],
features=_sample_to_features(sample),
calibration_path=args.calibration_file,
mode=ENGINE_MODE_EMOS_PRIMARY,
)
emos_mu = float(calibration.get("calibrated_mu") or raw_mu)
emos_sigma = max(0.1, float(calibration.get("calibrated_sigma") or raw_sigma))
emos_distribution = calibration.get("distribution") or []
emos_crps.append(_gaussian_crps(actual_high, emos_mu, emos_sigma))
emos_mae.append(abs(emos_mu - actual_high))
emos_bucket = _top_bucket_value(emos_distribution)
emos_bucket_hits.append(1.0 if emos_bucket == actual_bucket else 0.0)
row = by_city[city]
row["samples"] += 1
row["legacy_crps"].append(legacy_crps[-1])
row["emos_crps"].append(emos_crps[-1])
row["legacy_mae"].append(legacy_mae[-1])
row["emos_mae"].append(emos_mae[-1])
row["legacy_bucket_hits"].append(legacy_bucket_hits[-1])
row["emos_bucket_hits"].append(emos_bucket_hits[-1])
summary = {
"sample_count": len(samples),
"filled_actual_from_history": filled_actual_from_history,
"legacy": {
"mean_crps": round(_mean(legacy_crps), 6) if legacy_crps else None,
"mean_mae": round(_mean(legacy_mae), 6) if legacy_mae else None,
"bucket_hit_rate": round(_mean(legacy_bucket_hits), 6) if legacy_bucket_hits else None,
},
"emos": {
"mean_crps": round(_mean(emos_crps), 6) if emos_crps else None,
"mean_mae": round(_mean(emos_mae), 6) if emos_mae else None,
"bucket_hit_rate": round(_mean(emos_bucket_hits), 6) if emos_bucket_hits else None,
},
"delta": {
"crps": round((_mean(emos_crps) or 0.0) - (_mean(legacy_crps) or 0.0), 6),
"mae": round((_mean(emos_mae) or 0.0) - (_mean(legacy_mae) or 0.0), 6),
"bucket_hit_rate": round((_mean(emos_bucket_hits) or 0.0) - (_mean(legacy_bucket_hits) or 0.0), 6),
},
}
city_report = {}
for city, metrics in sorted(by_city.items()):
city_report[city] = {
"samples": metrics["samples"],
"legacy_mean_crps": round(_mean(metrics["legacy_crps"]), 6),
"emos_mean_crps": round(_mean(metrics["emos_crps"]), 6),
"legacy_mean_mae": round(_mean(metrics["legacy_mae"]), 6),
"emos_mean_mae": round(_mean(metrics["emos_mae"]), 6),
"legacy_bucket_hit_rate": round(_mean(metrics["legacy_bucket_hits"]), 6),
"emos_bucket_hit_rate": round(_mean(metrics["emos_bucket_hits"]), 6),
}
payload = {
"summary": summary,
"by_city": city_report,
}
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(payload, fh, ensure_ascii=False, indent=2)
print(json.dumps(summary, ensure_ascii=False, indent=2))
print(f"saved evaluation report to {args.output}")
_log(args.verbose, "done")
if __name__ == "__main__":
main()
scripts/export_probability_training_dataset.py
-78
View File
@@ -1,78 +0,0 @@
import argparse
import json
import os
import sys
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from scripts.fit_probability_calibration import ( # noqa: E402
_default_history_arg,
_default_snapshot_arg,
_extract_samples,
_load_history_with_fallback,
_load_json_if_exists,
_load_snapshot_rows,
)
def main():
parser = argparse.ArgumentParser(description="Export normalized probability calibration training samples.")
parser.add_argument(
"--history-file",
default=_default_history_arg(),
)
parser.add_argument(
"--settlement-history",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"settlement_history.json",
),
)
parser.add_argument(
"--output",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"training_samples.json",
),
)
parser.add_argument(
"--snapshot-file",
default=_default_snapshot_arg(),
)
args = parser.parse_args()
history = _load_history_with_fallback(args.history_file)
settlement_history = _load_json_if_exists(args.settlement_history)
snapshot_rows = _load_snapshot_rows(args.snapshot_file)
samples, filled_actual_from_history = _extract_samples(
history,
settlement_history=settlement_history,
snapshot_rows=snapshot_rows,
)
snapshot_count = sum(1 for sample in samples if sample.get("sample_source") == "snapshot")
daily_record_count = sum(1 for sample in samples if sample.get("sample_source") == "daily_record")
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
payload = {
"sample_count": len(samples),
"snapshot_sample_count": snapshot_count,
"daily_record_sample_count": daily_record_count,
"filled_actual_from_history": filled_actual_from_history,
"samples": samples,
}
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(payload, fh, ensure_ascii=False, indent=2)
print(f"exported {len(samples)} samples to {args.output}")
if __name__ == "__main__":
main()
scripts/fit_probability_calibration.py
-523
View File
@@ -1,523 +0,0 @@
import argparse
import json
import os
import sys
from datetime import datetime
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.probability_calibration import ( # noqa: E402
DEFAULT_CALIBRATION_FILE,
default_calibration_payload,
fit_calibration,
)
from src.analysis.deb_algorithm import load_history # noqa: E402
from src.database.runtime_state import ( # noqa: E402
DailyRecordRepository,
ProbabilitySnapshotRepository,
STATE_STORAGE_FILE,
STATE_STORAGE_SQLITE,
TrainingFeatureRecordRepository,
TruthRecordRepository,
get_state_storage_mode,
)
def _sf(value):
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _env_int(name, default=None):
try:
value = os.getenv(name)
if value is None or str(value).strip() == "":
return default
return int(value)
except Exception:
return default
def _log(enabled, message):
if enabled:
print(f"[fit_probability_calibration] {message}", flush=True)
def _load_json_if_exists(path):
if not path or not os.path.exists(path):
return {}
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
return data if isinstance(data, dict) else {}
def _legacy_training_samples_path():
return os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"training_samples.json",
)
def _legacy_history_path():
return os.path.join(PROJECT_ROOT, "data", "daily_records.json")
def _legacy_snapshot_path():
return os.path.join(PROJECT_ROOT, "data", "probability_training_snapshots.jsonl")
def _default_history_arg():
return _legacy_history_path() if get_state_storage_mode() == STATE_STORAGE_FILE else None
def _default_snapshot_arg():
return _legacy_snapshot_path() if get_state_storage_mode() == STATE_STORAGE_FILE else None
def _load_history_with_fallback(path):
if not path:
if get_state_storage_mode() == STATE_STORAGE_SQLITE:
return DailyRecordRepository().load_all()
return {}
data = _load_json_if_exists(path)
if data:
return data
return load_history(path)
def _load_truth_history():
if get_state_storage_mode() != STATE_STORAGE_SQLITE:
return {}
try:
return TruthRecordRepository().load_all()
except Exception:
return {}
def _load_training_feature_history():
if get_state_storage_mode() != STATE_STORAGE_SQLITE:
return {}
try:
return TrainingFeatureRecordRepository().load_all()
except Exception:
return {}
def _load_snapshot_rows(path, limit=None):
if get_state_storage_mode() == STATE_STORAGE_SQLITE:
repo = ProbabilitySnapshotRepository()
if limit is not None and int(limit) > 0:
with repo.db.connect() as conn:
rows = conn.execute(
"""
SELECT payload_json
FROM probability_training_snapshots_store
ORDER BY id DESC
LIMIT ?
""",
(int(limit),),
).fetchall()
out = []
for row in reversed(rows):
try:
out.append(json.loads(row["payload_json"]))
except Exception:
continue
return out
return repo.load_all_rows()
rows = []
if not path or not os.path.exists(path):
return rows
with open(path, "r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except Exception:
continue
if isinstance(row, dict):
rows.append(row)
return rows
def _load_legacy_training_samples(path=None):
payload = _load_json_if_exists(path or _legacy_training_samples_path())
rows = payload.get("samples") if isinstance(payload, dict) else None
if not isinstance(rows, list):
return []
return [row for row in rows if isinstance(row, dict)]
def _actual_high_for(history, truth_history, settlement_history, city, date_str):
city_rows = (history or {}).get(city) or {}
record = city_rows.get(date_str) or {}
actual_high = _sf(record.get("actual_high")) if isinstance(record, dict) else None
truth_record = ((truth_history.get(city) or {}).get(date_str) or {})
if actual_high is None and isinstance(truth_record, dict):
actual_high = _sf(truth_record.get("actual_high"))
filled = False
if actual_high is None:
actual_high = _sf(((settlement_history.get(city) or {}).get(date_str) or {}).get("max_temp"))
filled = actual_high is not None
metadata = {
"settlement_source": truth_record.get("settlement_source"),
"settlement_station_code": truth_record.get("settlement_station_code"),
"truth_version": truth_record.get("truth_version"),
"truth_updated_by": truth_record.get("updated_by"),
"truth_updated_at": truth_record.get("truth_updated_at"),
}
return actual_high, filled, metadata
def _extract_snapshot_samples(history, truth_history=None, snapshot_rows=None, settlement_history=None):
samples = []
filled_actual_from_history = 0
today = datetime.utcnow().strftime("%Y-%m-%d")
settlement_history = settlement_history or {}
for row in snapshot_rows or []:
city = str(row.get("city") or "").strip().lower()
date_str = str(row.get("date") or "").strip()
if not city or not date_str or date_str == today:
continue
actual_high, filled, truth_meta = _actual_high_for(
history,
truth_history or {},
settlement_history,
city,
date_str,
)
if actual_high is None:
continue
if filled:
filled_actual_from_history += 1
raw_mu = _sf(row.get("raw_mu"))
raw_sigma = _sf(row.get("raw_sigma"))
deb_prediction = _sf(row.get("deb_prediction"))
ensemble = row.get("ensemble") or {}
if not isinstance(ensemble, dict):
ensemble = {}
ens_median = _sf(ensemble.get("median"))
ensemble_spread = None
ens_p10 = _sf(ensemble.get("p10"))
ens_p90 = _sf(ensemble.get("p90"))
if ens_p10 is not None and ens_p90 is not None and ens_p90 >= ens_p10:
ensemble_spread = max(0.1, (ens_p90 - ens_p10) / 2.56)
multi_model = row.get("multi_model") or {}
if not isinstance(multi_model, dict):
multi_model = {}
forecast_values = [val for val in (_sf(v) for v in multi_model.values()) if val is not None]
forecast_values.sort()
if ensemble_spread is None:
if len(forecast_values) >= 2:
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
elif raw_sigma is not None:
ensemble_spread = raw_sigma
else:
ensemble_spread = 1.0
if raw_sigma is None:
raw_sigma = ensemble_spread
peak_status = str(row.get("peak_status") or "before").strip().lower()
peak_flag = 0.0
if peak_status == "in_window":
peak_flag = 0.5
elif peak_status == "past":
peak_flag = 1.0
max_so_far = _sf(row.get("max_so_far"))
max_so_far_gap = None
if deb_prediction is not None and max_so_far is not None:
max_so_far_gap = deb_prediction - max_so_far
if raw_mu is None:
continue
samples.append(
{
"city": city,
"date": date_str,
"timestamp": row.get("timestamp"),
"actual_high": actual_high,
"raw_mu": raw_mu,
"raw_sigma": raw_sigma or 1.0,
"deb_prediction": deb_prediction,
"ens_median": ens_median if ens_median is not None else raw_mu,
"ensemble_spread": ensemble_spread,
"max_so_far_gap": max_so_far_gap,
"peak_flag": peak_flag,
"sample_source": "snapshot",
**truth_meta,
}
)
return samples, filled_actual_from_history
def _extract_daily_record_samples(
history,
training_feature_history=None,
truth_history=None,
settlement_history=None,
excluded_keys=None,
):
samples = []
filled_actual_from_history = 0
today = datetime.utcnow().strftime("%Y-%m-%d")
settlement_history = settlement_history or {}
excluded_keys = excluded_keys or set()
for city, city_rows in (history or {}).items():
if not isinstance(city_rows, dict):
continue
city_settlement = settlement_history.get(city) or {}
for date_str, record in city_rows.items():
if date_str == today or not isinstance(record, dict):
continue
if (city, date_str) in excluded_keys:
continue
actual_high = _sf(record.get("actual_high"))
truth_meta = ((truth_history or {}).get(city) or {}).get(date_str) or {}
if actual_high is None:
actual_high = _sf(truth_meta.get("actual_high"))
if actual_high is None:
actual_high = _sf((city_settlement.get(date_str) or {}).get("max_temp"))
if actual_high is not None:
filled_actual_from_history += 1
feature_record = ((training_feature_history or {}).get(city) or {}).get(date_str) or {}
source_record = feature_record if isinstance(feature_record, dict) and feature_record else record
deb_prediction = _sf(source_record.get("deb_prediction"))
raw_mu = _sf(source_record.get("mu")) or deb_prediction
forecasts = source_record.get("forecasts") or {}
if not isinstance(forecasts, dict):
forecasts = {}
forecast_values = [val for val in (_sf(v) for v in forecasts.values()) if val is not None]
forecast_values.sort()
forecast_median = (
forecast_values[len(forecast_values) // 2] if forecast_values else None
)
feature_snapshot = source_record.get("probability_features") or {}
if not isinstance(feature_snapshot, dict):
feature_snapshot = {}
ens_median = _sf(feature_snapshot.get("ens_median")) or forecast_median or raw_mu
ensemble_spread = _sf(feature_snapshot.get("ensemble_spread"))
if ensemble_spread is None:
if len(forecast_values) >= 2:
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
else:
ensemble_spread = 1.0
raw_sigma = _sf(feature_snapshot.get("raw_sigma")) or ensemble_spread or 1.0
peak_status = str(feature_snapshot.get("peak_status") or "before").strip().lower()
peak_flag = 0.0
if peak_status == "in_window":
peak_flag = 0.5
elif peak_status == "past":
peak_flag = 1.0
if actual_high is None or raw_mu is None:
continue
max_so_far = _sf(feature_snapshot.get("max_so_far"))
max_so_far_gap = _sf(feature_snapshot.get("max_so_far_gap"))
if max_so_far_gap is None and max_so_far is not None and deb_prediction is not None:
max_so_far_gap = deb_prediction - max_so_far
samples.append(
{
"city": city,
"date": date_str,
"actual_high": actual_high,
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"deb_prediction": deb_prediction,
"ens_median": ens_median,
"ensemble_spread": ensemble_spread,
"max_so_far_gap": max_so_far_gap,
"peak_flag": peak_flag,
"sample_source": "daily_record",
"settlement_source": truth_meta.get("settlement_source"),
"settlement_station_code": truth_meta.get("settlement_station_code"),
"truth_version": truth_meta.get("truth_version"),
"truth_updated_by": truth_meta.get("updated_by"),
"truth_updated_at": truth_meta.get("truth_updated_at"),
}
)
return samples, filled_actual_from_history
def _extract_samples(history, training_feature_history=None, truth_history=None, settlement_history=None, snapshot_rows=None):
snapshot_samples, snapshot_filled = _extract_snapshot_samples(
history,
truth_history=truth_history,
snapshot_rows=snapshot_rows or [],
settlement_history=settlement_history,
)
excluded_keys = {
(sample["city"], sample["date"])
for sample in snapshot_samples
}
daily_samples, daily_filled = _extract_daily_record_samples(
history,
training_feature_history=training_feature_history,
truth_history=truth_history,
settlement_history=settlement_history,
excluded_keys=excluded_keys,
)
return snapshot_samples + daily_samples, snapshot_filled + daily_filled
def merge_samples_with_legacy_archive(samples, legacy_samples=None):
merged = []
seen = set()
for sample in samples or []:
if not isinstance(sample, dict):
continue
key = (
str(sample.get("city") or "").strip().lower(),
str(sample.get("date") or "").strip(),
str(sample.get("sample_source") or "").strip().lower(),
)
if not key[0] or not key[1]:
continue
if key in seen:
continue
merged.append(sample)
seen.add(key)
for sample in legacy_samples or []:
if not isinstance(sample, dict):
continue
key = (
str(sample.get("city") or "").strip().lower(),
str(sample.get("date") or "").strip(),
str(sample.get("sample_source") or "").strip().lower(),
)
if not key[0] or not key[1]:
continue
if key in seen:
continue
merged.append(sample)
seen.add(key)
return merged
def main():
parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.")
parser.add_argument(
"--history-file",
default=_default_history_arg(),
help="Optional legacy daily_records.json path. In sqlite mode this defaults to the runtime database.",
)
parser.add_argument(
"--output",
default=DEFAULT_CALIBRATION_FILE,
help="Output JSON file for fitted calibration parameters.",
)
parser.add_argument(
"--settlement-history",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"settlement_history.json",
),
help="Optional daily settlement history JSON built from historical CSV files.",
)
parser.add_argument(
"--snapshot-file",
default=_default_snapshot_arg(),
help="Optional legacy JSONL snapshot archive path. In sqlite mode this defaults to the runtime database.",
)
parser.add_argument(
"--version",
default=None,
help="Optional explicit calibration version.",
)
parser.add_argument(
"--snapshot-limit",
type=int,
default=_env_int("POLYWEATHER_EMOS_TRAINING_SNAPSHOT_LIMIT"),
help="Optional max number of recent probability snapshots to load from SQLite.",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Print data loading and fitting progress.",
)
args = parser.parse_args()
_log(args.verbose, "loading daily records")
history = _load_history_with_fallback(args.history_file)
_log(args.verbose, f"loaded daily record cities={len(history or {})}")
_log(args.verbose, "loading training feature history")
training_feature_history = _load_training_feature_history()
_log(args.verbose, f"loaded training feature cities={len(training_feature_history or {})}")
_log(args.verbose, "loading truth history")
truth_history = _load_truth_history()
_log(args.verbose, f"loaded truth cities={len(truth_history or {})}")
_log(args.verbose, "loading settlement history")
settlement_history = _load_json_if_exists(args.settlement_history)
_log(args.verbose, f"loaded settlement history cities={len(settlement_history or {})}")
_log(
args.verbose,
"loading probability snapshots"
+ (f" limit={args.snapshot_limit}" if args.snapshot_limit else ""),
)
snapshot_rows = _load_snapshot_rows(args.snapshot_file, limit=args.snapshot_limit)
_log(args.verbose, f"loaded probability snapshots={len(snapshot_rows or [])}")
_log(args.verbose, "loading legacy training archive")
legacy_training_samples = _load_legacy_training_samples()
_log(args.verbose, f"loaded legacy training samples={len(legacy_training_samples or [])}")
_log(args.verbose, "extracting EMOS samples")
samples, filled_actual_from_history = _extract_samples(
history,
training_feature_history=training_feature_history,
truth_history=truth_history,
settlement_history=settlement_history,
snapshot_rows=snapshot_rows,
)
samples = merge_samples_with_legacy_archive(samples, legacy_training_samples)
_log(args.verbose, f"fitting calibration samples={len(samples or [])}")
calibration = fit_calibration(samples, version=args.version)
if not samples:
calibration = default_calibration_payload(
version=args.version,
reason="no_samples",
)
calibration.setdefault("metrics", {})
calibration["metrics"]["filled_actual_from_history"] = filled_actual_from_history
calibration["metrics"]["settlement_history_city_count"] = len(settlement_history)
calibration["metrics"]["legacy_archive_samples"] = len(legacy_training_samples)
try:
calibration["source"] = os.path.relpath(args.output, PROJECT_ROOT)
except ValueError:
calibration["source"] = os.path.abspath(args.output)
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(calibration, fh, ensure_ascii=False, indent=2)
_log(args.verbose, "done")
print(
"saved calibration to {path} with {count} samples".format(
path=args.output,
count=calibration.get("metrics", {}).get("sample_count", 0),
)
)
if __name__ == "__main__":
main()
scripts/judge_probability_rollout.py
-56
View File
@@ -1,56 +0,0 @@
import argparse
import json
import os
import sys
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.probability_rollout import build_rollout_report # noqa: E402
def main():
parser = argparse.ArgumentParser(description="Judge whether EMOS is ready for primary rollout.")
parser.add_argument(
"--evaluation-report",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"evaluation_report.json",
),
)
parser.add_argument(
"--shadow-report",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"shadow_report.json",
),
)
parser.add_argument(
"--output",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"rollout_report.json",
),
)
args = parser.parse_args()
payload = build_rollout_report(args.evaluation_report, args.shadow_report)
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.output, "w", encoding="utf-8") as fh:
json.dump(payload, fh, ensure_ascii=False, indent=2)
print(json.dumps(payload["decision"], ensure_ascii=False, indent=2))
print(f"saved rollout report to {args.output}")
if __name__ == "__main__":
main()
scripts/report_lgbm_daily_high.py
-78
View File
@@ -1,78 +0,0 @@
from __future__ import annotations
import json
import os
import sys
from typing import Any, Dict
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SCHEMA_PATH = os.path.join(ROOT_DIR, "artifacts", "models", "lgbm_daily_high_schema.json")
def _load_schema(path: str) -> Dict[str, Any]:
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
if not isinstance(data, dict):
raise SystemExit(f"Invalid schema payload in {path}")
return data
def _fmt_metric(value: Any) -> str:
if value is None:
return "--"
try:
return f"{float(value):.3f}"
except Exception:
return str(value)
def _winner(metrics: Dict[str, Any]) -> str:
candidates = {
"LGBM": metrics.get("lgbm_mae"),
"DEB": metrics.get("deb_mae"),
"Best Single": metrics.get("best_single_mae"),
"Median": metrics.get("median_mae"),
}
filtered = {k: float(v) for k, v in candidates.items() if v is not None}
if not filtered:
return "--"
return min(filtered.items(), key=lambda item: item[1])[0]
def _print_block(label: str, metrics: Dict[str, Any]) -> None:
print(label)
print(f" Samples : {metrics.get('sample_count', 0)}")
print(f" LGBM MAE : {_fmt_metric(metrics.get('lgbm_mae'))}")
print(f" DEB MAE : {_fmt_metric(metrics.get('deb_mae'))}")
print(f" Best Single : {_fmt_metric(metrics.get('best_single_mae'))}")
print(f" Model Median : {_fmt_metric(metrics.get('median_mae'))}")
print(f" Winner : {_winner(metrics)}")
def main() -> int:
path = sys.argv[1] if len(sys.argv) > 1 else SCHEMA_PATH
if not os.path.exists(path):
raise SystemExit(f"Schema file not found: {path}")
schema = _load_schema(path)
metrics = schema.get("metrics") or {}
validation = metrics.get("validation") or {}
full_sample = metrics.get("full_sample") or {}
print("LightGBM Daily High Report")
print(f" Target : {schema.get('target', '--')}")
print(f" Horizon : {schema.get('horizon', '--')}")
print(f" Sample Count : {schema.get('sample_count', 0)}")
print(f" Train Count : {schema.get('train_count', 0)}")
print(f" Valid Count : {schema.get('validation_count', 0)}")
print(f" Trained At : {schema.get('trained_at', '--')}")
print("")
_print_block("Validation", validation)
print("")
_print_block("Full Sample", full_sample)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/train_lgbm_daily_high.py
-202
View File
@@ -1,202 +0,0 @@
from __future__ import annotations
import json
import os
import sys
from datetime import datetime
from typing import Any, Dict, List
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MODEL_PATH = os.path.join(ROOT_DIR, "artifacts", "models", "lgbm_daily_high.txt")
SCHEMA_PATH = os.path.join(ROOT_DIR, "artifacts", "models", "lgbm_daily_high_schema.json")
def _mae(pairs: List[tuple[float, float]]) -> float | None:
if not pairs:
return None
return round(sum(abs(pred - actual) for pred, actual in pairs) / len(pairs), 3)
def _best_single_forecast(sample: Dict[str, Any]) -> float | None:
target = float(sample["target"])
forecasts = sample.get("forecasts") or {}
best_value = None
best_error = None
for value in forecasts.values():
try:
numeric = float(value)
except Exception:
continue
error = abs(numeric - target)
if best_error is None or error < best_error:
best_error = error
best_value = numeric
return best_value
def _chronological_split(samples: List[Dict[str, Any]]) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
if len(samples) < 12:
return samples, []
ordered = sorted(samples, key=lambda row: (row["date"], row["city"]))
validation_count = max(12, int(round(len(ordered) * 0.2)))
validation_count = min(validation_count, len(ordered) - 1)
if validation_count <= 0:
return ordered, []
return ordered[:-validation_count], ordered[-validation_count:]
def _dataset_from_samples(samples: List[Dict[str, Any]], np: Any) -> tuple[Any, Any]:
features = np.asarray([row["vector"] for row in samples], dtype=np.float32)
targets = np.asarray([row["target"] for row in samples], dtype=np.float32)
return features, targets
def _train_model(
train_samples: List[Dict[str, Any]],
valid_samples: List[Dict[str, Any]],
lgb: Any,
np: Any,
feature_names: List[str],
) -> Any:
train_x, train_y = _dataset_from_samples(train_samples, np)
train_data = lgb.Dataset(train_x, label=train_y, feature_name=feature_names, free_raw_data=True)
valid_sets = [train_data]
valid_names = ["train"]
params = {
"objective": "regression",
"metric": "l1",
"learning_rate": 0.05,
"num_leaves": 15,
"feature_fraction": 0.9,
"bagging_fraction": 0.9,
"bagging_freq": 1,
"min_data_in_leaf": 4,
"verbosity": -1,
"seed": 42,
}
callbacks = []
if valid_samples:
valid_x, valid_y = _dataset_from_samples(valid_samples, np)
valid_data = lgb.Dataset(valid_x, label=valid_y, feature_name=feature_names, reference=train_data)
valid_sets.append(valid_data)
valid_names.append("valid")
callbacks.append(lgb.early_stopping(stopping_rounds=15, verbose=False))
return lgb.train(
params=params,
train_set=train_data,
num_boost_round=120,
valid_sets=valid_sets,
valid_names=valid_names,
callbacks=callbacks,
)
def _evaluate(booster: Any, samples: List[Dict[str, Any]], np: Any) -> Dict[str, Any]:
if not samples:
return {
"sample_count": 0,
"lgbm_mae": None,
"deb_mae": None,
"best_single_mae": None,
"median_mae": None,
}
features, _ = _dataset_from_samples(samples, np)
preds = booster.predict(features, num_iteration=booster.best_iteration)
lgbm_pairs: List[tuple[float, float]] = []
deb_pairs: List[tuple[float, float]] = []
best_single_pairs: List[tuple[float, float]] = []
median_pairs: List[tuple[float, float]] = []
for sample, pred in zip(samples, preds):
actual = float(sample["target"])
lgbm_pairs.append((float(pred), actual))
deb_prediction = sample.get("deb_prediction")
if deb_prediction is not None:
deb_pairs.append((float(deb_prediction), actual))
best_single = _best_single_forecast(sample)
if best_single is not None:
best_single_pairs.append((best_single, actual))
median_prediction = (sample.get("features") or {}).get("model_median")
if median_prediction is not None:
median_pairs.append((float(median_prediction), actual))
return {
"sample_count": len(samples),
"lgbm_mae": _mae(lgbm_pairs),
"deb_mae": _mae(deb_pairs),
"best_single_mae": _mae(best_single_pairs),
"median_mae": _mae(median_pairs),
}
def main() -> int:
if ROOT_DIR not in sys.path:
sys.path.insert(0, ROOT_DIR)
import lightgbm as lgb
import numpy as np
from src.models.lgbm_features import (
FEATURE_NAMES,
build_training_samples,
schema_payload,
)
samples = build_training_samples()
if len(samples) < 16:
raise SystemExit(f"Not enough supervised samples for LightGBM training: {len(samples)}")
train_samples, valid_samples = _chronological_split(samples)
booster = _train_model(train_samples, valid_samples, lgb, np, FEATURE_NAMES)
all_features, all_targets = _dataset_from_samples(samples, np)
final_train = lgb.Dataset(all_features, label=all_targets, feature_name=FEATURE_NAMES, free_raw_data=True)
final_booster = lgb.train(
params={
"objective": "regression",
"metric": "l1",
"learning_rate": 0.05,
"num_leaves": 15,
"feature_fraction": 0.9,
"bagging_fraction": 0.9,
"bagging_freq": 1,
"min_data_in_leaf": 4,
"verbosity": -1,
"seed": 42,
},
train_set=final_train,
num_boost_round=max(int(booster.best_iteration or 60), 20),
)
os.makedirs(os.path.dirname(MODEL_PATH), exist_ok=True)
final_booster.save_model(MODEL_PATH)
metrics = {
"validation": _evaluate(booster, valid_samples, np),
"full_sample": _evaluate(final_booster, samples, np),
}
schema = schema_payload(
model_path=os.path.relpath(MODEL_PATH, ROOT_DIR),
sample_count=len(samples),
train_count=len(train_samples),
validation_count=len(valid_samples),
metrics=metrics,
)
schema["trained_at"] = datetime.utcnow().isoformat() + "Z"
with open(SCHEMA_PATH, "w", encoding="utf-8") as fh:
json.dump(schema, fh, ensure_ascii=False, indent=2)
print(json.dumps({"model_path": MODEL_PATH, "schema_path": SCHEMA_PATH, "metrics": metrics}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
src/analysis/probability_calibration.py
-834
View File
@@ -1,834 +0,0 @@
from __future__ import annotations
import json
import math
import os
from datetime import datetime, timezone
from typing import Any, Dict, Iterable, List, Optional, Tuple
import numpy as np
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
ENGINE_MODE_LEGACY = "legacy"
ENGINE_MODE_EMOS_SHADOW = "emos_shadow"
ENGINE_MODE_EMOS_PRIMARY = "emos_primary"
DEFAULT_ENGINE_MODE = ENGINE_MODE_EMOS_PRIMARY
VALID_ENGINE_MODES = {
ENGINE_MODE_LEGACY,
ENGINE_MODE_EMOS_SHADOW,
ENGINE_MODE_EMOS_PRIMARY,
}
DEFAULT_CALIBRATION_FILE = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
"artifacts",
"probability_calibration",
"default.json",
)
_CALIBRATION_CACHE: Dict[str, Dict[str, Any]] = {}
_CALIBRATION_MTIME: Dict[str, float] = {}
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _coalesce_float(value: Any, default: float) -> float:
parsed = _sf(value)
return default if parsed is None else parsed
def _mean(values: Iterable[float]) -> Optional[float]:
values = list(values)
return (sum(values) / len(values)) if values else None
def resolve_probability_engine_mode(explicit_mode: Optional[str] = None) -> str:
mode = str(
explicit_mode
or os.getenv("POLYWEATHER_PROBABILITY_ENGINE")
or DEFAULT_ENGINE_MODE
).strip().lower()
if mode not in VALID_ENGINE_MODES:
return DEFAULT_ENGINE_MODE
return mode
def load_calibration(calibration_path: Optional[str] = None) -> Dict[str, Any]:
path = str(
calibration_path
or os.getenv("POLYWEATHER_PROBABILITY_CALIBRATION_FILE")
or DEFAULT_CALIBRATION_FILE
).strip()
if not path:
return {}
if not os.path.exists(path):
return {}
try:
mtime = os.path.getmtime(path)
cached = _CALIBRATION_CACHE.get(path)
if cached and _CALIBRATION_MTIME.get(path) == mtime:
return cached
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
if not isinstance(data, dict):
return {}
_CALIBRATION_CACHE[path] = data
_CALIBRATION_MTIME[path] = mtime
return data
except Exception:
return {}
def build_probability_features(
city_name: str,
raw_mu: Optional[float],
raw_sigma: Optional[float],
deb_prediction: Optional[float],
ens_data: Optional[Dict[str, Any]],
current_forecasts: Optional[Dict[str, Any]],
max_so_far: Optional[float],
peak_status: str,
local_hour_frac: Optional[float],
) -> Dict[str, Any]:
ens_data = ens_data or {}
current_forecasts = current_forecasts or {}
forecast_values = [
v for v in (_sf(val) for val in current_forecasts.values()) if v is not None
]
forecast_values.sort()
forecast_median = None
if forecast_values:
forecast_median = forecast_values[len(forecast_values) // 2]
ens_median = _sf(ens_data.get("median"))
ens_p10 = _sf(ens_data.get("p10"))
ens_p90 = _sf(ens_data.get("p90"))
ensemble_spread = None
if ens_p10 is not None and ens_p90 is not None and ens_p90 >= ens_p10:
ensemble_spread = max(0.1, (ens_p90 - ens_p10) / 2.56)
elif len(forecast_values) >= 2:
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
elif raw_sigma is not None:
ensemble_spread = max(0.1, raw_sigma)
baseline = deb_prediction if deb_prediction is not None else raw_mu
max_so_far_gap = None
if baseline is not None and max_so_far is not None:
max_so_far_gap = baseline - max_so_far
peak_flag = 0.0
if peak_status == "in_window":
peak_flag = 0.5
elif peak_status == "past":
peak_flag = 1.0
return {
"city": str(city_name or "").strip().lower(),
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"deb_prediction": deb_prediction,
"ens_median": ens_median,
"ens_p10": ens_p10,
"ens_p90": ens_p90,
"forecast_median": forecast_median,
"forecast_spread": forecast_values[-1] - forecast_values[0]
if len(forecast_values) >= 2
else None,
"ensemble_spread": ensemble_spread,
"max_so_far": max_so_far,
"max_so_far_gap": max_so_far_gap,
"peak_status": peak_status,
"peak_flag": peak_flag,
"local_hour_frac": local_hour_frac,
"model_count": len(forecast_values),
}
def _normal_cdf(x: float, mean: float, sigma: float) -> float:
return 0.5 * (1.0 + math.erf((x - mean) / (sigma * math.sqrt(2.0))))
def _normal_pdf(x: float) -> float:
return math.exp(-(x ** 2) / 2.0) / math.sqrt(2.0 * math.pi)
def _bucket_probabilities(
mu: float,
sigma: float,
max_so_far: Optional[float],
city_name: str,
) -> Tuple[List[Dict[str, Any]], List[Tuple[int, float]], List[Dict[str, Any]]]:
sigma = max(0.1, float(sigma))
min_possible = (
apply_city_settlement(city_name, max_so_far) if max_so_far is not None else -999
)
probs: Dict[int, float] = {}
search_range = max(2, int(sigma * 2.5))
is_exact = is_exact_settlement_city(city_name)
target_mu = apply_city_settlement(city_name, mu)
if is_exact:
target_mu = int(math.floor(mu))
for value in range(target_mu - search_range, target_mu + search_range + 1):
if value < min_possible:
continue
if is_exact:
prob = _normal_cdf(value + 1.0, mu, sigma) - _normal_cdf(value, mu, sigma)
else:
prob = _normal_cdf(value + 0.5, mu, sigma) - _normal_cdf(value - 0.5, mu, sigma)
if prob > 0.01:
probs[value] = prob
total = sum(probs.values())
if total <= 0:
return [], [], []
normalized = {key: val / total for key, val in probs.items()}
sorted_probs = sorted(normalized.items(), key=lambda item: item[1], reverse=True)
full_distribution = []
for value, prob in sorted_probs:
if is_exact:
bucket_range = "[{0}.0~{1}.0)".format(value, value + 1)
else:
bucket_range = "[{0}~{1})".format(value - 0.5, value + 0.5)
full_distribution.append(
{
"value": int(value),
"range": bucket_range,
"probability": round(prob, 3),
}
)
return full_distribution[:4], sorted_probs, full_distribution
def _top_bucket_value(distribution: Optional[List[Dict[str, Any]]]) -> Optional[int]:
if not distribution:
return None
top = max(
(row for row in distribution if isinstance(row, dict)),
key=lambda row: float(row.get("probability") or 0.0),
default=None,
)
if not top:
return None
value = top.get("value")
return int(value) if value is not None else None
def _bucket_brier_score(
distribution: Optional[List[Dict[str, Any]]],
city_name: str,
actual_high: float,
) -> float:
actual_bucket = apply_city_settlement(city_name, actual_high)
hit_prob = 0.0
total = 0.0
for row in distribution or []:
if not isinstance(row, dict):
continue
value = row.get("value")
try:
prob = float(row.get("probability") or 0.0)
except Exception:
prob = 0.0
if value == actual_bucket:
hit_prob = prob
else:
total += prob * prob
total += (1.0 - hit_prob) ** 2
return total
def _composite_score(
mean_crps: float,
mean_mae: float,
bucket_hit_rate: float,
bucket_brier: float,
) -> float:
return mean_crps + 0.1 * mean_mae + 1.5 * (1.0 - bucket_hit_rate) + 0.75 * bucket_brier
def _blend_value(raw_value: float, calibrated_value: float, alpha: float) -> float:
return (1.0 - alpha) * raw_value + alpha * calibrated_value
def _clamp_sigma(
raw_sigma: float,
calibrated_sigma: float,
constraints: Optional[Dict[str, Any]],
) -> float:
constraints = constraints or {}
min_ratio = max(0.25, _coalesce_float(constraints.get("min_ratio"), 0.85))
max_ratio = max(min_ratio, _coalesce_float(constraints.get("max_ratio"), 1.35))
absolute_min = max(0.1, _coalesce_float(constraints.get("absolute_min"), 0.25))
absolute_max = max(
absolute_min,
_coalesce_float(constraints.get("absolute_max"), raw_sigma * max_ratio),
)
sigma_floor = max(absolute_min, raw_sigma * min_ratio)
sigma_cap = min(absolute_max, raw_sigma * max_ratio)
return min(max(calibrated_sigma, sigma_floor), sigma_cap)
def apply_probability_calibration(
city_name: str,
temp_symbol: str,
raw_mu: Optional[float],
raw_sigma: Optional[float],
max_so_far: Optional[float],
legacy_distribution: Optional[List[Dict[str, Any]]],
features: Optional[Dict[str, Any]] = None,
calibration_path: Optional[str] = None,
mode: Optional[str] = None,
) -> Dict[str, Any]:
selected_mode = resolve_probability_engine_mode(mode)
if raw_mu is None or raw_sigma is None:
return {
"mode": selected_mode,
"engine": ENGINE_MODE_LEGACY,
"distribution": legacy_distribution or [],
"distribution_all": legacy_distribution or [],
"shadow_distribution": [],
"shadow_distribution_all": [],
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"calibrated_mu": None,
"calibrated_sigma": None,
"calibration_version": None,
"calibration_source": None,
}
calibration = load_calibration(calibration_path)
if not calibration:
return {
"mode": selected_mode,
"engine": ENGINE_MODE_LEGACY,
"distribution": legacy_distribution or [],
"distribution_all": legacy_distribution or [],
"shadow_distribution": [],
"shadow_distribution_all": [],
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"calibrated_mu": None,
"calibrated_sigma": None,
"calibration_version": None,
"calibration_source": None,
}
features = features or {}
city_key = str(city_name or "").strip().lower()
global_params = calibration.get("global", {}) or {}
city_params = (calibration.get("cities", {}) or {}).get(city_key, {}) or {}
blending_cfg = calibration.get("blending", {}) or {}
sigma_constraints = calibration.get("sigma_constraints", {}) or {}
mu_cfg = global_params.get("mu", {}) or {}
sigma_cfg = global_params.get("sigma", {}) or {}
city_confidence = max(0.0, min(1.0, _coalesce_float(city_params.get("confidence"), 1.0)))
city_mu_bias = _coalesce_float(city_params.get("mu_bias"), 0.0) * city_confidence
city_sigma_scale = 1.0 + (
(_coalesce_float(city_params.get("sigma_scale"), 1.0) - 1.0) * city_confidence
)
deb_prediction = _sf(features.get("deb_prediction"))
ens_median = _sf(features.get("ens_median"))
max_so_far_gap = _sf(features.get("max_so_far_gap"))
peak_flag = _sf(features.get("peak_flag")) or 0.0
ensemble_spread = _sf(features.get("ensemble_spread"))
mu_intercept = _coalesce_float(mu_cfg.get("intercept"), 0.0)
mu_raw_coef = _coalesce_float(mu_cfg.get("raw_mu_coef"), 1.0)
mu_deb_coef = _coalesce_float(mu_cfg.get("deb_coef"), 0.0)
mu_ens_coef = _coalesce_float(mu_cfg.get("ens_median_coef"), 0.0)
mu_gap_coef = _coalesce_float(mu_cfg.get("max_so_far_gap_coef"), 0.0)
sigma_intercept = _coalesce_float(
sigma_cfg.get("intercept"),
math.log(max(raw_sigma, 0.1)),
)
sigma_raw_coef = _coalesce_float(sigma_cfg.get("raw_sigma_coef"), 1.0)
sigma_spread_coef = _coalesce_float(sigma_cfg.get("spread_coef"), 0.0)
sigma_peak_coef = _coalesce_float(sigma_cfg.get("peak_flag_coef"), 0.0)
sigma_gap_coef = _coalesce_float(sigma_cfg.get("max_so_far_gap_coef"), 0.0)
calibrated_mu = (
mu_intercept
+ mu_raw_coef * raw_mu
+ mu_deb_coef * (deb_prediction if deb_prediction is not None else raw_mu)
+ mu_ens_coef * (ens_median if ens_median is not None else raw_mu)
+ mu_gap_coef * (max_so_far_gap if max_so_far_gap is not None else 0.0)
+ city_mu_bias
)
sigma_log = (
sigma_intercept
+ sigma_raw_coef * math.log(max(raw_sigma, 0.1))
+ sigma_spread_coef * math.log(max(ensemble_spread or raw_sigma, 0.1))
+ sigma_peak_coef * peak_flag
+ sigma_gap_coef * (max_so_far_gap if max_so_far_gap is not None else 0.0)
)
calibrated_sigma = max(0.1, math.exp(sigma_log) * city_sigma_scale)
blend_alpha_mu = max(0.0, min(1.0, _coalesce_float(blending_cfg.get("alpha_mu"), 1.0)))
blend_alpha_sigma = max(0.0, min(1.0, _coalesce_float(blending_cfg.get("alpha_sigma"), 1.0)))
calibrated_mu = _blend_value(raw_mu, calibrated_mu, blend_alpha_mu)
if max_so_far is not None:
observed_floor = _sf(max_so_far)
if observed_floor is not None and calibrated_mu < observed_floor:
calibrated_mu = observed_floor
calibrated_sigma = max(0.1, _blend_value(raw_sigma, calibrated_sigma, blend_alpha_sigma))
calibrated_sigma = _clamp_sigma(raw_sigma, calibrated_sigma, sigma_constraints)
calibrated_distribution, calibrated_sorted, calibrated_distribution_all = _bucket_probabilities(
calibrated_mu,
calibrated_sigma,
max_so_far=max_so_far,
city_name=city_key,
)
engine = ENGINE_MODE_LEGACY
selected_distribution = legacy_distribution or []
selected_distribution_all = legacy_distribution or []
selected_sorted: List[Tuple[int, float]] = []
shadow_distribution: List[Dict[str, Any]] = []
shadow_distribution_all: List[Dict[str, Any]] = []
shadow_sorted: List[Tuple[int, float]] = []
if selected_mode == ENGINE_MODE_EMOS_PRIMARY:
engine = "emos"
selected_distribution = calibrated_distribution
selected_distribution_all = calibrated_distribution_all
selected_sorted = calibrated_sorted
elif selected_mode == ENGINE_MODE_EMOS_SHADOW:
shadow_distribution = calibrated_distribution
shadow_distribution_all = calibrated_distribution_all
shadow_sorted = calibrated_sorted
return {
"mode": selected_mode,
"engine": engine,
"distribution": selected_distribution,
"distribution_all": selected_distribution_all,
"selected_sorted_probs": selected_sorted,
"shadow_distribution": shadow_distribution,
"shadow_distribution_all": shadow_distribution_all,
"shadow_sorted_probs": shadow_sorted,
"raw_mu": raw_mu,
"raw_sigma": raw_sigma,
"calibrated_mu": calibrated_mu,
"calibrated_sigma": calibrated_sigma,
"blend_alpha_mu": blend_alpha_mu,
"blend_alpha_sigma": blend_alpha_sigma,
"calibration_version": calibration.get("version"),
"calibration_source": calibration.get("source")
or os.path.relpath(
calibration_path or DEFAULT_CALIBRATION_FILE,
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
),
}
def _gaussian_crps(observation: float, mean: float, sigma: float) -> float:
sigma = max(0.1, float(sigma))
z = (observation - mean) / sigma
return sigma * (
z * (2.0 * _normal_cdf(z, 0.0, 1.0) - 1.0)
+ 2.0 * _normal_pdf(z)
- 1.0 / math.sqrt(math.pi)
)
def _fit_linear(xs: np.ndarray, ys: np.ndarray) -> np.ndarray:
if len(xs) == 0:
return np.zeros(xs.shape[1], dtype=float)
coeffs, _, _, _ = np.linalg.lstsq(xs, ys, rcond=None)
return coeffs
def fit_calibration(
samples: Iterable[Dict[str, Any]],
version: Optional[str] = None,
) -> Dict[str, Any]:
normalized_samples: List[Dict[str, Any]] = []
city_residuals: Dict[str, List[float]] = {}
city_sigma_ratios: Dict[str, List[float]] = {}
for raw_sample in samples:
actual = _sf(raw_sample.get("actual_high"))
raw_mu = _sf(raw_sample.get("raw_mu"))
raw_sigma = _sf(raw_sample.get("raw_sigma"))
if actual is None or raw_mu is None or raw_sigma is None:
continue
feature_row = {
"city": str(raw_sample.get("city") or "").strip().lower(),
"actual_high": actual,
"raw_mu": raw_mu,
"raw_sigma": max(0.1, raw_sigma),
"deb_prediction": _sf(raw_sample.get("deb_prediction")),
"ens_median": _sf(raw_sample.get("ens_median")),
"ensemble_spread": _sf(raw_sample.get("ensemble_spread")),
"max_so_far_gap": _sf(raw_sample.get("max_so_far_gap")),
"peak_flag": _sf(raw_sample.get("peak_flag")) or 0.0,
}
normalized_samples.append(feature_row)
if len(normalized_samples) < 3:
return default_calibration_payload(version=version, reason="insufficient_samples")
mu_rows = []
mu_targets = []
for sample in normalized_samples:
deb = sample["deb_prediction"] if sample["deb_prediction"] is not None else sample["raw_mu"]
ens_median = sample["ens_median"] if sample["ens_median"] is not None else sample["raw_mu"]
gap = sample["max_so_far_gap"] if sample["max_so_far_gap"] is not None else 0.0
mu_rows.append([1.0, sample["raw_mu"], deb, ens_median, gap])
mu_targets.append(sample["actual_high"])
mu_coeffs = _fit_linear(np.array(mu_rows, dtype=float), np.array(mu_targets, dtype=float))
sigma_rows = []
sigma_targets = []
mu_predictions = []
for idx, sample in enumerate(normalized_samples):
predicted_mu = float(np.dot(mu_coeffs, np.array(mu_rows[idx], dtype=float)))
mu_predictions.append(predicted_mu)
residual = max(abs(sample["actual_high"] - predicted_mu), 0.1)
spread = max(sample["ensemble_spread"] or sample["raw_sigma"], 0.1)
gap = sample["max_so_far_gap"] if sample["max_so_far_gap"] is not None else 0.0
sigma_rows.append([1.0, math.log(sample["raw_sigma"]), math.log(spread), sample["peak_flag"], gap])
sigma_targets.append(math.log(residual))
city_residuals.setdefault(sample["city"], []).append(sample["actual_high"] - predicted_mu)
city_sigma_ratios.setdefault(sample["city"], []).append(residual / max(sample["raw_sigma"], 0.1))
sigma_coeffs = _fit_linear(np.array(sigma_rows, dtype=float), np.array(sigma_targets, dtype=float))
crps_values = []
for idx, sample in enumerate(normalized_samples):
predicted_mu = mu_predictions[idx]
sigma_log = float(np.dot(sigma_coeffs, np.array(sigma_rows[idx], dtype=float)))
predicted_sigma = max(0.1, math.exp(sigma_log))
crps_values.append(_gaussian_crps(sample["actual_high"], predicted_mu, predicted_sigma))
city_params: Dict[str, Dict[str, Any]] = {}
for city, residuals in city_residuals.items():
if len(residuals) < 3:
continue
sigma_ratios = city_sigma_ratios.get(city) or [1.0]
confidence = max(0.25, min(1.0, len(residuals) / 8.0))
city_params[city] = {
"samples": len(residuals),
"mu_bias": round(sum(residuals) / len(residuals), 6),
"sigma_scale": round(
max(0.5, min(2.0, sum(sigma_ratios) / len(sigma_ratios))),
6,
),
"confidence": round(confidence, 6),
}
legacy_crps_values = []
legacy_mae_values = []
legacy_bucket_hits = []
legacy_bucket_briers = []
candidate_predictions = []
for idx, sample in enumerate(normalized_samples):
city = sample["city"]
city_meta = city_params.get(city, {})
city_confidence = max(
0.0,
min(1.0, _coalesce_float(city_meta.get("confidence"), 1.0)),
)
city_mu_bias = _coalesce_float(city_meta.get("mu_bias"), 0.0) * city_confidence
city_sigma_scale = 1.0 + (
(_coalesce_float(city_meta.get("sigma_scale"), 1.0) - 1.0) * city_confidence
)
legacy_mu = sample["raw_mu"]
legacy_sigma = sample["raw_sigma"]
actual_high = sample["actual_high"]
legacy_crps_values.append(_gaussian_crps(actual_high, legacy_mu, legacy_sigma))
legacy_mae_values.append(abs(legacy_mu - actual_high))
legacy_bucket_hits.append(
1.0
if apply_city_settlement(city, legacy_mu)
== apply_city_settlement(city, actual_high)
else 0.0
)
legacy_distribution, _, _ = _bucket_probabilities(
legacy_mu,
legacy_sigma,
max_so_far=None,
city_name=city,
)
legacy_bucket_briers.append(
_bucket_brier_score(legacy_distribution, city, actual_high)
)
calibrated_mu = mu_predictions[idx] + city_mu_bias
sigma_log = float(np.dot(sigma_coeffs, np.array(sigma_rows[idx], dtype=float)))
calibrated_sigma = max(0.1, math.exp(sigma_log) * city_sigma_scale)
candidate_predictions.append(
{
"city": city,
"actual_high": actual_high,
"raw_mu": legacy_mu,
"raw_sigma": legacy_sigma,
"calibrated_mu": calibrated_mu,
"calibrated_sigma": calibrated_sigma,
}
)
legacy_mean_crps = _mean(legacy_crps_values) or 0.0
legacy_mean_mae = _mean(legacy_mae_values) or 0.0
legacy_bucket_hit_rate = _mean(legacy_bucket_hits) or 0.0
legacy_bucket_brier = _mean(legacy_bucket_briers) or 0.0
sigma_constraints = {
"min_ratio": 0.85,
"max_ratio": 1.35,
"absolute_min": 0.25,
"absolute_max": 3.0,
}
legacy_score = _composite_score(
legacy_mean_crps,
legacy_mean_mae,
legacy_bucket_hit_rate,
legacy_bucket_brier,
)
guardrails = {
"max_mae_increase": 0.02,
"max_bucket_hit_drop": 0.01,
"max_bucket_brier_increase": 0.05,
}
best_alpha_mu = 0.0
best_alpha_sigma = 0.0
best_score = legacy_score
best_metrics = {
"mean_crps": legacy_mean_crps,
"mean_mae": legacy_mean_mae,
"bucket_hit_rate": legacy_bucket_hit_rate,
"bucket_brier": legacy_bucket_brier,
}
alpha_grid = [step / 20.0 for step in range(21)]
for alpha_mu in alpha_grid:
for alpha_sigma in alpha_grid:
crps_values = []
mae_values = []
bucket_hits = []
bucket_briers = []
for row in candidate_predictions:
mu_hat = _blend_value(row["raw_mu"], row["calibrated_mu"], alpha_mu)
sigma_hat = _clamp_sigma(
row["raw_sigma"],
max(
0.1,
_blend_value(row["raw_sigma"], row["calibrated_sigma"], alpha_sigma),
),
sigma_constraints,
)
actual_high = row["actual_high"]
city = row["city"]
crps_values.append(_gaussian_crps(actual_high, mu_hat, sigma_hat))
mae_values.append(abs(mu_hat - actual_high))
distribution, _, _ = _bucket_probabilities(
mu_hat,
sigma_hat,
max_so_far=None,
city_name=city,
)
predicted_bucket = _top_bucket_value(distribution)
actual_bucket = apply_city_settlement(city, actual_high)
bucket_hits.append(1.0 if predicted_bucket == actual_bucket else 0.0)
bucket_briers.append(
_bucket_brier_score(distribution, city, actual_high)
)
mean_crps = _mean(crps_values) or 0.0
mean_mae = _mean(mae_values) or 0.0
bucket_hit_rate = _mean(bucket_hits) or 0.0
bucket_brier = _mean(bucket_briers) or 0.0
if mean_mae > legacy_mean_mae + guardrails["max_mae_increase"]:
continue
if bucket_hit_rate + guardrails["max_bucket_hit_drop"] < legacy_bucket_hit_rate:
continue
if bucket_brier > legacy_bucket_brier + guardrails["max_bucket_brier_increase"]:
continue
score = _composite_score(mean_crps, mean_mae, bucket_hit_rate, bucket_brier)
if score + 1e-9 < best_score:
best_score = score
best_alpha_mu = alpha_mu
best_alpha_sigma = alpha_sigma
best_metrics = {
"mean_crps": mean_crps,
"mean_mae": mean_mae,
"bucket_hit_rate": bucket_hit_rate,
"bucket_brier": bucket_brier,
}
return {
"version": version or datetime.now(timezone.utc).strftime("emos-%Y%m%d%H%M%S"),
"trained_at": datetime.now(timezone.utc).isoformat(),
"global": {
"mu": {
"intercept": round(float(mu_coeffs[0]), 8),
"raw_mu_coef": round(float(mu_coeffs[1]), 8),
"deb_coef": round(float(mu_coeffs[2]), 8),
"ens_median_coef": round(float(mu_coeffs[3]), 8),
"max_so_far_gap_coef": round(float(mu_coeffs[4]), 8),
},
"sigma": {
"intercept": round(float(sigma_coeffs[0]), 8),
"raw_sigma_coef": round(float(sigma_coeffs[1]), 8),
"spread_coef": round(float(sigma_coeffs[2]), 8),
"peak_flag_coef": round(float(sigma_coeffs[3]), 8),
"max_so_far_gap_coef": round(float(sigma_coeffs[4]), 8),
},
},
"sigma_constraints": sigma_constraints,
"selection_guardrails": guardrails,
"blending": {
"alpha_mu": round(best_alpha_mu, 6),
"alpha_sigma": round(best_alpha_sigma, 6),
},
"cities": city_params,
"metrics": {
"sample_count": len(normalized_samples),
"mean_crps": round(sum(crps_values) / len(crps_values), 6),
"legacy_mean_crps": round(legacy_mean_crps, 6),
"legacy_mean_mae": round(legacy_mean_mae, 6),
"legacy_bucket_hit_rate": round(legacy_bucket_hit_rate, 6),
"legacy_bucket_brier": round(legacy_bucket_brier, 6),
"selected_mean_crps": round(best_metrics["mean_crps"], 6),
"selected_mean_mae": round(best_metrics["mean_mae"], 6),
"selected_bucket_hit_rate": round(best_metrics["bucket_hit_rate"], 6),
"selected_bucket_brier": round(best_metrics["bucket_brier"], 6),
"selected_score": round(best_score, 6),
"legacy_score": round(legacy_score, 6),
},
}
def default_calibration_payload(
version: Optional[str] = None,
reason: str = "bootstrap",
) -> Dict[str, Any]:
return {
"version": version or "emos-bootstrap-v1",
"trained_at": datetime.now(timezone.utc).isoformat(),
"global": {
"mu": {
"intercept": 0.0,
"raw_mu_coef": 1.0,
"deb_coef": 0.0,
"ens_median_coef": 0.0,
"max_so_far_gap_coef": 0.0,
},
"sigma": {
"intercept": 0.0,
"raw_sigma_coef": 1.0,
"spread_coef": 0.0,
"peak_flag_coef": 0.0,
"max_so_far_gap_coef": 0.0,
},
},
"sigma_constraints": {
"min_ratio": 0.85,
"max_ratio": 1.35,
"absolute_min": 0.25,
"absolute_max": 3.0,
},
"selection_guardrails": {
"max_mae_increase": 0.02,
"max_bucket_hit_drop": 0.01,
"max_bucket_brier_increase": 0.05,
},
"blending": {
"alpha_mu": 1.0,
"alpha_sigma": 1.0,
},
"cities": {},
"metrics": {
"sample_count": 0,
"mean_crps": None,
"reason": reason,
},
}
def check_calibration_drift(records: list[dict[str, Any]], calibration_path: str | None = None) -> dict[str, Any]:
"""Compare recent CRPS against calibration baseline to detect drift.
Returns a dict with keys: drifted (bool), current_crps (float),
baseline_crps (float), delta_pct (float), sample_count (int), warning (str|None).
A positive delta_pct means the model is performing worse than baseline.
"""
import json
import os
if len(records) < 5:
return {"drifted": False, "sample_count": len(records), "warning": "Insufficient samples"}
path = calibration_path or DEFAULT_CALIBRATION_FILE
baseline_crps = None
try:
with open(path, "r", encoding="utf-8") as fh:
json.load(fh) # calibration file
eval_path = os.path.join(
os.path.dirname(path) if os.path.dirname(path) else os.path.join(os.path.dirname(DEFAULT_CALIBRATION_FILE)),
"evaluation_report.json",
)
with open(eval_path, "r", encoding="utf-8") as fh:
report = json.load(fh)
baseline_crps = float(report.get("metrics", {}).get("selected_mean_crps") or 0) or None
except Exception:
pass
current_crps_values: list[float] = []
for r in records:
actual = float(r.get("actual_high") or r.get("observed") or 0)
mu = float(r.get("deb_prediction") or r.get("mu") or 0)
sigma = float(r.get("sigma") or r.get("ensemble_std") or 2.0)
if actual and mu and sigma > 0:
current_crps_values.append(_gaussian_crps(actual, mu, sigma))
if not current_crps_values:
return {"drifted": False, "sample_count": 0, "warning": "No valid records for CRPS"}
current_crps = round(sum(current_crps_values) / len(current_crps_values), 6)
if baseline_crps is None or baseline_crps <= 0:
return {
"drifted": False,
"current_crps": current_crps,
"baseline_crps": baseline_crps,
"sample_count": len(records),
"warning": "No baseline available",
}
delta_pct = round((current_crps - baseline_crps) / baseline_crps * 100, 1)
DRIFT_THRESHOLD_PCT = 15.0 # warn if CRPS degraded by >15%
if delta_pct > DRIFT_THRESHOLD_PCT:
return {
"drifted": True,
"current_crps": current_crps,
"baseline_crps": baseline_crps,
"delta_pct": delta_pct,
"sample_count": len(records),
"warning": f"CRPS degraded {delta_pct}% vs baseline; consider re-running fit_calibration()",
}
return {
"drifted": False,
"current_crps": current_crps,
"baseline_crps": baseline_crps,
"delta_pct": delta_pct,
"sample_count": len(records),
}
src/analysis/probability_rollout.py
-217
View File
@@ -1,217 +0,0 @@
from __future__ import annotations
import json
import os
from typing import Any, Dict, List, Optional
def _load_json_file(path: str) -> Dict[str, Any]:
try:
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
return data if isinstance(data, dict) else {}
except Exception:
return {}
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _append_reason(reasons: List[str], condition: bool, message: str) -> None:
if condition:
reasons.append(message)
def _top_shadow_regressions(by_city: Dict[str, Any], limit: int = 5) -> List[Dict[str, Any]]:
rows: List[Dict[str, Any]] = []
for city, metrics in (by_city or {}).items():
if not isinstance(metrics, dict):
continue
rows.append(
{
"city": city,
"samples": int(metrics.get("samples") or 0),
"delta_mae": _sf(metrics.get("delta_mae")),
"delta_bucket_hit_rate": _sf(metrics.get("delta_bucket_hit_rate")),
"delta_bucket_brier": _sf(metrics.get("delta_bucket_brier")),
}
)
rows.sort(
key=lambda row: (
-(row["delta_bucket_brier"] or 0.0),
row["delta_bucket_hit_rate"] or 0.0,
-(row["delta_mae"] or 0.0),
)
)
return rows[:limit]
def judge_probability_rollout(
evaluation_report: Dict[str, Any],
shadow_report: Dict[str, Any],
) -> Dict[str, Any]:
thresholds = {
"evaluation_min_samples": 80,
"shadow_min_samples": 50,
"max_delta_mae": 0.05,
"min_delta_crps": -0.02,
"min_delta_bucket_hit_rate": 0.0,
"max_delta_bucket_brier_promote": 0.02,
"max_delta_bucket_brier_observe": 0.15,
}
eval_summary = (evaluation_report or {}).get("summary") or {}
eval_delta = eval_summary.get("delta") or {}
shadow_summary = (shadow_report or {}).get("summary") or {}
eval_samples = int(eval_summary.get("sample_count") or 0)
shadow_samples = int(shadow_summary.get("samples") or 0)
delta_crps = _sf(eval_delta.get("crps"))
delta_mae = _sf(eval_delta.get("mae"))
delta_hit = _sf(eval_delta.get("bucket_hit_rate"))
shadow_delta_mae = _sf(shadow_summary.get("delta_mae"))
shadow_delta_hit = _sf(shadow_summary.get("delta_bucket_hit_rate"))
shadow_delta_brier = _sf(shadow_summary.get("delta_bucket_brier"))
promote_reasons: List[str] = []
_append_reason(
promote_reasons,
eval_samples < thresholds["evaluation_min_samples"],
f"离线评估样本不足：{eval_samples} < {thresholds['evaluation_min_samples']}",
)
_append_reason(
promote_reasons,
shadow_samples < thresholds["shadow_min_samples"],
f"shadow 样本不足：{shadow_samples} < {thresholds['shadow_min_samples']}",
)
_append_reason(
promote_reasons,
delta_crps is None or delta_crps > thresholds["min_delta_crps"],
f"离线 CRPS 改善不足：delta={delta_crps}",
)
_append_reason(
promote_reasons,
delta_mae is None or delta_mae > thresholds["max_delta_mae"],
f"离线 MAE 退化超限：delta={delta_mae}",
)
_append_reason(
promote_reasons,
delta_hit is None or delta_hit < thresholds["min_delta_bucket_hit_rate"],
f"离线 bucket 命中率下降：delta={delta_hit}",
)
_append_reason(
promote_reasons,
shadow_delta_mae is None or shadow_delta_mae > thresholds["max_delta_mae"],
f"shadow MAE 退化超限：delta={shadow_delta_mae}",
)
_append_reason(
promote_reasons,
shadow_delta_hit is None or shadow_delta_hit < thresholds["min_delta_bucket_hit_rate"],
f"shadow bucket 命中率下降：delta={shadow_delta_hit}",
)
_append_reason(
promote_reasons,
shadow_delta_brier is None
or shadow_delta_brier > thresholds["max_delta_bucket_brier_promote"],
f"shadow bucket brier 退化超限：delta={shadow_delta_brier}",
)
if not promote_reasons:
decision = "promote"
summary = "离线与 shadow 指标均达标，可以考虑切换 emos_primary。"
else:
observe_reasons: List[str] = []
_append_reason(
observe_reasons,
eval_samples < thresholds["evaluation_min_samples"],
f"离线评估样本不足：{eval_samples}",
)
_append_reason(
observe_reasons,
delta_crps is None or delta_crps > thresholds["min_delta_crps"],
f"离线 CRPS 改善不足：delta={delta_crps}",
)
_append_reason(
observe_reasons,
delta_mae is None or delta_mae > thresholds["max_delta_mae"],
f"离线 MAE 退化超限：delta={delta_mae}",
)
_append_reason(
observe_reasons,
delta_hit is None or delta_hit < thresholds["min_delta_bucket_hit_rate"],
f"离线 bucket 命中率下降：delta={delta_hit}",
)
_append_reason(
observe_reasons,
shadow_samples < thresholds["shadow_min_samples"],
f"shadow 样本不足：{shadow_samples}",
)
_append_reason(
observe_reasons,
shadow_delta_mae is None or shadow_delta_mae > thresholds["max_delta_mae"],
f"shadow MAE 退化超限：delta={shadow_delta_mae}",
)
_append_reason(
observe_reasons,
shadow_delta_hit is None or shadow_delta_hit < thresholds["min_delta_bucket_hit_rate"],
f"shadow bucket 命中率下降：delta={shadow_delta_hit}",
)
_append_reason(
observe_reasons,
shadow_delta_brier is None
or shadow_delta_brier > thresholds["max_delta_bucket_brier_observe"],
f"shadow bucket brier 退化偏大：delta={shadow_delta_brier}",
)
if not observe_reasons:
decision = "observe"
summary = "离线评估达标，但 shadow 仍需继续观察，暂不切主路径。"
else:
decision = "hold"
summary = "当前指标不足以切换 emos_primary，应继续保持 shadow。"
return {
"decision": decision,
"ready_for_primary": decision == "promote",
"summary": summary,
"thresholds": thresholds,
"evaluation": {
"sample_count": eval_samples,
"delta_crps": delta_crps,
"delta_mae": delta_mae,
"delta_bucket_hit_rate": delta_hit,
},
"shadow": {
"sample_count": shadow_samples,
"delta_mae": shadow_delta_mae,
"delta_bucket_hit_rate": shadow_delta_hit,
"delta_bucket_brier": shadow_delta_brier,
},
"blocking_reasons": promote_reasons,
"worst_shadow_regressions": _top_shadow_regressions(
(shadow_report or {}).get("by_city") or {}
),
}
def build_rollout_report(
evaluation_report_path: str,
shadow_report_path: str,
) -> Dict[str, Any]:
evaluation_report = _load_json_file(evaluation_report_path)
shadow_report = _load_json_file(shadow_report_path)
decision = judge_probability_rollout(evaluation_report, shadow_report)
return {
"evaluation_report_path": evaluation_report_path,
"shadow_report_path": shadow_report_path,
"evaluation_report_exists": os.path.exists(evaluation_report_path),
"shadow_report_exists": os.path.exists(shadow_report_path),
"decision": decision,
}
src/analysis/probability_snapshot_archive.py
-297
View File
@@ -1,297 +0,0 @@
from __future__ import annotations
import json
import os
from datetime import datetime
from typing import Any, Dict, List, Optional
from src.database.runtime_state import (
ProbabilitySnapshotRepository,
STATE_STORAGE_SQLITE,
TrainingFeatureRecordRepository,
get_state_storage_mode,
)
DEDUP_SCAN_LINES = 200
MU_THRESHOLD = 0.2
SIGMA_THRESHOLD = 0.15
MAX_SO_FAR_THRESHOLD = 0.2
_snapshot_repo = ProbabilitySnapshotRepository()
_training_feature_repo = TrainingFeatureRecordRepository()
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _compact_snapshot(distribution: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
compact: List[Dict[str, Any]] = []
for row in distribution or []:
if not isinstance(row, dict):
continue
value = row.get("value")
probability = row.get("probability")
if value is None or probability is None:
continue
try:
compact.append(
{
"v": int(value),
"p": round(float(probability), 3),
}
)
except Exception:
continue
if len(compact) >= 4:
break
return compact
def _top_bucket(snapshot: Optional[List[Dict[str, Any]]]) -> Optional[int]:
best_value = None
best_prob = -1.0
for row in snapshot or []:
if not isinstance(row, dict):
continue
value = row.get("v")
prob = _sf(row.get("p"))
if value is None or prob is None:
continue
if prob > best_prob:
best_value = int(value)
best_prob = prob
return best_value
def _load_recent_rows(path: str, max_lines: int = DEDUP_SCAN_LINES) -> List[Dict[str, Any]]:
if not os.path.exists(path):
return []
with open(path, "r", encoding="utf-8") as fh:
lines = fh.readlines()[-max_lines:]
rows = []
for line in lines:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except Exception:
continue
if isinstance(row, dict):
rows.append(row)
return rows
def load_snapshot_rows_for_day(
city_name: str,
target_date: str,
archive_path: Optional[str] = None,
) -> List[Dict[str, Any]]:
city_key = str(city_name or "").strip().lower()
date_key = str(target_date or "").strip()
if not city_key or not date_key:
return []
mode = get_state_storage_mode()
if mode == STATE_STORAGE_SQLITE:
return _snapshot_repo.load_rows_by_city_date(city_key, date_key)
root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
path = archive_path or os.path.join(root_dir, "data", "probability_training_snapshots.jsonl")
if not os.path.exists(path):
if mode == STATE_STORAGE_SQLITE:
return _snapshot_repo.load_rows_by_city_date(city_key, date_key)
return []
rows: List[Dict[str, Any]] = []
try:
with open(path, "r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except Exception:
continue
if not isinstance(row, dict):
continue
if str(row.get("city") or "").strip().lower() != city_key:
continue
if str(row.get("date") or "").strip() != date_key:
continue
rows.append(row)
except Exception:
return []
rows.sort(key=lambda row: str(row.get("timestamp") or ""))
return rows
def _should_skip_append(path: str, payload: Dict[str, Any]) -> bool:
mode = get_state_storage_mode()
if mode == STATE_STORAGE_SQLITE:
recent_rows = _snapshot_repo.load_recent_rows(
str(payload.get("city") or ""),
str(payload.get("date") or ""),
DEDUP_SCAN_LINES,
)
else:
recent_rows = _load_recent_rows(path)
city = payload.get("city")
date_str = payload.get("date")
if not city or not date_str:
return False
for row in reversed(recent_rows):
if row.get("city") != city or row.get("date") != date_str:
continue
if row.get("peak_status") != payload.get("peak_status"):
return False
if row.get("probability_mode") != payload.get("probability_mode"):
return False
current_top = _top_bucket(payload.get("prob_snapshot"))
previous_top = _top_bucket(row.get("prob_snapshot"))
current_shadow_top = _top_bucket(payload.get("shadow_prob_snapshot"))
previous_shadow_top = _top_bucket(row.get("shadow_prob_snapshot"))
if current_top != previous_top or current_shadow_top != previous_shadow_top:
return False
if abs((_sf(payload.get("raw_mu")) or 0.0) - (_sf(row.get("raw_mu")) or 0.0)) > MU_THRESHOLD:
return False
if abs((_sf(payload.get("raw_sigma")) or 0.0) - (_sf(row.get("raw_sigma")) or 0.0)) > SIGMA_THRESHOLD:
return False
if abs((_sf(payload.get("max_so_far")) or 0.0) - (_sf(row.get("max_so_far")) or 0.0)) > MAX_SO_FAR_THRESHOLD:
return False
return True
return False
def append_probability_snapshot(
city_name: str,
*,
local_date: str,
observation_time: Optional[str],
temp_symbol: str,
raw_mu: Optional[float],
raw_sigma: Optional[float],
deb_prediction: Optional[float],
ens_data: Optional[Dict[str, Any]],
current_forecasts: Optional[Dict[str, Any]],
max_so_far: Optional[float],
current_temp: Optional[float] = None,
humidity: Optional[float] = None,
wind_speed_kt: Optional[float] = None,
visibility_mi: Optional[float] = None,
local_hour: Optional[float] = None,
peak_status: Optional[str],
probabilities: Optional[List[Dict[str, Any]]],
shadow_probabilities: Optional[List[Dict[str, Any]]],
calibration_summary: Optional[Dict[str, Any]],
archive_path: Optional[str] = None,
) -> None:
city_key = str(city_name or "").strip().lower()
if not city_key:
return
root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
path = archive_path or os.path.join(
root_dir,
"data",
"probability_training_snapshots.jsonl",
)
calibration_summary = calibration_summary or {}
ens_data = ens_data or {}
current_forecasts = current_forecasts or {}
timestamp = str(observation_time or datetime.utcnow().isoformat() + "Z").strip()
payload = {
"city": city_key,
"timestamp": timestamp,
"date": local_date,
"temp_symbol": temp_symbol,
"raw_mu": _sf(raw_mu),
"raw_sigma": _sf(raw_sigma),
"deb_prediction": _sf(deb_prediction),
"ensemble": {
"p10": _sf(ens_data.get("p10")),
"median": _sf(ens_data.get("median")),
"p90": _sf(ens_data.get("p90")),
},
"multi_model": {
key: _sf(value)
for key, value in current_forecasts.items()
if _sf(value) is not None
},
"max_so_far": _sf(max_so_far),
"observation": {
"current_temp": _sf(current_temp),
"humidity": _sf(humidity),
"wind_speed_kt": _sf(wind_speed_kt),
"visibility_mi": _sf(visibility_mi),
"local_hour": _sf(local_hour),
},
"peak_status": peak_status,
"prob_snapshot": _compact_snapshot(probabilities),
"shadow_prob_snapshot": _compact_snapshot(shadow_probabilities),
"probability_engine": calibration_summary.get("engine"),
"probability_mode": calibration_summary.get("mode"),
"calibration_version": calibration_summary.get("calibration_version"),
"calibration_source": calibration_summary.get("calibration_source"),
"calibrated_mu": _sf(calibration_summary.get("calibrated_mu")),
"calibrated_sigma": _sf(calibration_summary.get("calibrated_sigma")),
}
parent = os.path.dirname(os.path.abspath(path))
if parent:
os.makedirs(parent, exist_ok=True)
if _should_skip_append(path, payload):
return
mode = get_state_storage_mode()
if mode == STATE_STORAGE_SQLITE:
_snapshot_repo.append_snapshot(payload)
_training_feature_repo.upsert_record(
city_key,
local_date,
{
"forecasts": payload.get("multi_model") or {},
"deb_prediction": payload.get("deb_prediction"),
"mu": payload.get("raw_mu"),
"probability_features": {
"raw_mu": payload.get("raw_mu"),
"raw_sigma": payload.get("raw_sigma"),
"deb_prediction": payload.get("deb_prediction"),
"ens_median": (payload.get("ensemble") or {}).get("median"),
"ensemble_spread": None,
"max_so_far": payload.get("max_so_far"),
"peak_status": payload.get("peak_status"),
},
"prob_snapshot": payload.get("prob_snapshot") or [],
"shadow_prob_snapshot": payload.get("shadow_prob_snapshot") or [],
"probability_calibration": {
"engine": payload.get("probability_engine"),
"mode": payload.get("probability_mode"),
"calibration_version": payload.get("calibration_version"),
"calibration_source": payload.get("calibration_source"),
"calibrated_mu": payload.get("calibrated_mu"),
"calibrated_sigma": payload.get("calibrated_sigma"),
},
"observation": payload.get("observation") or {},
"snapshot_timestamp": payload.get("timestamp"),
},
)
if mode != STATE_STORAGE_SQLITE:
with open(path, "a", encoding="utf-8") as fh:
fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
src/analysis/trend_engine.py
+2 -103
View File
@@ -15,11 +15,6 @@ from src.analysis.deb_algorithm import (
update_daily_record,
_is_excluded_model_name,
)
from src.analysis.probability_calibration import (
apply_probability_calibration,
build_probability_features,
)
from src.analysis.probability_snapshot_archive import append_probability_snapshot
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
from src.data_collection.city_registry import CITY_REGISTRY
from src.data_collection.city_risk_profiles import get_city_risk_profile
@@ -595,20 +590,7 @@ def analyze_weather_trend(
# === Probability Engine ===
probabilities: List[Dict[str, Any]] = []
probabilities_all: List[Dict[str, Any]] = []
shadow_probabilities: List[Dict[str, Any]] = []
shadow_probabilities_all: List[Dict[str, Any]] = []
forecast_miss_deg = 0.0
probability_features = None
calibration_summary = {
"mode": "legacy",
"engine": "legacy",
"raw_mu": None,
"raw_sigma": sigma,
"calibrated_mu": None,
"calibrated_sigma": None,
"calibration_version": None,
"calibration_source": None,
}
if is_dead_market:
settled_wu = apply_city_settlement(city_name, max_so_far) if max_so_far is not None else 0
@@ -662,7 +644,7 @@ def analyze_weather_trend(
f"实测最高 {max_so_far}{temp_symbol}，偏差 {forecast_miss_deg}°。当前趋势: {_trend_dir}。"
)
# Probability Engine
# Probability (legacy Gaussian buckets)
probs_result = calculate_prob_distribution(
mu, sigma, max_so_far, temp_symbol, city_name
)
@@ -671,45 +653,6 @@ def analyze_weather_trend(
probabilities_all = probs_result.get("probabilities_all", probabilities)
sorted_probs = probs_result.get("sorted_probs", [])
probability_features = build_probability_features(
city_name=city_name or "",
raw_mu=mu,
raw_sigma=sigma,
deb_prediction=deb_prediction,
ens_data=ens_data,
current_forecasts=current_forecasts,
max_so_far=max_so_far,
peak_status=peak_status,
local_hour_frac=local_hour_frac,
)
calibration_result = apply_probability_calibration(
city_name=city_name or "",
temp_symbol=temp_symbol,
raw_mu=mu,
raw_sigma=sigma,
max_so_far=max_so_far,
legacy_distribution=probabilities,
features=probability_features,
)
calibration_summary = {
"mode": calibration_result.get("mode", "legacy"),
"engine": calibration_result.get("engine", "legacy"),
"raw_mu": calibration_result.get("raw_mu"),
"raw_sigma": calibration_result.get("raw_sigma"),
"calibrated_mu": calibration_result.get("calibrated_mu"),
"calibrated_sigma": calibration_result.get("calibrated_sigma"),
"calibration_version": calibration_result.get("calibration_version"),
"calibration_source": calibration_result.get("calibration_source"),
}
shadow_probabilities = calibration_result.get("shadow_distribution") or []
shadow_probabilities_all = calibration_result.get("shadow_distribution_all") or shadow_probabilities
if calibration_result.get("engine") == "emos":
mu = calibration_result.get("calibrated_mu", mu)
sigma = calibration_result.get("calibrated_sigma", sigma)
probabilities = calibration_result.get("distribution") or probabilities
probabilities_all = calibration_result.get("distribution_all") or probabilities_all or probabilities
sorted_probs = calibration_result.get("selected_sorted_probs") or sorted_probs
if sorted_probs:
prob_parts = [
f"{int(t)}{temp_symbol} [{t - 0.5}~{t + 0.5}) {p * 100:.0f}%"
@@ -868,7 +811,6 @@ def analyze_weather_trend(
# === Save daily record (with μ + prob snapshot) ===
try:
_prob_list = None
_shadow_prob_list = None
if sorted_probs:
_prob_list = [
{"value": int(t), "probability": round(p, 3)}
@@ -876,12 +818,6 @@ def analyze_weather_trend(
]
elif is_dead_market and max_so_far is not None:
_prob_list = [{"value": apply_city_settlement(city_name, max_so_far), "probability": 1.0}]
if shadow_probabilities:
_shadow_prob_list = [
{"value": int(row.get("value")), "probability": round(float(row.get("probability") or 0.0), 3)}
for row in shadow_probabilities[:4]
if row.get("value") is not None
]
update_daily_record(
city_name,
@@ -891,34 +827,6 @@ def analyze_weather_trend(
deb_prediction=_deb_to_save,
mu=mu,
probabilities=_prob_list,
probability_features=probability_features,
shadow_probabilities=_shadow_prob_list,
calibration_summary=calibration_summary,
)
except Exception:
pass
try:
append_probability_snapshot(
city_name=city_name or "",
local_date=local_date_str,
observation_time=obs_time_raw or local_time_full or None,
temp_symbol=temp_symbol,
raw_mu=calibration_summary.get("raw_mu"),
raw_sigma=calibration_summary.get("raw_sigma"),
deb_prediction=_deb_to_save,
ens_data=ens_data,
current_forecasts=current_forecasts,
max_so_far=max_so_far,
current_temp=cur_temp,
humidity=_sf(primary_current.get("humidity")),
wind_speed_kt=_sf(primary_current.get("wind_speed_kt")),
visibility_mi=_sf(primary_current.get("visibility_mi")),
local_hour=local_hour_frac,
peak_status=peak_status,
probabilities=_prob_list,
shadow_probabilities=_shadow_prob_list,
calibration_summary=calibration_summary,
)
except Exception:
pass
@@ -933,16 +841,7 @@ def analyze_weather_trend(
"mu": mu,
"probabilities": probabilities,
"probabilities_all": probabilities_all or probabilities,
"shadow_probabilities": shadow_probabilities,
"shadow_probabilities_all": shadow_probabilities_all or shadow_probabilities,
"probability_engine": calibration_summary["engine"],
"probability_calibration_mode": calibration_summary["mode"],
"probability_calibration_version": calibration_summary["calibration_version"],
"probability_calibration_source": calibration_summary["calibration_source"],
"probability_raw_mu": calibration_summary["raw_mu"],
"probability_raw_sigma": calibration_summary["raw_sigma"],
"probability_calibrated_mu": calibration_summary["calibrated_mu"],
"probability_calibrated_sigma": calibration_summary["calibrated_sigma"],
"probability_engine": "legacy",
"trend_info": {
"direction": trend_direction if 'trend_direction' in dir() else "unknown",
"recent": recent_list,
src/models/lgbm_daily_high.py
-171
View File
@@ -1,171 +0,0 @@
from __future__ import annotations
import json
import os
from typing import Any, Dict, List, Optional, Tuple
from loguru import logger
from src.models.lgbm_features import (
FEATURE_NAMES,
build_runtime_feature_map,
)
_MODEL_CACHE: Dict[str, Any] = {"path": None, "mtime": None, "booster": None}
_SCHEMA_CACHE: Dict[str, Any] = {"path": None, "mtime": None, "schema": None}
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _truthy_env(name: str, default: str = "false") -> bool:
return str(os.getenv(name, default)).strip().lower() in {"1", "true", "yes", "on"}
def lgbm_model_path() -> str:
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
return str(
os.getenv(
"POLYWEATHER_LGBM_MODEL_PATH",
os.path.join(root, "artifacts", "models", "lgbm_daily_high.txt"),
)
).strip()
def lgbm_schema_path() -> str:
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
return str(
os.getenv(
"POLYWEATHER_LGBM_SCHEMA_PATH",
os.path.join(root, "artifacts", "models", "lgbm_daily_high_schema.json"),
)
).strip()
def lgbm_min_history_points() -> int:
try:
return max(1, int(os.getenv("POLYWEATHER_LGBM_MIN_HISTORY_POINTS", "3")))
except Exception:
return 3
def is_lgbm_enabled() -> bool:
return _truthy_env("POLYWEATHER_LGBM_ENABLED", "false")
def _load_schema(schema_path: str) -> Optional[Dict[str, Any]]:
if not schema_path or not os.path.exists(schema_path):
return None
mtime = os.path.getmtime(schema_path)
if (
_SCHEMA_CACHE["schema"] is not None
and _SCHEMA_CACHE["path"] == schema_path
and _SCHEMA_CACHE["mtime"] == mtime
):
return _SCHEMA_CACHE["schema"]
with open(schema_path, "r", encoding="utf-8") as fh:
data = json.load(fh)
if not isinstance(data, dict):
return None
_SCHEMA_CACHE.update({"path": schema_path, "mtime": mtime, "schema": data})
return data
def _load_booster(model_path: str):
if not model_path or not os.path.exists(model_path):
return None
mtime = os.path.getmtime(model_path)
if (
_MODEL_CACHE["booster"] is not None
and _MODEL_CACHE["path"] == model_path
and _MODEL_CACHE["mtime"] == mtime
):
return _MODEL_CACHE["booster"]
try:
import lightgbm as lgb
except Exception as exc:
logger.warning(f"LGBM runtime dependency missing: {exc}")
return None
booster = lgb.Booster(model_file=model_path)
_MODEL_CACHE.update({"path": model_path, "mtime": mtime, "booster": booster})
return booster
def _vector_from_features(
feature_map: Dict[str, Optional[float]],
schema: Optional[Dict[str, Any]],
) -> List[float]:
feature_names = schema.get("feature_names") if isinstance(schema, dict) else None
ordered_names = feature_names if isinstance(feature_names, list) and feature_names else FEATURE_NAMES
vector: List[float] = []
for name in ordered_names:
value = feature_map.get(str(name))
vector.append(float("nan") if value is None else float(value))
return vector
def predict_lgbm_daily_high(
*,
city_name: str,
current_forecasts: Dict[str, Any],
deb_prediction: Optional[float],
current_temp: Optional[float],
max_so_far: Optional[float],
humidity: Optional[float],
wind_speed_kt: Optional[float],
visibility_mi: Optional[float],
local_hour: int,
local_date: str,
peak_status: str,
history_data: Optional[Dict[str, Any]] = None,
) -> Tuple[Optional[float], Dict[str, Any]]:
if not is_lgbm_enabled():
return None, {"reason": "disabled"}
schema = _load_schema(lgbm_schema_path())
booster = _load_booster(lgbm_model_path())
if schema is None or booster is None:
return None, {"reason": "artifact_missing"}
feature_map, meta = build_runtime_feature_map(
city_name=city_name,
current_forecasts=current_forecasts,
deb_prediction=deb_prediction,
current_temp=current_temp,
max_so_far=max_so_far,
humidity=humidity,
wind_speed_kt=wind_speed_kt,
visibility_mi=visibility_mi,
local_hour=local_hour,
local_date=local_date,
peak_status=peak_status,
history_data=history_data,
)
if not feature_map:
return None, meta
if int(meta.get("history_count") or 0) < lgbm_min_history_points():
return None, {
"reason": "insufficient_history",
"history_count": int(meta.get("history_count") or 0),
}
try:
vector = _vector_from_features(feature_map, schema)
prediction = booster.predict([vector], num_iteration=booster.best_iteration)
value = _sf(prediction[0] if prediction is not None else None)
if value is None:
return None, {"reason": "empty_prediction"}
return round(float(value), 1), {
"reason": "ok",
"history_count": int(meta.get("history_count") or 0),
}
except Exception as exc:
logger.warning(f"LGBM prediction failed for {city_name}: {exc}")
return None, {"reason": "predict_failed", "error": str(exc)}
src/models/lgbm_features.py
-472
View File
@@ -1,472 +0,0 @@
from __future__ import annotations
import json
import os
from datetime import datetime
from statistics import mean
from typing import Any, Dict, List, Optional, Tuple
from src.analysis.deb_algorithm import load_history
from src.data_collection.city_registry import ALIASES
from src.database.runtime_state import (
DailyRecordRepository,
ProbabilitySnapshotRepository,
STATE_STORAGE_FILE,
STATE_STORAGE_SQLITE,
TrainingFeatureRecordRepository,
TruthRecordRepository,
get_state_storage_mode,
)
BASE_MODEL_COLUMNS: List[Tuple[str, str]] = [
("Open-Meteo", "open_meteo"),
("ECMWF", "ecmwf"),
("GFS", "gfs"),
("GEM", "gem"),
("JMA", "jma"),
("ICON", "icon"),
("MGM", "mgm"),
("NWS", "nws"),
]
FEATURE_NAMES: List[str] = [
"actual_high_lag_1",
"actual_high_lag_2",
"actual_high_lag_3",
"actual_high_lag_7",
"actual_high_mean_7",
"actual_high_mean_14",
"actual_high_trend_3",
*[column for _, column in BASE_MODEL_COLUMNS],
"deb_prediction",
"model_median",
"model_spread",
"current_temp",
"max_so_far",
"humidity",
"wind_speed_kt",
"visibility_mi",
"local_hour",
"month",
"weekday",
"peak_status_code",
]
PEAK_STATUS_CODES = {
"before": 0.0,
"in_window": 1.0,
"past": 2.0,
}
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _parse_date(value: Any) -> Optional[datetime]:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.strptime(text, "%Y-%m-%d")
except Exception:
return None
def _parse_timestamp(value: Any) -> Optional[datetime]:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.fromisoformat(text.replace("Z", "+00:00"))
except Exception:
return None
def _history_file_path() -> str:
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if get_state_storage_mode() == STATE_STORAGE_FILE:
return os.path.join(root, "data", "daily_records.json")
return ""
def _snapshot_archive_path() -> str:
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if get_state_storage_mode() == STATE_STORAGE_FILE:
return os.path.join(root, "data", "probability_training_snapshots.jsonl")
return ""
def _normalized_city_key(city_name: str) -> str:
city_key = str(city_name or "").strip().lower()
return ALIASES.get(city_key, city_key)
def _safe_mean(values: List[Optional[float]]) -> Optional[float]:
valid = [float(v) for v in values if v is not None]
if not valid:
return None
return float(mean(valid))
def _peak_status_code(value: Any) -> Optional[float]:
status = str(value or "").strip().lower()
if not status:
return None
return PEAK_STATUS_CODES.get(status, -1.0)
def _compute_model_summary(features: Dict[str, Optional[float]]) -> Tuple[Optional[float], Optional[float]]:
values = [
features.get(column)
for _, column in BASE_MODEL_COLUMNS
if features.get(column) is not None
]
values = [float(v) for v in values if v is not None]
if not values:
return None, None
ordered = sorted(values)
median_value = ordered[len(ordered) // 2]
spread_value = ordered[-1] - ordered[0] if len(ordered) >= 2 else 0.0
return float(median_value), float(spread_value)
def load_snapshot_index(archive_path: Optional[str] = None) -> Dict[Tuple[str, str], Dict[str, Any]]:
mode = get_state_storage_mode()
if mode == STATE_STORAGE_SQLITE:
latest_rows: Dict[Tuple[str, str], Dict[str, Any]] = {}
for row in ProbabilitySnapshotRepository().load_all_rows():
if not isinstance(row, dict):
continue
city = _normalized_city_key(str(row.get("city") or ""))
date_str = str(row.get("date") or "").strip()
if not city or not date_str:
continue
key = (city, date_str)
current_best = latest_rows.get(key)
if current_best is None or str(row.get("timestamp") or "") >= str(
current_best.get("timestamp") or ""
):
latest_rows[key] = row
return latest_rows
path = archive_path or _snapshot_archive_path()
if not path or not os.path.exists(path):
return {}
latest_rows: Dict[Tuple[str, str], Dict[str, Any]] = {}
with open(path, "r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except Exception:
continue
if not isinstance(row, dict):
continue
city = _normalized_city_key(str(row.get("city") or ""))
date_str = str(row.get("date") or "").strip()
if not city or not date_str:
continue
key = (city, date_str)
current_best = latest_rows.get(key)
if current_best is None or str(row.get("timestamp") or "") >= str(
current_best.get("timestamp") or ""
):
latest_rows[key] = row
return latest_rows
def _extract_history_rows(
history_data: Dict[str, Any],
city_name: str,
exclude_date: Optional[str] = None,
) -> List[Tuple[str, float]]:
city_key = _normalized_city_key(city_name)
city_rows = history_data.get(city_key) if isinstance(history_data, dict) else None
if not isinstance(city_rows, dict):
return []
rows: List[Tuple[str, float]] = []
for date_str, record in city_rows.items():
if exclude_date and str(date_str) >= str(exclude_date):
continue
if not isinstance(record, dict):
continue
actual = _sf(record.get("actual_high"))
if actual is None:
continue
rows.append((str(date_str), float(actual)))
rows.sort(key=lambda item: item[0])
return rows
def _lag(values: List[float], distance: int) -> Optional[float]:
if len(values) < distance:
return None
return float(values[-distance])
def build_runtime_feature_map(
*,
city_name: str,
current_forecasts: Dict[str, Any],
deb_prediction: Optional[float],
current_temp: Optional[float],
max_so_far: Optional[float],
humidity: Optional[float],
wind_speed_kt: Optional[float],
visibility_mi: Optional[float],
local_hour: int,
local_date: str,
peak_status: str,
history_data: Optional[Dict[str, Any]] = None,
) -> Tuple[Optional[Dict[str, Optional[float]]], Dict[str, Any]]:
data = history_data if isinstance(history_data, dict) else load_history(_history_file_path())
history_rows = _extract_history_rows(data, city_name, exclude_date=local_date)
history_values = [value for _, value in history_rows]
if not history_values:
return None, {"reason": "no_history", "history_count": 0}
date_obj = _parse_date(local_date)
if date_obj is None:
return None, {"reason": "invalid_date", "history_count": len(history_values)}
features: Dict[str, Optional[float]] = {
"actual_high_lag_1": _lag(history_values, 1),
"actual_high_lag_2": _lag(history_values, 2),
"actual_high_lag_3": _lag(history_values, 3),
"actual_high_lag_7": _lag(history_values, 7),
"actual_high_mean_7": _safe_mean(history_values[-7:]),
"actual_high_mean_14": _safe_mean(history_values[-14:]),
"actual_high_trend_3": (
history_values[-1] - history_values[-3] if len(history_values) >= 3 else None
),
"deb_prediction": _sf(deb_prediction),
"current_temp": _sf(current_temp),
"max_so_far": _sf(max_so_far),
"humidity": _sf(humidity),
"wind_speed_kt": _sf(wind_speed_kt),
"visibility_mi": _sf(visibility_mi),
"local_hour": float(local_hour),
"month": float(date_obj.month),
"weekday": float(date_obj.weekday()),
"peak_status_code": _peak_status_code(peak_status),
}
for model_name, column in BASE_MODEL_COLUMNS:
features[column] = _sf(current_forecasts.get(model_name))
model_median, model_spread = _compute_model_summary(features)
features["model_median"] = model_median
features["model_spread"] = model_spread
return features, {
"reason": "ok",
"history_count": len(history_values),
}
def _features_to_vector(features: Dict[str, Optional[float]], feature_names: Optional[List[str]] = None) -> List[float]:
ordered_names = feature_names or FEATURE_NAMES
vector: List[float] = []
for name in ordered_names:
value = features.get(name)
vector.append(float("nan") if value is None else float(value))
return vector
def build_training_samples(
history_data: Optional[Dict[str, Any]] = None,
snapshot_index: Optional[Dict[Tuple[str, str], Dict[str, Any]]] = None,
) -> List[Dict[str, Any]]:
mode = get_state_storage_mode()
if isinstance(history_data, dict):
runtime_history = history_data
elif mode == STATE_STORAGE_SQLITE:
runtime_history = DailyRecordRepository().load_all()
else:
runtime_history = load_history(_history_file_path())
if isinstance(history_data, dict):
truth_history = runtime_history
training_feature_history = {}
elif mode == STATE_STORAGE_SQLITE:
truth_history = TruthRecordRepository().load_all()
training_feature_history = TrainingFeatureRecordRepository().load_all()
else:
truth_history = runtime_history
training_feature_history = {}
snapshots = snapshot_index if isinstance(snapshot_index, dict) else load_snapshot_index()
samples: List[Dict[str, Any]] = []
excluded_keys: set[tuple[str, str]] = set()
for (city_name, date_str), snapshot in (snapshots or {}).items():
if not isinstance(snapshot, dict):
continue
truth_row = ((truth_history.get(city_name) or {}).get(str(date_str)) or {})
target = _sf(truth_row.get("actual_high"))
if target is None:
runtime_record = ((runtime_history.get(city_name) or {}).get(str(date_str)) or {})
target = _sf(runtime_record.get("actual_high"))
if target is None:
continue
observation = snapshot.get("observation") if isinstance(snapshot.get("observation"), dict) else {}
current_forecasts = snapshot.get("multi_model") if isinstance(snapshot.get("multi_model"), dict) else {}
local_hour = _sf(observation.get("local_hour"))
if local_hour is None:
timestamp = _parse_timestamp(snapshot.get("timestamp"))
local_hour = float(timestamp.hour) if timestamp is not None else 12.0
feature_map, meta = build_runtime_feature_map(
city_name=city_name,
current_forecasts=current_forecasts,
deb_prediction=_sf(snapshot.get("deb_prediction")) or _sf(snapshot.get("raw_mu")),
current_temp=_sf(observation.get("current_temp")),
max_so_far=_sf(snapshot.get("max_so_far")),
humidity=_sf(observation.get("humidity")),
wind_speed_kt=_sf(observation.get("wind_speed_kt")),
visibility_mi=_sf(observation.get("visibility_mi")),
local_hour=int(local_hour),
local_date=str(date_str),
peak_status=str(snapshot.get("peak_status") or "before"),
history_data=truth_history,
)
if not feature_map:
continue
samples.append(
{
"city": _normalized_city_key(city_name),
"date": str(date_str),
"target": float(target),
"features": feature_map,
"vector": _features_to_vector(feature_map),
"history_count": int(meta.get("history_count") or 0),
"deb_prediction": _sf(snapshot.get("deb_prediction")) or _sf(snapshot.get("raw_mu")),
"forecasts": {
key: _sf(value)
for key, value in current_forecasts.items()
if _sf(value) is not None
},
"sample_source": "snapshot",
"settlement_source": truth_row.get("settlement_source"),
"settlement_station_code": truth_row.get("settlement_station_code"),
"truth_version": truth_row.get("truth_version"),
"truth_updated_by": truth_row.get("updated_by"),
"truth_updated_at": truth_row.get("truth_updated_at"),
}
)
excluded_keys.add((_normalized_city_key(city_name), str(date_str)))
training_source = training_feature_history or runtime_history or {}
for city_name, city_records in training_source.items():
if not isinstance(city_records, dict):
continue
ordered_dates = sorted(city_records.keys())
for date_str in ordered_dates:
normalized_city = _normalized_city_key(city_name)
if (normalized_city, str(date_str)) in excluded_keys:
continue
record = city_records.get(date_str)
if not isinstance(record, dict):
continue
truth_row = ((truth_history.get(normalized_city) or {}).get(str(date_str)) or {})
target = _sf(truth_row.get("actual_high"))
if target is None:
target = _sf(((runtime_history.get(normalized_city) or {}).get(str(date_str)) or {}).get("actual_high"))
forecasts = record.get("forecasts") if isinstance(record.get("forecasts"), dict) else {}
if target is None or not forecasts:
continue
feature_map, meta = build_runtime_feature_map(
city_name=city_name,
current_forecasts=forecasts,
deb_prediction=_sf(record.get("deb_prediction")),
current_temp=None,
max_so_far=None,
humidity=None,
wind_speed_kt=None,
visibility_mi=None,
local_hour=12,
local_date=str(date_str),
peak_status="before",
history_data=truth_history,
)
if not feature_map:
continue
snapshot = snapshots.get((_normalized_city_key(city_name), str(date_str))) or {}
observation = snapshot.get("observation") if isinstance(snapshot.get("observation"), dict) else {}
feature_map["max_so_far"] = _sf(snapshot.get("max_so_far"))
feature_map["current_temp"] = _sf(observation.get("current_temp"))
feature_map["humidity"] = _sf(observation.get("humidity"))
feature_map["wind_speed_kt"] = _sf(observation.get("wind_speed_kt"))
feature_map["visibility_mi"] = _sf(observation.get("visibility_mi"))
local_hour = _sf(observation.get("local_hour"))
if local_hour is not None:
feature_map["local_hour"] = local_hour
else:
timestamp = _parse_timestamp(snapshot.get("timestamp"))
if timestamp is not None:
feature_map["local_hour"] = float(timestamp.hour)
peak_code = _peak_status_code(snapshot.get("peak_status"))
if peak_code is not None:
feature_map["peak_status_code"] = peak_code
model_median, model_spread = _compute_model_summary(feature_map)
feature_map["model_median"] = model_median
feature_map["model_spread"] = model_spread
samples.append(
{
"city": _normalized_city_key(city_name),
"date": str(date_str),
"target": float(target),
"features": feature_map,
"vector": _features_to_vector(feature_map),
"history_count": int(meta.get("history_count") or 0),
"deb_prediction": _sf(record.get("deb_prediction")),
"forecasts": {
key: _sf(value)
for key, value in forecasts.items()
if _sf(value) is not None
},
"sample_source": "daily_record",
"settlement_source": truth_row.get("settlement_source"),
"settlement_station_code": truth_row.get("settlement_station_code"),
"truth_version": truth_row.get("truth_version"),
"truth_updated_by": truth_row.get("updated_by"),
"truth_updated_at": truth_row.get("truth_updated_at"),
}
)
samples.sort(key=lambda row: (row["date"], row["city"]))
return samples
def schema_payload(
*,
model_path: str,
sample_count: int,
train_count: int,
validation_count: int,
metrics: Dict[str, Any],
) -> Dict[str, Any]:
return {
"model_type": "LightGBMRegressor",
"target": "actual_high",
"horizon": "D0",
"feature_names": FEATURE_NAMES,
"base_model_columns": [column for _, column in BASE_MODEL_COLUMNS],
"model_path": model_path,
"sample_count": sample_count,
"train_count": train_count,
"validation_count": validation_count,
"metrics": metrics,
"generated_at": datetime.utcnow().isoformat() + "Z",
}
web/analysis_service.py
+1 -48
View File
@@ -29,7 +29,6 @@ from src.data_collection.city_registry import ALIASES, CITY_REGISTRY
from src.data_collection.city_time import get_city_utc_offset_seconds
from src.data_collection.nmc_sources import NMC_CITY_REFERENCES
from src.database.runtime_state import IntradayPathSnapshotRepository
from src.models.lgbm_daily_high import predict_lgbm_daily_high
from web.services.groq_commentary import (
build_groq_commentary_context as _groq_context_builder,
clean_commentary_text as _groq_clean_text,
@@ -2187,24 +2186,6 @@ def _analyze(
else:
peak_status = "before"
lgbm_val = None
if current_forecasts and deb_val is not None:
lgbm_val, _ = predict_lgbm_daily_high(
city_name=city,
current_forecasts=current_forecasts,
deb_prediction=deb_val,
current_temp=cur_temp,
max_so_far=max_so_far,
humidity=_sf(primary_current.get("humidity")),
wind_speed_kt=_sf(primary_current.get("wind_speed_kt")),
visibility_mi=_sf(primary_current.get("visibility_mi")),
local_hour=local_hour,
local_date=local_date_str,
peak_status=peak_status,
)
# LGBM is kept as an independent reference (lgbm.prediction),
# not fed back into DEB to avoid circular dependency
deviation_monitor = _build_deviation_monitor(
current_temp=cur_temp,
deb_prediction=deb_val,
@@ -2225,33 +2206,14 @@ def _analyze(
probabilities = []
probabilities_all = []
shadow_probabilities = []
shadow_probabilities_all = []
mu = None
probability_engine = "legacy"
probability_calibration_mode = "legacy"
probability_calibration_version = None
probability_raw_mu = None
probability_raw_sigma = None
probability_calibrated_mu = None
probability_calibrated_sigma = None
dynamic_commentary = {"summary": "", "notes": []}
try:
_, _ai_context, sd = _trend_analyze(raw, sym, city)
# Use structured data from shared engine
mu = sd.get("mu")
probabilities = sd.get("probabilities", [])
probabilities_all = sd.get("probabilities_all", probabilities)
shadow_probabilities = sd.get("shadow_probabilities", [])
shadow_probabilities_all = sd.get("shadow_probabilities_all", shadow_probabilities)
probability_engine = sd.get("probability_engine", "legacy")
probability_calibration_mode = sd.get("probability_calibration_mode", "legacy")
probability_calibration_version = sd.get("probability_calibration_version")
probability_raw_mu = sd.get("probability_raw_mu")
probability_raw_sigma = sd.get("probability_raw_sigma")
probability_calibrated_mu = sd.get("probability_calibrated_mu")
probability_calibrated_sigma = sd.get("probability_calibrated_sigma")
dynamic_commentary = sd.get("dynamic_commentary") or dynamic_commentary
trend_info["is_dead_market"] = sd.get("trend_info", {}).get("is_dead_market", False)
trend_info["direction"] = sd.get("trend_info", {}).get("direction", trend_info.get("direction", "unknown"))
@@ -2632,22 +2594,13 @@ def _analyze(
"multi_model": {k: v for k, v in current_forecasts.items() if v is not None},
"multi_model_daily": multi_model_daily,
"deb": {"prediction": deb_val, "weights_info": deb_weights},
"lgbm": {"prediction": lgbm_val},
"deviation_monitor": deviation_monitor,
"ensemble": ens_data,
"probabilities": {
"mu": round(mu, 1) if mu is not None else None,
"distribution": probabilities,
"distribution_all": probabilities_all or probabilities,
"engine": probability_engine,
"calibration_mode": probability_calibration_mode,
"calibration_version": probability_calibration_version,
"raw_mu": probability_raw_mu,
"raw_sigma": probability_raw_sigma,
"calibrated_mu": probability_calibrated_mu,
"calibrated_sigma": probability_calibrated_sigma,
"shadow_distribution": shadow_probabilities,
"shadow_distribution_all": shadow_probabilities_all or shadow_probabilities,
"engine": "legacy",
},
"trend": trend_info,
"peak": {
web/core.py
+2 -123
View File
@@ -27,12 +27,6 @@ from src.utils.metrics import (
gauge_set,
histogram_observe,
)
from src.analysis.probability_calibration import (
DEFAULT_CALIBRATION_FILE,
check_calibration_drift,
resolve_probability_engine_mode,
)
from src.analysis.probability_rollout import build_rollout_report
from src.database.db_manager import DBManager
from src.database.runtime_state import get_state_storage_mode
from src.payments import PAYMENT_CHECKOUT, PaymentCheckoutError # noqa: F401
@@ -87,12 +81,6 @@ CACHE_TTL = 300
CACHE_TTL_ANKARA = 60
CACHE_TTL_KOREAN_AMOS = 60
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_PROBABILITY_EVALUATION_REPORT = os.path.join(
_PROJECT_ROOT,
"artifacts",
"probability_calibration",
"evaluation_report.json",
)
@app.middleware("http")
@@ -167,26 +155,6 @@ async def _etag_middleware(request: Request, call_next):
response.headers["Cache-Control"] = "private, max-age=30"
return response
_PROBABILITY_SHADOW_REPORT = os.path.join(
_PROJECT_ROOT,
"artifacts",
"probability_calibration",
"shadow_report.json",
)
_PROBABILITY_TRAINING_SAMPLES = os.path.join(
_PROJECT_ROOT,
"artifacts",
"probability_calibration",
"training_samples.json",
)
_LGBM_SCHEMA_REPORT = os.path.join(
_PROJECT_ROOT,
"artifacts",
"models",
"lgbm_daily_high_schema.json",
)
def _env_bool(name: str, default: bool = False) -> bool:
raw = os.getenv(name)
if raw is None:
@@ -508,44 +476,8 @@ def _integration_summary() -> Dict[str, Any]:
def _probability_summary() -> Dict[str, Any]:
rollout = build_rollout_report(
_PROBABILITY_EVALUATION_REPORT,
_PROBABILITY_SHADOW_REPORT,
)
drift = {"drifted": False, "sample_count": 0, "warning": "not checked"}
try:
import sqlite3
conn = sqlite3.connect(_account_db.db_path)
rows = conn.execute(
"SELECT city, target_date, actual_high, deb_prediction, extra_json FROM daily_records_store ORDER BY target_date DESC LIMIT 200"
).fetchall()
conn.close()
records: list[dict[str, Any]] = []
for row in rows:
rec = {"city": row[0], "target_date": row[1], "actual_high": row[2], "deb_prediction": row[3]}
try:
extra = json.loads(row[4] or "{}") if row[4] else {}
rec["mu"] = extra.get("mu")
rec["sigma"] = extra.get("sigma")
except Exception:
pass
records.append(rec)
if records:
drift = check_calibration_drift(records)
if drift.get("drifted"):
logger.warning(
"Calibration drift detected: {} — {}",
drift.get("warning") or "CRPS degraded",
f"delta={drift.get('delta_pct')}% samples={drift.get('sample_count')}",
)
except Exception:
pass
return {
"engine_mode": resolve_probability_engine_mode(),
"calibration_file": os.getenv("POLYWEATHER_PROBABILITY_CALIBRATION_FILE")
or DEFAULT_CALIBRATION_FILE,
"rollout": rollout,
"drift": drift,
"engine_mode": "legacy",
}
@@ -698,27 +630,10 @@ def _city_coverage_summary(conn: sqlite3.Connection) -> Dict[str, Any]:
def _model_city_coverage_summary(
city_entries: Any,
training_samples_payload: Dict[str, Any],
evaluation_report: Dict[str, Any],
) -> Dict[str, Any]:
training_samples = training_samples_payload.get("samples") or []
emos_training_counts: Dict[str, int] = {}
emos_snapshot_counts: Dict[str, int] = {}
for item in training_samples:
if not isinstance(item, dict):
continue
city = str(item.get("city") or "").strip().lower()
if not city:
continue
emos_training_counts[city] = emos_training_counts.get(city, 0) + 1
if str(item.get("sample_source") or "").strip().lower() == "snapshot":
emos_snapshot_counts[city] = emos_snapshot_counts.get(city, 0) + 1
evaluation_by_city = (evaluation_report.get("by_city") or {}) if isinstance(evaluation_report, dict) else {}
rows = []
for entry in city_entries or []:
city = str(entry.get("city") or "").strip().lower()
emos_eval = evaluation_by_city.get(city) or {}
rows.append(
{
"city": city,
@@ -726,22 +641,13 @@ def _model_city_coverage_summary(
"settlement_source": entry.get("settlement_source"),
"truth_rows": int(entry.get("truth_rows") or 0),
"feature_rows": int(entry.get("feature_rows") or 0),
"emos_training_samples": int(emos_training_counts.get(city, 0)),
"emos_snapshot_samples": int(emos_snapshot_counts.get(city, 0)),
"emos_evaluation_samples": int(emos_eval.get("samples") or 0),
"emos_delta_crps": emos_eval.get("emos_mean_crps"),
"lgbm_candidate_rows": int(entry.get("feature_rows") or 0),
}
)
weakest = sorted(
rows,
key=lambda row: (
row["emos_training_samples"] > 0,
row["lgbm_candidate_rows"] > 0,
row["truth_rows"] > 0,
row["emos_training_samples"],
row["lgbm_candidate_rows"],
row["truth_rows"],
row["city"],
),
@@ -749,15 +655,11 @@ def _model_city_coverage_summary(
strongest = sorted(
rows,
key=lambda row: (
-row["emos_training_samples"],
-row["lgbm_candidate_rows"],
-row["truth_rows"],
row["city"],
),
)[:8]
return {
"cities_with_emos_training": sum(1 for row in rows if row["emos_training_samples"] > 0),
"cities_with_lgbm_candidates": sum(1 for row in rows if row["lgbm_candidate_rows"] > 0),
"weakest": weakest,
"strongest": strongest,
}
@@ -786,17 +688,9 @@ def _training_data_summary() -> Dict[str, Any]:
"truth_revisions": truth_revisions,
"training_features": training_features,
"city_coverage": {},
"artifacts": {},
"model_city_coverage": {},
}
evaluation_report = _read_json_file(_PROBABILITY_EVALUATION_REPORT) or {}
shadow_report = _read_json_file(_PROBABILITY_SHADOW_REPORT) or {}
training_samples = _read_json_file(_PROBABILITY_TRAINING_SAMPLES) or {}
lgbm_report = _read_json_file(_LGBM_SCHEMA_REPORT) or {}
evaluation_summary = (evaluation_report.get("summary") or {}) if isinstance(evaluation_report, dict) else {}
shadow_summary = shadow_report.get("summary") or {}
lgbm_validation = ((lgbm_report.get("metrics") or {}).get("validation") or {})
return {
"db_path": db_path,
"db_ok": True,
@@ -806,22 +700,7 @@ def _training_data_summary() -> Dict[str, Any]:
"city_coverage": city_coverage,
"model_city_coverage": _model_city_coverage_summary(
city_coverage.get("entries") or [],
training_samples,
evaluation_report,
),
"artifacts": {
"emos_training_samples": training_samples.get("sample_count"),
"emos_snapshot_samples": training_samples.get("snapshot_sample_count"),
"emos_daily_record_samples": training_samples.get("daily_record_sample_count"),
"emos_evaluation_samples": evaluation_summary.get("sample_count"),
"emos_shadow_samples": shadow_summary.get("sample_count"),
"emos_delta_crps": (evaluation_summary.get("delta") or {}).get("crps"),
"lgbm_sample_count": lgbm_report.get("sample_count"),
"lgbm_train_count": lgbm_report.get("train_count"),
"lgbm_validation_count": lgbm_report.get("validation_count"),
"lgbm_validation_mae": lgbm_validation.get("lgbm_mae"),
"lgbm_validation_deb_mae": lgbm_validation.get("deb_mae"),
},
}