gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Archive probability snapshots and wire them into training

Browse Source
This commit is contained in:
2569718930@qq.com
2026-03-20 21:30:52 +08:00
parent 03dcb4329b
commit 3196552c78
11 changed files with 1129 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
artifacts/probability_calibration/default.json
+2 -2
View File
@@ -1,6 +1,6 @@
{
"version": "emos-20260320130245",
"trained_at": "2026-03-20T13:02:45.903772+00:00",
"version": "emos-20260320132525",
"trained_at": "2026-03-20T13:25:25.836021+00:00",
"global": {
"mu": {
"intercept": -1.57406048,
artifacts/probability_calibration/evaluation_report.json
+60 -60
View File
@@ -8,239 +8,239 @@
"bucket_hit_rate": 0.695238
},
"emos": {
"mean_crps": 2.650216,
"mean_mae": 2.722829,
"bucket_hit_rate": 0.666667
"mean_crps": 2.700275,
"mean_mae": 2.721143,
"bucket_hit_rate": 0.695238
},
"delta": {
"crps": -0.143722,
"mae": 0.001686,
"bucket_hit_rate": -0.028571
"crps": -0.093663,
"mae": 0.0,
"bucket_hit_rate": 0.0
}
},
"by_city": {
"ankara": {
"samples": 7,
"legacy_mean_crps": 2.023242,
"emos_mean_crps": 2.015458,
"emos_mean_crps": 2.023242,
"legacy_mean_mae": 1.984286,
"emos_mean_mae": 1.977721,
"emos_mean_mae": 1.984286,
"legacy_bucket_hit_rate": 0.714286,
"emos_bucket_hit_rate": 0.714286
},
"atlanta": {
"samples": 5,
"legacy_mean_crps": 12.792034,
"emos_mean_crps": 12.526108,
"emos_mean_crps": 12.694543,
"legacy_mean_mae": 12.806,
"emos_mean_mae": 12.919677,
"emos_mean_mae": 12.806,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.4
"emos_bucket_hit_rate": 0.6
},
"buenos aires": {
"samples": 5,
"legacy_mean_crps": 3.846144,
"emos_mean_crps": 3.743967,
"emos_mean_crps": 3.846144,
"legacy_mean_mae": 4.168,
"emos_mean_mae": 4.086663,
"emos_mean_mae": 4.168,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"chicago": {
"samples": 4,
"legacy_mean_crps": 1.346667,
"emos_mean_crps": 0.603128,
"emos_mean_crps": 0.601765,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.075265,
"emos_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"dallas": {
"samples": 4,
"legacy_mean_crps": 1.256111,
"emos_mean_crps": 0.65514,
"emos_mean_crps": 0.651425,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.114807,
"emos_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"hong kong": {
"samples": 3,
"legacy_mean_crps": 0.261027,
"emos_mean_crps": 0.269856,
"emos_mean_crps": 0.261027,
"legacy_mean_mae": 0.1,
"emos_mean_mae": 0.174879,
"emos_mean_mae": 0.1,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.666667
},
"london": {
"samples": 6,
"legacy_mean_crps": 2.079624,
"emos_mean_crps": 2.005311,
"emos_mean_crps": 2.079624,
"legacy_mean_mae": 2.451667,
"emos_mean_mae": 2.327304,
"emos_mean_mae": 2.451667,
"legacy_bucket_hit_rate": 0.166667,
"emos_bucket_hit_rate": 0.333333
"emos_bucket_hit_rate": 0.166667
},
"lucknow": {
"samples": 4,
"legacy_mean_crps": 1.468528,
"emos_mean_crps": 1.443159,
"emos_mean_crps": 1.468528,
"legacy_mean_mae": 1.6025,
"emos_mean_mae": 1.682697,
"emos_mean_mae": 1.6025,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 0.5
},
"madrid": {
"samples": 2,
"legacy_mean_crps": 6.27726,
"emos_mean_crps": 6.090563,
"emos_mean_crps": 6.27726,
"legacy_mean_mae": 7.33,
"emos_mean_mae": 7.141649,
"emos_mean_mae": 7.33,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"miami": {
"samples": 5,
"legacy_mean_crps": 11.665378,
"emos_mean_crps": 11.493094,
"emos_mean_crps": 11.665378,
"legacy_mean_mae": 12.07,
"emos_mean_mae": 12.157623,
"emos_mean_mae": 12.07,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.2
"emos_bucket_hit_rate": 0.6
},
"milan": {
"samples": 3,
"legacy_mean_crps": 4.401392,
"emos_mean_crps": 3.837715,
"emos_mean_crps": 3.928883,
"legacy_mean_mae": 4.06,
"emos_mean_mae": 3.942288,
"emos_mean_mae": 4.06,
"legacy_bucket_hit_rate": 0.666667,
"emos_bucket_hit_rate": 0.666667
},
"munich": {
"samples": 6,
"legacy_mean_crps": 2.988583,
"emos_mean_crps": 2.918299,
"emos_mean_crps": 2.988583,
"legacy_mean_mae": 3.143333,
"emos_mean_mae": 3.126482,
"emos_mean_mae": 3.143333,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 0.5
},
"new york": {
"samples": 4,
"legacy_mean_crps": 1.861101,
"emos_mean_crps": 1.363203,
"emos_mean_crps": 1.409393,
"legacy_mean_mae": 1.3725,
"emos_mean_mae": 1.369301,
"emos_mean_mae": 1.3725,
"legacy_bucket_hit_rate": 0.75,
"emos_bucket_hit_rate": 0.75
},
"paris": {
"samples": 7,
"legacy_mean_crps": 2.430082,
"emos_mean_crps": 2.365416,
"emos_mean_crps": 2.430082,
"legacy_mean_mae": 2.518571,
"emos_mean_mae": 2.48437,
"emos_mean_mae": 2.518571,
"legacy_bucket_hit_rate": 0.571429,
"emos_bucket_hit_rate": 0.571429
},
"sao paulo": {
"samples": 5,
"legacy_mean_crps": 2.454756,
"emos_mean_crps": 2.410624,
"emos_mean_crps": 2.454756,
"legacy_mean_mae": 2.628,
"emos_mean_mae": 2.689909,
"emos_mean_mae": 2.628,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"seattle": {
"samples": 4,
"legacy_mean_crps": 0.531656,
"emos_mean_crps": 0.457257,
"emos_mean_crps": 0.452784,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.11262,
"emos_mean_mae": 0.0,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"seoul": {
"samples": 6,
"legacy_mean_crps": 0.328088,
"emos_mean_crps": 0.332722,
"emos_mean_crps": 0.328088,
"legacy_mean_mae": 0.2,
"emos_mean_mae": 0.246659,
"emos_mean_mae": 0.2,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"shanghai": {
"samples": 2,
"legacy_mean_crps": 0.250034,
"emos_mean_crps": 0.249952,
"emos_mean_crps": 0.250034,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 0.204362,
"emos_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"singapore": {
"samples": 2,
"legacy_mean_crps": 0.281993,
"emos_mean_crps": 0.270875,
"emos_mean_crps": 0.281993,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 0.070547,
"emos_mean_mae": 0.15,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"taipei": {
"samples": 3,
"legacy_mean_crps": 0.356996,
"emos_mean_crps": 0.351432,
"emos_mean_crps": 0.356996,
"legacy_mean_mae": 0.1,
"emos_mean_mae": 0.084784,
"emos_mean_mae": 0.1,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.666667
"emos_bucket_hit_rate": 1.0
},
"tel aviv": {
"samples": 2,
"legacy_mean_crps": 0.446758,
"emos_mean_crps": 0.434936,
"emos_mean_crps": 0.446758,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 0.196589,
"emos_mean_mae": 0.3,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"tokyo": {
"samples": 2,
"legacy_mean_crps": 0.450128,
"emos_mean_crps": 0.429389,
"emos_mean_crps": 0.450128,
"legacy_mean_mae": 0.25,
"emos_mean_mae": 0.100782,
"emos_mean_mae": 0.25,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 1.0
},
"toronto": {
"samples": 5,
"legacy_mean_crps": 2.647861,
"emos_mean_crps": 2.512236,
"emos_mean_crps": 2.566068,
"legacy_mean_mae": 2.532,
"emos_mean_mae": 2.535078,
"emos_mean_mae": 2.532,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.6
},
"warsaw": {
"samples": 3,
"legacy_mean_crps": 1.618875,
"emos_mean_crps": 1.545129,
"emos_mean_crps": 1.618875,
"legacy_mean_mae": 2.056667,
"emos_mean_mae": 1.942917,
"emos_mean_mae": 2.056667,
"legacy_bucket_hit_rate": 0.333333,
"emos_bucket_hit_rate": 0.333333
},
"wellington": {
"samples": 6,
"legacy_mean_crps": 0.266349,
"emos_mean_crps": 0.264506,
"emos_mean_crps": 0.266349,
"legacy_mean_mae": 0.2,
"emos_mean_mae": 0.205623,
"emos_mean_mae": 0.2,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
}
artifacts/probability_calibration/training_samples.json
+212 -105
View File
@@ -1,5 +1,7 @@
{
"sample_count": 105,
"snapshot_sample_count": 0,
"daily_record_sample_count": 105,
"filled_actual_from_history": 2,
"samples": [
{
@@ -12,7 +14,8 @@
"ens_median": 5.9,
"ensemble_spread": 1.5499999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "ankara",
@@ -24,7 +27,8 @@
"ens_median": 7.6,
"ensemble_spread": 1.75,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "ankara",
@@ -36,7 +40,8 @@
"ens_median": 7.9,
"ensemble_spread": 1.5499999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "ankara",
@@ -48,7 +53,8 @@
"ens_median": 10.0,
"ensemble_spread": 0.8500000000000005,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "ankara",
@@ -60,7 +66,8 @@
"ens_median": 13.9,
"ensemble_spread": 1.75,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "ankara",
@@ -72,7 +79,8 @@
"ens_median": 15.8,
"ensemble_spread": 1.2000000000000002,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "ankara",
@@ -84,7 +92,8 @@
"ens_median": 10.1,
"ensemble_spread": 2.05,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "london",
@@ -96,7 +105,8 @@
"ens_median": 10.1,
"ensemble_spread": 0.9000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "london",
@@ -108,7 +118,8 @@
"ens_median": 11.4,
"ensemble_spread": 2.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "london",
@@ -120,7 +131,8 @@
"ens_median": 15.4,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "london",
@@ -132,7 +144,8 @@
"ens_median": 13.3,
"ensemble_spread": 0.9500000000000002,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "london",
@@ -144,7 +157,8 @@
"ens_median": 16.8,
"ensemble_spread": 1.299999999999999,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "london",
@@ -156,7 +170,8 @@
"ens_median": 15.8,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "new york",
@@ -168,7 +183,8 @@
"ens_median": 46.7,
"ensemble_spread": 9.7,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "new york",
@@ -180,7 +196,8 @@
"ens_median": 67.2,
"ensemble_spread": 2.1499999999999986,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "new york",
@@ -192,7 +209,8 @@
"ens_median": 66.6,
"ensemble_spread": 4.149999999999999,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "new york",
@@ -204,7 +222,8 @@
"ens_median": 37.0,
"ensemble_spread": 2.25,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "paris",
@@ -216,7 +235,8 @@
"ens_median": 16.7,
"ensemble_spread": 0.7000000000000011,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "paris",
@@ -228,7 +248,8 @@
"ens_median": 16.2,
"ensemble_spread": 1.5999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "paris",
@@ -240,7 +261,8 @@
"ens_median": 17.4,
"ensemble_spread": 0.6999999999999993,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "paris",
@@ -252,7 +274,8 @@
"ens_median": 15.3,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "paris",
@@ -264,7 +287,8 @@
"ens_median": 13.1,
"ensemble_spread": 0.8499999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "paris",
@@ -276,7 +300,8 @@
"ens_median": 14.8,
"ensemble_spread": 0.9000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "paris",
@@ -288,7 +313,8 @@
"ens_median": 16.3,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seoul",
@@ -300,7 +326,8 @@
"ens_median": 4.0,
"ensemble_spread": 0.7999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seoul",
@@ -312,7 +339,8 @@
"ens_median": 6.3,
"ensemble_spread": 1.4,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seoul",
@@ -324,7 +352,8 @@
"ens_median": 4.9,
"ensemble_spread": 1.35,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seoul",
@@ -336,7 +365,8 @@
"ens_median": 9.1,
"ensemble_spread": 1.7999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seoul",
@@ -348,7 +378,8 @@
"ens_median": 7.4,
"ensemble_spread": 0.7999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seoul",
@@ -360,7 +391,8 @@
"ens_median": 6.3,
"ensemble_spread": 1.7999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "toronto",
@@ -372,7 +404,8 @@
"ens_median": 16.9,
"ensemble_spread": 2.8500000000000005,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "toronto",
@@ -384,7 +417,8 @@
"ens_median": 12.3,
"ensemble_spread": 2.0,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "toronto",
@@ -396,7 +430,8 @@
"ens_median": 17.6,
"ensemble_spread": 4.75,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "toronto",
@@ -408,7 +443,8 @@
"ens_median": -1.1,
"ensemble_spread": 0.8,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "toronto",
@@ -420,7 +456,8 @@
"ens_median": 6.3,
"ensemble_spread": 2.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "buenos aires",
@@ -432,7 +469,8 @@
"ens_median": 26.0,
"ensemble_spread": 1.3999999999999986,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "buenos aires",
@@ -444,7 +482,8 @@
"ens_median": 26.4,
"ensemble_spread": 1.200000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "buenos aires",
@@ -456,7 +495,8 @@
"ens_median": 26.2,
"ensemble_spread": 1.5999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "buenos aires",
@@ -468,7 +508,8 @@
"ens_median": 26.0,
"ensemble_spread": 2.0,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "buenos aires",
@@ -480,7 +521,8 @@
"ens_median": 27.9,
"ensemble_spread": 2.0999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "wellington",
@@ -492,7 +534,8 @@
"ens_median": 18.4,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "wellington",
@@ -504,7 +547,8 @@
"ens_median": 17.2,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "wellington",
@@ -516,7 +560,8 @@
"ens_median": 15.7,
"ensemble_spread": 0.9000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "wellington",
@@ -528,7 +573,8 @@
"ens_median": 18.7,
"ensemble_spread": 1.049999999999999,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "wellington",
@@ -540,7 +586,8 @@
"ens_median": 19.1,
"ensemble_spread": 0.9500000000000011,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "wellington",
@@ -552,7 +599,8 @@
"ens_median": 17.1,
"ensemble_spread": 2.0999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "chicago",
@@ -564,7 +612,8 @@
"ens_median": 62.8,
"ensemble_spread": 1.3000000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "chicago",
@@ -576,7 +625,8 @@
"ens_median": 60.1,
"ensemble_spread": 5.600000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "chicago",
@@ -588,7 +638,8 @@
"ens_median": 70.8,
"ensemble_spread": 10.799999999999997,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "chicago",
@@ -600,7 +651,8 @@
"ens_median": 44.2,
"ensemble_spread": 5.350000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "sao paulo",
@@ -612,7 +664,8 @@
"ens_median": 27.4,
"ensemble_spread": 1.5,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "sao paulo",
@@ -624,7 +677,8 @@
"ens_median": 28.0,
"ensemble_spread": 2.1999999999999993,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "sao paulo",
@@ -636,7 +690,8 @@
"ens_median": 23.5,
"ensemble_spread": 1.4000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "sao paulo",
@@ -648,7 +703,8 @@
"ens_median": 29.4,
"ensemble_spread": 2.450000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "sao paulo",
@@ -660,7 +716,8 @@
"ens_median": 26.5,
"ensemble_spread": 1.200000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "dallas",
@@ -672,7 +729,8 @@
"ens_median": 72.7,
"ensemble_spread": 2.1499999999999986,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "dallas",
@@ -684,7 +742,8 @@
"ens_median": 71.5,
"ensemble_spread": 6.200000000000003,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "dallas",
@@ -696,7 +755,8 @@
"ens_median": 81.0,
"ensemble_spread": 3.8500000000000014,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "dallas",
@@ -708,7 +768,8 @@
"ens_median": 74.4,
"ensemble_spread": 9.299999999999997,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "miami",
@@ -720,7 +781,8 @@
"ens_median": 81.6,
"ensemble_spread": 1.3999999999999986,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "miami",
@@ -732,7 +794,8 @@
"ens_median": 82.7,
"ensemble_spread": 1.5500000000000043,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "miami",
@@ -744,7 +807,8 @@
"ens_median": 83.9,
"ensemble_spread": 1.5499999999999972,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "miami",
@@ -756,7 +820,8 @@
"ens_median": 70.4,
"ensemble_spread": 2.8999999999999986,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "miami",
@@ -768,7 +833,8 @@
"ens_median": 74.0,
"ensemble_spread": 2.5500000000000043,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "atlanta",
@@ -780,7 +846,8 @@
"ens_median": 79.7,
"ensemble_spread": 4.600000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "atlanta",
@@ -792,7 +859,8 @@
"ens_median": 67.7,
"ensemble_spread": 5.899999999999999,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "atlanta",
@@ -804,7 +872,8 @@
"ens_median": 77.4,
"ensemble_spread": 2.6000000000000014,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "atlanta",
@@ -816,7 +885,8 @@
"ens_median": 52.2,
"ensemble_spread": 4.0,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "atlanta",
@@ -828,7 +898,8 @@
"ens_median": 65.9,
"ensemble_spread": 1.5500000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seattle",
@@ -840,7 +911,8 @@
"ens_median": 52.8,
"ensemble_spread": 1.3999999999999986,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seattle",
@@ -852,7 +924,8 @@
"ens_median": 50.9,
"ensemble_spread": 4.350000000000001,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seattle",
@@ -864,7 +937,8 @@
"ens_median": 45.1,
"ensemble_spread": 2.0,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "seattle",
@@ -876,7 +950,8 @@
"ens_median": 55.9,
"ensemble_spread": 1.3500000000000014,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "lucknow",
@@ -888,7 +963,8 @@
"ens_median": 33.9,
"ensemble_spread": 1.8500000000000014,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "lucknow",
@@ -900,7 +976,8 @@
"ens_median": 34.6,
"ensemble_spread": 2.0,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "lucknow",
@@ -912,7 +989,8 @@
"ens_median": 34.6,
"ensemble_spread": 1.3000000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "lucknow",
@@ -924,7 +1002,8 @@
"ens_median": 34.7,
"ensemble_spread": 1.75,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "munich",
@@ -936,7 +1015,8 @@
"ens_median": 15.1,
"ensemble_spread": 1.6000000000000005,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "munich",
@@ -948,7 +1028,8 @@
"ens_median": 14.3,
"ensemble_spread": 0.9000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "munich",
@@ -960,7 +1041,8 @@
"ens_median": 14.5,
"ensemble_spread": 0.8999999999999995,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "munich",
@@ -972,7 +1054,8 @@
"ens_median": 15.7,
"ensemble_spread": 1.2999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "munich",
@@ -984,7 +1067,8 @@
"ens_median": 10.6,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "munich",
@@ -996,7 +1080,8 @@
"ens_median": 11.8,
"ensemble_spread": 1.3000000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "hong kong",
@@ -1008,7 +1093,8 @@
"ens_median": 23.1,
"ensemble_spread": 1.9000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "hong kong",
@@ -1020,7 +1106,8 @@
"ens_median": 24.9,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "hong kong",
@@ -1032,7 +1119,8 @@
"ens_median": 25.0,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "taipei",
@@ -1044,7 +1132,8 @@
"ens_median": 25.4,
"ensemble_spread": 2.25,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "taipei",
@@ -1056,7 +1145,8 @@
"ens_median": 27.5,
"ensemble_spread": 1.0500000000000007,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "taipei",
@@ -1068,7 +1158,8 @@
"ens_median": 21.3,
"ensemble_spread": 1.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "milan",
@@ -1080,7 +1171,8 @@
"ens_median": 15.1,
"ensemble_spread": 9.1,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "milan",
@@ -1092,7 +1184,8 @@
"ens_median": 13.8,
"ensemble_spread": 0.6,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "milan",
@@ -1104,7 +1197,8 @@
"ens_median": 17.0,
"ensemble_spread": 1.25,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "warsaw",
@@ -1116,7 +1210,8 @@
"ens_median": 10.5,
"ensemble_spread": 0.6499999999999995,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "warsaw",
@@ -1128,7 +1223,8 @@
"ens_median": 14.2,
"ensemble_spread": 1.4000000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "warsaw",
@@ -1140,7 +1236,8 @@
"ens_median": 12.3,
"ensemble_spread": 1.5999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "singapore",
@@ -1152,7 +1249,8 @@
"ens_median": 30.1,
"ensemble_spread": 0.9500000000000011,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "singapore",
@@ -1164,7 +1262,8 @@
"ens_median": 32.1,
"ensemble_spread": 1.3499999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "shanghai",
@@ -1176,7 +1275,8 @@
"ens_median": 13.0,
"ensemble_spread": 1.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "shanghai",
@@ -1188,7 +1288,8 @@
"ens_median": 11.2,
"ensemble_spread": 0.7999999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "tokyo",
@@ -1200,7 +1301,8 @@
"ens_median": 15.8,
"ensemble_spread": 1.5499999999999998,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "tokyo",
@@ -1212,7 +1314,8 @@
"ens_median": 18.5,
"ensemble_spread": 2.0999999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "tel aviv",
@@ -1224,7 +1327,8 @@
"ens_median": 28.9,
"ensemble_spread": 2.1500000000000004,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "tel aviv",
@@ -1236,7 +1340,8 @@
"ens_median": 21.1,
"ensemble_spread": 1.5,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "madrid",
@@ -1248,7 +1353,8 @@
"ens_median": 18.7,
"ensemble_spread": 1.8499999999999996,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
},
{
"city": "madrid",
@@ -1260,7 +1366,8 @@
"ens_median": 18.8,
"ensemble_spread": 1.8999999999999995,
"max_so_far_gap": null,
"peak_flag": 0.0
"peak_flag": 0.0,
"sample_source": "daily_record"
}
]
}
data/probability_training_snapshots.jsonl
+1
View File
@@ -0,0 +1 @@
{"city": "ankara", "timestamp": "2026-03-20T12:00:00+03:00", "date": "2026-03-20", "temp_symbol": "°C", "raw_mu": 15.2, "raw_sigma": 1.2, "deb_prediction": 15.4, "ensemble": {"p10": 14.8, "median": 15.8, "p90": 17.9}, "multi_model": {"ECMWF": 15.8, "GFS": 14.1, "ICON": 15.9}, "max_so_far": 15.0, "peak_status": "before", "prob_snapshot": [{"v": 15, "p": 0.552}, {"v": 16, "p": 0.377}], "shadow_prob_snapshot": [{"v": 15, "p": 0.324}, {"v": 16, "p": 0.238}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320130245", "calibration_source": "artifacts/probability_calibration/default.json", "calibrated_mu": 15.1, "calibrated_sigma": 1.25}
docs/PROBABILITY_SNAPSHOT_ARCHIVE_ZH.md
+324
View File
@@ -0,0 +1,324 @@
# 概率训练样本归档说明（中文）
## 1. 目的
这份文档说明两件事：
1. 为什么 `EMOS` 训练不能只依赖历史实测天气
2. 未来如何持续沉淀“历史预测记录”，让概率引擎越训越稳
一句话结论：
- 历史实测天气只能补 `actual_high`
- 真正决定 `EMOS` 训练质量的是“当时那一刻的预测快照”
## 2. 什么是“历史预测记录”
对 PolyWeather 来说，一条可训练的历史预测记录，至少应该包含这些字段：
- `city`
- `timestamp`
- `date`
- `raw_mu`
- `raw_sigma`
- `deb_prediction`
- `ensemble p10 / p50 / p90`
- `multi-model forecasts`
- `max_so_far`
- `peak_status`
- `prob_snapshot`
- 当天最终 `actual_high`
- 当天最终 `settlement bucket`
这类记录的核心价值是：
- 还原“当时系统实际看到什么”
- 再对照“后来真实发生了什么”
只有这两者成对，`EMOS` 才能学习偏差。
## 3. 为什么不能只用历史天气实测
历史天气 CSV 只能告诉你：
- 当天最高温是多少
- 某小时温度是多少
但它不能告诉你：
- 当天早上 09:00 时，系统的 `mu` 是多少
- 当时的 `ensemble spread` 是多少
- 当时 `DEB` 怎么看
- 当时的 top bucket 是什么
所以：
- 历史实测天气是标签
- 历史预测记录才是训练输入
缺少后者，EMOS 只能学到很有限的东西。
## 4. 当前项目里已经有的基础
### 4.1 已有历史日记录
文件：
- [daily_records.json](/E:/web/PolyWeather/data/daily_records.json)
当前已经保存了一部分训练相关字段，例如：
- `forecasts`
- `actual_high`
- `deb_prediction`
- `mu`
- `prob_snapshot`
- `shadow_prob_snapshot`
- `probability_calibration`
- `probability_features`
这已经是“历史预测记录”的雏形。
### 4.2 已有历史天气 CSV
目录：
- [data/historical](/E:/web/PolyWeather/data/historical)
它们可以帮助补：
- `actual_high`
- `settlement history`
但不能替代预测快照归档。
## 5. 未来应该怎么存历史预测记录
推荐做法是：
### 5.1 固定时点归档
每天为每个重点城市固定存几次快照，例如：
- 当地 `09:00`
- 当地 `12:00`
- 当地 `15:00`
这样能确保每个交易日都有稳定可比样本。
### 5.2 关键变化时补充归档
除了固定时点，还应该在以下情况额外存一次：
- `max_so_far` 创新高
- `mu` 变化超过阈值
- `top bucket` 发生变化
- `shadow top bucket` 发生变化
这样能捕捉真正有训练价值的转折点。
### 5.3 建议的存储格式
建议新增一个文件，例如：
- `data/probability_training_snapshots.jsonl`
每一行保存一条 JSON 记录。
优点：
- 追加写入简单
- 后续导出训练集方便
- 不容易因为单个大 JSON 文件损坏而全盘受影响
## 6. 一条建议的快照结构
示例：
```json
{
"city": "ankara",
"timestamp": "2026-03-20T12:00:00+03:00",
"date": "2026-03-20",
"raw_mu": 15.2,
"raw_sigma": 1.2,
"deb_prediction": 15.4,
"ensemble": {
"p10": 14.8,
"median": 15.8,
"p90": 17.9
},
"multi_model": {
"ECMWF": 15.8,
"GFS": 14.1,
"ICON": 15.9,
"GEM": 16.5,
"JMA": 14.5
},
"max_so_far": 15.0,
"peak_status": "before",
"prob_snapshot": [
{"v": 15, "p": 0.552},
{"v": 16, "p": 0.377}
],
"shadow_prob_snapshot": [
{"v": 15, "p": 0.324},
{"v": 16, "p": 0.238}
],
"probability_engine": "legacy",
"probability_mode": "emos_shadow",
"calibration_version": "emos-20260320130245"
}
```
当天结束后，再由后处理脚本回填：
- `actual_high`
- `settlement_bucket`
## 7. 现阶段你可以执行的命令
### 7.1 回填历史天气 CSV
```bash
python scripts/backfill_historical_weather.py
```
作用：
- 补全 30 城市历史天气时序 CSV
### 7.2 从历史 CSV 构建日级结算标签
```bash
python scripts/build_settlement_history_from_csv.py
```
作用：
- 生成 [settlement_history.json](/E:/web/PolyWeather/artifacts/probability_calibration/settlement_history.json)
### 7.3 导出当前训练样本
```bash
python scripts/export_probability_training_dataset.py
```
作用：
- 生成 [training_samples.json](/E:/web/PolyWeather/artifacts/probability_calibration/training_samples.json)
### 7.4 重训 EMOS
```bash
python scripts/fit_probability_calibration.py
```
作用：
- 生成新的 [default.json](/E:/web/PolyWeather/artifacts/probability_calibration/default.json)
### 7.5 离线评估训练效果
```bash
python scripts/evaluate_probability_calibration.py
```
作用：
- 生成 [evaluation_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/evaluation_report.json)
### 7.6 回填 shadow 结果到历史记录
```bash
python scripts/backfill_probability_shadow_history.py
```
作用：
- 把 `shadow_prob_snapshot` 和 `probability_calibration` 回填到 [daily_records.json](/E:/web/PolyWeather/data/daily_records.json)
### 7.7 生成线上 shadow 滚动报表
```bash
python scripts/build_probability_shadow_report.py
```
作用：
- 生成 [shadow_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/shadow_report.json)
## 8. 推荐的一整套重训流程
如果过了十天、半个月，想重新训练一次，建议按这个顺序执行：
```bash
python scripts/build_settlement_history_from_csv.py
python scripts/export_probability_training_dataset.py
python scripts/fit_probability_calibration.py
python scripts/evaluate_probability_calibration.py
python scripts/backfill_probability_shadow_history.py
python scripts/build_probability_shadow_report.py
```
如果历史天气 CSV 还没补全，再先执行：
```bash
python scripts/backfill_historical_weather.py
```
## 9. 怎么判断这次训练有没有进步
重训后，不要只看一个指标。
至少看这 4 个：
1. `CRPS`
- 越低越好
2. `MAE`
- 越低越好
- 至少不要明显变差
3. `Bucket Hit Rate`
- 越高越好
- 这是业务上非常关键的指标
4. `Bucket Brier`
- 越低越好
- 反映概率分布质量
只有同时满足下面条件，才可以说训练效果真的进步：
- `CRPS` 下降
- `MAE` 不上升
- `Bucket Hit Rate` 不下降
- `Bucket Brier` 不上升
## 10. 当前最重要的现实判断
过去的“完整历史预测记录”通常没法完全补出来，除非：
1. 你之前就存过
2. 你接入了支持 forecast archive 的商业数据源
所以现实里最重要的不是“把过去全补齐”，而是：
- 从现在开始系统化归档
- 每天稳定沉淀可训练样本
- 定期离线重训
## 11. 推荐的下一步
最值得做的改造是：
1. 新增 `probability_training_snapshots.jsonl`
2. 每次分析时自动追加一条快照
3. 当天结束后自动回填 `actual_high`
4. 每 1-2 周重新训练一次
## 12. 总结
如果只记住一句话，就记这个：
**EMOS 要想越训越好，关键不是多下载一点历史天气，而是持续保存“当时系统看到的预测快照”。**
scripts/export_probability_training_dataset.py
+13 -2
View File
@@ -7,10 +7,11 @@ PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.deb_algorithm import load_history # noqa: E402
from scripts.fit_probability_calibration import ( # noqa: E402
_extract_samples,
_load_history_with_fallback,
_load_json_if_exists,
_load_snapshot_rows,
)
@@ -38,20 +39,30 @@ def main():
"training_samples.json",
),
)
parser.add_argument(
"--snapshot-file",
default=os.path.join(PROJECT_ROOT, "data", "probability_training_snapshots.jsonl"),
)
args = parser.parse_args()
history = load_history(args.history_file)
history = _load_history_with_fallback(args.history_file)
settlement_history = _load_json_if_exists(args.settlement_history)
snapshot_rows = _load_snapshot_rows(args.snapshot_file)
samples, filled_actual_from_history = _extract_samples(
history,
settlement_history=settlement_history,
snapshot_rows=snapshot_rows,
)
snapshot_count = sum(1 for sample in samples if sample.get("sample_source") == "snapshot")
daily_record_count = sum(1 for sample in samples if sample.get("sample_source") == "daily_record")
output_dir = os.path.dirname(os.path.abspath(args.output))
if output_dir:
os.makedirs(output_dir, exist_ok=True)
payload = {
"sample_count": len(samples),
"snapshot_sample_count": snapshot_count,
"daily_record_sample_count": daily_record_count,
"filled_actual_from_history": filled_actual_from_history,
"samples": samples,
}
scripts/fit_probability_calibration.py
+147 -2
View File
@@ -33,11 +33,128 @@ def _load_json_if_exists(path):
return data if isinstance(data, dict) else {}
def _extract_samples(history, settlement_history=None):
def _load_history_with_fallback(path):
data = load_history(path)
if data:
return data
return _load_json_if_exists(path)
def _load_snapshot_rows(path):
rows = []
if not path or not os.path.exists(path):
return rows
with open(path, "r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except Exception:
continue
if isinstance(row, dict):
rows.append(row)
return rows
def _actual_high_for(history, settlement_history, city, date_str):
city_rows = (history or {}).get(city) or {}
record = city_rows.get(date_str) or {}
actual_high = _sf(record.get("actual_high")) if isinstance(record, dict) else None
filled = False
if actual_high is None:
actual_high = _sf(((settlement_history.get(city) or {}).get(date_str) or {}).get("max_temp"))
filled = actual_high is not None
return actual_high, filled
def _extract_snapshot_samples(history, snapshot_rows, settlement_history=None):
samples = []
filled_actual_from_history = 0
today = datetime.utcnow().strftime("%Y-%m-%d")
settlement_history = settlement_history or {}
for row in snapshot_rows or []:
city = str(row.get("city") or "").strip().lower()
date_str = str(row.get("date") or "").strip()
if not city or not date_str or date_str == today:
continue
actual_high, filled = _actual_high_for(history, settlement_history, city, date_str)
if actual_high is None:
continue
if filled:
filled_actual_from_history += 1
raw_mu = _sf(row.get("raw_mu"))
raw_sigma = _sf(row.get("raw_sigma"))
deb_prediction = _sf(row.get("deb_prediction"))
ensemble = row.get("ensemble") or {}
if not isinstance(ensemble, dict):
ensemble = {}
ens_median = _sf(ensemble.get("median"))
ensemble_spread = None
ens_p10 = _sf(ensemble.get("p10"))
ens_p90 = _sf(ensemble.get("p90"))
if ens_p10 is not None and ens_p90 is not None and ens_p90 >= ens_p10:
ensemble_spread = max(0.1, (ens_p90 - ens_p10) / 2.56)
multi_model = row.get("multi_model") or {}
if not isinstance(multi_model, dict):
multi_model = {}
forecast_values = [val for val in (_sf(v) for v in multi_model.values()) if val is not None]
forecast_values.sort()
if ensemble_spread is None:
if len(forecast_values) >= 2:
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
elif raw_sigma is not None:
ensemble_spread = raw_sigma
else:
ensemble_spread = 1.0
if raw_sigma is None:
raw_sigma = ensemble_spread
peak_status = str(row.get("peak_status") or "before").strip().lower()
peak_flag = 0.0
if peak_status == "in_window":
peak_flag = 0.5
elif peak_status == "past":
peak_flag = 1.0
max_so_far = _sf(row.get("max_so_far"))
max_so_far_gap = None
if deb_prediction is not None and max_so_far is not None:
max_so_far_gap = deb_prediction - max_so_far
if raw_mu is None:
continue
samples.append(
{
"city": city,
"date": date_str,
"timestamp": row.get("timestamp"),
"actual_high": actual_high,
"raw_mu": raw_mu,
"raw_sigma": raw_sigma or 1.0,
"deb_prediction": deb_prediction,
"ens_median": ens_median if ens_median is not None else raw_mu,
"ensemble_spread": ensemble_spread,
"max_so_far_gap": max_so_far_gap,
"peak_flag": peak_flag,
"sample_source": "snapshot",
}
)
return samples, filled_actual_from_history
def _extract_daily_record_samples(history, settlement_history=None, excluded_keys=None):
samples = []
filled_actual_from_history = 0
today = datetime.utcnow().strftime("%Y-%m-%d")
settlement_history = settlement_history or {}
excluded_keys = excluded_keys or set()
for city, city_rows in (history or {}).items():
if not isinstance(city_rows, dict):
continue
@@ -45,6 +162,8 @@ def _extract_samples(history, settlement_history=None):
for date_str, record in city_rows.items():
if date_str == today or not isinstance(record, dict):
continue
if (city, date_str) in excluded_keys:
continue
actual_high = _sf(record.get("actual_high"))
if actual_high is None:
actual_high = _sf((city_settlement.get(date_str) or {}).get("max_temp"))
@@ -99,11 +218,30 @@ def _extract_samples(history, settlement_history=None):
"ensemble_spread": ensemble_spread,
"max_so_far_gap": max_so_far_gap,
"peak_flag": peak_flag,
"sample_source": "daily_record",
}
)
return samples, filled_actual_from_history
def _extract_samples(history, settlement_history=None, snapshot_rows=None):
snapshot_samples, snapshot_filled = _extract_snapshot_samples(
history,
snapshot_rows or [],
settlement_history=settlement_history,
)
excluded_keys = {
(sample["city"], sample["date"])
for sample in snapshot_samples
}
daily_samples, daily_filled = _extract_daily_record_samples(
history,
settlement_history=settlement_history,
excluded_keys=excluded_keys,
)
return snapshot_samples + daily_samples, snapshot_filled + daily_filled
def main():
parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.")
parser.add_argument(
@@ -126,6 +264,11 @@ def main():
),
help="Optional daily settlement history JSON built from historical CSV files.",
)
parser.add_argument(
"--snapshot-file",
default=os.path.join(PROJECT_ROOT, "data", "probability_training_snapshots.jsonl"),
help="Optional JSONL file with archived probability snapshots.",
)
parser.add_argument(
"--version",
default=None,
@@ -133,11 +276,13 @@ def main():
)
args = parser.parse_args()
history = load_history(args.history_file)
history = _load_history_with_fallback(args.history_file)
settlement_history = _load_json_if_exists(args.settlement_history)
snapshot_rows = _load_snapshot_rows(args.snapshot_file)
samples, filled_actual_from_history = _extract_samples(
history,
settlement_history=settlement_history,
snapshot_rows=snapshot_rows,
)
calibration = fit_calibration(samples, version=args.version)
if not samples:
src/analysis/probability_snapshot_archive.py
+187
View File
@@ -0,0 +1,187 @@
from __future__ import annotations
import json
import os
from datetime import datetime
from typing import Any, Dict, List, Optional
DEDUP_SCAN_LINES = 200
MU_THRESHOLD = 0.2
SIGMA_THRESHOLD = 0.15
MAX_SO_FAR_THRESHOLD = 0.2
def _sf(value: Any) -> Optional[float]:
if value is None:
return None
try:
return float(value)
except Exception:
return None
def _compact_snapshot(distribution: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
compact: List[Dict[str, Any]] = []
for row in distribution or []:
if not isinstance(row, dict):
continue
value = row.get("value")
probability = row.get("probability")
if value is None or probability is None:
continue
try:
compact.append(
{
"v": int(value),
"p": round(float(probability), 3),
}
)
except Exception:
continue
if len(compact) >= 4:
break
return compact
def _top_bucket(snapshot: Optional[List[Dict[str, Any]]]) -> Optional[int]:
best_value = None
best_prob = -1.0
for row in snapshot or []:
if not isinstance(row, dict):
continue
value = row.get("v")
prob = _sf(row.get("p"))
if value is None or prob is None:
continue
if prob > best_prob:
best_value = int(value)
best_prob = prob
return best_value
def _load_recent_rows(path: str, max_lines: int = DEDUP_SCAN_LINES) -> List[Dict[str, Any]]:
if not os.path.exists(path):
return []
with open(path, "r", encoding="utf-8") as fh:
lines = fh.readlines()[-max_lines:]
rows = []
for line in lines:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except Exception:
continue
if isinstance(row, dict):
rows.append(row)
return rows
def _should_skip_append(path: str, payload: Dict[str, Any]) -> bool:
recent_rows = _load_recent_rows(path)
city = payload.get("city")
date_str = payload.get("date")
if not city or not date_str:
return False
for row in reversed(recent_rows):
if row.get("city") != city or row.get("date") != date_str:
continue
if row.get("peak_status") != payload.get("peak_status"):
return False
if row.get("probability_mode") != payload.get("probability_mode"):
return False
current_top = _top_bucket(payload.get("prob_snapshot"))
previous_top = _top_bucket(row.get("prob_snapshot"))
current_shadow_top = _top_bucket(payload.get("shadow_prob_snapshot"))
previous_shadow_top = _top_bucket(row.get("shadow_prob_snapshot"))
if current_top != previous_top or current_shadow_top != previous_shadow_top:
return False
if abs((_sf(payload.get("raw_mu")) or 0.0) - (_sf(row.get("raw_mu")) or 0.0)) > MU_THRESHOLD:
return False
if abs((_sf(payload.get("raw_sigma")) or 0.0) - (_sf(row.get("raw_sigma")) or 0.0)) > SIGMA_THRESHOLD:
return False
if abs((_sf(payload.get("max_so_far")) or 0.0) - (_sf(row.get("max_so_far")) or 0.0)) > MAX_SO_FAR_THRESHOLD:
return False
return True
return False
def append_probability_snapshot(
city_name: str,
*,
local_date: str,
observation_time: Optional[str],
temp_symbol: str,
raw_mu: Optional[float],
raw_sigma: Optional[float],
deb_prediction: Optional[float],
ens_data: Optional[Dict[str, Any]],
current_forecasts: Optional[Dict[str, Any]],
max_so_far: Optional[float],
peak_status: Optional[str],
probabilities: Optional[List[Dict[str, Any]]],
shadow_probabilities: Optional[List[Dict[str, Any]]],
calibration_summary: Optional[Dict[str, Any]],
archive_path: Optional[str] = None,
) -> None:
city_key = str(city_name or "").strip().lower()
if not city_key:
return
root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
path = archive_path or os.path.join(
root_dir,
"data",
"probability_training_snapshots.jsonl",
)
calibration_summary = calibration_summary or {}
ens_data = ens_data or {}
current_forecasts = current_forecasts or {}
timestamp = str(observation_time or datetime.utcnow().isoformat() + "Z").strip()
payload = {
"city": city_key,
"timestamp": timestamp,
"date": local_date,
"temp_symbol": temp_symbol,
"raw_mu": _sf(raw_mu),
"raw_sigma": _sf(raw_sigma),
"deb_prediction": _sf(deb_prediction),
"ensemble": {
"p10": _sf(ens_data.get("p10")),
"median": _sf(ens_data.get("median")),
"p90": _sf(ens_data.get("p90")),
},
"multi_model": {
key: _sf(value)
for key, value in current_forecasts.items()
if _sf(value) is not None
},
"max_so_far": _sf(max_so_far),
"peak_status": peak_status,
"prob_snapshot": _compact_snapshot(probabilities),
"shadow_prob_snapshot": _compact_snapshot(shadow_probabilities),
"probability_engine": calibration_summary.get("engine"),
"probability_mode": calibration_summary.get("mode"),
"calibration_version": calibration_summary.get("calibration_version"),
"calibration_source": calibration_summary.get("calibration_source"),
"calibrated_mu": _sf(calibration_summary.get("calibrated_mu")),
"calibrated_sigma": _sf(calibration_summary.get("calibrated_sigma")),
}
parent = os.path.dirname(os.path.abspath(path))
if parent:
os.makedirs(parent, exist_ok=True)
if _should_skip_append(path, payload):
return
with open(path, "a", encoding="utf-8") as fh:
fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
src/analysis/trend_engine.py
+21
View File
@@ -19,6 +19,7 @@ from src.analysis.probability_calibration import (
apply_probability_calibration,
build_probability_features,
)
from src.analysis.probability_snapshot_archive import append_probability_snapshot
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
from src.data_collection.city_registry import CITY_REGISTRY
from src.data_collection.city_risk_profiles import get_city_risk_profile
@@ -733,6 +734,26 @@ def analyze_weather_trend(
except Exception:
pass
try:
append_probability_snapshot(
city_name=city_name or "",
local_date=local_date_str,
observation_time=obs_time_raw or local_time_full or None,
temp_symbol=temp_symbol,
raw_mu=calibration_summary.get("raw_mu"),
raw_sigma=calibration_summary.get("raw_sigma"),
deb_prediction=_deb_to_save,
ens_data=ens_data,
current_forecasts=current_forecasts,
max_so_far=max_so_far,
peak_status=peak_status,
probabilities=_prob_list,
shadow_probabilities=_shadow_prob_list,
calibration_summary=calibration_summary,
)
except Exception:
pass
# === Build recent list for trend_info ===
recent_list = []
for tm, t in recent_temps[:4]:
tests/test_probability_snapshot_archive.py
+117
View File
@@ -0,0 +1,117 @@
import json
from pathlib import Path
from src.analysis.probability_snapshot_archive import append_probability_snapshot
def test_append_probability_snapshot_writes_jsonl(tmp_path: Path):
archive_path = tmp_path / "probability_training_snapshots.jsonl"
append_probability_snapshot(
city_name="ankara",
local_date="2026-03-20",
observation_time="2026-03-20T12:00:00+03:00",
temp_symbol="°C",
raw_mu=15.2,
raw_sigma=1.2,
deb_prediction=15.4,
ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9},
current_forecasts={"ECMWF": 15.8, "GFS": 14.1},
max_so_far=15.0,
peak_status="before",
probabilities=[{"value": 15, "probability": 0.552}],
shadow_probabilities=[{"value": 15, "probability": 0.324}],
calibration_summary={
"engine": "legacy",
"mode": "emos_shadow",
"calibration_version": "emos-test",
"calibration_source": "artifacts/probability_calibration/default.json",
"calibrated_mu": 15.1,
"calibrated_sigma": 1.25,
},
archive_path=str(archive_path),
)
lines = archive_path.read_text(encoding="utf-8").strip().splitlines()
assert len(lines) == 1
payload = json.loads(lines[0])
assert payload["city"] == "ankara"
assert payload["date"] == "2026-03-20"
assert payload["raw_mu"] == 15.2
assert payload["ensemble"]["median"] == 15.8
assert payload["prob_snapshot"][0]["v"] == 15
assert payload["shadow_prob_snapshot"][0]["v"] == 15
assert payload["calibration_version"] == "emos-test"
def test_append_probability_snapshot_skips_near_duplicate(tmp_path: Path):
archive_path = tmp_path / "probability_training_snapshots.jsonl"
kwargs = dict(
city_name="ankara",
local_date="2026-03-20",
observation_time="2026-03-20T12:00:00+03:00",
temp_symbol="°C",
raw_mu=15.2,
raw_sigma=1.2,
deb_prediction=15.4,
ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9},
current_forecasts={"ECMWF": 15.8, "GFS": 14.1},
max_so_far=15.0,
peak_status="before",
probabilities=[{"value": 15, "probability": 0.552}],
shadow_probabilities=[{"value": 15, "probability": 0.324}],
calibration_summary={
"engine": "legacy",
"mode": "emos_shadow",
"calibration_version": "emos-test",
"calibration_source": "artifacts/probability_calibration/default.json",
"calibrated_mu": 15.1,
"calibrated_sigma": 1.25,
},
archive_path=str(archive_path),
)
append_probability_snapshot(**kwargs)
append_probability_snapshot(**kwargs)
lines = archive_path.read_text(encoding="utf-8").strip().splitlines()
assert len(lines) == 1
def test_append_probability_snapshot_writes_on_bucket_change(tmp_path: Path):
archive_path = tmp_path / "probability_training_snapshots.jsonl"
base_kwargs = dict(
city_name="ankara",
local_date="2026-03-20",
observation_time="2026-03-20T12:00:00+03:00",
temp_symbol="°C",
raw_mu=15.2,
raw_sigma=1.2,
deb_prediction=15.4,
ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9},
current_forecasts={"ECMWF": 15.8, "GFS": 14.1},
max_so_far=15.0,
peak_status="before",
shadow_probabilities=[{"value": 15, "probability": 0.324}],
calibration_summary={
"engine": "legacy",
"mode": "emos_shadow",
"calibration_version": "emos-test",
"calibration_source": "artifacts/probability_calibration/default.json",
"calibrated_mu": 15.1,
"calibrated_sigma": 1.25,
},
archive_path=str(archive_path),
)
append_probability_snapshot(
probabilities=[{"value": 15, "probability": 0.552}],
**base_kwargs,
)
append_probability_snapshot(
probabilities=[{"value": 16, "probability": 0.552}],
**base_kwargs,
)
lines = archive_path.read_text(encoding="utf-8").strip().splitlines()
assert len(lines) == 2
tests/test_probability_training_dataset.py
+45
View File
@@ -0,0 +1,45 @@
from scripts.fit_probability_calibration import _extract_samples
def test_extract_samples_prefers_snapshot_rows_for_same_city_day():
history = {
"ankara": {
"2026-03-19": {
"actual_high": 11.0,
"mu": 10.8,
"deb_prediction": 10.9,
"forecasts": {"ECMWF": 10.5, "GFS": 11.2},
"probability_features": {
"ens_median": 10.7,
"ensemble_spread": 0.8,
"peak_status": "before",
},
}
}
}
snapshot_rows = [
{
"city": "ankara",
"date": "2026-03-19",
"timestamp": "2026-03-19T12:00:00+03:00",
"raw_mu": 11.2,
"raw_sigma": 1.1,
"deb_prediction": 11.0,
"ensemble": {"p10": 10.0, "median": 11.1, "p90": 12.2},
"multi_model": {"ECMWF": 10.5, "GFS": 11.2},
"max_so_far": 10.9,
"peak_status": "in_window",
}
]
samples, filled = _extract_samples(
history,
settlement_history={},
snapshot_rows=snapshot_rows,
)
assert filled == 0
assert len(samples) == 1
assert samples[0]["sample_source"] == "snapshot"
assert samples[0]["raw_mu"] == 11.2
assert samples[0]["peak_flag"] == 0.5