From 3196552c781a5f7ac144c89a63cfb6b16377167c Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Fri, 20 Mar 2026 21:30:52 +0800 Subject: [PATCH] Archive probability snapshots and wire them into training --- .../probability_calibration/default.json | 4 +- .../evaluation_report.json | 120 +++---- .../training_samples.json | 317 +++++++++++------ data/probability_training_snapshots.jsonl | 1 + docs/PROBABILITY_SNAPSHOT_ARCHIVE_ZH.md | 324 ++++++++++++++++++ .../export_probability_training_dataset.py | 15 +- scripts/fit_probability_calibration.py | 149 +++++++- src/analysis/probability_snapshot_archive.py | 187 ++++++++++ src/analysis/trend_engine.py | 21 ++ tests/test_probability_snapshot_archive.py | 117 +++++++ tests/test_probability_training_dataset.py | 45 +++ 11 files changed, 1129 insertions(+), 171 deletions(-) create mode 100644 data/probability_training_snapshots.jsonl create mode 100644 docs/PROBABILITY_SNAPSHOT_ARCHIVE_ZH.md create mode 100644 src/analysis/probability_snapshot_archive.py create mode 100644 tests/test_probability_snapshot_archive.py create mode 100644 tests/test_probability_training_dataset.py diff --git a/artifacts/probability_calibration/default.json b/artifacts/probability_calibration/default.json index 80ebc4b4..281fbf86 100644 --- a/artifacts/probability_calibration/default.json +++ b/artifacts/probability_calibration/default.json @@ -1,6 +1,6 @@ { - "version": "emos-20260320130245", - "trained_at": "2026-03-20T13:02:45.903772+00:00", + "version": "emos-20260320132525", + "trained_at": "2026-03-20T13:25:25.836021+00:00", "global": { "mu": { "intercept": -1.57406048, diff --git a/artifacts/probability_calibration/evaluation_report.json b/artifacts/probability_calibration/evaluation_report.json index 9bb1db0c..f3b787d1 100644 --- a/artifacts/probability_calibration/evaluation_report.json +++ b/artifacts/probability_calibration/evaluation_report.json @@ -8,239 +8,239 @@ "bucket_hit_rate": 0.695238 }, "emos": { - "mean_crps": 2.650216, - "mean_mae": 2.722829, - "bucket_hit_rate": 0.666667 + "mean_crps": 2.700275, + "mean_mae": 2.721143, + "bucket_hit_rate": 0.695238 }, "delta": { - "crps": -0.143722, - "mae": 0.001686, - "bucket_hit_rate": -0.028571 + "crps": -0.093663, + "mae": 0.0, + "bucket_hit_rate": 0.0 } }, "by_city": { "ankara": { "samples": 7, "legacy_mean_crps": 2.023242, - "emos_mean_crps": 2.015458, + "emos_mean_crps": 2.023242, "legacy_mean_mae": 1.984286, - "emos_mean_mae": 1.977721, + "emos_mean_mae": 1.984286, "legacy_bucket_hit_rate": 0.714286, "emos_bucket_hit_rate": 0.714286 }, "atlanta": { "samples": 5, "legacy_mean_crps": 12.792034, - "emos_mean_crps": 12.526108, + "emos_mean_crps": 12.694543, "legacy_mean_mae": 12.806, - "emos_mean_mae": 12.919677, + "emos_mean_mae": 12.806, "legacy_bucket_hit_rate": 0.6, - "emos_bucket_hit_rate": 0.4 + "emos_bucket_hit_rate": 0.6 }, "buenos aires": { "samples": 5, "legacy_mean_crps": 3.846144, - "emos_mean_crps": 3.743967, + "emos_mean_crps": 3.846144, "legacy_mean_mae": 4.168, - "emos_mean_mae": 4.086663, + "emos_mean_mae": 4.168, "legacy_bucket_hit_rate": 0.6, "emos_bucket_hit_rate": 0.6 }, "chicago": { "samples": 4, "legacy_mean_crps": 1.346667, - "emos_mean_crps": 0.603128, + "emos_mean_crps": 0.601765, "legacy_mean_mae": 0.0, - "emos_mean_mae": 0.075265, + "emos_mean_mae": 0.0, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 1.0 }, "dallas": { "samples": 4, "legacy_mean_crps": 1.256111, - "emos_mean_crps": 0.65514, + "emos_mean_crps": 0.651425, "legacy_mean_mae": 0.0, - "emos_mean_mae": 0.114807, + "emos_mean_mae": 0.0, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 1.0 }, "hong kong": { "samples": 3, "legacy_mean_crps": 0.261027, - "emos_mean_crps": 0.269856, + "emos_mean_crps": 0.261027, "legacy_mean_mae": 0.1, - "emos_mean_mae": 0.174879, + "emos_mean_mae": 0.1, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 0.666667 }, "london": { "samples": 6, "legacy_mean_crps": 2.079624, - "emos_mean_crps": 2.005311, + "emos_mean_crps": 2.079624, "legacy_mean_mae": 2.451667, - "emos_mean_mae": 2.327304, + "emos_mean_mae": 2.451667, "legacy_bucket_hit_rate": 0.166667, - "emos_bucket_hit_rate": 0.333333 + "emos_bucket_hit_rate": 0.166667 }, "lucknow": { "samples": 4, "legacy_mean_crps": 1.468528, - "emos_mean_crps": 1.443159, + "emos_mean_crps": 1.468528, "legacy_mean_mae": 1.6025, - "emos_mean_mae": 1.682697, + "emos_mean_mae": 1.6025, "legacy_bucket_hit_rate": 0.5, "emos_bucket_hit_rate": 0.5 }, "madrid": { "samples": 2, "legacy_mean_crps": 6.27726, - "emos_mean_crps": 6.090563, + "emos_mean_crps": 6.27726, "legacy_mean_mae": 7.33, - "emos_mean_mae": 7.141649, + "emos_mean_mae": 7.33, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "miami": { "samples": 5, "legacy_mean_crps": 11.665378, - "emos_mean_crps": 11.493094, + "emos_mean_crps": 11.665378, "legacy_mean_mae": 12.07, - "emos_mean_mae": 12.157623, + "emos_mean_mae": 12.07, "legacy_bucket_hit_rate": 0.6, - "emos_bucket_hit_rate": 0.2 + "emos_bucket_hit_rate": 0.6 }, "milan": { "samples": 3, "legacy_mean_crps": 4.401392, - "emos_mean_crps": 3.837715, + "emos_mean_crps": 3.928883, "legacy_mean_mae": 4.06, - "emos_mean_mae": 3.942288, + "emos_mean_mae": 4.06, "legacy_bucket_hit_rate": 0.666667, "emos_bucket_hit_rate": 0.666667 }, "munich": { "samples": 6, "legacy_mean_crps": 2.988583, - "emos_mean_crps": 2.918299, + "emos_mean_crps": 2.988583, "legacy_mean_mae": 3.143333, - "emos_mean_mae": 3.126482, + "emos_mean_mae": 3.143333, "legacy_bucket_hit_rate": 0.5, "emos_bucket_hit_rate": 0.5 }, "new york": { "samples": 4, "legacy_mean_crps": 1.861101, - "emos_mean_crps": 1.363203, + "emos_mean_crps": 1.409393, "legacy_mean_mae": 1.3725, - "emos_mean_mae": 1.369301, + "emos_mean_mae": 1.3725, "legacy_bucket_hit_rate": 0.75, "emos_bucket_hit_rate": 0.75 }, "paris": { "samples": 7, "legacy_mean_crps": 2.430082, - "emos_mean_crps": 2.365416, + "emos_mean_crps": 2.430082, "legacy_mean_mae": 2.518571, - "emos_mean_mae": 2.48437, + "emos_mean_mae": 2.518571, "legacy_bucket_hit_rate": 0.571429, "emos_bucket_hit_rate": 0.571429 }, "sao paulo": { "samples": 5, "legacy_mean_crps": 2.454756, - "emos_mean_crps": 2.410624, + "emos_mean_crps": 2.454756, "legacy_mean_mae": 2.628, - "emos_mean_mae": 2.689909, + "emos_mean_mae": 2.628, "legacy_bucket_hit_rate": 0.6, "emos_bucket_hit_rate": 0.6 }, "seattle": { "samples": 4, "legacy_mean_crps": 0.531656, - "emos_mean_crps": 0.457257, + "emos_mean_crps": 0.452784, "legacy_mean_mae": 0.0, - "emos_mean_mae": 0.11262, + "emos_mean_mae": 0.0, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 1.0 }, "seoul": { "samples": 6, "legacy_mean_crps": 0.328088, - "emos_mean_crps": 0.332722, + "emos_mean_crps": 0.328088, "legacy_mean_mae": 0.2, - "emos_mean_mae": 0.246659, + "emos_mean_mae": 0.2, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 1.0 }, "shanghai": { "samples": 2, "legacy_mean_crps": 0.250034, - "emos_mean_crps": 0.249952, + "emos_mean_crps": 0.250034, "legacy_mean_mae": 0.15, - "emos_mean_mae": 0.204362, + "emos_mean_mae": 0.15, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 1.0 }, "singapore": { "samples": 2, "legacy_mean_crps": 0.281993, - "emos_mean_crps": 0.270875, + "emos_mean_crps": 0.281993, "legacy_mean_mae": 0.15, - "emos_mean_mae": 0.070547, + "emos_mean_mae": 0.15, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 1.0 }, "taipei": { "samples": 3, "legacy_mean_crps": 0.356996, - "emos_mean_crps": 0.351432, + "emos_mean_crps": 0.356996, "legacy_mean_mae": 0.1, - "emos_mean_mae": 0.084784, + "emos_mean_mae": 0.1, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.666667 + "emos_bucket_hit_rate": 1.0 }, "tel aviv": { "samples": 2, "legacy_mean_crps": 0.446758, - "emos_mean_crps": 0.434936, + "emos_mean_crps": 0.446758, "legacy_mean_mae": 0.3, - "emos_mean_mae": 0.196589, + "emos_mean_mae": 0.3, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 1.0 }, "tokyo": { "samples": 2, "legacy_mean_crps": 0.450128, - "emos_mean_crps": 0.429389, + "emos_mean_crps": 0.450128, "legacy_mean_mae": 0.25, - "emos_mean_mae": 0.100782, + "emos_mean_mae": 0.25, "legacy_bucket_hit_rate": 0.5, "emos_bucket_hit_rate": 1.0 }, "toronto": { "samples": 5, "legacy_mean_crps": 2.647861, - "emos_mean_crps": 2.512236, + "emos_mean_crps": 2.566068, "legacy_mean_mae": 2.532, - "emos_mean_mae": 2.535078, + "emos_mean_mae": 2.532, "legacy_bucket_hit_rate": 0.6, "emos_bucket_hit_rate": 0.6 }, "warsaw": { "samples": 3, "legacy_mean_crps": 1.618875, - "emos_mean_crps": 1.545129, + "emos_mean_crps": 1.618875, "legacy_mean_mae": 2.056667, - "emos_mean_mae": 1.942917, + "emos_mean_mae": 2.056667, "legacy_bucket_hit_rate": 0.333333, "emos_bucket_hit_rate": 0.333333 }, "wellington": { "samples": 6, "legacy_mean_crps": 0.266349, - "emos_mean_crps": 0.264506, + "emos_mean_crps": 0.266349, "legacy_mean_mae": 0.2, - "emos_mean_mae": 0.205623, + "emos_mean_mae": 0.2, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 1.0 } diff --git a/artifacts/probability_calibration/training_samples.json b/artifacts/probability_calibration/training_samples.json index 721c2ddb..f9991ffa 100644 --- a/artifacts/probability_calibration/training_samples.json +++ b/artifacts/probability_calibration/training_samples.json @@ -1,5 +1,7 @@ { "sample_count": 105, + "snapshot_sample_count": 0, + "daily_record_sample_count": 105, "filled_actual_from_history": 2, "samples": [ { @@ -12,7 +14,8 @@ "ens_median": 5.9, "ensemble_spread": 1.5499999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "ankara", @@ -24,7 +27,8 @@ "ens_median": 7.6, "ensemble_spread": 1.75, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "ankara", @@ -36,7 +40,8 @@ "ens_median": 7.9, "ensemble_spread": 1.5499999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "ankara", @@ -48,7 +53,8 @@ "ens_median": 10.0, "ensemble_spread": 0.8500000000000005, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "ankara", @@ -60,7 +66,8 @@ "ens_median": 13.9, "ensemble_spread": 1.75, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "ankara", @@ -72,7 +79,8 @@ "ens_median": 15.8, "ensemble_spread": 1.2000000000000002, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "ankara", @@ -84,7 +92,8 @@ "ens_median": 10.1, "ensemble_spread": 2.05, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "london", @@ -96,7 +105,8 @@ "ens_median": 10.1, "ensemble_spread": 0.9000000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "london", @@ -108,7 +118,8 @@ "ens_median": 11.4, "ensemble_spread": 2.1500000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "london", @@ -120,7 +131,8 @@ "ens_median": 15.4, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "london", @@ -132,7 +144,8 @@ "ens_median": 13.3, "ensemble_spread": 0.9500000000000002, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "london", @@ -144,7 +157,8 @@ "ens_median": 16.8, "ensemble_spread": 1.299999999999999, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "london", @@ -156,7 +170,8 @@ "ens_median": 15.8, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "new york", @@ -168,7 +183,8 @@ "ens_median": 46.7, "ensemble_spread": 9.7, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "new york", @@ -180,7 +196,8 @@ "ens_median": 67.2, "ensemble_spread": 2.1499999999999986, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "new york", @@ -192,7 +209,8 @@ "ens_median": 66.6, "ensemble_spread": 4.149999999999999, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "new york", @@ -204,7 +222,8 @@ "ens_median": 37.0, "ensemble_spread": 2.25, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "paris", @@ -216,7 +235,8 @@ "ens_median": 16.7, "ensemble_spread": 0.7000000000000011, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "paris", @@ -228,7 +248,8 @@ "ens_median": 16.2, "ensemble_spread": 1.5999999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "paris", @@ -240,7 +261,8 @@ "ens_median": 17.4, "ensemble_spread": 0.6999999999999993, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "paris", @@ -252,7 +274,8 @@ "ens_median": 15.3, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "paris", @@ -264,7 +287,8 @@ "ens_median": 13.1, "ensemble_spread": 0.8499999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "paris", @@ -276,7 +300,8 @@ "ens_median": 14.8, "ensemble_spread": 0.9000000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "paris", @@ -288,7 +313,8 @@ "ens_median": 16.3, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seoul", @@ -300,7 +326,8 @@ "ens_median": 4.0, "ensemble_spread": 0.7999999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seoul", @@ -312,7 +339,8 @@ "ens_median": 6.3, "ensemble_spread": 1.4, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seoul", @@ -324,7 +352,8 @@ "ens_median": 4.9, "ensemble_spread": 1.35, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seoul", @@ -336,7 +365,8 @@ "ens_median": 9.1, "ensemble_spread": 1.7999999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seoul", @@ -348,7 +378,8 @@ "ens_median": 7.4, "ensemble_spread": 0.7999999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seoul", @@ -360,7 +391,8 @@ "ens_median": 6.3, "ensemble_spread": 1.7999999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "toronto", @@ -372,7 +404,8 @@ "ens_median": 16.9, "ensemble_spread": 2.8500000000000005, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "toronto", @@ -384,7 +417,8 @@ "ens_median": 12.3, "ensemble_spread": 2.0, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "toronto", @@ -396,7 +430,8 @@ "ens_median": 17.6, "ensemble_spread": 4.75, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "toronto", @@ -408,7 +443,8 @@ "ens_median": -1.1, "ensemble_spread": 0.8, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "toronto", @@ -420,7 +456,8 @@ "ens_median": 6.3, "ensemble_spread": 2.1500000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "buenos aires", @@ -432,7 +469,8 @@ "ens_median": 26.0, "ensemble_spread": 1.3999999999999986, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "buenos aires", @@ -444,7 +482,8 @@ "ens_median": 26.4, "ensemble_spread": 1.200000000000001, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "buenos aires", @@ -456,7 +495,8 @@ "ens_median": 26.2, "ensemble_spread": 1.5999999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "buenos aires", @@ -468,7 +508,8 @@ "ens_median": 26.0, "ensemble_spread": 2.0, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "buenos aires", @@ -480,7 +521,8 @@ "ens_median": 27.9, "ensemble_spread": 2.0999999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "wellington", @@ -492,7 +534,8 @@ "ens_median": 18.4, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "wellington", @@ -504,7 +547,8 @@ "ens_median": 17.2, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "wellington", @@ -516,7 +560,8 @@ "ens_median": 15.7, "ensemble_spread": 0.9000000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "wellington", @@ -528,7 +573,8 @@ "ens_median": 18.7, "ensemble_spread": 1.049999999999999, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "wellington", @@ -540,7 +586,8 @@ "ens_median": 19.1, "ensemble_spread": 0.9500000000000011, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "wellington", @@ -552,7 +599,8 @@ "ens_median": 17.1, "ensemble_spread": 2.0999999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "chicago", @@ -564,7 +612,8 @@ "ens_median": 62.8, "ensemble_spread": 1.3000000000000007, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "chicago", @@ -576,7 +625,8 @@ "ens_median": 60.1, "ensemble_spread": 5.600000000000001, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "chicago", @@ -588,7 +638,8 @@ "ens_median": 70.8, "ensemble_spread": 10.799999999999997, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "chicago", @@ -600,7 +651,8 @@ "ens_median": 44.2, "ensemble_spread": 5.350000000000001, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "sao paulo", @@ -612,7 +664,8 @@ "ens_median": 27.4, "ensemble_spread": 1.5, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "sao paulo", @@ -624,7 +677,8 @@ "ens_median": 28.0, "ensemble_spread": 2.1999999999999993, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "sao paulo", @@ -636,7 +690,8 @@ "ens_median": 23.5, "ensemble_spread": 1.4000000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "sao paulo", @@ -648,7 +703,8 @@ "ens_median": 29.4, "ensemble_spread": 2.450000000000001, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "sao paulo", @@ -660,7 +716,8 @@ "ens_median": 26.5, "ensemble_spread": 1.200000000000001, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "dallas", @@ -672,7 +729,8 @@ "ens_median": 72.7, "ensemble_spread": 2.1499999999999986, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "dallas", @@ -684,7 +742,8 @@ "ens_median": 71.5, "ensemble_spread": 6.200000000000003, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "dallas", @@ -696,7 +755,8 @@ "ens_median": 81.0, "ensemble_spread": 3.8500000000000014, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "dallas", @@ -708,7 +768,8 @@ "ens_median": 74.4, "ensemble_spread": 9.299999999999997, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "miami", @@ -720,7 +781,8 @@ "ens_median": 81.6, "ensemble_spread": 1.3999999999999986, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "miami", @@ -732,7 +794,8 @@ "ens_median": 82.7, "ensemble_spread": 1.5500000000000043, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "miami", @@ -744,7 +807,8 @@ "ens_median": 83.9, "ensemble_spread": 1.5499999999999972, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "miami", @@ -756,7 +820,8 @@ "ens_median": 70.4, "ensemble_spread": 2.8999999999999986, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "miami", @@ -768,7 +833,8 @@ "ens_median": 74.0, "ensemble_spread": 2.5500000000000043, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "atlanta", @@ -780,7 +846,8 @@ "ens_median": 79.7, "ensemble_spread": 4.600000000000001, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "atlanta", @@ -792,7 +859,8 @@ "ens_median": 67.7, "ensemble_spread": 5.899999999999999, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "atlanta", @@ -804,7 +872,8 @@ "ens_median": 77.4, "ensemble_spread": 2.6000000000000014, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "atlanta", @@ -816,7 +885,8 @@ "ens_median": 52.2, "ensemble_spread": 4.0, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "atlanta", @@ -828,7 +898,8 @@ "ens_median": 65.9, "ensemble_spread": 1.5500000000000007, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seattle", @@ -840,7 +911,8 @@ "ens_median": 52.8, "ensemble_spread": 1.3999999999999986, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seattle", @@ -852,7 +924,8 @@ "ens_median": 50.9, "ensemble_spread": 4.350000000000001, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seattle", @@ -864,7 +937,8 @@ "ens_median": 45.1, "ensemble_spread": 2.0, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "seattle", @@ -876,7 +950,8 @@ "ens_median": 55.9, "ensemble_spread": 1.3500000000000014, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "lucknow", @@ -888,7 +963,8 @@ "ens_median": 33.9, "ensemble_spread": 1.8500000000000014, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "lucknow", @@ -900,7 +976,8 @@ "ens_median": 34.6, "ensemble_spread": 2.0, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "lucknow", @@ -912,7 +989,8 @@ "ens_median": 34.6, "ensemble_spread": 1.3000000000000007, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "lucknow", @@ -924,7 +1002,8 @@ "ens_median": 34.7, "ensemble_spread": 1.75, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "munich", @@ -936,7 +1015,8 @@ "ens_median": 15.1, "ensemble_spread": 1.6000000000000005, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "munich", @@ -948,7 +1028,8 @@ "ens_median": 14.3, "ensemble_spread": 0.9000000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "munich", @@ -960,7 +1041,8 @@ "ens_median": 14.5, "ensemble_spread": 0.8999999999999995, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "munich", @@ -972,7 +1054,8 @@ "ens_median": 15.7, "ensemble_spread": 1.2999999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "munich", @@ -984,7 +1067,8 @@ "ens_median": 10.6, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "munich", @@ -996,7 +1080,8 @@ "ens_median": 11.8, "ensemble_spread": 1.3000000000000007, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "hong kong", @@ -1008,7 +1093,8 @@ "ens_median": 23.1, "ensemble_spread": 1.9000000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "hong kong", @@ -1020,7 +1106,8 @@ "ens_median": 24.9, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "hong kong", @@ -1032,7 +1119,8 @@ "ens_median": 25.0, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "taipei", @@ -1044,7 +1132,8 @@ "ens_median": 25.4, "ensemble_spread": 2.25, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "taipei", @@ -1056,7 +1145,8 @@ "ens_median": 27.5, "ensemble_spread": 1.0500000000000007, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "taipei", @@ -1068,7 +1158,8 @@ "ens_median": 21.3, "ensemble_spread": 1.1500000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "milan", @@ -1080,7 +1171,8 @@ "ens_median": 15.1, "ensemble_spread": 9.1, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "milan", @@ -1092,7 +1184,8 @@ "ens_median": 13.8, "ensemble_spread": 0.6, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "milan", @@ -1104,7 +1197,8 @@ "ens_median": 17.0, "ensemble_spread": 1.25, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "warsaw", @@ -1116,7 +1210,8 @@ "ens_median": 10.5, "ensemble_spread": 0.6499999999999995, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "warsaw", @@ -1128,7 +1223,8 @@ "ens_median": 14.2, "ensemble_spread": 1.4000000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "warsaw", @@ -1140,7 +1236,8 @@ "ens_median": 12.3, "ensemble_spread": 1.5999999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "singapore", @@ -1152,7 +1249,8 @@ "ens_median": 30.1, "ensemble_spread": 0.9500000000000011, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "singapore", @@ -1164,7 +1262,8 @@ "ens_median": 32.1, "ensemble_spread": 1.3499999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "shanghai", @@ -1176,7 +1275,8 @@ "ens_median": 13.0, "ensemble_spread": 1.1500000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "shanghai", @@ -1188,7 +1288,8 @@ "ens_median": 11.2, "ensemble_spread": 0.7999999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "tokyo", @@ -1200,7 +1301,8 @@ "ens_median": 15.8, "ensemble_spread": 1.5499999999999998, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "tokyo", @@ -1212,7 +1314,8 @@ "ens_median": 18.5, "ensemble_spread": 2.0999999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "tel aviv", @@ -1224,7 +1327,8 @@ "ens_median": 28.9, "ensemble_spread": 2.1500000000000004, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "tel aviv", @@ -1236,7 +1340,8 @@ "ens_median": 21.1, "ensemble_spread": 1.5, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "madrid", @@ -1248,7 +1353,8 @@ "ens_median": 18.7, "ensemble_spread": 1.8499999999999996, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" }, { "city": "madrid", @@ -1260,7 +1366,8 @@ "ens_median": 18.8, "ensemble_spread": 1.8999999999999995, "max_so_far_gap": null, - "peak_flag": 0.0 + "peak_flag": 0.0, + "sample_source": "daily_record" } ] } \ No newline at end of file diff --git a/data/probability_training_snapshots.jsonl b/data/probability_training_snapshots.jsonl new file mode 100644 index 00000000..bb34e8e7 --- /dev/null +++ b/data/probability_training_snapshots.jsonl @@ -0,0 +1 @@ +{"city": "ankara", "timestamp": "2026-03-20T12:00:00+03:00", "date": "2026-03-20", "temp_symbol": "°C", "raw_mu": 15.2, "raw_sigma": 1.2, "deb_prediction": 15.4, "ensemble": {"p10": 14.8, "median": 15.8, "p90": 17.9}, "multi_model": {"ECMWF": 15.8, "GFS": 14.1, "ICON": 15.9}, "max_so_far": 15.0, "peak_status": "before", "prob_snapshot": [{"v": 15, "p": 0.552}, {"v": 16, "p": 0.377}], "shadow_prob_snapshot": [{"v": 15, "p": 0.324}, {"v": 16, "p": 0.238}], "probability_engine": "legacy", "probability_mode": "emos_shadow", "calibration_version": "emos-20260320130245", "calibration_source": "artifacts/probability_calibration/default.json", "calibrated_mu": 15.1, "calibrated_sigma": 1.25} diff --git a/docs/PROBABILITY_SNAPSHOT_ARCHIVE_ZH.md b/docs/PROBABILITY_SNAPSHOT_ARCHIVE_ZH.md new file mode 100644 index 00000000..00471ed5 --- /dev/null +++ b/docs/PROBABILITY_SNAPSHOT_ARCHIVE_ZH.md @@ -0,0 +1,324 @@ +# 概率训练样本归档说明(中文) + +## 1. 目的 + +这份文档说明两件事: + +1. 为什么 `EMOS` 训练不能只依赖历史实测天气 +2. 未来如何持续沉淀“历史预测记录”,让概率引擎越训越稳 + +一句话结论: + +- 历史实测天气只能补 `actual_high` +- 真正决定 `EMOS` 训练质量的是“当时那一刻的预测快照” + +## 2. 什么是“历史预测记录” + +对 PolyWeather 来说,一条可训练的历史预测记录,至少应该包含这些字段: + +- `city` +- `timestamp` +- `date` +- `raw_mu` +- `raw_sigma` +- `deb_prediction` +- `ensemble p10 / p50 / p90` +- `multi-model forecasts` +- `max_so_far` +- `peak_status` +- `prob_snapshot` +- 当天最终 `actual_high` +- 当天最终 `settlement bucket` + +这类记录的核心价值是: + +- 还原“当时系统实际看到什么” +- 再对照“后来真实发生了什么” + +只有这两者成对,`EMOS` 才能学习偏差。 + +## 3. 为什么不能只用历史天气实测 + +历史天气 CSV 只能告诉你: + +- 当天最高温是多少 +- 某小时温度是多少 + +但它不能告诉你: + +- 当天早上 09:00 时,系统的 `mu` 是多少 +- 当时的 `ensemble spread` 是多少 +- 当时 `DEB` 怎么看 +- 当时的 top bucket 是什么 + +所以: + +- 历史实测天气是标签 +- 历史预测记录才是训练输入 + +缺少后者,EMOS 只能学到很有限的东西。 + +## 4. 当前项目里已经有的基础 + +### 4.1 已有历史日记录 + +文件: + +- [daily_records.json](/E:/web/PolyWeather/data/daily_records.json) + +当前已经保存了一部分训练相关字段,例如: + +- `forecasts` +- `actual_high` +- `deb_prediction` +- `mu` +- `prob_snapshot` +- `shadow_prob_snapshot` +- `probability_calibration` +- `probability_features` + +这已经是“历史预测记录”的雏形。 + +### 4.2 已有历史天气 CSV + +目录: + +- [data/historical](/E:/web/PolyWeather/data/historical) + +它们可以帮助补: + +- `actual_high` +- `settlement history` + +但不能替代预测快照归档。 + +## 5. 未来应该怎么存历史预测记录 + +推荐做法是: + +### 5.1 固定时点归档 + +每天为每个重点城市固定存几次快照,例如: + +- 当地 `09:00` +- 当地 `12:00` +- 当地 `15:00` + +这样能确保每个交易日都有稳定可比样本。 + +### 5.2 关键变化时补充归档 + +除了固定时点,还应该在以下情况额外存一次: + +- `max_so_far` 创新高 +- `mu` 变化超过阈值 +- `top bucket` 发生变化 +- `shadow top bucket` 发生变化 + +这样能捕捉真正有训练价值的转折点。 + +### 5.3 建议的存储格式 + +建议新增一个文件,例如: + +- `data/probability_training_snapshots.jsonl` + +每一行保存一条 JSON 记录。 + +优点: + +- 追加写入简单 +- 后续导出训练集方便 +- 不容易因为单个大 JSON 文件损坏而全盘受影响 + +## 6. 一条建议的快照结构 + +示例: + +```json +{ + "city": "ankara", + "timestamp": "2026-03-20T12:00:00+03:00", + "date": "2026-03-20", + "raw_mu": 15.2, + "raw_sigma": 1.2, + "deb_prediction": 15.4, + "ensemble": { + "p10": 14.8, + "median": 15.8, + "p90": 17.9 + }, + "multi_model": { + "ECMWF": 15.8, + "GFS": 14.1, + "ICON": 15.9, + "GEM": 16.5, + "JMA": 14.5 + }, + "max_so_far": 15.0, + "peak_status": "before", + "prob_snapshot": [ + {"v": 15, "p": 0.552}, + {"v": 16, "p": 0.377} + ], + "shadow_prob_snapshot": [ + {"v": 15, "p": 0.324}, + {"v": 16, "p": 0.238} + ], + "probability_engine": "legacy", + "probability_mode": "emos_shadow", + "calibration_version": "emos-20260320130245" +} +``` + +当天结束后,再由后处理脚本回填: + +- `actual_high` +- `settlement_bucket` + +## 7. 现阶段你可以执行的命令 + +### 7.1 回填历史天气 CSV + +```bash +python scripts/backfill_historical_weather.py +``` + +作用: + +- 补全 30 城市历史天气时序 CSV + +### 7.2 从历史 CSV 构建日级结算标签 + +```bash +python scripts/build_settlement_history_from_csv.py +``` + +作用: + +- 生成 [settlement_history.json](/E:/web/PolyWeather/artifacts/probability_calibration/settlement_history.json) + +### 7.3 导出当前训练样本 + +```bash +python scripts/export_probability_training_dataset.py +``` + +作用: + +- 生成 [training_samples.json](/E:/web/PolyWeather/artifacts/probability_calibration/training_samples.json) + +### 7.4 重训 EMOS + +```bash +python scripts/fit_probability_calibration.py +``` + +作用: + +- 生成新的 [default.json](/E:/web/PolyWeather/artifacts/probability_calibration/default.json) + +### 7.5 离线评估训练效果 + +```bash +python scripts/evaluate_probability_calibration.py +``` + +作用: + +- 生成 [evaluation_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/evaluation_report.json) + +### 7.6 回填 shadow 结果到历史记录 + +```bash +python scripts/backfill_probability_shadow_history.py +``` + +作用: + +- 把 `shadow_prob_snapshot` 和 `probability_calibration` 回填到 [daily_records.json](/E:/web/PolyWeather/data/daily_records.json) + +### 7.7 生成线上 shadow 滚动报表 + +```bash +python scripts/build_probability_shadow_report.py +``` + +作用: + +- 生成 [shadow_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/shadow_report.json) + +## 8. 推荐的一整套重训流程 + +如果过了十天、半个月,想重新训练一次,建议按这个顺序执行: + +```bash +python scripts/build_settlement_history_from_csv.py +python scripts/export_probability_training_dataset.py +python scripts/fit_probability_calibration.py +python scripts/evaluate_probability_calibration.py +python scripts/backfill_probability_shadow_history.py +python scripts/build_probability_shadow_report.py +``` + +如果历史天气 CSV 还没补全,再先执行: + +```bash +python scripts/backfill_historical_weather.py +``` + +## 9. 怎么判断这次训练有没有进步 + +重训后,不要只看一个指标。 + +至少看这 4 个: + +1. `CRPS` +- 越低越好 + +2. `MAE` +- 越低越好 +- 至少不要明显变差 + +3. `Bucket Hit Rate` +- 越高越好 +- 这是业务上非常关键的指标 + +4. `Bucket Brier` +- 越低越好 +- 反映概率分布质量 + +只有同时满足下面条件,才可以说训练效果真的进步: + +- `CRPS` 下降 +- `MAE` 不上升 +- `Bucket Hit Rate` 不下降 +- `Bucket Brier` 不上升 + +## 10. 当前最重要的现实判断 + +过去的“完整历史预测记录”通常没法完全补出来,除非: + +1. 你之前就存过 +2. 你接入了支持 forecast archive 的商业数据源 + +所以现实里最重要的不是“把过去全补齐”,而是: + +- 从现在开始系统化归档 +- 每天稳定沉淀可训练样本 +- 定期离线重训 + +## 11. 推荐的下一步 + +最值得做的改造是: + +1. 新增 `probability_training_snapshots.jsonl` +2. 每次分析时自动追加一条快照 +3. 当天结束后自动回填 `actual_high` +4. 每 1-2 周重新训练一次 + +## 12. 总结 + +如果只记住一句话,就记这个: + +**EMOS 要想越训越好,关键不是多下载一点历史天气,而是持续保存“当时系统看到的预测快照”。** diff --git a/scripts/export_probability_training_dataset.py b/scripts/export_probability_training_dataset.py index 420f8785..69890a5a 100644 --- a/scripts/export_probability_training_dataset.py +++ b/scripts/export_probability_training_dataset.py @@ -7,10 +7,11 @@ PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if PROJECT_ROOT not in sys.path: sys.path.insert(0, PROJECT_ROOT) -from src.analysis.deb_algorithm import load_history # noqa: E402 from scripts.fit_probability_calibration import ( # noqa: E402 _extract_samples, + _load_history_with_fallback, _load_json_if_exists, + _load_snapshot_rows, ) @@ -38,20 +39,30 @@ def main(): "training_samples.json", ), ) + parser.add_argument( + "--snapshot-file", + default=os.path.join(PROJECT_ROOT, "data", "probability_training_snapshots.jsonl"), + ) args = parser.parse_args() - history = load_history(args.history_file) + history = _load_history_with_fallback(args.history_file) settlement_history = _load_json_if_exists(args.settlement_history) + snapshot_rows = _load_snapshot_rows(args.snapshot_file) samples, filled_actual_from_history = _extract_samples( history, settlement_history=settlement_history, + snapshot_rows=snapshot_rows, ) + snapshot_count = sum(1 for sample in samples if sample.get("sample_source") == "snapshot") + daily_record_count = sum(1 for sample in samples if sample.get("sample_source") == "daily_record") output_dir = os.path.dirname(os.path.abspath(args.output)) if output_dir: os.makedirs(output_dir, exist_ok=True) payload = { "sample_count": len(samples), + "snapshot_sample_count": snapshot_count, + "daily_record_sample_count": daily_record_count, "filled_actual_from_history": filled_actual_from_history, "samples": samples, } diff --git a/scripts/fit_probability_calibration.py b/scripts/fit_probability_calibration.py index b67095f5..7a27a4c3 100644 --- a/scripts/fit_probability_calibration.py +++ b/scripts/fit_probability_calibration.py @@ -33,11 +33,128 @@ def _load_json_if_exists(path): return data if isinstance(data, dict) else {} -def _extract_samples(history, settlement_history=None): +def _load_history_with_fallback(path): + data = load_history(path) + if data: + return data + return _load_json_if_exists(path) + + +def _load_snapshot_rows(path): + rows = [] + if not path or not os.path.exists(path): + return rows + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except Exception: + continue + if isinstance(row, dict): + rows.append(row) + return rows + + +def _actual_high_for(history, settlement_history, city, date_str): + city_rows = (history or {}).get(city) or {} + record = city_rows.get(date_str) or {} + actual_high = _sf(record.get("actual_high")) if isinstance(record, dict) else None + filled = False + if actual_high is None: + actual_high = _sf(((settlement_history.get(city) or {}).get(date_str) or {}).get("max_temp")) + filled = actual_high is not None + return actual_high, filled + + +def _extract_snapshot_samples(history, snapshot_rows, settlement_history=None): samples = [] filled_actual_from_history = 0 today = datetime.utcnow().strftime("%Y-%m-%d") settlement_history = settlement_history or {} + + for row in snapshot_rows or []: + city = str(row.get("city") or "").strip().lower() + date_str = str(row.get("date") or "").strip() + if not city or not date_str or date_str == today: + continue + + actual_high, filled = _actual_high_for(history, settlement_history, city, date_str) + if actual_high is None: + continue + if filled: + filled_actual_from_history += 1 + + raw_mu = _sf(row.get("raw_mu")) + raw_sigma = _sf(row.get("raw_sigma")) + deb_prediction = _sf(row.get("deb_prediction")) + ensemble = row.get("ensemble") or {} + if not isinstance(ensemble, dict): + ensemble = {} + ens_median = _sf(ensemble.get("median")) + ensemble_spread = None + ens_p10 = _sf(ensemble.get("p10")) + ens_p90 = _sf(ensemble.get("p90")) + if ens_p10 is not None and ens_p90 is not None and ens_p90 >= ens_p10: + ensemble_spread = max(0.1, (ens_p90 - ens_p10) / 2.56) + multi_model = row.get("multi_model") or {} + if not isinstance(multi_model, dict): + multi_model = {} + forecast_values = [val for val in (_sf(v) for v in multi_model.values()) if val is not None] + forecast_values.sort() + if ensemble_spread is None: + if len(forecast_values) >= 2: + ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0) + elif raw_sigma is not None: + ensemble_spread = raw_sigma + else: + ensemble_spread = 1.0 + if raw_sigma is None: + raw_sigma = ensemble_spread + + peak_status = str(row.get("peak_status") or "before").strip().lower() + peak_flag = 0.0 + if peak_status == "in_window": + peak_flag = 0.5 + elif peak_status == "past": + peak_flag = 1.0 + + max_so_far = _sf(row.get("max_so_far")) + max_so_far_gap = None + if deb_prediction is not None and max_so_far is not None: + max_so_far_gap = deb_prediction - max_so_far + + if raw_mu is None: + continue + + samples.append( + { + "city": city, + "date": date_str, + "timestamp": row.get("timestamp"), + "actual_high": actual_high, + "raw_mu": raw_mu, + "raw_sigma": raw_sigma or 1.0, + "deb_prediction": deb_prediction, + "ens_median": ens_median if ens_median is not None else raw_mu, + "ensemble_spread": ensemble_spread, + "max_so_far_gap": max_so_far_gap, + "peak_flag": peak_flag, + "sample_source": "snapshot", + } + ) + + return samples, filled_actual_from_history + + +def _extract_daily_record_samples(history, settlement_history=None, excluded_keys=None): + samples = [] + filled_actual_from_history = 0 + today = datetime.utcnow().strftime("%Y-%m-%d") + settlement_history = settlement_history or {} + excluded_keys = excluded_keys or set() for city, city_rows in (history or {}).items(): if not isinstance(city_rows, dict): continue @@ -45,6 +162,8 @@ def _extract_samples(history, settlement_history=None): for date_str, record in city_rows.items(): if date_str == today or not isinstance(record, dict): continue + if (city, date_str) in excluded_keys: + continue actual_high = _sf(record.get("actual_high")) if actual_high is None: actual_high = _sf((city_settlement.get(date_str) or {}).get("max_temp")) @@ -99,11 +218,30 @@ def _extract_samples(history, settlement_history=None): "ensemble_spread": ensemble_spread, "max_so_far_gap": max_so_far_gap, "peak_flag": peak_flag, + "sample_source": "daily_record", } ) return samples, filled_actual_from_history +def _extract_samples(history, settlement_history=None, snapshot_rows=None): + snapshot_samples, snapshot_filled = _extract_snapshot_samples( + history, + snapshot_rows or [], + settlement_history=settlement_history, + ) + excluded_keys = { + (sample["city"], sample["date"]) + for sample in snapshot_samples + } + daily_samples, daily_filled = _extract_daily_record_samples( + history, + settlement_history=settlement_history, + excluded_keys=excluded_keys, + ) + return snapshot_samples + daily_samples, snapshot_filled + daily_filled + + def main(): parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.") parser.add_argument( @@ -126,6 +264,11 @@ def main(): ), help="Optional daily settlement history JSON built from historical CSV files.", ) + parser.add_argument( + "--snapshot-file", + default=os.path.join(PROJECT_ROOT, "data", "probability_training_snapshots.jsonl"), + help="Optional JSONL file with archived probability snapshots.", + ) parser.add_argument( "--version", default=None, @@ -133,11 +276,13 @@ def main(): ) args = parser.parse_args() - history = load_history(args.history_file) + history = _load_history_with_fallback(args.history_file) settlement_history = _load_json_if_exists(args.settlement_history) + snapshot_rows = _load_snapshot_rows(args.snapshot_file) samples, filled_actual_from_history = _extract_samples( history, settlement_history=settlement_history, + snapshot_rows=snapshot_rows, ) calibration = fit_calibration(samples, version=args.version) if not samples: diff --git a/src/analysis/probability_snapshot_archive.py b/src/analysis/probability_snapshot_archive.py new file mode 100644 index 00000000..c5964d32 --- /dev/null +++ b/src/analysis/probability_snapshot_archive.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import json +import os +from datetime import datetime +from typing import Any, Dict, List, Optional + +DEDUP_SCAN_LINES = 200 +MU_THRESHOLD = 0.2 +SIGMA_THRESHOLD = 0.15 +MAX_SO_FAR_THRESHOLD = 0.2 + + +def _sf(value: Any) -> Optional[float]: + if value is None: + return None + try: + return float(value) + except Exception: + return None + + +def _compact_snapshot(distribution: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + compact: List[Dict[str, Any]] = [] + for row in distribution or []: + if not isinstance(row, dict): + continue + value = row.get("value") + probability = row.get("probability") + if value is None or probability is None: + continue + try: + compact.append( + { + "v": int(value), + "p": round(float(probability), 3), + } + ) + except Exception: + continue + if len(compact) >= 4: + break + return compact + + +def _top_bucket(snapshot: Optional[List[Dict[str, Any]]]) -> Optional[int]: + best_value = None + best_prob = -1.0 + for row in snapshot or []: + if not isinstance(row, dict): + continue + value = row.get("v") + prob = _sf(row.get("p")) + if value is None or prob is None: + continue + if prob > best_prob: + best_value = int(value) + best_prob = prob + return best_value + + +def _load_recent_rows(path: str, max_lines: int = DEDUP_SCAN_LINES) -> List[Dict[str, Any]]: + if not os.path.exists(path): + return [] + with open(path, "r", encoding="utf-8") as fh: + lines = fh.readlines()[-max_lines:] + rows = [] + for line in lines: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except Exception: + continue + if isinstance(row, dict): + rows.append(row) + return rows + + +def _should_skip_append(path: str, payload: Dict[str, Any]) -> bool: + recent_rows = _load_recent_rows(path) + city = payload.get("city") + date_str = payload.get("date") + if not city or not date_str: + return False + + for row in reversed(recent_rows): + if row.get("city") != city or row.get("date") != date_str: + continue + if row.get("peak_status") != payload.get("peak_status"): + return False + if row.get("probability_mode") != payload.get("probability_mode"): + return False + + current_top = _top_bucket(payload.get("prob_snapshot")) + previous_top = _top_bucket(row.get("prob_snapshot")) + current_shadow_top = _top_bucket(payload.get("shadow_prob_snapshot")) + previous_shadow_top = _top_bucket(row.get("shadow_prob_snapshot")) + if current_top != previous_top or current_shadow_top != previous_shadow_top: + return False + + if abs((_sf(payload.get("raw_mu")) or 0.0) - (_sf(row.get("raw_mu")) or 0.0)) > MU_THRESHOLD: + return False + if abs((_sf(payload.get("raw_sigma")) or 0.0) - (_sf(row.get("raw_sigma")) or 0.0)) > SIGMA_THRESHOLD: + return False + if abs((_sf(payload.get("max_so_far")) or 0.0) - (_sf(row.get("max_so_far")) or 0.0)) > MAX_SO_FAR_THRESHOLD: + return False + + return True + + return False + + +def append_probability_snapshot( + city_name: str, + *, + local_date: str, + observation_time: Optional[str], + temp_symbol: str, + raw_mu: Optional[float], + raw_sigma: Optional[float], + deb_prediction: Optional[float], + ens_data: Optional[Dict[str, Any]], + current_forecasts: Optional[Dict[str, Any]], + max_so_far: Optional[float], + peak_status: Optional[str], + probabilities: Optional[List[Dict[str, Any]]], + shadow_probabilities: Optional[List[Dict[str, Any]]], + calibration_summary: Optional[Dict[str, Any]], + archive_path: Optional[str] = None, +) -> None: + city_key = str(city_name or "").strip().lower() + if not city_key: + return + + root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + path = archive_path or os.path.join( + root_dir, + "data", + "probability_training_snapshots.jsonl", + ) + + calibration_summary = calibration_summary or {} + ens_data = ens_data or {} + current_forecasts = current_forecasts or {} + timestamp = str(observation_time or datetime.utcnow().isoformat() + "Z").strip() + + payload = { + "city": city_key, + "timestamp": timestamp, + "date": local_date, + "temp_symbol": temp_symbol, + "raw_mu": _sf(raw_mu), + "raw_sigma": _sf(raw_sigma), + "deb_prediction": _sf(deb_prediction), + "ensemble": { + "p10": _sf(ens_data.get("p10")), + "median": _sf(ens_data.get("median")), + "p90": _sf(ens_data.get("p90")), + }, + "multi_model": { + key: _sf(value) + for key, value in current_forecasts.items() + if _sf(value) is not None + }, + "max_so_far": _sf(max_so_far), + "peak_status": peak_status, + "prob_snapshot": _compact_snapshot(probabilities), + "shadow_prob_snapshot": _compact_snapshot(shadow_probabilities), + "probability_engine": calibration_summary.get("engine"), + "probability_mode": calibration_summary.get("mode"), + "calibration_version": calibration_summary.get("calibration_version"), + "calibration_source": calibration_summary.get("calibration_source"), + "calibrated_mu": _sf(calibration_summary.get("calibrated_mu")), + "calibrated_sigma": _sf(calibration_summary.get("calibrated_sigma")), + } + + parent = os.path.dirname(os.path.abspath(path)) + if parent: + os.makedirs(parent, exist_ok=True) + + if _should_skip_append(path, payload): + return + + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(payload, ensure_ascii=False) + "\n") diff --git a/src/analysis/trend_engine.py b/src/analysis/trend_engine.py index 7adf8a28..10c47b31 100644 --- a/src/analysis/trend_engine.py +++ b/src/analysis/trend_engine.py @@ -19,6 +19,7 @@ from src.analysis.probability_calibration import ( apply_probability_calibration, build_probability_features, ) +from src.analysis.probability_snapshot_archive import append_probability_snapshot from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city from src.data_collection.city_registry import CITY_REGISTRY from src.data_collection.city_risk_profiles import get_city_risk_profile @@ -733,6 +734,26 @@ def analyze_weather_trend( except Exception: pass + try: + append_probability_snapshot( + city_name=city_name or "", + local_date=local_date_str, + observation_time=obs_time_raw or local_time_full or None, + temp_symbol=temp_symbol, + raw_mu=calibration_summary.get("raw_mu"), + raw_sigma=calibration_summary.get("raw_sigma"), + deb_prediction=_deb_to_save, + ens_data=ens_data, + current_forecasts=current_forecasts, + max_so_far=max_so_far, + peak_status=peak_status, + probabilities=_prob_list, + shadow_probabilities=_shadow_prob_list, + calibration_summary=calibration_summary, + ) + except Exception: + pass + # === Build recent list for trend_info === recent_list = [] for tm, t in recent_temps[:4]: diff --git a/tests/test_probability_snapshot_archive.py b/tests/test_probability_snapshot_archive.py new file mode 100644 index 00000000..9be1d041 --- /dev/null +++ b/tests/test_probability_snapshot_archive.py @@ -0,0 +1,117 @@ +import json +from pathlib import Path + +from src.analysis.probability_snapshot_archive import append_probability_snapshot + + +def test_append_probability_snapshot_writes_jsonl(tmp_path: Path): + archive_path = tmp_path / "probability_training_snapshots.jsonl" + + append_probability_snapshot( + city_name="ankara", + local_date="2026-03-20", + observation_time="2026-03-20T12:00:00+03:00", + temp_symbol="°C", + raw_mu=15.2, + raw_sigma=1.2, + deb_prediction=15.4, + ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9}, + current_forecasts={"ECMWF": 15.8, "GFS": 14.1}, + max_so_far=15.0, + peak_status="before", + probabilities=[{"value": 15, "probability": 0.552}], + shadow_probabilities=[{"value": 15, "probability": 0.324}], + calibration_summary={ + "engine": "legacy", + "mode": "emos_shadow", + "calibration_version": "emos-test", + "calibration_source": "artifacts/probability_calibration/default.json", + "calibrated_mu": 15.1, + "calibrated_sigma": 1.25, + }, + archive_path=str(archive_path), + ) + + lines = archive_path.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 1 + payload = json.loads(lines[0]) + assert payload["city"] == "ankara" + assert payload["date"] == "2026-03-20" + assert payload["raw_mu"] == 15.2 + assert payload["ensemble"]["median"] == 15.8 + assert payload["prob_snapshot"][0]["v"] == 15 + assert payload["shadow_prob_snapshot"][0]["v"] == 15 + assert payload["calibration_version"] == "emos-test" + + +def test_append_probability_snapshot_skips_near_duplicate(tmp_path: Path): + archive_path = tmp_path / "probability_training_snapshots.jsonl" + kwargs = dict( + city_name="ankara", + local_date="2026-03-20", + observation_time="2026-03-20T12:00:00+03:00", + temp_symbol="°C", + raw_mu=15.2, + raw_sigma=1.2, + deb_prediction=15.4, + ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9}, + current_forecasts={"ECMWF": 15.8, "GFS": 14.1}, + max_so_far=15.0, + peak_status="before", + probabilities=[{"value": 15, "probability": 0.552}], + shadow_probabilities=[{"value": 15, "probability": 0.324}], + calibration_summary={ + "engine": "legacy", + "mode": "emos_shadow", + "calibration_version": "emos-test", + "calibration_source": "artifacts/probability_calibration/default.json", + "calibrated_mu": 15.1, + "calibrated_sigma": 1.25, + }, + archive_path=str(archive_path), + ) + + append_probability_snapshot(**kwargs) + append_probability_snapshot(**kwargs) + + lines = archive_path.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 1 + + +def test_append_probability_snapshot_writes_on_bucket_change(tmp_path: Path): + archive_path = tmp_path / "probability_training_snapshots.jsonl" + base_kwargs = dict( + city_name="ankara", + local_date="2026-03-20", + observation_time="2026-03-20T12:00:00+03:00", + temp_symbol="°C", + raw_mu=15.2, + raw_sigma=1.2, + deb_prediction=15.4, + ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9}, + current_forecasts={"ECMWF": 15.8, "GFS": 14.1}, + max_so_far=15.0, + peak_status="before", + shadow_probabilities=[{"value": 15, "probability": 0.324}], + calibration_summary={ + "engine": "legacy", + "mode": "emos_shadow", + "calibration_version": "emos-test", + "calibration_source": "artifacts/probability_calibration/default.json", + "calibrated_mu": 15.1, + "calibrated_sigma": 1.25, + }, + archive_path=str(archive_path), + ) + + append_probability_snapshot( + probabilities=[{"value": 15, "probability": 0.552}], + **base_kwargs, + ) + append_probability_snapshot( + probabilities=[{"value": 16, "probability": 0.552}], + **base_kwargs, + ) + + lines = archive_path.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 2 diff --git a/tests/test_probability_training_dataset.py b/tests/test_probability_training_dataset.py new file mode 100644 index 00000000..68b4602c --- /dev/null +++ b/tests/test_probability_training_dataset.py @@ -0,0 +1,45 @@ +from scripts.fit_probability_calibration import _extract_samples + + +def test_extract_samples_prefers_snapshot_rows_for_same_city_day(): + history = { + "ankara": { + "2026-03-19": { + "actual_high": 11.0, + "mu": 10.8, + "deb_prediction": 10.9, + "forecasts": {"ECMWF": 10.5, "GFS": 11.2}, + "probability_features": { + "ens_median": 10.7, + "ensemble_spread": 0.8, + "peak_status": "before", + }, + } + } + } + snapshot_rows = [ + { + "city": "ankara", + "date": "2026-03-19", + "timestamp": "2026-03-19T12:00:00+03:00", + "raw_mu": 11.2, + "raw_sigma": 1.1, + "deb_prediction": 11.0, + "ensemble": {"p10": 10.0, "median": 11.1, "p90": 12.2}, + "multi_model": {"ECMWF": 10.5, "GFS": 11.2}, + "max_so_far": 10.9, + "peak_status": "in_window", + } + ] + + samples, filled = _extract_samples( + history, + settlement_history={}, + snapshot_rows=snapshot_rows, + ) + + assert filled == 0 + assert len(samples) == 1 + assert samples[0]["sample_source"] == "snapshot" + assert samples[0]["raw_mu"] == 11.2 + assert samples[0]["peak_flag"] == 0.5