From 1892d638fa071da7e93c645d6c543a5d29e1fead Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Sun, 19 Apr 2026 03:36:26 +0800 Subject: [PATCH] Promote EMOS to the primary probability engine --- .env.example | 1 + README.md | 4 +- README_ZH.md | 4 +- artifacts/models/lgbm_daily_high.txt | 490 +++++++++--------- artifacts/models/lgbm_daily_high_schema.json | 26 +- .../probability_calibration/default.json | 84 +-- .../evaluation_report.json | 183 +++---- docs/EMOS_TRAINING_REPORT_ZH.md | 351 +++---------- docs/TECH_DEBT.md | 12 +- docs/TECH_DEBT_ZH.md | 12 +- src/analysis/probability_calibration.py | 5 +- tests/test_probability_calibration.py | 12 + 12 files changed, 509 insertions(+), 675 deletions(-) diff --git a/.env.example b/.env.example index a5903a7c..8cc0f09b 100644 --- a/.env.example +++ b/.env.example @@ -55,6 +55,7 @@ POLYWEATHER_METAR_CLUSTER_TIMEOUT_SEC=3.5 METAR_CACHE_TTL_SEC=600 JMA_AMEDAS_CACHE_TTL_SEC=120 METEOBLUE_CACHE_TTL_SEC=7200 +POLYWEATHER_PROBABILITY_ENGINE=emos_primary POLYWEATHER_LGBM_ENABLED=false POLYWEATHER_LGBM_MODEL_PATH=/app/artifacts/models/lgbm_daily_high.txt POLYWEATHER_LGBM_SCHEMA_PATH=/app/artifacts/models/lgbm_daily_high_schema.json diff --git a/README.md b/README.md index c73efa25..12cb8736 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Public docs center: `/docs/intro` on the main site (bilingual product documentat ![PolyWeather Ankara analysis](docs/images/demo_ankara.png) -## Product Status (2026-04-18) +## Product Status (2026-04-19) - Subscription live: `Pro Monthly 5 USDC`. - Points redemption live: `500 points = 1 USDC`, max `3 USDC` off. @@ -26,7 +26,7 @@ Public docs center: `/docs/intro` on the main site (bilingual product documentat - Ops dashboard live: `/ops` for memberships, leaderboard, manual point grants, and payment incident triage. - Lightweight observability live: `/healthz`, `/api/system/status`, `/metrics`. - Runtime state, cache, and core offline training/backfill flows now use SQLite as the primary path; legacy JSON/JSONL files remain only for migration, export, and explicit fallback input. -- EMOS/CRPS pipeline is integrated in `shadow` mode with rollout gating. +- EMOS/CRPS calibrated probability is now the default primary probability engine (`emos_primary`); set `POLYWEATHER_PROBABILITY_ENGINE=emos_shadow` or `legacy` to roll back. - Intraday analysis is now positioned as a professional meteorology read: headline, confidence, base/upside/downside paths, next observation point, evidence chain, failure modes, and confirmation rules. - Intraday modal now blocks stale cached detail during refresh, so users do not briefly trade off old city/date data before full detail arrives. - Calibrated model probability is now the primary probability panel. `LGBM` is shown as a calibrated probability engine when available; model consensus and market prices are secondary references. diff --git a/README_ZH.md b/README_ZH.md index 7286e61b..08de7fb9 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -14,7 +14,7 @@ ![PolyWeather Ankara 分析页](docs/images/demo_ankara.png) -## 当前产品状态（2026-04-18） +## 当前产品状态（2026-04-19） - 已上线订阅制：`Pro 月付 5 USDC`。 - 已上线积分抵扣：`500 积分 = 1 USDC`，最多抵扣 `3 USDC`。 @@ -25,7 +25,7 @@ - 已上线轻量可观测性：`/healthz`、`/api/system/status`、`/metrics`。 - 已补最小外部监控栈：Prometheus + Alertmanager + Grafana + Telegram 告警 relay。 - 运行态状态、缓存与核心离线训练/回填链路已完成 SQLite 主路径收口；legacy JSON/JSONL 仅保留给迁移、导出与显式回退输入。 -- 已接入 EMOS/CRPS 校准链路，但当前仍保持 `emos_shadow`。 +- EMOS/CRPS 校准概率已切为默认主路径（`emos_primary`）；如需回滚可设置 `POLYWEATHER_PROBABILITY_ENGINE=emos_shadow` 或 `legacy`。 - 官方增强站网已统一接入： - `MGM`（土耳其） - `CMA/NMC`（中国内地） diff --git a/artifacts/models/lgbm_daily_high.txt b/artifacts/models/lgbm_daily_high.txt index 255f926f..799712b2 100644 --- a/artifacts/models/lgbm_daily_high.txt +++ b/artifacts/models/lgbm_daily_high.txt @@ -6,24 +6,24 @@ label_index=0 max_feature_idx=26 objective=regression feature_names=actual_high_lag_1 actual_high_lag_2 actual_high_lag_3 actual_high_lag_7 actual_high_mean_7 actual_high_mean_14 actual_high_trend_3 open_meteo ecmwf gfs gem jma icon mgm nws deb_prediction model_median model_spread current_temp max_so_far humidity wind_speed_kt visibility_mi local_hour month weekday peak_status_code -feature_infos=[2:33] [5.5999999046325684:28] [5.5999999046325684:28] [5.4000000953674316:27.100000381469727] [7.5714287757873535:27.871429443359375] [8.9142856597900391:26.335714340209961] [-14:11] [7.5:30.899999618530273] [7.5:30.899999618530273] [8.1999998092651367:32.900001525878906] [8.1999998092651367:30.700000762939453] [8.3000001907348633:30.799999237060547] [7.5:30.899999618530273] none none [7.9000000953674316:30.899999618530273] [8.1999998092651367:30.899999618530273] [0.30000001192092896:7.5999999046325684] [0:29.200000762939453] [2:32.099998474121094] [46.900001525878906:81] [1:17] none [5:20] none [0:6] [0:2] -tree_sizes=411 401 404 404 426 314 427 422 423 424 430 427 519 431 427 518 427 429 500 427 +feature_infos=[2:33] [5.5999999046325684:28] [5.5999999046325684:28] [5.4000000953674316:27.100000381469727] [7.5714287757873535:27.857143402099609] [8.9142856597900391:26.328571319580078] [-14:11] [7.5:30.899999618530273] [7.5:30.899999618530273] [8.1999998092651367:32.900001525878906] [8.1999998092651367:30.700000762939453] [8.3000001907348633:30.799999237060547] [7.5:30.899999618530273] none none [7.9000000953674316:30.899999618530273] [8.1999998092651367:30.899999618530273] [0.30000001192092896:7.5999999046325684] [0:29.200000762939453] [2:32.099998474121094] [46.900001525878906:81] [1:17] none [5:20] none [0:6] [0:2] +tree_sizes=415 427 423 408 502 425 426 426 501 503 523 521 524 524 526 522 524 521 525 521 Tree=0 num_leaves=3 num_cat=0 -split_feature=10 2 -split_gain=1001.8 123.037 -threshold=20.04999923706055 19.94999980926514 +split_feature=10 19 +split_gain=959.069 145.855 +threshold=19.000000000000004 23.500000000000004 decision_type=10 10 left_child=-1 -2 right_child=1 -3 -leaf_value=19.431389017899832 20.051389029050867 20.38722237745921 -leaf_weight=6.0000000000000027 4.9999999999999982 6 -leaf_count=6 5 6 -internal_value=19.9511 20.2346 -internal_weight=17 11 -internal_count=17 11 +leaf_value=19.533250135183334 20.11658346115922 20.452535862582071 +leaf_weight=6.0000000000000027 5.9999999999999982 7 +leaf_count=6 6 7 +internal_value=20.0561 20.2975 +internal_weight=19 13 +internal_count=19 13 is_linear=0 shrinkage=1 @@ -31,18 +31,18 @@ shrinkage=1 Tree=1 num_leaves=3 num_cat=0 -split_feature=7 0 -split_gain=858.905 67.8176 -threshold=15.950000286102297 25.85000038146973 -decision_type=8 2 +split_feature=7 20 +split_gain=781.384 128.779 +threshold=15.950000286102297 inf +decision_type=8 8 left_child=-1 -2 right_child=1 -3 -leaf_value=-0.50156944513320922 0.1882326394319534 0.47938890457153316 -leaf_weight=4.9999999999999991 4 4 -leaf_count=5 4 4 -internal_value=0.0125107 0.333811 -internal_weight=13 8 -internal_count=13 8 +leaf_value=-0.58666250705718992 0.39862325191497805 0.032365272504587962 +leaf_weight=4.9999999999999991 3.9999999999999991 6.0000000000000009 +leaf_count=5 4 6 +internal_value=-0.0763085 0.178868 +internal_weight=15 10 +internal_count=15 10 is_linear=0 shrinkage=0.05 @@ -50,18 +50,18 @@ shrinkage=0.05 Tree=2 num_leaves=3 num_cat=0 -split_feature=10 0 -split_gain=763.646 100.764 -threshold=20.04999923706055 25.85000038146973 -decision_type=10 2 +split_feature=7 1 +split_gain=714.365 143.208 +threshold=15.950000286102297 20.500000000000004 +decision_type=8 10 left_child=-1 -2 right_child=1 -3 -leaf_value=-0.43713240964072081 0.10051891766488551 0.45541944503784171 -leaf_weight=7.0000000000000009 4 4 -leaf_count=7 4 4 -internal_value=-0.0557449 0.277969 -internal_weight=15 8 -internal_count=15 8 +leaf_value=-0.55982939004898025 0.065748141493116127 0.41610464572906491 +leaf_weight=4.0000000000000027 6.9999999999999982 5 +leaf_count=4 7 5 +internal_value=0.0188402 0.21173 +internal_weight=16 12 +internal_count=16 12 is_linear=0 shrinkage=0.05 @@ -69,56 +69,56 @@ shrinkage=0.05 Tree=3 num_leaves=3 num_cat=0 -split_feature=10 0 -split_gain=466.421 90.9391 -threshold=20.04999923706055 25.85000038146973 -decision_type=10 2 +split_feature=10 1 +split_gain=481.327 111.706 +threshold=19.000000000000004 20.500000000000004 +decision_type=8 10 left_child=-1 -2 right_child=1 -3 -leaf_value=-0.35153238058090203 0.095492970943450911 0.43264847993850702 -leaf_weight=5.0000000000000009 4 4 -leaf_count=5 4 4 -internal_value=0.0273003 0.264071 -internal_weight=13 8 -internal_count=13 8 +leaf_value=-0.39178395509719849 0.07259848924974599 0.41371505260467523 +leaf_weight=4.9999999999999991 6 4 +leaf_count=5 6 4 +internal_value=0.00876876 0.209045 +internal_weight=15 10 +internal_count=15 10 is_linear=0 shrinkage=0.05 Tree=4 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=8 20 -split_gain=680.381 113.467 -threshold=16.19999980926514 inf -decision_type=2 8 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.44872442086537651 0.38909818649292011 0.066590625792741762 -leaf_weight=6.0000000000000027 4.9999999999999973 6.0000000000000009 -leaf_count=6 5 6 -internal_value=-0.0204301 0.213185 -internal_weight=17 11 -internal_count=17 11 +split_feature=9 1 9 +split_gain=692.581 166.399 14.5487 +threshold=16.150000095367435 20.500000000000004 21.35000038146973 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.50974870681762663 -0.040919089317321793 0.37483656803766885 0.093935826048254956 +leaf_weight=5.0000000000000027 3.9999999999999982 6 4 +leaf_count=5 4 6 4 +internal_value=-0.00461354 0.175792 0.0265084 +internal_weight=19 14 8 +internal_count=19 14 8 is_linear=0 shrinkage=0.05 Tree=5 -num_leaves=2 +num_leaves=3 num_cat=0 -split_feature=3 -split_gain=658.731 -threshold=13.449999809265138 -decision_type=10 -left_child=-1 -right_child=-2 -leaf_value=-0.3789437881537846 0.33501165111859638 -leaf_weight=7 6 -leaf_count=7 6 -internal_value=-0.0494259 -internal_weight=13 -internal_count=13 +split_feature=10 19 +split_gain=180.29 52.216 +threshold=19.000000000000004 23.500000000000004 +decision_type=10 10 +left_child=-1 -2 +right_child=1 -3 +leaf_value=-0.23234437108039849 0.071550059318542497 0.30477037429809561 +leaf_weight=4.0000000000000009 5.9999999999999982 4 +leaf_count=4 6 4 +internal_value=0.0513575 0.164838 +internal_weight=14 10 +internal_count=14 10 is_linear=0 shrinkage=0.05 @@ -126,18 +126,18 @@ shrinkage=0.05 Tree=6 num_leaves=3 num_cat=0 -split_feature=7 20 -split_gain=619.423 90.8198 -threshold=15.950000286102297 inf -decision_type=8 8 +split_feature=10 1 +split_gain=514.677 88.2508 +threshold=19.000000000000004 20.500000000000004 +decision_type=10 10 left_child=-1 -2 right_child=1 -3 -leaf_value=-0.47567430973052949 0.35289269924163835 0.064359399676322912 -leaf_weight=5.0000000000000027 4.9999999999999973 6.0000000000000009 -leaf_count=5 5 6 -internal_value=-0.0142345 0.195511 -internal_weight=16 11 -internal_count=16 11 +leaf_value=-0.35969944695631645 0.079533948642866961 0.34085621833801266 +leaf_weight=6.0000000000000027 6.9999999999999982 6 +leaf_count=6 7 6 +internal_value=0.0233515 0.200144 +internal_weight=19 13 +internal_count=19 13 is_linear=0 shrinkage=0.05 @@ -145,94 +145,94 @@ shrinkage=0.05 Tree=7 num_leaves=3 num_cat=0 -split_feature=10 0 -split_gain=520.638 78.456 -threshold=20.04999923706055 25.85000038146973 -decision_type=8 2 +split_feature=7 19 +split_gain=536.147 90.9495 +threshold=15.950000286102297 23.500000000000004 +decision_type=8 10 left_child=-1 -2 right_child=1 -3 -leaf_value=-0.38976687689622219 0.067351022859414439 0.33552634239196771 -leaf_weight=6.0000000000000027 5.9999999999999982 5 -leaf_count=6 6 5 -internal_value=-0.0151096 0.189249 -internal_weight=17 11 -internal_count=17 11 +leaf_value=-0.39410252372423787 0.043377760052680979 0.29825801338468277 +leaf_weight=6.0000000000000027 6.9999999999999982 7 +leaf_count=6 7 7 +internal_value=0.00134176 0.170818 +internal_weight=20 14 +internal_count=20 14 is_linear=0 shrinkage=0.05 Tree=8 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=10 2 -split_gain=499.553 74.407 -threshold=20.04999923706055 19.94999980926514 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.37027853230635305 0.063983469704786952 0.31299318869908649 -leaf_weight=6.0000000000000027 5.9999999999999982 6 -leaf_count=6 6 6 -internal_value=0.00223271 0.188488 -internal_weight=18 12 -internal_count=18 12 +split_feature=10 1 19 +split_gain=488.557 89.3384 13.4331 +threshold=19.000000000000004 20.500000000000004 21.500000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.37621160050233188 -0.011120753735303883 0.30890050729115803 0.11846074610948561 +leaf_weight=6.0000000000000027 3.9999999999999982 6 4 +leaf_count=6 4 6 4 +internal_value=0.00127467 0.163055 0.05367 +internal_weight=20 14 8 +internal_count=20 14 8 is_linear=0 shrinkage=0.05 Tree=9 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=7 21 -split_gain=453.956 67.1434 -threshold=15.950000286102297 3.6000000238418584 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.3529831647872923 0.31961423397064215 0.07971531493323189 -leaf_weight=6.0000000000000027 4.9999999999999982 7 -leaf_count=6 5 7 -internal_value=0.00212108 0.179673 -internal_weight=18 12 -internal_count=18 12 +split_feature=7 1 19 +split_gain=443.948 80.0165 11.7632 +threshold=15.950000286102297 20.500000000000004 21.500000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.35862923761208831 -0.0087223708629608175 0.29345548550287875 0.11253771036863325 +leaf_weight=6.0000000000000027 3.9999999999999982 6 4 +leaf_count=6 4 6 4 +internal_value=0.00121094 0.155428 0.0519077 +internal_weight=20 14 8 +internal_count=20 14 8 is_linear=0 shrinkage=0.05 Tree=10 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=18 19 -split_gain=417.969 98.6176 -threshold=10.500000000000002 23.500000000000004 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.44873919486999481 -0.0019017819847379415 0.26350565637860973 -leaf_weight=4.0000000000000027 6.9999999999999982 7 -leaf_count=4 7 7 -internal_value=0.00201502 0.130802 -internal_weight=18 14 -internal_count=18 14 +split_feature=18 1 19 +split_gain=408.359 104.122 22.3723 +threshold=10.500000000000002 20.500000000000004 21.500000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.45071191787719683 -0.045747331281503052 0.27878270943959554 0.10691082179546355 +leaf_weight=4.0000000000000027 5.9999999999999982 5.9999999999999991 4 +leaf_count=4 6 6 4 +internal_value=0.00115039 0.114116 0.0153159 +internal_weight=20 16 10 +internal_count=20 16 10 is_linear=0 shrinkage=0.05 Tree=11 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=10 10 -split_gain=379.422 54.6231 -threshold=20.04999923706055 27.150000572204593 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.32273159126440665 0.074079313874244698 0.29045824527740471 -leaf_weight=6.0000000000000027 6.9999999999999982 5 -leaf_count=6 7 5 -internal_value=0.00191427 0.164237 -internal_weight=18 12 -internal_count=18 12 +split_feature=18 1 19 +split_gain=368.544 93.9702 20.191 +threshold=10.500000000000002 20.500000000000004 21.500000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.42817632555961577 -0.043459965785344445 0.26484357118606566 0.10156528204679488 +leaf_weight=4.0000000000000027 5.9999999999999982 5.9999999999999991 4 +leaf_count=4 6 6 4 +internal_value=0.00109287 0.10841 0.0145501 +internal_weight=20 16 10 +internal_count=20 16 10 is_linear=0 shrinkage=0.05 @@ -240,56 +240,56 @@ shrinkage=0.05 Tree=12 num_leaves=4 num_cat=0 -split_feature=18 2 6 -split_gain=349.161 79.9137 12.1022 -threshold=10.500000000000002 19.94999980926514 1.5500000119209292 -decision_type=8 10 8 +split_feature=10 3 19 +split_gain=335.123 58.0754 13.7912 +threshold=19.000000000000004 21.500000000000004 21.500000000000004 +decision_type=8 10 10 left_child=-1 2 -2 right_child=1 -3 -4 -leaf_value=-0.41016564369201625 0.077571737766265914 0.25746693611145016 -0.045422995835542657 -leaf_weight=4.0000000000000027 3.9999999999999973 6 4.0000000000000009 -leaf_count=4 4 6 4 -internal_value=0.00181856 0.119528 0.0160744 -internal_weight=18 14 8 -internal_count=18 14 8 +leaf_value=-0.31160237689812964 -0.0012943979352712635 0.29604412913322448 0.11856331576903661 +leaf_weight=6.0000000000000027 3.9999999999999982 3.9999999999999991 6 +leaf_count=6 4 4 6 +internal_value=0.00103823 0.135027 0.0706202 +internal_weight=20 14 10 +internal_count=20 14 10 is_linear=0 shrinkage=0.05 Tree=13 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=18 19 -split_gain=315.118 72.4693 -threshold=10.500000000000002 23.500000000000004 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.38965735435485804 -0.00020638235977717812 0.22731021472385948 -leaf_weight=4.0000000000000027 6.9999999999999982 7 -leaf_count=4 7 7 -internal_value=0.00172763 0.113552 -internal_weight=18 14 -internal_count=18 14 +split_feature=18 1 19 +split_gain=307.6 76.0482 15.3887 +threshold=10.500000000000002 20.500000000000004 21.500000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.39118738174438444 -0.036050447821617135 0.23975720206896464 0.090558854490518551 +leaf_weight=4.0000000000000027 5.9999999999999982 5.9999999999999991 4 +leaf_count=4 6 6 4 +internal_value=0.000986321 0.0990297 0.0145933 +internal_weight=20 16 10 +internal_count=20 16 10 is_linear=0 shrinkage=0.05 Tree=14 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=7 21 -split_gain=287.831 40.4927 -threshold=15.950000286102297 3.6000000238418584 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.28111823151508952 0.25169647216796881 0.065395631534712659 -leaf_weight=6.0000000000000027 4.9999999999999982 7 -leaf_count=6 5 7 -internal_value=0.00164124 0.143021 -internal_weight=18 12 -internal_count=18 12 +split_feature=18 1 19 +split_gain=277.609 68.6335 13.8883 +threshold=10.500000000000002 20.500000000000004 21.500000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.37162801623344394 -0.034247924884160362 0.22776933908462527 0.086030908674001677 +leaf_weight=4.0000000000000027 5.9999999999999982 5.9999999999999991 4 +leaf_count=4 6 6 4 +internal_value=0.000937003 0.0940783 0.0138636 +internal_weight=20 16 10 +internal_count=20 16 10 is_linear=0 shrinkage=0.05 @@ -297,56 +297,56 @@ shrinkage=0.05 Tree=15 num_leaves=4 num_cat=0 -split_feature=18 2 6 -split_gain=263.177 58.9207 10.397 -threshold=10.500000000000002 19.94999980926514 1.5500000119209292 +split_feature=7 10 6 +split_gain=252.191 44.3705 10.4139 +threshold=15.950000286102297 27.450000762939457 3.6000000238418584 decision_type=8 10 8 left_child=-1 2 -2 right_child=1 -3 -4 -leaf_value=-0.35611858367919891 0.071921156719327015 0.22219576636950175 -0.042079883068799957 -leaf_weight=4.0000000000000027 3.9999999999999973 6 4.0000000000000009 -leaf_count=4 4 6 4 -internal_value=0.00155919 0.103753 0.0149206 -internal_weight=18 14 8 -internal_count=18 14 8 +leaf_value=-0.27032166918118777 0.11185126662254335 0.25786558985710145 0.0098028752207756035 +leaf_weight=6.0000000000000027 4.9999999999999982 3.9999999999999991 5 +leaf_count=6 5 4 5 +internal_value=0.000890153 0.117124 0.0608271 +internal_weight=20 14 10 +internal_count=20 14 10 is_linear=0 shrinkage=0.05 Tree=16 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=7 21 -split_gain=237.631 32.9682 -threshold=15.950000286102297 3.6000000238418584 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.25544037123521152 0.22800186157226565 0.059899282455444326 -leaf_weight=6.0000000000000027 4.9999999999999982 7 -leaf_count=6 5 7 -internal_value=0.00148123 0.129942 -internal_weight=18 12 -internal_count=18 12 +split_feature=18 1 19 +split_gain=231.712 55.2789 10.7397 +threshold=10.500000000000002 20.500000000000004 21.500000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.3395305335521695 -0.028356930613517767 0.2059211631615957 0.077412404865026457 +leaf_weight=4.0000000000000027 5.9999999999999982 5.9999999999999991 4 +leaf_count=4 6 6 4 +internal_value=0.000845644 0.0859397 0.0139508 +internal_weight=20 16 10 +internal_count=20 16 10 is_linear=0 shrinkage=0.05 Tree=17 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=18 19 -split_gain=219.898 47.5691 -threshold=10.500000000000002 23.500000000000004 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.3255406379699704 0.0026553549936839515 0.18698629140853881 -leaf_weight=4.0000000000000027 6.9999999999999982 7 -leaf_count=4 7 7 -internal_value=0.00140717 0.0948208 -internal_weight=18 14 -internal_count=18 14 +split_feature=10 3 18 +split_gain=209.605 36.6466 6.77648 +threshold=19.000000000000004 21.500000000000004 22.000000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.24645142853260027 0.022000332176685337 0.23467625677585602 0.10601718723773955 +leaf_weight=6.0000000000000027 5.9999999999999982 3.9999999999999991 4 +leaf_count=6 6 4 4 +internal_value=0.00080336 0.10677 0.0556071 +internal_weight=20 14 10 +internal_count=20 14 10 is_linear=0 shrinkage=0.05 @@ -354,37 +354,37 @@ shrinkage=0.05 Tree=18 num_leaves=4 num_cat=0 -split_feature=11 21 12 -split_gain=200.198 25.8458 5.04583 -threshold=16.19999980926514 3.6000000238418584 22.10000038146973 -decision_type=8 8 10 -left_child=-1 -2 -3 -right_child=1 2 -4 -leaf_value=-0.23448222527901319 0.22302051782608043 0.027650000527501103 0.10706844031810758 -leaf_weight=6.0000000000000027 3.9999999999999973 4 4 -leaf_count=6 4 4 4 -internal_value=0.0013368 0.119246 0.0673592 -internal_weight=18 12 8 -internal_count=18 12 8 +split_feature=18 3 19 +split_gain=193.435 44.5018 13.9622 +threshold=10.500000000000002 21.500000000000004 21.500000000000004 +decision_type=8 10 10 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.31023144423961613 -0.02356490443150203 0.22294244468212129 0.084301546216011047 +leaf_weight=4.0000000000000027 5.9999999999999982 3.9999999999999991 6 +leaf_count=4 6 4 6 +internal_value=0.000763193 0.0785119 0.0303683 +internal_weight=20 16 12 +internal_count=20 16 12 is_linear=0 shrinkage=0.05 Tree=19 -num_leaves=3 +num_leaves=4 num_cat=0 -split_feature=10 10 -split_gain=180.679 24.1787 -threshold=20.04999923706055 27.150000572204593 -decision_type=8 10 -left_child=-1 -2 -right_child=1 -3 -leaf_value=-0.22275811135768878 0.053300471178122935 0.19726094961166379 -leaf_weight=6.0000000000000027 6.9999999999999982 5 -leaf_count=6 7 5 -internal_value=0.00126997 0.113284 -internal_weight=18 12 -internal_count=18 12 +split_feature=7 3 6 +split_gain=173.45 29.4571 6.36567 +threshold=15.950000286102297 21.500000000000004 3.6000000238418584 +decision_type=8 10 8 +left_child=-1 2 -2 +right_child=1 -3 -4 +leaf_value=-0.22419602423906315 0.091142139434814473 0.21179532408714297 0.011356965601444243 +leaf_weight=6.0000000000000027 4.9999999999999982 3.9999999999999991 5 +leaf_count=6 5 4 5 +internal_value=0.000725034 0.0971198 0.0512496 +internal_weight=20 14 10 +internal_count=20 14 10 is_linear=0 shrinkage=0.05 @@ -392,19 +392,15 @@ shrinkage=0.05 end of trees feature_importances: -gem=9 -open_meteo=5 -current_temp=5 -actual_high_lag_1=4 -actual_high_lag_3=4 -wind_speed_kt=4 -max_so_far=3 +max_so_far=12 +actual_high_lag_2=11 +gem=8 +current_temp=7 +open_meteo=6 +actual_high_lag_7=4 actual_high_trend_3=2 -humidity=2 -actual_high_lag_7=1 -ecmwf=1 -jma=1 -icon=1 +gfs=2 +humidity=1 parameters: [boosting: gbdt] diff --git a/artifacts/models/lgbm_daily_high_schema.json b/artifacts/models/lgbm_daily_high_schema.json index f4638a29..9f2460ab 100644 --- a/artifacts/models/lgbm_daily_high_schema.json +++ b/artifacts/models/lgbm_daily_high_schema.json @@ -42,25 +42,25 @@ "nws" ], "model_path": "artifacts\\models\\lgbm_daily_high.txt", - "sample_count": 18, - "train_count": 6, + "sample_count": 20, + "train_count": 8, "validation_count": 12, "metrics": { "validation": { "sample_count": 12, - "lgbm_mae": 7.992, - "deb_mae": 2.375, - "best_single_mae": 1.05, - "median_mae": 2.292 + "lgbm_mae": 5.867, + "deb_mae": 1.825, + "best_single_mae": 0.567, + "median_mae": 1.7 }, "full_sample": { - "sample_count": 18, - "lgbm_mae": 3.034, - "deb_mae": 1.689, - "best_single_mae": 0.822, - "median_mae": 1.711 + "sample_count": 20, + "lgbm_mae": 2.66, + "deb_mae": 1.68, + "best_single_mae": 0.775, + "median_mae": 1.695 } }, - "generated_at": "2026-04-15T16:41:56.814421Z", - "trained_at": "2026-04-15T16:41:56.814421Z" + "generated_at": "2026-04-18T19:27:39.228210Z", + "trained_at": "2026-04-18T19:27:39.228210Z" } \ No newline at end of file diff --git a/artifacts/probability_calibration/default.json b/artifacts/probability_calibration/default.json index d41b189c..a3796c3e 100644 --- a/artifacts/probability_calibration/default.json +++ b/artifacts/probability_calibration/default.json @@ -1,20 +1,20 @@ { - "version": "emos-20260415170349", - "trained_at": "2026-04-15T17:03:49.122865+00:00", + "version": "emos-20260418192717", + "trained_at": "2026-04-18T19:27:17.348931+00:00", "global": { "mu": { - "intercept": 2.45853035, - "raw_mu_coef": 2.03689031, - "deb_coef": -1.41207502, - "ens_median_coef": 0.08616811, - "max_so_far_gap_coef": -0.5886898 + "intercept": 2.60074353, + "raw_mu_coef": 1.97765258, + "deb_coef": -1.37457626, + "ens_median_coef": 0.10693472, + "max_so_far_gap_coef": -0.58256593 }, "sigma": { - "intercept": 0.61260118, - "raw_sigma_coef": -1.48747774, - "spread_coef": 2.24562634, - "peak_flag_coef": -0.15906747, - "max_so_far_gap_coef": -0.01835251 + "intercept": 0.6373609, + "raw_sigma_coef": -1.41537225, + "spread_coef": 2.12997577, + "peak_flag_coef": -0.08633203, + "max_so_far_gap_coef": -0.00716383 } }, "sigma_constraints": { @@ -30,77 +30,83 @@ }, "blending": { "alpha_mu": 0.05, - "alpha_sigma": 1.0 + "alpha_sigma": 0.7 }, "cities": { "tokyo": { "samples": 5, - "mu_bias": 2.223946, + "mu_bias": 2.101101, "sigma_scale": 2.0, "confidence": 0.625 }, "ankara": { "samples": 6, - "mu_bias": -0.396953, - "sigma_scale": 1.094474, + "mu_bias": -0.523145, + "sigma_scale": 1.139849, "confidence": 0.75 }, "istanbul": { "samples": 3, - "mu_bias": -0.163153, - "sigma_scale": 0.5, + "mu_bias": -0.285183, + "sigma_scale": 0.550171, "confidence": 0.375 }, "hong kong": { "samples": 7, - "mu_bias": 3.267553, + "mu_bias": 3.234676, "sigma_scale": 2.0, "confidence": 0.875 }, "taipei": { "samples": 5, - "mu_bias": 4.80044, + "mu_bias": 4.731246, "sigma_scale": 2.0, "confidence": 0.625 }, + "busan": { + "samples": 3, + "mu_bias": 0.177772, + "sigma_scale": 2.0, + "confidence": 0.375 + }, "seoul": { "samples": 3, - "mu_bias": -3.080962, - "sigma_scale": 1.437353, + "mu_bias": -3.070325, + "sigma_scale": 1.462196, "confidence": 0.375 }, "milan": { "samples": 3, - "mu_bias": -4.20255, + "mu_bias": -4.249866, "sigma_scale": 2.0, "confidence": 0.375 }, "shanghai": { "samples": 3, - "mu_bias": 1.185335, - "sigma_scale": 1.68386, + "mu_bias": 1.080039, + "sigma_scale": 1.638119, "confidence": 0.375 }, "warsaw": { "samples": 3, - "mu_bias": -1.574246, - "sigma_scale": 1.343402, + "mu_bias": -1.68611, + "sigma_scale": 1.38114, "confidence": 0.375 } }, "metrics": { - "sample_count": 71, - "mean_crps": 3.226613, - "legacy_mean_crps": 3.608885, - "legacy_mean_mae": 3.81507, - "legacy_bucket_hit_rate": 0.492958, - "legacy_bucket_brier": 0.873776, - "selected_mean_crps": 3.44742, - "selected_mean_mae": 3.749078, - "selected_bucket_hit_rate": 0.492958, - "selected_bucket_brier": 0.878621, - "selected_score": 5.241857, - "legacy_score": 5.406287, + "sample_count": 74, + "mean_crps": 3.181501, + "legacy_mean_crps": 3.474108, + "legacy_mean_mae": 3.679324, + "legacy_bucket_hit_rate": 0.5, + "legacy_bucket_brier": 0.855395, + "selected_mean_crps": 3.322469, + "selected_mean_mae": 3.618254, + "selected_bucket_hit_rate": 0.5, + "selected_bucket_brier": 0.863048, + "selected_score": 5.08158, + "legacy_score": 5.233586, "filled_actual_from_history": 0, "settlement_history_city_count": 30, "legacy_archive_samples": 54 diff --git a/artifacts/probability_calibration/evaluation_report.json b/artifacts/probability_calibration/evaluation_report.json index 9fba7f43..f574035d 100644 --- a/artifacts/probability_calibration/evaluation_report.json +++ b/artifacts/probability_calibration/evaluation_report.json @@ -1,284 +1,293 @@ { "summary": { - "sample_count": 71, + "sample_count": 74, "filled_actual_from_history": 0, "legacy": { - "mean_crps": 3.608885, - "mean_mae": 3.81507, - "bucket_hit_rate": 0.492958 + "mean_crps": 3.474108, + "mean_mae": 3.679324, + "bucket_hit_rate": 0.5 }, "emos": { - "mean_crps": 3.881798, - "mean_mae": 4.378282, - "bucket_hit_rate": 0.225352 + "mean_crps": 3.33124, + "mean_mae": 3.622584, + "bucket_hit_rate": 0.5 }, "delta": { - "crps": 0.272913, - "mae": 0.563211, - "bucket_hit_rate": -0.267606 + "crps": -0.142868, + "mae": -0.056741, + "bucket_hit_rate": 0.0 } }, "by_city": { "ankara": { "samples": 6, "legacy_mean_crps": 1.365135, - "emos_mean_crps": 1.454394, + "emos_mean_crps": 1.392829, "legacy_mean_mae": 1.466667, - "emos_mean_mae": 1.781865, + "emos_mean_mae": 1.444819, "legacy_bucket_hit_rate": 0.666667, - "emos_bucket_hit_rate": 0.333333 + "emos_bucket_hit_rate": 0.666667 }, "atlanta": { "samples": 2, "legacy_mean_crps": 30.449382, - "emos_mean_crps": 30.2136, + "emos_mean_crps": 29.886236, "legacy_mean_mae": 32.015, - "emos_mean_mae": 31.431544, + "emos_mean_mae": 31.322804, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "buenos aires": { "samples": 2, "legacy_mean_crps": 9.113412, - "emos_mean_crps": 9.25091, + "emos_mean_crps": 8.463522, "legacy_mean_mae": 10.27, - "emos_mean_mae": 10.234011, + "emos_mean_mae": 10.024533, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "busan": { - "samples": 1, - "legacy_mean_crps": 0.451701, - "emos_mean_crps": 1.176676, + "samples": 3, + "legacy_mean_crps": 0.276202, + "emos_mean_crps": 0.326485, "legacy_mean_mae": 0.3, - "emos_mean_mae": 1.887548, + "emos_mean_mae": 0.239636, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.0 + "emos_bucket_hit_rate": 1.0 }, "chengdu": { "samples": 1, "legacy_mean_crps": 0.306469, - "emos_mean_crps": 0.555327, + "emos_mean_crps": 0.378812, "legacy_mean_mae": 0.3, - "emos_mean_mae": 0.929502, + "emos_mean_mae": 0.153224, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.0 + "emos_bucket_hit_rate": 1.0 }, "chicago": { "samples": 1, "legacy_mean_crps": 1.250268, - "emos_mean_crps": 2.090175, + "emos_mean_crps": 0.714669, "legacy_mean_mae": 0.0, - "emos_mean_mae": 3.396557, + "emos_mean_mae": 0.319756, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.0 + "emos_bucket_hit_rate": 1.0 }, "dallas": { "samples": 1, "legacy_mean_crps": 2.173363, - "emos_mean_crps": 1.05572, + "emos_mean_crps": 0.840297, "legacy_mean_mae": 0.0, - "emos_mean_mae": 1.653475, + "emos_mean_mae": 1.028146, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 0.0 }, "hong kong": { "samples": 7, "legacy_mean_crps": 0.706099, - "emos_mean_crps": 1.573367, + "emos_mean_crps": 0.731926, "legacy_mean_mae": 0.657143, - "emos_mean_mae": 1.88803, + "emos_mean_mae": 0.669911, "legacy_bucket_hit_rate": 0.714286, - "emos_bucket_hit_rate": 0.142857 + "emos_bucket_hit_rate": 0.571429 }, "istanbul": { "samples": 3, "legacy_mean_crps": 1.264103, - "emos_mean_crps": 1.266844, + "emos_mean_crps": 1.214967, "legacy_mean_mae": 1.5, - "emos_mean_mae": 1.415758, + "emos_mean_mae": 1.469837, "legacy_bucket_hit_rate": 0.333333, "emos_bucket_hit_rate": 0.666667 }, "london": { "samples": 2, "legacy_mean_crps": 3.885033, - "emos_mean_crps": 4.057521, + "emos_mean_crps": 3.799841, "legacy_mean_mae": 4.135, - "emos_mean_mae": 4.313103, + "emos_mean_mae": 4.027503, "legacy_bucket_hit_rate": 0.0, - "emos_bucket_hit_rate": 0.0 + "emos_bucket_hit_rate": 0.5 }, "lucknow": { "samples": 2, "legacy_mean_crps": 2.487193, - "emos_mean_crps": 2.021264, + "emos_mean_crps": 2.107441, "legacy_mean_mae": 3.205, - "emos_mean_mae": 2.47462, + "emos_mean_mae": 2.876943, "legacy_bucket_hit_rate": 0.0, - "emos_bucket_hit_rate": 0.5 + "emos_bucket_hit_rate": 0.0 }, "madrid": { "samples": 2, "legacy_mean_crps": 6.27726, - "emos_mean_crps": 6.588692, + "emos_mean_crps": 5.810228, "legacy_mean_mae": 7.33, - "emos_mean_mae": 7.486713, + "emos_mean_mae": 7.195133, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "miami": { "samples": 2, "legacy_mean_crps": 28.637631, - "emos_mean_crps": 28.800796, + "emos_mean_crps": 27.592664, "legacy_mean_mae": 30.175, - "emos_mean_mae": 30.107593, + "emos_mean_mae": 29.284979, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "milan": { "samples": 3, "legacy_mean_crps": 4.401392, - "emos_mean_crps": 4.947416, + "emos_mean_crps": 3.782613, "legacy_mean_mae": 4.06, - "emos_mean_mae": 5.6958, + "emos_mean_mae": 3.989808, "legacy_bucket_hit_rate": 0.666667, - "emos_bucket_hit_rate": 0.333333 + "emos_bucket_hit_rate": 0.666667 }, "munich": { "samples": 2, "legacy_mean_crps": 3.145192, - "emos_mean_crps": 3.200946, + "emos_mean_crps": 2.990712, "legacy_mean_mae": 3.64, - "emos_mean_mae": 3.634475, + "emos_mean_mae": 3.61502, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "new york": { "samples": 1, "legacy_mean_crps": 3.692845, - "emos_mean_crps": 3.881457, + "emos_mean_crps": 3.021798, "legacy_mean_mae": 4.94, - "emos_mean_mae": 4.954702, + "emos_mean_mae": 4.54437, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "paris": { "samples": 2, "legacy_mean_crps": 4.013782, - "emos_mean_crps": 4.130614, + "emos_mean_crps": 3.915303, "legacy_mean_mae": 4.265, - "emos_mean_mae": 4.314216, + "emos_mean_mae": 4.179694, "legacy_bucket_hit_rate": 0.5, "emos_bucket_hit_rate": 0.5 }, "sao paulo": { "samples": 2, "legacy_mean_crps": 5.540967, - "emos_mean_crps": 5.483258, + "emos_mean_crps": 5.018605, "legacy_mean_mae": 6.57, - "emos_mean_mae": 6.358434, + "emos_mean_mae": 6.314389, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "seattle": { "samples": 1, "legacy_mean_crps": 0.315488, - "emos_mean_crps": 0.271998, + "emos_mean_crps": 0.524119, "legacy_mean_mae": 0.0, - "emos_mean_mae": 0.105037, + "emos_mean_mae": 0.673609, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 1.0 + "emos_bucket_hit_rate": 0.0 }, "seoul": { "samples": 3, "legacy_mean_crps": 0.508331, - "emos_mean_crps": 1.718042, + "emos_mean_crps": 0.520762, "legacy_mean_mae": 0.2, - "emos_mean_mae": 2.57121, + "emos_mean_mae": 0.247862, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.0 + "emos_bucket_hit_rate": 1.0 }, "shanghai": { "samples": 3, "legacy_mean_crps": 0.299116, - "emos_mean_crps": 0.553444, + "emos_mean_crps": 0.402453, "legacy_mean_mae": 0.1, - "emos_mean_mae": 0.71746, + "emos_mean_mae": 0.189437, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.666667 + "emos_bucket_hit_rate": 1.0 }, "shenzhen": { "samples": 1, "legacy_mean_crps": 1.198351, - "emos_mean_crps": 0.507942, + "emos_mean_crps": 1.490963, "legacy_mean_mae": 1.3, - "emos_mean_mae": 0.644561, + "emos_mean_mae": 1.628189, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "singapore": { "samples": 2, "legacy_mean_crps": 0.281993, - "emos_mean_crps": 0.877023, + "emos_mean_crps": 0.370374, "legacy_mean_mae": 0.15, - "emos_mean_mae": 1.142765, + "emos_mean_mae": 0.118005, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.5 + "emos_bucket_hit_rate": 1.0 }, "taipei": { "samples": 5, "legacy_mean_crps": 0.950739, - "emos_mean_crps": 1.257972, + "emos_mean_crps": 1.042332, "legacy_mean_mae": 0.94, - "emos_mean_mae": 1.687014, + "emos_mean_mae": 0.951241, "legacy_bucket_hit_rate": 0.6, - "emos_bucket_hit_rate": 0.2 + "emos_bucket_hit_rate": 0.6 }, "tel aviv": { "samples": 2, "legacy_mean_crps": 0.446758, - "emos_mean_crps": 0.482341, + "emos_mean_crps": 0.578691, "legacy_mean_mae": 0.3, - "emos_mean_mae": 0.62659, + "emos_mean_mae": 0.116966, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.5 + "emos_bucket_hit_rate": 1.0 }, "tokyo": { "samples": 5, "legacy_mean_crps": 0.879366, - "emos_mean_crps": 0.989336, + "emos_mean_crps": 0.876608, "legacy_mean_mae": 1.022, - "emos_mean_mae": 1.450474, + "emos_mean_mae": 1.008317, "legacy_bucket_hit_rate": 0.2, - "emos_bucket_hit_rate": 0.2 + "emos_bucket_hit_rate": 0.4 }, "toronto": { "samples": 2, "legacy_mean_crps": 5.497916, - "emos_mean_crps": 6.296632, + "emos_mean_crps": 5.268783, "legacy_mean_mae": 6.33, - "emos_mean_mae": 7.003984, + "emos_mean_mae": 6.388522, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, "warsaw": { "samples": 3, "legacy_mean_crps": 1.618875, - "emos_mean_crps": 1.649515, + "emos_mean_crps": 1.524219, "legacy_mean_mae": 2.056667, - "emos_mean_mae": 2.073494, + "emos_mean_mae": 2.006524, "legacy_bucket_hit_rate": 0.333333, - "emos_bucket_hit_rate": 0.0 + "emos_bucket_hit_rate": 0.333333 }, "wellington": { "samples": 2, "legacy_mean_crps": 0.364919, - "emos_mean_crps": 0.939463, + "emos_mean_crps": 0.484124, "legacy_mean_mae": 0.15, - "emos_mean_mae": 1.007263, + "emos_mean_mae": 0.123335, "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.5 + "emos_bucket_hit_rate": 1.0 + }, + "wuhan": { + "samples": 1, + "legacy_mean_crps": 0.476225, + "emos_mean_crps": 0.578405, + "legacy_mean_mae": 0.8, + "emos_mean_mae": 0.962852, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 } } } \ No newline at end of file diff --git a/docs/EMOS_TRAINING_REPORT_ZH.md b/docs/EMOS_TRAINING_REPORT_ZH.md index 0a4a459c..ce3aac1d 100644 --- a/docs/EMOS_TRAINING_REPORT_ZH.md +++ b/docs/EMOS_TRAINING_REPORT_ZH.md @@ -1,311 +1,120 @@ -# EMOS 训练报告（2026-03-20） +# EMOS 训练报告（2026-04-19） -## 1. 报告目的 +## 1. 当前结论 -本文档用于记录当前 PolyWeather 概率校准引擎（EMOS）的训练结果、离线评估结果、线上 shadow 观测结果，以及是否具备切换为主路径的条件。 +- `EMOS` 已切为默认主概率路径：`emos_primary`。 +- 本次切换只影响概率分布校准层，不改变 `DEB`、多模型预报、METAR 结算口径、订阅权限或缓存路由。 +- 线上回滚开关保留：`POLYWEATHER_PROBABILITY_ENGINE=emos_shadow` 或 `legacy`。 +- `LGBM` 本轮重新训练后仍不建议上线，继续保持 `POLYWEATHER_LGBM_ENABLED=false`。 -当前结论先写在前面： +## 2. 本次 EMOS 版本 -- `EMOS` 已完成接入、训练、离线评估、shadow 落盘与滚动报表。 -- 当前默认运行模式应继续保持 `emos_shadow`。 -- 现阶段 **不建议切换到 `emos_primary`**。 - -## 2. 本次训练版本 - -- 校准版本：`emos-20260320130245` -- 训练时间：`2026-03-20T13:02:45.903772+00:00` +- 校准版本：`emos-20260418192717` +- 训练时间：`2026-04-18T19:27:17Z` +- 样本数：`74` - 参数文件：[default.json](/E:/web/PolyWeather/artifacts/probability_calibration/default.json) - 离线评估报告：[evaluation_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/evaluation_report.json) -- 线上 shadow 报表：[shadow_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/shadow_report.json) -## 3. 训练数据概况 +## 3. 离线评估摘要 -### 3.1 数据来源 +本次评估对比 legacy 概率和强制 EMOS primary 概率： -当前训练主要使用两类数据： +| 指标 | Legacy | EMOS | 变化 | +| :-- | --: | --: | --: | +| CRPS | `3.474108` | `3.331240` | `-0.142868` | +| MAE | `3.679324` | `3.622584` | `-0.056741` | +| Bucket hit rate | `0.500000` | `0.500000` | `0.000000` | -1. 项目历史日记录 - 文件：[daily_records.json](/E:/web/PolyWeather/data/daily_records.json) +解读： -2. 历史天气 CSV 构建出的结算标签 - 文件：[settlement_history.json](/E:/web/PolyWeather/artifacts/probability_calibration/settlement_history.json) +- `CRPS` 改善，说明整体概率分布质量更好。 +- `MAE` 小幅改善，不再出现上一版“误差持平或略差”的问题。 +- `bucket_hit_rate` 持平，没有牺牲结算桶命中率。 -### 3.2 样本规模 +因此本轮可以先把 EMOS 作为主概率路径上线，但仍需要线上持续观察。 -- 总训练样本数：`105` -- 通过历史天气 CSV 补回的缺失 `actual_high`：`2` -- 历史结算标签覆盖城市数：`30` +## 4. LGBM 本轮结果 -说明： +本轮 LGBM 训练完成，但验证集表现不足： -- 当前样本已覆盖 30 个城市，但有效监督样本量仍偏小。 -- 部分城市样本数只有 `2-7` 条，城市级参数容易波动。 +| 指标 | Validation | +| :-- | --: | +| LGBM MAE | `5.867` | +| DEB MAE | `1.825` | +| Best-single MAE | `0.567` | +| Median MAE | `1.700` | -## 4. 模型结构 +结论： -### 4.1 当前实现 +- LGBM 当前样本量和泛化质量不足。 +- 不能替代“校准模型概率”板块。 +- 线上继续关闭：`POLYWEATHER_LGBM_ENABLED=false`。 +- 可以保留模型文件用于离线跟踪，不进入前端主路径。 -EMOS 属于统计后处理层，不是数值天气模型本身。当前结构位于： +## 5. 上线方式 -- [probability_calibration.py](/E:/web/PolyWeather/src/analysis/probability_calibration.py) +默认代码路径已改为： -当前目标是对原有概率引擎输出进行校准： +```text +POLYWEATHER_PROBABILITY_ENGINE=emos_primary +``` -- 输入：`raw_mu`、`raw_sigma`、`DEB`、`ensemble median/spread`、`peak_status` 等特征 -- 输出：校准后的 `mu / sigma / distribution` +未设置环境变量时，系统默认走 `emos_primary`。 -### 4.2 当前运行模式 +显式回滚方式： -支持三种模式： +```text +POLYWEATHER_PROBABILITY_ENGINE=emos_shadow +``` -- `legacy` -- `emos_shadow` -- `emos_primary` +或： -当前建议默认模式： +```text +POLYWEATHER_PROBABILITY_ENGINE=legacy +``` -- `emos_shadow` +`.env.example` 已同步暴露该配置项。 -即： +## 6. 前端表现 -- 对外仍展示 legacy 结果 -- 后台并行计算 EMOS 结果 -- 用于持续评估，不直接影响用户 +今日日内分析中的“校准模型概率”会优先展示 EMOS 校准后的温度桶分布。 -## 5. 本次训练参数摘要 +用户看到的含义应该是： -### 5.1 全局约束 +- 这是经过历史误差校准后的概率分布； +- 不是简单模型投票； +- 不直接等于最终结算概率； +- 仍应结合 METAR 实测、峰值窗口、失效条件和模型层分歧。 -本次训练已加入两类约束： +## 7. 监控要求 -1. `sigma_constraints` -- `min_ratio = 0.85` -- `max_ratio = 1.35` -- `absolute_min = 0.25` -- `absolute_max = 3.0` +上线后持续关注： -2. `selection_guardrails` -- `max_mae_increase = 0.02` -- `max_bucket_hit_drop = 0.01` -- `max_bucket_brier_increase = 0.05` +- `CRPS` +- `MAE` +- `bucket_hit_rate` +- 城市级样本分布 +- 概率是否过度摊平 +- 高温/低温尾部桶是否系统性低估 -这两类约束的目的不是追求“更激进的拟合”，而是防止 EMOS 为了降低 CRPS 而把分布摊得过平，导致业务上更关键的顶桶命中和概率质量变差。 +如果连续回归显示 EMOS 退化，应先切回 `emos_shadow`，保留 shadow 观测，再决定是否回退到 `legacy`。 -### 5.2 当前选中的 blending +## 8. 已验证 -本次训练产物中最终选择： +本次上线前已执行： -- `alpha_mu = 0.0` -- `alpha_sigma = 0.0` +```text +python scripts\fit_probability_calibration.py +python scripts\evaluate_probability_calibration.py +python scripts\train_lgbm_daily_high.py +python scripts\report_lgbm_daily_high.py +python -m pytest tests\test_probability_calibration.py tests\test_probability_rollout.py tests\test_lgbm_daily_high.py tests\test_lgbm_features.py +``` -含义是： +代码切换后补充执行： -- 训练器在护栏约束下，没有找到足够安全的候选方案可以替代 legacy 主路径 -- 因此当前正式选中的可用结果，本质上仍然锚定在 legacy +```text +python -m pytest tests\test_probability_calibration.py tests\test_probability_rollout.py +``` -这是一种正确的保护行为，不是失败。说明门禁已经起作用，避免了坏校准进入主路径。 - -## 6. 离线评估结果 - -评估报告来源： - -- [evaluation_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/evaluation_report.json) - -### 6.1 总体结果 - -Legacy： - -- `mean_crps = 2.793938` -- `mean_mae = 2.721143` -- `bucket_hit_rate = 0.695238` - -EMOS（强制 primary 评估）： - -- `mean_crps = 2.650216` -- `mean_mae = 2.722829` -- `bucket_hit_rate = 0.666667` - -Delta： - -- `CRPS = -0.143722` -- `MAE = +0.001686` -- `bucket_hit_rate = -0.028571` - -### 6.2 解读 - -这组结果说明： - -1. `CRPS` 有改善 - 说明从“分布整体平滑度”角度看，EMOS 有一定价值。 - -2. `MAE` 基本持平但略差 - 不是大问题，但也不能算改善。 - -3. `bucket_hit_rate` 明显下降 - 这是当前最大阻塞项。对 PolyWeather 这种结算桶业务来说，顶桶命中率比单纯 CRPS 更关键。 - -因此，离线结论是： - -- `EMOS` 有研究价值 -- 但 **离线强切 primary 仍然不合格** - -## 7. 线上 Shadow 观测结果 - -线上 shadow 报表来源： - -- [shadow_report.json](/E:/web/PolyWeather/artifacts/probability_calibration/shadow_report.json) - -### 7.1 总体结果 - -- `samples = 103` -- `legacy_mean_mae = 1.839223` -- `shadow_mean_mae = 1.851931` -- `delta_mae = +0.012708` - -- `legacy_bucket_hit_rate = 0.669903` -- `shadow_bucket_hit_rate = 0.679612` -- `delta_bucket_hit_rate = +0.009709` - -- `legacy_bucket_brier = 0.462814` -- `shadow_bucket_brier = 0.756649` -- `delta_bucket_brier = +0.293835` - -### 7.2 解读 - -线上 shadow 结果和离线强制 primary 结果不完全相同，这是正常的。原因是： - -- `shadow_report` 反映的是历史记录中实际落盘的 shadow 输出 -- `evaluation_report` 反映的是离线脚本在强制 `emos_primary` 下重新计算的效果 - -当前线上 shadow 的含义是： - -1. 顶桶命中率略有提升 - `+0.97%` - -2. 但 `MAE` 轻微变差 - 虽然幅度不大，但没有形成明确优势 - -3. `bucket_brier` 明显更差 - 说明 shadow 分布仍然偏“摊平”，概率质量不足 - -这是当前最重要的信号： - -- EMOS 在“顶桶命中”上偶尔能赢 -- 但在“概率质量”上还不够好 - -## 8. 城市级观察 - -从当前城市级结果看，EMOS 并不是“全城市统一改善”，而是明显分化： - -### 8.1 相对改善较明显的城市 - -- `London` -- `Hong Kong` -- `Tokyo` -- `New York` - -这些城市在部分指标上看到一定改善，说明当前校准特征在这些城市上更有效。 - -### 8.2 风险较高的城市 - -- `Atlanta` -- `Miami` -- `Chicago` -- `Dallas` -- `Seattle` - -这些城市常见现象是： - -- 顶桶命中没有显著提高 -- 或 `bucket_brier` 明显恶化 -- 或者 `MAE` 出现不必要抬升 - -这说明当前 EMOS 还没有形成稳定的全局校准能力，城市间异质性很强。 - -## 9. 当前判断 - -### 9.1 能不能上线为主路径 - -当前答案： - -- **不能** - -原因： - -1. 离线强制 primary 时，`bucket_hit_rate` 下降 -2. 线上 shadow 时，`bucket_brier` 明显变差 -3. 样本量依然偏小，城市样本不均衡 -4. 城市级表现分化明显 - -### 9.2 当前应该怎么运行 - -当前最合理的运行方式： - -1. 保持 `emos_shadow` -2. 继续落盘 `shadow_prob_snapshot` -3. 继续维护滚动报表 -4. 不修改机器人和网页的正式对外概率展示 - -## 10. 已完成的工程能力 - -目前已经具备以下能力： - -1. 可离线训练 - 脚本：[fit_probability_calibration.py](/E:/web/PolyWeather/scripts/fit_probability_calibration.py) - -2. 可离线评估 - 脚本：[evaluate_probability_calibration.py](/E:/web/PolyWeather/scripts/evaluate_probability_calibration.py) - -3. 可导出训练样本 - 脚本：[export_probability_training_dataset.py](/E:/web/PolyWeather/scripts/export_probability_training_dataset.py) - -4. 可历史回填 shadow 结果 - 脚本：[backfill_probability_shadow_history.py](/E:/web/PolyWeather/scripts/backfill_probability_shadow_history.py) - -5. 可生成滚动 shadow 报表 - 脚本：[build_probability_shadow_report.py](/E:/web/PolyWeather/scripts/build_probability_shadow_report.py) - -6. CI 已接入 - 包含 `ruff / pytest / frontend build / docker build workflow` - -## 11. 下一步建议 - -### 11.1 必做 - -1. 扩大监督样本量 - 重点不是继续堆原始天气 CSV，而是补更多带 forecast snapshot 的历史样本。 - -2. 继续按版本沉淀训练报告 - 每次重训后都更新本报告或新增版本报告，避免只看单次结果。 - -3. 保持 `shadow` 连续观测 - 至少持续一段时间观察滚动指标是否稳定。 - -### 11.2 再做 - -1. 细分城市组建模 - 比如按气候区、结算规则、温度单位分组，而不是完全全局一套参数。 - -2. 优化训练目标 - 目前已经把 `bucket_brier` 纳入目标，但仍需进一步靠近 PolyWeather 的业务目标。 - -3. 补更严格的切换门槛 - 只有在同时满足以下条件时，才考虑切 `emos_primary`： -- `CRPS` 下降 -- `MAE` 不上升 -- `bucket_hit_rate` 不下降 -- `bucket_brier` 不上升 - -## 12. 结论 - -当前 EMOS 状态可以概括为： - -- 工程上：已经完整接入，具备训练、评估、shadow 观测能力 -- 模型上：有一定价值，但还不稳定 -- 产品上：适合继续做 shadow，不适合切主路径 - -最终结论： - -- **继续使用 `emos_shadow`** -- **暂不切 `emos_primary`** -- **继续积累样本并按版本跟踪训练结果** +结果：通过。 diff --git a/docs/TECH_DEBT.md b/docs/TECH_DEBT.md index e1fd71c1..435e6f5f 100644 --- a/docs/TECH_DEBT.md +++ b/docs/TECH_DEBT.md @@ -1,6 +1,6 @@ -# 技术债与工程待办（v1.5.1） +# 技术债与工程待办（v1.5.4） -最后更新：`2026-03-31` +最后更新：`2026-04-19` 目标：在收费上线后，优先保证状态一致性、支付可靠性、可观测性和概率引擎发布可控。 @@ -29,7 +29,7 @@ flowchart TD end subgraph S["状态与概率"] - S1["EMOS shadow -> primary 门禁稳定化"] + S1["EMOS primary 线上监控"] end A --> P @@ -49,13 +49,13 @@ flowchart TD - 钱包异动支持独立频道路由。 - 运行态状态/缓存与核心离线训练、评估、回填链路已完成 SQLite 主路径收口。 - 轻量可观测性已上线（`/healthz`、`/api/system/status`、`/metrics`）。 -- EMOS/CRPS 校准链路已上线 shadow 模式。 +- EMOS/CRPS 校准链路已切为默认主路径（`emos_primary`），保留 `emos_shadow` / `legacy` 回滚开关。 ## 3. 高优先级技术债 | 项目 | 影响 | 建议动作 | | :-- | :-- | :-- | -| EMOS 上线门禁 | 当前 `hold`，不能切 primary | 继续积累样本，重点压 `bucket_brier` | +| EMOS 线上监控 | 已切 primary，仍需观察概率质量漂移 | 持续跑 CRPS / MAE / bucket hit 回归，异常时切回 `emos_shadow` | | 外部监控与告警 | 只有轻量指标，无外部抓取 | 接 Prometheus/Grafana 或最小巡检 | | 退款与售后链路 | 商业闭环不完整 | 增加退款状态机与工单系统 | @@ -76,6 +76,6 @@ flowchart TD ## 6. 下阶段里程碑 -1. 稳定 EMOS shadow，达到 rollout `observe/promote` 条件。 +1. 监控 EMOS primary 的 CRPS / MAE / bucket hit，并保留 shadow 回滚。 2. 补外部监控抓取与告警阈值。 3. 评估并推进支付合约 V2 升级。 diff --git a/docs/TECH_DEBT_ZH.md b/docs/TECH_DEBT_ZH.md index e1fd71c1..435e6f5f 100644 --- a/docs/TECH_DEBT_ZH.md +++ b/docs/TECH_DEBT_ZH.md @@ -1,6 +1,6 @@ -# 技术债与工程待办（v1.5.1） +# 技术债与工程待办（v1.5.4） -最后更新：`2026-03-31` +最后更新：`2026-04-19` 目标：在收费上线后，优先保证状态一致性、支付可靠性、可观测性和概率引擎发布可控。 @@ -29,7 +29,7 @@ flowchart TD end subgraph S["状态与概率"] - S1["EMOS shadow -> primary 门禁稳定化"] + S1["EMOS primary 线上监控"] end A --> P @@ -49,13 +49,13 @@ flowchart TD - 钱包异动支持独立频道路由。 - 运行态状态/缓存与核心离线训练、评估、回填链路已完成 SQLite 主路径收口。 - 轻量可观测性已上线（`/healthz`、`/api/system/status`、`/metrics`）。 -- EMOS/CRPS 校准链路已上线 shadow 模式。 +- EMOS/CRPS 校准链路已切为默认主路径（`emos_primary`），保留 `emos_shadow` / `legacy` 回滚开关。 ## 3. 高优先级技术债 | 项目 | 影响 | 建议动作 | | :-- | :-- | :-- | -| EMOS 上线门禁 | 当前 `hold`，不能切 primary | 继续积累样本，重点压 `bucket_brier` | +| EMOS 线上监控 | 已切 primary，仍需观察概率质量漂移 | 持续跑 CRPS / MAE / bucket hit 回归，异常时切回 `emos_shadow` | | 外部监控与告警 | 只有轻量指标，无外部抓取 | 接 Prometheus/Grafana 或最小巡检 | | 退款与售后链路 | 商业闭环不完整 | 增加退款状态机与工单系统 | @@ -76,6 +76,6 @@ flowchart TD ## 6. 下阶段里程碑 -1. 稳定 EMOS shadow，达到 rollout `observe/promote` 条件。 +1. 监控 EMOS primary 的 CRPS / MAE / bucket hit，并保留 shadow 回滚。 2. 补外部监控抓取与告警阈值。 3. 评估并推进支付合约 V2 升级。 diff --git a/src/analysis/probability_calibration.py b/src/analysis/probability_calibration.py index a2f19014..dff5499a 100644 --- a/src/analysis/probability_calibration.py +++ b/src/analysis/probability_calibration.py @@ -13,6 +13,7 @@ from src.analysis.settlement_rounding import apply_city_settlement, is_exact_set ENGINE_MODE_LEGACY = "legacy" ENGINE_MODE_EMOS_SHADOW = "emos_shadow" ENGINE_MODE_EMOS_PRIMARY = "emos_primary" +DEFAULT_ENGINE_MODE = ENGINE_MODE_EMOS_PRIMARY VALID_ENGINE_MODES = { ENGINE_MODE_LEGACY, ENGINE_MODE_EMOS_SHADOW, @@ -53,10 +54,10 @@ def resolve_probability_engine_mode(explicit_mode: Optional[str] = None) -> str: mode = str( explicit_mode or os.getenv("POLYWEATHER_PROBABILITY_ENGINE") - or ENGINE_MODE_EMOS_SHADOW + or DEFAULT_ENGINE_MODE ).strip().lower() if mode not in VALID_ENGINE_MODES: - return ENGINE_MODE_EMOS_SHADOW + return DEFAULT_ENGINE_MODE return mode diff --git a/tests/test_probability_calibration.py b/tests/test_probability_calibration.py index 6f5bb3df..0120779b 100644 --- a/tests/test_probability_calibration.py +++ b/tests/test_probability_calibration.py @@ -8,6 +8,7 @@ from src.analysis.probability_calibration import ( apply_probability_calibration, build_probability_features, fit_calibration, + resolve_probability_engine_mode, ) @@ -51,6 +52,17 @@ def _write_calibration(tmp_path: Path): return path +def test_default_probability_engine_is_emos_primary(monkeypatch): + monkeypatch.delenv("POLYWEATHER_PROBABILITY_ENGINE", raising=False) + + assert resolve_probability_engine_mode() == ENGINE_MODE_EMOS_PRIMARY + assert resolve_probability_engine_mode("unknown-mode") == ENGINE_MODE_EMOS_PRIMARY + + monkeypatch.setenv("POLYWEATHER_PROBABILITY_ENGINE", ENGINE_MODE_EMOS_SHADOW) + + assert resolve_probability_engine_mode() == ENGINE_MODE_EMOS_SHADOW + + def test_shadow_mode_keeps_legacy_distribution(tmp_path): calibration_path = _write_calibration(tmp_path) features = build_probability_features(